diff --git a/contraceptive/lct_gan/eval.csv b/contraceptive/lct_gan/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..43de02b3ab2713df089561a03f4f71e77135db47 --- /dev/null +++ b/contraceptive/lct_gan/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +lct_gan,0.007988620376563631,0.006744927319876765,0.001855382380545052,1.6703412532806396,0.010601414367556572,0.34547173976898193,0.01630617491900921,2.5985364118241705e-06,2.7118337154388428,0.03212312236428261,0.09255968779325485,0.04307414963841438,0.08756907284259796,0.016603846102952957,4.382174968719482 diff --git a/contraceptive/lct_gan/history.csv b/contraceptive/lct_gan/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..bbc9b7eec8e0492357f8bb5382041a573f071fdb --- /dev/null +++ b/contraceptive/lct_gan/history.csv @@ -0,0 +1,10 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.019665115947168993,1.3259430897919648,0.0016351528701719095,0.12722583417470257,0.0,0.0,0.0,0.0,0.0199518630333801,900,225,270.090327501297,1.20040145556132,0.30010036389033,0.11211167593641827,0.005123136696509189,1.9587845133610828,3.225599730537245e-05,0.0,0.0,0.0,0.0,0.0,0.005123136696509189,450,113,87.16603398323059,0.7713808317100053,0.19370229774051242,0.048274144109964896 +1,0.004459613269524804,0.5696247978427467,4.493151174229456e-05,0.02243374695568087,0.0,0.0,0.0,0.0,0.004538138056490829,900,225,271.9333186149597,1.2085925271775988,0.3021481317943997,0.10111145882142915,0.003131905284826644,1.191799519885096,1.3011600207322418e-05,0.0,0.0,0.0,0.0,0.0,0.003131905284826644,450,113,92.20275020599365,0.8159535416459616,0.20489500045776368,0.05275093076289096 +2,0.0035123983815234777,0.4977147144650333,2.859574940254904e-05,0.018076600913837965,0.0,0.0,0.0,0.0,0.003567821487797321,900,225,275.1772153377533,1.2230098459455703,0.30575246148639257,0.1008471870360275,0.0031233079556032962,1.3221473698086115,4.6298641943619236e-05,0.0,0.0,0.0,0.0,0.0,0.0031233079556032962,450,113,88.90858387947083,0.7868016272519542,0.1975746308432685,0.05913279911882318 +3,0.003161510648399902,0.5058521567848904,2.4743936033731546e-05,0.014727871190666014,0.0,0.0,0.0,0.0,0.00322171315592843,900,225,272.91378474235535,1.2129501544104682,0.30323753860261704,0.09905971649413307,0.004386183474238755,2.076401953705954,2.7025729084248062e-05,0.0,0.0,0.0,0.0,0.0,0.004386183474238755,450,113,88.29646849632263,0.7813846769586074,0.19621437443627252,0.03421469124248334 +4,0.0026987572804662505,0.35000921227432324,1.5105901600198398e-05,0.015139500791944253,0.0,0.0,0.0,0.0,0.0027399578163426162,900,225,272.80935049057007,1.2124860021803114,0.30312150054507786,0.09980787922110823,0.002216681500544332,1.5294503328664861,5.906305432745928e-06,0.0,0.0,0.0,0.0,0.0,0.002216681500544332,450,113,88.42259788513184,0.7825008662401047,0.19649466196695964,0.05325051893447155 +5,0.002033243282720731,0.23602186178530385,7.030283134612784e-06,0.015872358748973865,0.0,0.0,0.0,0.0,0.0020601135703246756,900,225,275.2858557701111,1.2234926923116047,0.3058731730779012,0.10312151636100478,0.002322478660401733,1.320404739686366,9.749587738599938e-06,0.0,0.0,0.0,0.0,0.0,0.002322478660401733,450,113,90.18929839134216,0.7981353839941784,0.20042066309187148,0.0515262810760631 +6,0.001614498706185259,0.1809647554011328,2.9324602819437286e-06,0.01615508396980456,0.0,0.0,0.0,0.0,0.0016328314457981225,900,225,275.734769821167,1.2254878658718533,0.30637196646796333,0.10563728055192365,0.0019716585887686556,1.9351633055870725,4.450378878835354e-06,0.0,0.0,0.0,0.0,0.0,0.0019716585887686556,450,113,88.78998851776123,0.7857521107766481,0.19731108559502494,0.04906956392414538 +7,0.0014726841067183867,0.21683113766619005,2.8254440730661005e-06,0.016018104005423892,0.0,0.0,0.0,0.0,0.0014894471139349562,900,225,272.83155059814453,1.2125846693250868,0.3031461673312717,0.10693864568240112,0.0019270910127347127,1.2538913027706378,3.4612182754591007e-06,0.0,0.0,0.0,0.0,0.0,0.0019270910127347127,450,113,89.2393867969513,0.789729086698684,0.19830974843766955,0.050700269499381556 +8,0.001313657746925148,0.1492658925630795,3.4148349597811518e-06,0.016415953670317927,0.0,0.0,0.0,0.0,0.0013283096294455592,900,225,286.74962306022644,1.2744427691565618,0.31861069228914046,0.1061259970565637,0.0018737697858078818,1.1538138667777444,4.4807563205433855e-06,0.0,0.0,0.0,0.0,0.0,0.0018737697858078818,450,113,99.40478444099426,0.879688357884905,0.22089952097998725,0.05413557932530818 diff --git a/contraceptive/lct_gan/mlu-eval-load.ipynb b/contraceptive/lct_gan/mlu-eval-load.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..71c01e1cb7c49db04fb9590ad032708274f4e53e --- /dev/null +++ b/contraceptive/lct_gan/mlu-eval-load.ipynb @@ -0,0 +1,1891 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:12.417443Z", + "iopub.status.busy": "2024-03-26T07:02:12.416703Z", + "iopub.status.idle": "2024-03-26T07:02:12.449971Z", + "shell.execute_reply": "2024-03-26T07:02:12.449270Z" + }, + "papermill": { + "duration": 0.047215, + "end_time": "2024-03-26T07:02:12.451992", + "exception": false, + "start_time": "2024-03-26T07:02:12.404777", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:12.474839Z", + "iopub.status.busy": "2024-03-26T07:02:12.474507Z", + "iopub.status.idle": "2024-03-26T07:02:12.481011Z", + "shell.execute_reply": "2024-03-26T07:02:12.480100Z" + }, + "papermill": { + "duration": 0.020096, + "end_time": "2024-03-26T07:02:12.483029", + "exception": false, + "start_time": "2024-03-26T07:02:12.462933", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:12.504021Z", + "iopub.status.busy": "2024-03-26T07:02:12.503779Z", + "iopub.status.idle": "2024-03-26T07:02:12.507801Z", + "shell.execute_reply": "2024-03-26T07:02:12.506969Z" + }, + "papermill": { + "duration": 0.016766, + "end_time": "2024-03-26T07:02:12.509678", + "exception": false, + "start_time": "2024-03-26T07:02:12.492912", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:12.532080Z", + "iopub.status.busy": "2024-03-26T07:02:12.531842Z", + "iopub.status.idle": "2024-03-26T07:02:12.535367Z", + "shell.execute_reply": "2024-03-26T07:02:12.534699Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.016239, + "end_time": "2024-03-26T07:02:12.537370", + "exception": false, + "start_time": "2024-03-26T07:02:12.521131", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:12.558226Z", + "iopub.status.busy": "2024-03-26T07:02:12.557923Z", + "iopub.status.idle": "2024-03-26T07:02:12.564108Z", + "shell.execute_reply": "2024-03-26T07:02:12.563406Z" + }, + "papermill": { + "duration": 0.018993, + "end_time": "2024-03-26T07:02:12.566082", + "exception": false, + "start_time": "2024-03-26T07:02:12.547089", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "b858079c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:12.588874Z", + "iopub.status.busy": "2024-03-26T07:02:12.588317Z", + "iopub.status.idle": "2024-03-26T07:02:12.593261Z", + "shell.execute_reply": "2024-03-26T07:02:12.592443Z" + }, + "papermill": { + "duration": 0.018287, + "end_time": "2024-03-26T07:02:12.595095", + "exception": false, + "start_time": "2024-03-26T07:02:12.576808", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"lct_gan\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 42\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/lct_gan/42\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.009863, + "end_time": "2024-03-26T07:02:12.614868", + "exception": false, + "start_time": "2024-03-26T07:02:12.605005", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:12.636298Z", + "iopub.status.busy": "2024-03-26T07:02:12.635672Z", + "iopub.status.idle": "2024-03-26T07:02:12.644408Z", + "shell.execute_reply": "2024-03-26T07:02:12.643640Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.021529, + "end_time": "2024-03-26T07:02:12.646287", + "exception": false, + "start_time": "2024-03-26T07:02:12.624758", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/lct_gan/42\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:12.668084Z", + "iopub.status.busy": "2024-03-26T07:02:12.667457Z", + "iopub.status.idle": "2024-03-26T07:02:14.615406Z", + "shell.execute_reply": "2024-03-26T07:02:14.614489Z" + }, + "papermill": { + "duration": 1.961215, + "end_time": "2024-03-26T07:02:14.617625", + "exception": false, + "start_time": "2024-03-26T07:02:12.656410", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:14.641191Z", + "iopub.status.busy": "2024-03-26T07:02:14.640268Z", + "iopub.status.idle": "2024-03-26T07:02:14.653315Z", + "shell.execute_reply": "2024-03-26T07:02:14.652601Z" + }, + "papermill": { + "duration": 0.026634, + "end_time": "2024-03-26T07:02:14.655264", + "exception": false, + "start_time": "2024-03-26T07:02:14.628630", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:14.676819Z", + "iopub.status.busy": "2024-03-26T07:02:14.676539Z", + "iopub.status.idle": "2024-03-26T07:02:14.683573Z", + "shell.execute_reply": "2024-03-26T07:02:14.682770Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.020205, + "end_time": "2024-03-26T07:02:14.685389", + "exception": false, + "start_time": "2024-03-26T07:02:14.665184", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:14.706766Z", + "iopub.status.busy": "2024-03-26T07:02:14.706499Z", + "iopub.status.idle": "2024-03-26T07:02:14.801732Z", + "shell.execute_reply": "2024-03-26T07:02:14.800750Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.108319, + "end_time": "2024-03-26T07:02:14.803852", + "exception": false, + "start_time": "2024-03-26T07:02:14.695533", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:14.827250Z", + "iopub.status.busy": "2024-03-26T07:02:14.826979Z", + "iopub.status.idle": "2024-03-26T07:02:19.478683Z", + "shell.execute_reply": "2024-03-26T07:02:19.477856Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.666057, + "end_time": "2024-03-26T07:02:19.481151", + "exception": false, + "start_time": "2024-03-26T07:02:14.815094", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-26 07:02:17.052860: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-26 07:02:17.052915: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-26 07:02:17.054585: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:19.503687Z", + "iopub.status.busy": "2024-03-26T07:02:19.503148Z", + "iopub.status.idle": "2024-03-26T07:02:19.508962Z", + "shell.execute_reply": "2024-03-26T07:02:19.508236Z" + }, + "papermill": { + "duration": 0.018995, + "end_time": "2024-03-26T07:02:19.510852", + "exception": false, + "start_time": "2024-03-26T07:02:19.491857", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:19.534311Z", + "iopub.status.busy": "2024-03-26T07:02:19.534035Z", + "iopub.status.idle": "2024-03-26T07:02:27.867879Z", + "shell.execute_reply": "2024-03-26T07:02:27.866611Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.349106, + "end_time": "2024-03-26T07:02:27.871099", + "exception": false, + "start_time": "2024-03-26T07:02:19.521993", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-26T07:02:27.899172Z", + "iopub.status.busy": "2024-03-26T07:02:27.898744Z", + "iopub.status.idle": "2024-03-26T07:02:27.906135Z", + "shell.execute_reply": "2024-03-26T07:02:27.905319Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.022906, + "end_time": "2024-03-26T07:02:27.907989", + "exception": false, + "start_time": "2024-03-26T07:02:27.885083", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 46,\n", + " 'realtabformer': (24, 72, Embedding(72, 672), True),\n", + " 'lct_gan': 40,\n", + " 'tab_ddpm_concat': 10}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:27.930393Z", + "iopub.status.busy": "2024-03-26T07:02:27.929860Z", + "iopub.status.idle": "2024-03-26T07:02:27.996610Z", + "shell.execute_reply": "2024-03-26T07:02:27.995783Z" + }, + "papermill": { + "duration": 0.079913, + "end_time": "2024-03-26T07:02:27.998534", + "exception": false, + "start_time": "2024-03-26T07:02:27.918621", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/aug_test/contraceptive 0\n", + "Caching in ../../../../contraceptive/_cache_bs_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/bs_test/contraceptive 0\n", + "Caching in ../../../../contraceptive/_cache_synth_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/synthetics/contraceptive 600\n", + "600\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:28.023439Z", + "iopub.status.busy": "2024-03-26T07:02:28.022748Z", + "iopub.status.idle": "2024-03-26T07:02:28.349942Z", + "shell.execute_reply": "2024-03-26T07:02:28.348993Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.342343, + "end_time": "2024-03-26T07:02:28.352048", + "exception": false, + "start_time": "2024-03-26T07:02:28.009705", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'loss_balancer_beta': 0.7999999999999999,\n", + " 'loss_balancer_r': 0.95,\n", + " 'tf_pma_low': 8,\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7,\n", + " 'bias_weight_decay': 0.05,\n", + " 'head_activation': torch.nn.modules.activation.ReLU6,\n", + " 'tf_activation': torch.nn.modules.activation.Tanh,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.07,\n", + " 'n_warmup_steps': 200,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'fixed_role_model': 'lct_gan',\n", + " 'g_loss_mul': 0.1,\n", + " 'd_model': 256,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 512,\n", + " 'ada_n_layers': 9,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 512,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'single_model': True,\n", + " 'models': ['lct_gan'],\n", + " 'max_seconds': 3600,\n", + " 'Body': 'twin_encoder',\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 128,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.5, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "# params = {\n", + "# **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "# }\n", + "params = getattr(PARAMS, dataset_name).BEST_DICT[gp][gp_multiply][single_model]\n", + "if isinstance(params, (list, tuple)):\n", + " params = params[param_index]\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-26T07:02:28.377372Z", + "iopub.status.busy": "2024-03-26T07:02:28.377075Z", + "iopub.status.idle": "2024-03-26T07:02:28.805846Z", + "shell.execute_reply": "2024-03-26T07:02:28.804741Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.443746, + "end_time": "2024-03-26T07:02:28.807872", + "exception": false, + "start_time": "2024-03-26T07:02:28.364126", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n", + "[*] Embedding False True\n", + "['lct_gan'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:28.836626Z", + "iopub.status.busy": "2024-03-26T07:02:28.836204Z", + "iopub.status.idle": "2024-03-26T07:02:28.840905Z", + "shell.execute_reply": "2024-03-26T07:02:28.839966Z" + }, + "papermill": { + "duration": 0.022416, + "end_time": "2024-03-26T07:02:28.843382", + "exception": false, + "start_time": "2024-03-26T07:02:28.820966", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:28.870881Z", + "iopub.status.busy": "2024-03-26T07:02:28.870225Z", + "iopub.status.idle": "2024-03-26T07:02:28.878302Z", + "shell.execute_reply": "2024-03-26T07:02:28.877297Z" + }, + "papermill": { + "duration": 0.024476, + "end_time": "2024-03-26T07:02:28.880618", + "exception": false, + "start_time": "2024-03-26T07:02:28.856142", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "7604993" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:28.909498Z", + "iopub.status.busy": "2024-03-26T07:02:28.908629Z", + "iopub.status.idle": "2024-03-26T07:02:28.994190Z", + "shell.execute_reply": "2024-03-26T07:02:28.993322Z" + }, + "papermill": { + "duration": 0.10216, + "end_time": "2024-03-26T07:02:28.996229", + "exception": false, + "start_time": "2024-03-26T07:02:28.894069", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 40] --\n", + "├─Adapter: 1-1 [2, 1179, 40] --\n", + "│ └─Sequential: 2-1 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 512] 20,992\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-16 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-9 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-17 [2, 1179, 256] 131,328\n", + "│ │ │ └─LeakyHardsigmoid: 4-18 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 40] (recursive)\n", + "│ └─Sequential: 2-2 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-10 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-32 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-17 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-33 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-34 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-18 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-36 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 2048] --\n", + "│ └─Encoder: 2-3 [2, 8, 256] --\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-6 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1179, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-12 [2, 1179, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-18 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1179, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-24 [2, 1179, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-39 [2, 8, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-30 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1179, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-36 [2, 1179, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 8, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 8, 256] 2,048\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 8, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 8, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 8, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 8, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 8, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-42 [2, 8, 256] --\n", + "│ └─Encoder: 2-4 [2, 8, 256] (recursive)\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-40 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-8 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-21 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-22 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-48 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-23 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 294, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-54 [2, 294, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-25 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-60 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 294, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-66 [2, 294, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-31 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-42 [2, 8, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-72 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 294, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-78 [2, 294, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-14 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-40 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 8, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 8, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-84 [2, 8, 256] --\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-21 [2, 512] --\n", + "│ │ │ └─Linear: 4-43 [2, 512] 1,049,088\n", + "│ │ │ └─ReLU6: 4-44 [2, 512] --\n", + "│ │ └─FeedForward: 3-22 [2, 512] --\n", + "│ │ │ └─Linear: 4-45 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-46 [2, 512] --\n", + "│ │ └─FeedForward: 3-23 [2, 512] --\n", + "│ │ │ └─Linear: 4-47 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-48 [2, 512] --\n", + "│ │ └─FeedForward: 3-24 [2, 512] --\n", + "│ │ │ └─Linear: 4-49 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-50 [2, 512] --\n", + "│ │ └─FeedForward: 3-25 [2, 512] --\n", + "│ │ │ └─Linear: 4-51 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-52 [2, 512] --\n", + "│ │ └─FeedForward: 3-26 [2, 512] --\n", + "│ │ │ └─Linear: 4-53 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-54 [2, 512] --\n", + "│ │ └─FeedForward: 3-27 [2, 512] --\n", + "│ │ │ └─Linear: 4-55 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-56 [2, 512] --\n", + "│ │ └─FeedForward: 3-28 [2, 512] --\n", + "│ │ │ └─Linear: 4-57 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-58 [2, 512] --\n", + "│ │ └─FeedForward: 3-29 [2, 1] --\n", + "│ │ │ └─Linear: 4-59 [2, 1] 513\n", + "│ │ │ └─LeakyHardsigmoid: 4-60 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 7,604,993\n", + "Trainable params: 7,604,993\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 24.24\n", + "========================================================================================================================\n", + "Input size (MB): 0.47\n", + "Forward/backward pass size (MB): 257.33\n", + "Params size (MB): 30.42\n", + "Estimated Total Size (MB): 288.22\n", + "========================================================================================================================" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = test_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "357e54f8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:29.023730Z", + "iopub.status.busy": "2024-03-26T07:02:29.022983Z", + "iopub.status.idle": "2024-03-26T07:02:29.076384Z", + "shell.execute_reply": "2024-03-26T07:02:29.075484Z" + }, + "papermill": { + "duration": 0.069356, + "end_time": "2024-03-26T07:02:29.078454", + "exception": false, + "start_time": "2024-03-26T07:02:29.009098", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torch\n", + "model.load_state_dict(torch.load(\"model.pt\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:02:29.104859Z", + "iopub.status.busy": "2024-03-26T07:02:29.104570Z", + "iopub.status.idle": "2024-03-26T07:04:43.125983Z", + "shell.execute_reply": "2024-03-26T07:04:43.124916Z" + }, + "papermill": { + "duration": 134.03742, + "end_time": "2024-03-26T07:04:43.128535", + "exception": false, + "start_time": "2024-03-26T07:02:29.091115", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:04:43.157466Z", + "iopub.status.busy": "2024-03-26T07:04:43.157125Z", + "iopub.status.idle": "2024-03-26T07:04:43.178957Z", + "shell.execute_reply": "2024-03-26T07:04:43.178165Z" + }, + "papermill": { + "duration": 0.03883, + "end_time": "2024-03-26T07:04:43.181158", + "exception": false, + "start_time": "2024-03-26T07:04:43.142328", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
lct_gan0.0079890.0067450.0018551.6703410.0106010.3454720.0163060.0000032.7118340.0321230.092560.0430740.0875690.0166044.382175
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "lct_gan 0.007989 0.006745 0.001855 1.670341 0.010601 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "lct_gan 0.345472 0.016306 0.000003 2.711834 0.032123 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "lct_gan 0.09256 0.043074 0.087569 0.016604 4.382175 " + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:04:43.207461Z", + "iopub.status.busy": "2024-03-26T07:04:43.207140Z", + "iopub.status.idle": "2024-03-26T07:04:43.664078Z", + "shell.execute_reply": "2024-03-26T07:04:43.663180Z" + }, + "papermill": { + "duration": 0.472335, + "end_time": "2024-03-26T07:04:43.666225", + "exception": false, + "start_time": "2024-03-26T07:04:43.193890", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:04:43.694954Z", + "iopub.status.busy": "2024-03-26T07:04:43.694087Z", + "iopub.status.idle": "2024-03-26T07:07:10.524565Z", + "shell.execute_reply": "2024-03-26T07:07:10.523643Z" + }, + "papermill": { + "duration": 146.847692, + "end_time": "2024-03-26T07:07:10.527325", + "exception": false, + "start_time": "2024-03-26T07:04:43.679633", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/lct_gan/all inf False\n", + "Caching in ../../../../contraceptive/_cache_bs_test/lct_gan/all inf False\n", + "Caching in ../../../../contraceptive/_cache_synth_test/lct_gan/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:07:10.555967Z", + "iopub.status.busy": "2024-03-26T07:07:10.555657Z", + "iopub.status.idle": "2024-03-26T07:07:10.575846Z", + "shell.execute_reply": "2024-03-26T07:07:10.575130Z" + }, + "papermill": { + "duration": 0.036416, + "end_time": "2024-03-26T07:07:10.577639", + "exception": false, + "start_time": "2024-03-26T07:07:10.541223", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:07:10.603605Z", + "iopub.status.busy": "2024-03-26T07:07:10.603305Z", + "iopub.status.idle": "2024-03-26T07:07:10.608506Z", + "shell.execute_reply": "2024-03-26T07:07:10.607674Z" + }, + "papermill": { + "duration": 0.020724, + "end_time": "2024-03-26T07:07:10.610366", + "exception": false, + "start_time": "2024-03-26T07:07:10.589642", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'lct_gan': 0.40845790738860766}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:07:10.635276Z", + "iopub.status.busy": "2024-03-26T07:07:10.635012Z", + "iopub.status.idle": "2024-03-26T07:07:11.081476Z", + "shell.execute_reply": "2024-03-26T07:07:11.080563Z" + }, + "papermill": { + "duration": 0.461541, + "end_time": "2024-03-26T07:07:11.083670", + "exception": false, + "start_time": "2024-03-26T07:07:10.622129", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCPklEQVR4nO3deXhU9b0/8PeZfZLMkn0hK4uArGEtIgJCWaQItaVarRIRtbdYtSnPo7S/gqgt6HWjrZfr9WoijwIuxeVeriC1ApXKvi+yhpCV7LPv5/v748xMMmQhmczMyUw+r+cJkzlzcuYzzOST7/7lGGMMhBAiIonYARBCCCUiQojoKBERQkRHiYgQIjpKRIQQ0VEiIoSIjhIRIUR0lIgIIaKjREQIER0lItKl0tJScByHq1evih0KiWGUiEjIbd68GW+88YbYYZAoQomIhBwlItJTlIgIIaKjRER67Msvv8T06dOh0Wig1WoxceJEbN68GQAwY8YMbN++HeXl5eA4DhzHIT8/v9vX5nkezz33HLKyshAXF4eZM2fi7NmzyM/PR1FRkf+8pqYmrFy5EqNGjUJCQgK0Wi3mz5+PEydOBFxv9+7d4DgOH330Ef74xz8iOzsbKpUKs2bNwqVLl0Lx30FCQCZ2ACS6lJaWYtmyZRgxYgRWrVoFvV6PY8eOYceOHbj//vvx+9//HgaDAZWVlXj99dcBAAkJCd2+/qpVq/Dyyy9j4cKFmDt3Lk6cOIG5c+fCbrcHnHflyhV89tlnWLJkCQoKCnD9+nW89dZbmD59Os6ePYusrKyA89evXw+JRIKVK1fCYDDg5ZdfxgMPPIADBw70/j+F9B4jpAslJSUMACsrK2MtLS1Mo9GwyZMnM5vNFnAez/P+7xcsWMDy8vJ6/Fy1tbVMJpOxxYsXBxx/7rnnGAC2dOlS/zG73c48Hk/AeWVlZUypVLLnn3/ef+ybb75hANjw4cOZw+HwH9+wYQMDwE6dOtXjOEnoUdWMdNuuXbtgMpnw7LPPQqVSBTzGcVyvr//111/D7XbjV7/6VcDxX//61+3OVSqVkEiEj6/H40FjYyMSEhIwdOhQHD16tN35Dz/8MBQKhf/+tGnTAAglKyI+SkSk2y5fvgwAGDlyZFiuX15eDgAYPHhwwPGkpCQkJiYGHON5Hq+//jqGDBkCpVKJlJQUpKam4uTJkzAYDO2unZubG3Dfd73m5uZQvgQSJEpEJCr96U9/QnFxMe644w68//772LlzJ3bt2oURI0aA5/l250ul0g6vw2il5D6BGqtJtw0aNAgAcPr06XallraCrabl5eUBAC5duoSCggL/8cbGxnYll08++QQzZ87EO++8E3C8paUFKSkpQT0/EQ+ViEi3zZkzBxqNBuvWrWvXi9W2ZBEfH99h9ehmZs2aBZlMho0bNwYc/+tf/9ruXKlU2q408/HHH6OqqqrHz0vERyUi0m1arRavv/46li9fjokTJ+L+++9HYmIiTpw4AavVivfeew8AMH78eHz44YcoLi7GxIkTkZCQgIULF970+unp6Xjqqafw6quv4u6778a8efNw4sQJfPnll0hJSQkoaf3oRz/C888/j4cffhi33XYbTp06hQ8++AADBw4M2+snYSRyrx3p49p23/t88cUX7LbbbmNqtZpptVo2adIktmXLFv/jZrOZ3X///Uyv1zMAPerKd7vd7A9/+APLyMhgarWa3XnnnezcuXMsOTmZ/fKXv/SfZ7fb2W9/+1uWmZnJ1Go1mzp1Kvvuu+/Y9OnT2fTp0/3n+brvP/7444DnKSsrYwBYSUlJT/9LSBhwjFFrHenbWlpakJiYiBdffBG///3vxQ6HhAG1EZE+xWaztTvmm0A7Y8aMyAZDIobaiEhE1NfXw+PxdPq4QqFAUlISPvzwQ5SWluKuu+5CQkICvv32W2zZsgVz5szB1KlTIxgxiSRKRCQiJk6c6B+w2JHp06dj9+7dGD16NGQyGV5++WUYjUZ/A/aLL74YwWhJpFEbEYmIffv2dVjt8klMTMT48eMjGBHpSygREUJER43VhBDRRXUbEc/zqK6uhkajCcnsb0JIaDHGYDKZkJWV5V8toSNRnYiqq6uRk5MjdhiEkJuoqKhAdnZ2p49HdSLSaDQAhBep1WpFjoYQciOj0YicnBz/72pnojoR+apjWq2WEhEhfdjNmk6osZoQIjpKRIQQ0VEiIoSILqrbiLqDMQa3293lPCfSMalUCplMRkMjSNjFdCJyOp2oqamB1WoVO5SoFRcXh8zMzIAdMAgJtZhNRDzPo6ysDFKpFFlZWVAoFPSXvQcYY3A6naivr0dZWRmGDBnS5YA0QnojZhOR0+kEz/PIyclBXFyc2OFEJbVaDblcjvLycjidznZ7mUUjxhgOljWhvNGKW7O0GDlAJ3ZIBDGciHzor3jvxNr/37GKFvzrciMAoKrFBn2cHNmJ9IdKbLH1KSOkC043j4NlTQAAhUz46B+91iJiRMQn5ktEhPhcqjPD5vRAp5bjx4UDcK3JimGZXU89IJFBJaJ+Lj8/378mdKy7WGcCAAzP1CIxXoExOXooZR3vAEsiS/REVFVVhV/84hdITk6GWq3GqFGjcPjwYbHDIjFoRJYWt6RrcEt6gtihkBuIWjVrbm7G1KlTMXPmTHz55ZdITU3FxYsXkZiYKGZYUcfpdNI4n24YnKbB4LTWqpjV6capSgNsLg9mDE0TMTIiaonopZdeQk5ODkpKSjBp0iQUFBRgzpw5/j3Ww8Xp5jv9cnv4bp/r6sa5wZgxYwaeeOIJPPHEE9DpdEhJScEf/vAH/xbL+fn5eOGFF/DQQw9Bq9XiscceAwB8++23mDZtGtRqNXJycvDkk0/CYrH4r1tXV4eFCxdCrVajoKAAH3zwQVDxxQoPz/Cvy404UWFo916SyBK1RPTFF19g7ty5WLJkCfbs2YMBAwbgV7/6FR599NEOz3c4HHA4HP77RqMxqOd985tLnT5WkBKPxYUD/Pf/a+9luDwdL+udnajGkgmtC7O9u68MNmfgVJLf/PCWoGJ877338Mgjj+DgwYM4fPgwHnvsMeTm5vr/b1555RWsXr0aa9asAQBcvnwZ8+bNw4svvoh3330X9fX1/mRWUlICACgqKkJ1dTW++eYbyOVyPPnkk6irqwsqvmjzfa0RSfEKpCYo/QNbE5QyxCulsDg8aDA7kKlTixxl/yVqIrpy5Qo2btyI4uJi/O53v8OhQ4fw5JNPQqFQYOnSpe3OX7duHdauXStCpJGXk5OD119/HRzHYejQoTh16hRef/11fyK688478dvf/tZ//vLly/HAAw/g6aefBgAMGTIEf/7znzF9+nRs3LgR165dw5dffomDBw9i4sSJAIB33nkHw4cPj/hrizSnm8fO09fBM4ZltxdAp5YDENbISY5XwuKwotHspEQkIlETEc/zmDBhAv70pz8BAAoLC3H69Gn853/+Z4eJaNWqVSguLvbf963+1lMrZg7u9DHJDbNAHruj82rijTNGlk0t6HEsnfnBD34QMCVlypQpePXVV/2TdydMmBBw/okTJ3Dy5MmA6hZjzD/V5cKFC5DJZAFb9gwbNgx6vT5kMfdV14128IxBo5L5k5BPUrwC15qsaLG6RIqOACInoszMTNx6660Bx4YPH46//e1vHZ6vVCqhVCp7/by+wWxinttb8fHxAffNZjMef/xxPPnkk+3Ozc3NxYULFyIVWp9z3WgHAGTo2k9R0ccJianJ6oxoTCSQqIlo6tSpOH/+fMCxCxcuIC8vT6SI+o4DBw4E3N+/fz+GDBkCqbTjcS/jxo3D2bNnMXhwx6W9YcOGwe1248iRI/6q2fnz59HS0hLSuPuiOpPQrpimaZ+IkuKF3sZmCyUiMYnaa/ab3/wG+/fvx5/+9CdcunQJmzdvxn/9139hxYoVYobVJ1y7dg3FxcU4f/48tmzZgr/85S946qmnOj3/mWeewb/+9S888cQTOH78OC5evIjPP/8cTzzxBABg6NChmDdvHh5//HEcOHAAR44cwfLly6FWx367iK9ElK5tX5pO9CYio80Fnqe9RsUiaiKaOHEiPv30U2zZsgUjR47ECy+8gDfeeAMPPPCAmGH1CQ899BBsNhsmTZqEFStW4KmnnvJ303dk9OjR2LNnDy5cuIBp06ahsLAQq1evRlZWlv+ckpISZGVlYfr06bjnnnvw2GOPIS0ttsfP2F0ef/tPRyUijVKGZVML8KuZgyG5sYGQRExUbzltNBqh0+lgMBja7eJht9tRVlaGgoKCqFu+YsaMGRg7dmyfmHoRzf+PAFDRZMUnRyqhVcvxyO2h60wg3dPV72hbNOmVxLR0rQo/GZcNJy0V3KdRIiIxTSGTIDe56/WGLtWZcKnOgrzkOAzPpP3xxECJqA/avXu32CH0K3UmB87VGCGTcJSIRCL67HtCwoUxhn9dasC5GiM8XfSIaVXCWCKTgwY1ioVKRCRmGW1uHChrgkzCYWh65wug+ROR3R2p0MgNqEREYlaDRRjImBiv6LJrXqMS/h6b7G5EcSdyVKNERGJWo1kYLZ2S0PVaTQneROR083AEuXQL6R1KRCRmNZqFElFSfNfzE+VSiX+eoNVJ3fxioEREYlajd/5Y8k1KRICwNhEgrNpIIo8aq0lM4nmGJm8iSrlJiQgAlkzIhkIqgUxKf5vFQImIxKQWmwsenkEu5aBV3/xjHqegXwUx0f8+iUmJcXIsm1oAk8MVsMAc6Zv6VzmUMcDtjPxXD7qEN23ahOTk5IC1uQFg8eLFePDBB0P9PxKzOI6DrgfbSVc0WbHjdC2OlDeFOTLSkf5VIvK4gH++GvnnnfZbQNa97X6WLFmCJ598El988QWWLFkCQNh9Y/v27fjqq6/CGWW/ZrS7cK7GCJsrDuPzksQOp9/pXyWiKKBWq3H//ff7d94AgPfffx+5ubmYMWOGeIFFmX9dbsB3lxthtHdv2ka8t43I4qDuezH0rxKRVC6UTsR43h549NFHMXHiRFRVVWHAgAEoLS1FUVERtXV0E2MMxyta4HDxGJyWAHRjGaU4pbAEr8VB3fdi6F+JiOO6XUUSU2FhIcaMGYNNmzZhzpw5OHPmDLZv3y52WFHD5vLA4RJGSPsWx78ZX4nI5vKA5xmt1hhh/SsRRZHly5fjjTfeQFVVFWbPnh3Utkn9VbN3aViNSgb5jeOCGANaygFOAuhy/HtCqeVScJzwsNXl8Q9wJJFBbUR91P3334/Kykq8/fbbWLZsmdjhRBXfjhy+HTr8GAPObAOObwGOfQCc+x9/j6ZEwkEtF6pnN+7WS8KPElEfpdPp8JOf/AQJCQlYvHix2OFElWbvHmWJcTckoqqjQP0FoRTEccD1M0DdOf/DaoWQiOwuSkSRRuXPPqyqqgoPPPBASDaV7E98VbOA9iGPGyjfJ3w/+IeA2waU/RO4+k8gbTjAcVgyPgdyKUfTPERAiagPam5uxu7du7F79278x3/8h9jhRB2DTUhEASWihguA0wIoNUDWWID3ANf2A9YmwFAB6HP9JSISeZSI+qDCwkI0NzfjpZdewtChQ8UOJ+rcPykXRpsL8W0bnOvOCrcZowCJVPhKGw7UnBSqZ/pccYIlAERuI3ruuefAcVzA17Bhw8QMqU+4evUqDAYDVq5cKXYoUUkq4ZAYr/CvMQS3A2i6InyfdmvriSm3CLeNlwHGUNZgwY7TNThR0RLReEkfKBGNGDECf//73/33ZTLRQyKxpuWaUBVTJwIJqa3H9XmARAbYDYC1Ec1WKc7VmMAzYEyOXrRw+yPRf+tlMhkyMjK6da7D4QiYDGo0Gm/6M7QGce9E2//fhesmXKm3YFBqPIb4FsxvvircJuYHnixTALps4fGWcqjlQwBQ970YRO8euHjxIrKysjBw4EA88MADuHbtWqfnrlu3Djqdzv/V1SA/uVzoMbFarSGPuT/x/f/5/j/7uspmK87VGFFnarN6QWeJCBASEQAYKlvHEVH3fcSJWiKaPHkySktLMXToUNTU1GDt2rWYNm0aTp8+DY2m/fYvq1atQnFxsf++0WjsNBlJpVLo9XrU1dUBAOLi4miuVg8wxmC1WlFXVwe9Xg+pNDp6lJotN3TdO8yApUEYN5SY1/4H/ImoCupsGkckFlET0fz58/3fjx49GpMnT0ZeXh4++ugjPPLII+3OVyqVPRpT46vy+ZIR6Tm9Xt/tqnNf4BvMqPd13Rurhdv4FECubv8D2iwhSdkNUPFC6c/m9IAxRn+4Ikj0NqK29Ho9brnlFly6dCkk1+M4DpmZmUhLS4PLRbt49pRcLo+akhAAuD08zN7Z83q1t0Rk8iYiTVbHPyRTAnHJgKUBakeDcB2eweVhUMgoEUVKn0pEZrMZly9fDvlKhFKpNKp+oUhwjHY3GAMUMgnifIMTjTXCrTaz8x9MSAMsDZDb6iGTpMHNM9hcntbufxJ2oiailStXYuHChcjLy0N1dTXWrFkDqVSKn//852KGRaKUb0S1Ti0XqlWMASZvItJ0lYjSgetnwVnqsOz2yVDJpZDSMiARJWoiqqysxM9//nM0NjYiNTUVt99+O/bv34/U1NSb/zAhN7A43OA4IREBAGzNwmBGiQyI7+IzlZAm3JrrAkdjk4gR9X9969atYj49iTEjB+gwLEMDl8c79snXUK1JF6Z0dCbem4hszcJmB1GweF6sofRPYopMKoHMl3PM14XbhJv0+ikTAEU84LTg/JUrKHPqMCRdg0GpCWGNlbSi1jgSuyxCLxjiU25+rvecloYanKsxoc7ouMkPkFCiRERiAmMMHx+uwM4ztXC4vQMSLfXCra8NqCtxyQCAeI8wbcjupkGNkUSJiMQEk8ONymYbvq8xQS6RAC4b4DAJD3bVUO2jFvYyU7sNAAAHja6OKEpEJCYYvKsyatUyYQcOX2lIpRUGLd5MnDcRuVoAAA43H44wSScoEZGY4BtD5J9j5ktE8d2olgH+qpnSbQAYT/PNIowSEYkJ/kSk9na9m32JqBsN1QCg0gESGWRgULpNsLuoRBRJlIhITGjxV81uLBF1c3AsxwFxiZBJOajdLa0N3iQiaBwRiQktNt+se7l3l0Rf130PRunHJUNtqsN9g+OhKBgYhihJZygRkZjA86x1eofLBrjswgPeRuhuUSdBwnFQe0wAzTWLKEpEJCY8OCUfbg8PCccBpirhoFIDSHuwsqRaL9zaWkIdHrkJaiMiMUMmlQhd97Zm4UBPSkOA0GAN4EpFJXacrvEPCSDhR4mIxB5rk3CrTuzZz6n0AABDcwPOVRtgtFMiihSqmpGod7KyBd/XmjA8Q4tR2brWEpG6hyUipRbgJJBLGBQeK/WcRRCViEjUqzc5UNVsg8nhLcHYvCWinlbNJBJApYNMwkHlNtJYogiiRESinq8KpVV5u+79JaIeVs0AQK2HTMJB6TZRiSiCKBGRqGeyCwvma1VywGkRFjfjOH+bT4+o9JBKJVQiijBKRCSqMcZg9E7v0KhkrdUypRaQBtEE2qZERPPNIocSEYlqdhfvXxpWSERBdt37qPT+NiKagR851GtGopqvfShBKYNMKgm+695HrUeqRonpSXJIR0TPxpLRjhIRiWpON48EpQxatfejHGzXvY9KB5lEAnjsAO8CJLSQfiRQIiJRLScpDo/eMRCMeXfusPWyRCRTCbt4uJ3CCo+y5NAESrrUZ9qI1q9fD47j8PTTT4sdColC/g0Ve9tGxHFwyjQoa7Rg35kroQuQdKlPJKJDhw7hrbfewujRo8UOhUQzhwnwuAFO4p83FgxeoUGtwY6r1TWtJS0SVqInIrPZjAceeABvv/02EhODLE6TfuuzY1X4+HAF6k2O1tKQStf1hoo3IY8TkpjSZaKeswgRPRGtWLECCxYswOzZs296rsPhgNFoDPgi/Vu1wYbKZpuwfFBv24e8pHF6SDlA4THDQYMaI0L0LaePHj2KQ4cOdev8devWYe3atWGOikQLu8vjTxQalRy43sv2IR+VDlKJBEq3GXa3Bzr0YE0jEhTRSkQVFRV46qmn8MEHH0ClUnXrZ1atWgWDweD/qqioCHOUpC/zTe1QK6RQyNqOIeplIlJqIZNyUFKJKGJEKxEdOXIEdXV1GDdunP+Yx+PB3r178de//hUOhwNSaWA9X6lUQqnsxh5VpF8ImOwKtBlDpO/dhVVayCQcFC4LHC53765FukW0RDRr1iycOnUq4NjDDz+MYcOG4ZlnnmmXhAi5UcAcM8Zal3jtbdVMoYFEKgXH3HBaTQC0vbseuSnREpFGo8HIkSMDjsXHxyM5ObndcUI64p91r5YDDiPAu4XeMmXwXfcAAIkEAwdkgHMYIU2i7vtIEL3XjJBgSSWcML1DJWttH1LphQXOekkZr4dCKoHUQT2zkdCnpnjs3r1b7BBIFJk6OAVTB3t3cq3yjoLuZde9n0oHGCqFkhYJu6D+dFy5QkPfSR/jn9oRmkTU6FGhrMGCy5XVIbke6VpQiWjw4MGYOXMm3n//fdjt9lDHREjPWXuxPGwHDLwatUY7GhvqQ3I90rWgEtHRo0cxevRoFBcXIyMjA48//jgOHjwY6tgI6VSTxYl3vi3D58e9myn2dvmPG8i80zzgMIXkeqRrQSWisWPHYsOGDaiursa7776Lmpoa3H777Rg5ciRee+011NfTXxESXkaby/8FngfsLcIDISoRyeP1AAAJtRFFRK+6F2QyGe655x58/PHHeOmll3Dp0iWsXLkSOTk5eOihh1BTUxOqOAkJ4B/MqJYDDgPAewCJrFez7ttSxOkBAMxlA9yOkFyTdK5Xiejw4cP41a9+hczMTLz22mtYuXIlLl++jF27dqG6uhqLFi0KVZyEBAjYucM/tUMv7N4RAkp1HDwSBTw8A7NTqSjcguq+f+2111BSUoLz58/jrrvuwqZNm3DXXXdB4h2/UVBQgNLSUuTn54cyVkL8AnfuaBEOhqrrHoBKJoFDmgAp3wSnpQXKhNSQXZu0F1Qi2rhxI5YtW4aioiJkZmZ2eE5aWhreeeedXgVHSGcCqmbGIHd27YJMKoFbrgFcTXBaW0AzHMMrqES0a9cu5Obm+ktAPowxVFRUIDc3FwqFAkuXLg1JkITcyFc106hkrct/hLBEBAC3jSiAss4IGW8J6XVJe0G1EQ0aNAgNDQ3tjjc1NaGgoKDXQRHSFcYYtCq5d3qHPHTLf9xAo0uGQiqBxGkO6XVJe0GViDpbx9dsNnd7bSFCgsVxHH42MUe4w3sAu0H4PsQlIig1wi114YddjxJRcXExAOGDsHr1asTFxfkf83g8OHDgAMaOHRvSAAnpkt0AMF7YXtqXOELkolECWYMFetYAWk09vHqUiI4dOwZAKBGdOnUKCkXr5nMKhQJjxozBypUrQxshIV2xtWkfClHXvU+5WYp4ox0KRTMSGQv59UmrHiWib775BoCwgNmGDRug1dKCUSTyDlxpxOlqI8bm6DBe4usxC/1GiFK1FgAHj9sNOC2AMiHkz0EEQbURlZSUhDoOQrqtxTu1w8MDcISnoRoAlAo5nNI4eHi3MOeMElHYdDsR3XPPPSgtLYVWq8U999zT5bnbtm3rdWCEdCZgMKMhNFsIdUQll8IijYebb/E2WHc8Zo70XrcTkU6nE7b19X5PiFgCloi1hX4wo49SJoFDlgA33wzQNI+w6nYialsdo6oZEQvPs9bBjHLWmiDCUTWTSeGQJsDtZMLEWhI2QQ1otNlssFqt/vvl5eV444038NVXX4UsMEI6Yna6wTMGCcchweNdK0iuAhRxXf9gEFRyCZyyBHh4nkpEYRZUIlq0aBE2bdoEAGhpacGkSZPw6quvYtGiRdi4cWNIAySkrbZTOyT20C6GdqN0rQrzJ9yCEVk6WiAtzIJeoXHatGkAgE8++QQZGRkoLy/Hpk2b8Oc//zmkARJyoyy9CulaVVjbhwBALpVAr0+GXCqh0dVhFlT3vdVqhUYjjGL96quvcM8990AikeAHP/gBysvLQxogIW0N0Ktx78Rc4c65I8JtmEpEAACVd6yc0+JdfI02/gyHoBfP/+yzz1BRUYGdO3dizpw5AIC6ujoa5EgiJ8wlIgDYX2HDlSY7XG4PlYrCKKhEtHr1aqxcuRL5+fmYPHkypkyZAkAoHRUWFnb7Ohs3bsTo0aOh1Wqh1WoxZcoUfPnll8GERPqJgAnXYZp139aJSgPKzTI4PTy1E4VRUFWzn/70p7j99ttRU1ODMWPG+I/PmjULP/7xj7t9nezsbKxfvx5DhgwBYwzvvfceFi1ahGPHjmHEiBHBhEZi3Pv7y+H0MCwcrkeayyYcDMNgRh+lTAKnLB5uj5l6zsIo6J1eMzIykJGREXBs0qRJPbrGwoULA+7/8Y9/xMaNG7F//35KRKQdxhharC64eQaV25sUlBpApuj6B3tBJRfGEnl4E1XNwiioRGSxWLB+/Xp8/fXXqKurA8/zAY8HsxOsx+PBxx9/DIvF4q/q3cjhcMDhaN1RwWikD0Z/YnV64OYZOA6I97QIB8PYPgQASrmwdrWbxhKFVVCJaPny5dizZw8efPBBZGZm+qd+BOPUqVOYMmUK7HY7EhIS8Omnn+LWW2/t8Nx169Zh7dq1QT8XiW6+daoTlDJIbY3CwbiUsD6nSiZFkywBbp5RiSiMgkpEX375JbZv346pU6f2OoChQ4fi+PHjMBgM+OSTT7B06VLs2bOnw2S0atUq/+JsgFAiysnJ6XUMJDoYbW3mmFm8iSg+9Mt/tOUrEXnclIjCKahElJiYiKSk0BSJFQoFBg8eDAAYP348Dh06hA0bNuCtt95qd65SqYRSSfsp9Ff+nTtUcsDsXTM9AiUipywebgejqlkYBdV9/8ILL2D16tUB881Chef5gHYgQnx8y3/olKx1ner48Cai0Tl6/Oz2EchJUgs7vtKur2ERVIno1VdfxeXLl5Geno78/HzI5fKAx48ePdqt66xatQrz589Hbm4uTCYTNm/ejN27d2Pnzp3BhEVinFYtR6ZOhVSJBWAMkKsBRXxYnzNBKRMWRFPGAS67UCqizRZDLqhEtHjx4pA8eV1dHR566CHU1NRAp9Nh9OjR2LlzJ374wx+G5PoktkzMT8LE/CSg9rRwIMyloQBKrZCIHJSIwiGoRLRmzZqQPDntBEuCYo1M+xAAmOwunKoyINXAYYgU1GAdJkG1EQHC8h///d//jVWrVqGpSRhqf/ToUVRVVYUsOEJ8eJ6B573TOyzeRBSBEpHN5cGBK024ZPT+qlCDdVgEVSI6efIkZs+eDZ1Oh6tXr+LRRx9FUlIStm3bhmvXrvnXKiIkVK6b7PjoUCUydSr8jPd13Yc/ESllwmx7M+LBwMBRiSgsgioRFRcXo6ioCBcvXgzY2fWuu+7C3r17QxYcIT5Gm7AyI8dcrXuZRaBqppILvyI2STx4BioRhUlQiejQoUN4/PHH2x0fMGAAamtrex0UITfyjSFK4UxCj5kiLuw9ZgCgkEog4Tg4ZQlwe3hqIwqToBKRUqnscJ7XhQsXkJpKPQok9HxjiJKYd+mPhPSI7LzKcRyUcgmc0nh4eCYsBdJ2KRISEkElorvvvhvPP/88XC7hw8FxHK5du4ZnnnkGP/nJT0IaICEAYPAmIr27TSKKEKVMAqc0Di4ewiqNTkvEnru/CCoRvfrqqzCbzUhNTYXNZsP06dMxePBgaDQa/PGPfwx1jIT4E5FGhESkkkvBOClcMu9OIVQ9C7mges10Oh127dqFffv24cSJEzCbzRg3bhxmz54d6vgIgYdnwoRXxiPe3QRwiGgimjtCWHdLey4FMFULDdbarIg9f3/Q40TE8zxKS0uxbds2XL16FRzHoaCgABkZGWCM9WpJEEI64vLwuCU9AU5DHRQuDyCVh3VVxhslxXsXXlPrhERES8aGXI+qZowx3H333Vi+fDmqqqowatQojBgxAuXl5SgqKurRMrGEdJdKLsX8UZlYNEQGDhwQnwZIgh6LGzylsHMN7foaej0qEZWWlmLv3r34+uuvMXPmzIDH/vGPf2Dx4sXYtGkTHnrooZAGSQgAwFQj3Goyuj4vxKpabChvtCDbKkMuQGOJwqBHf1a2bNmC3/3ud+2SEADceeedePbZZ/HBBx+ELDhCAMDu8gjTO4zVwoEIt8/UtNhw4EoTys3ev9vUWB1yPUpEJ0+exLx58zp9fP78+Thx4kSvgyKkrR2na/HmP86jvsa7ead2QESfXyUXpnlYJN4BlFQiCrkeJaKmpiakp3feW5Geno7m5uZeB0VIWy1WJ1T2eig47xpEEWyoBoRxRABgYWrhgNMCeNwRjSHW9SgReTweyGSdNytJpVK43fQGkdDheQaj3Y0EZ50w70s7ICIjqtvyl4iYApB4P/9O6jkLpR41VjPGUFRU1Om60bTEKwk1k90ND8+gd9ZBIZOIMn7HVyJyuBmg0go7zNqNES+ZxbIeJaKlS5fe9BzqMSOh1GhxAIwhjdWBgwLQZUc8Bt9SIHaXB9B6ExE1WIdUjxJRSUlJuOIgpENNFifUrmZoJQ5AGhfxhmpA2FIIANw8g0ehgRSgBusQC3rLaUIiodHihM5RBbVcCuhyAWnkP7JKmQT3TcqBUiYFV+sdy0Sjq0OKEhHp0wbo1UhTNkLDyYHEfFFi4DgOmTpvj5laJ9xS1SykKBGRPm1kZgJw2Qh4xEtEAVRa4dZO0zxCiRIR6duarwIel7C3WEKaaGF8X2tEk9mJoVoVkgEqEYWYCDMHCekes8MNS+UZMDAg5ZaIjx9q60yVEQfKmlDn9M7EdzuFfc5ISIiaiNatW4eJEydCo9EgLS0Nixcvxvnz58UMifQhpyqacfToQZTVW4DUoaLG4hvUaOelgNy7YQSVikJG1ES0Z88erFixAvv378euXbvgcrkwZ84cWCy0FCcBrDXfQ8bboYzXCD1mIvINarS7eEDlbbCmLvyQEbWNaMeOHQH3S0tLkZaWhiNHjuCOO+4QKSrSV3A1wgRqZfZYcdYfakOtaDOoUaUHTNcBe4uoMcWSPtVYbTAIPRFJSUkdPu5wOAKmkXS0kwiJDeaWeqhN5eAAaAsmiB2OPxHZXB4gTi8ctLWIFk+s6TON1TzP4+mnn8bUqVMxcuTIDs9Zt24ddDqd/ysnJyfCUZJIMX2/GwCDW58PhTb8GynejNrbRmRzektEAJWIQqjPJKIVK1bg9OnT2Lp1a6fnrFq1CgaDwf9VUVERwQhJxFgaYK86CQDgCqaJHIzAn4hcHkCtFw7aaMmbUOkTVbMnnngC//u//4u9e/ciO7vzSY1KpbLTmf8kRvA88P12mKxONKnzMWTAQLEjAgBk6FS4d2IO4hUyAN7pHfYWYbNF2jCi10RNRIwx/PrXv8ann36K3bt3o6CgQMxwSF9w5R+AsRrZqXoYc+7CAL1a7IgACN33Wb5YeJ2QfDxuYZE0ZYK4wcUAURPRihUrsHnzZnz++efQaDSora0FIOybplb3jQ8giRDGgCu7gYpDAICkcYtxe1rfKA21I5ECSq0wzcPeQokoBERtI9q4cSMMBgNmzJiBzMxM/9eHH34oZlgk0txO4OznwLX9wv1BdwJpw8WNqQMnKlrwr0sNsDrdbdqJWsQMKWaIXjUj/Zy1CTj9N8DSAHASOAbNwX5bNgY1WzFAr+5TG3YeutoEk92NgakJiFPpAZRTz1mI9JleM9IPtVQAR0qEJKSIB8b+HKf5fBwtb8Y35+vFjq4d31giKhGFXp/oNSP9kKESOLlVaPDVDQBG/BgOaRyOnhS2DBqbre9TpSHghi58GksUUpSISOTZDUJ1zOMGkgYCI+8Bz8nwjzO1MDvc0MfJMTxTI3aU7cS1neaRoBcOUokoJCgRkchiDDj7BeC0AglpMA9ZiJNlBlxusKDB5ICE4zB7eDpk0r7XaqDyj67mW0tEDpOQUEVYwjaW0P8eiayqo0K1TCoHRt4DxslxoKwJAKCQSTDn1nTkJMWJHGTHfFUzq9MtbPQoUwg9fnYDEJ8scnTRjRIRiRynBe7L38Bsc0I/Zg6gToQGwPi8ROjj5BiSpvE3CPdFARNfOU4oFZnrhKkelIh6hRIRiRh2dR+u1DShwqVFimcgxniP33FLqqhxddfA1ASkJCiRoPL+2qj1rYmI9AolIhIZtmY0XTqARosTFemTMUyrEjuiHktQypCgbPMrE+ctBdmaxAkohvS9FkESk9xXv0N5vQkGVTZuGTaqz8wh6xW1d90sa6O4ccQASkQk/JwW1F88DIebR0vqBEws6Hjhu76O5xmOXWvGtxcb4PbwQJwvEVGJqLeoakbCzlN5BDXNZpgVqRg+bATkfbBrvjs4DvjnxQZ4eIZR2TrofFUzh0noPZMpxA0wikXnJ4JED48LDee/E0pDyYUYMUAndkRB4zjOP6jR34Uv91YxqZ2oVygRkfCqPQXOZQVUeuQMLeyTAxV7It7bWG1xeIQDvlIRVc96hapmJHx4Hqg8hDSNCiljZ8MzIPrH2vgSkdXpFg7EJQkDNKnBulcoEZHwabwolBTkKkiyxkAS5aUhAIj3Vs38JSJfzxlVzXol+j8ZpM/ylO9Hk9UJPqMwZhpy4xS+qpmvRERVs1CgRETCo6UCTTVl+P66BZ/Wp4sdTcjEK70lorZVM0ComtFCf0GjqhkJj4oDqDPaUR83BAPS08SOJmQGpiYgVaNsHWGt0nsX0ncBTjOg7HvLl0QDKhGR0LM0wlpzHka7GzW6MRiRpRU7opBJUMqQqVNDo5ILB6Sy1iVBqME6aJSISOhVHkS9yYZmdR4yMwe0/tLGqnjvTrSWBnHjiGKUiEgghxkw1QpVjWDYWsDXnES9yYFq7RiMyIreAYwdYax1mofD7e05i/euHmCuEy+wKEdtRETgdgKXdgG1p4RGV5kCyJ0C5PwAkPTg71X5PrSYbWiQZ4HXDkBBSnz4YhYBx3HYf6UJdpcHwzI1UCZIgQRvG5iFElGwqEREAN4jrCFdc9KbhJRCYrqyBzj7qfB4d1gagNpTMNjdqNBNwK2ZOkglfWsB/FDwrUdktnt7znwlIksD9ZwFSdREtHfvXixcuBBZWVngOA6fffaZmOH0X9f2w3r9Es7U2bDZMRV/Uy9BY84PhR1N6y8Ap7fdPBkxBlz8CmAMBUPHYtEdEzEmJ7aqZT4ab4+Z2TeWSJ0ESGRCdZYWSQuKqInIYrFgzJgxePPNN8UMo3+zNsF6cQ/OVBtxVDkZ16UZuNZsw+ZyHapzfiT8gjVeEnZi5fnOr1N7EmguF3qRBs9CmlYVs43UGm+JyGj3tqNJJK3jiajBOiiiJqL58+fjxRdfxI9//ONune9wOGA0GgO+SO/wZf/EpVoDGhQDIM8ciSUTspGfEgc3z/DZNQVsQxd5S0bngfPbO656mOuBi1+BZwyO7NsAdWLkX0gE+RKsyVc1A6idqJeiqo1o3bp10Ol0/q+cnByxQ4pulkag7iwydCrYsqfiR2MHIDsxDgtHZyEnKQ53DEmFKn0IcOtigJMAtaeB8/8nbJ/jv0aDf6PEKi4db5elYP+V2B5P4ysRBSQifztR39uhNhpEVa/ZqlWrUFxc7L9vNBopGfVG5UFIAKQVjMKPR07w76wqk0rwk3EDWndaTb0FGL4QOPeF0KBtqALSRwAuG1BzDPC4wcclY6/rNrhsXNQufNZdrYmozRAHfxc+JaJgRFUiUiqVUCqVYocRG9wO4PoZ4fvsie22d2573+nmwScNhWrUEuD7/xVGEJftbT05MQ8Xk+9E3fdGKOWSmBpJ3ZFUjRL3TszxJyQArVUzWxOt1hiEqEpEJISun0ZZXQsU2lSkJmSjs1+biiYrdp6pRaZOjQWjBwGTHgeunwZMNYBEDiQPAp84CN/tF/asH5+b6N8RNVYpZVJk3bj4v1IjfDlMgLkW0OeKE1yUokTUHzEG69XDqDXYUS7NwXw3D0UnyUMpk8Di8ODCdROG1iVgcJoGyJ4QcM6x8mY0W11QK6QYm6uPwAvoozQZQiIy1lAi6iFRK/NmsxnHjx/H8ePHAQBlZWU4fvw4rl27JmZYsc9QiabrleA5GVTZo6FTd97NnqZVYUK+0Av2j+/rWlcm9GqxOvHdZaHL+vbBKVDKYrs05HPxugn/vFiPGoOt9aA2S7g11YgTVBQTNREdPnwYhYWFKCwsBAAUFxejsLAQq1evFjOs2Fd9DE0WJxriBmFw1s13WZ1ckITkBAUsDg++OF4Np7t1PFGL1QWO4zAgUR3zbUNtXaoz4/DVZlQ2t0lEmgzhlhJRj4laNZsxYwYYDYmPLKcVtuozsDg9qE+6FXPTEm76IzKpBD8anYUPD1WgxmDHloPXMGNoKvKS45GfEo+fTciBWiFt1+Ady/RxQqtai7VNz5kmU7i1tQBOK6CIi3xgUSq2+1lJe9dPo8lkg1WejKSM3G43LCfFK7C4MAvxSqlQmjI7/I8FLBTWT+jjhOpsi9XZelCubh3MSaWiHqFE1J8wBlQfR6PFgesJwzEkvWdVqUydGg9Nyccdt6S27zXqZxI7KhEBre1ExuoIRxTdKBH1J4YK8JYGyORKNGuGYFBaz5foUMmlGJ+XiExd/05EvhKR2eEOaDODLlu4NVSIEFX06l/l6f6u+hgkHIdbx07B4EHDoZDR36FgqeRSqORS2F0etNicSNOohAf0ecKtoUqYCiOlX7HuoE9if+EwCRNXASBzLCWhEEj0looMbatncUlCIzXvpnaiHqB03V9UHoLb7YIrYQDU2kyxo4kJc0ZkQCGT+DddBCDs6KHLEZJ+yzVAT3Mhu4P+LPYHLjtQfQyNZic+rM3CzjO1YkcUE5LiFUhQytoPW/BVz1rKIx9UlKJE1B9UHQbcTtS6E9CszvP3+JAwSSoQblsqhMnF5KYoEcU6hxm4th9unscp+UiA4zAoNbYWtBeLh2fYd6kBnx+vgsvTpucsLkn4YjzQVCZegFGEElGsu7Ib8LjQIElBvWogEuPkSIqnElEoSDjgdJUBV+otaLI4Ax9MGiTcNl2OfGBRiBJRLKu/IGwPxHE4GzdRKA2lJfSrqRjhxHEcUhKE9bHqjDdUwZK9iajxctdrfRMAlIhil7VJWNYVgCtzAs5ZhVHUg7sxt4x0X7pWGD8UMAsfEJYBkasAp4UarbuBElEsshuAU58IS7lq0nFFMx5ONw+dWo4M7y8OCY0svS8R2QMfkEiB1GHC93VnIxxV9KFxRLGmpQI4s02Y/a3SAqOWoEAaj/kSKRgDVctCzDfVpcnihM3pgbrtmKL0EUD1caD+e2DIXBpl3QUqEcWSqqPAiS1CEkpIBcY+ACg1UMgkGJahxfDM/rNeUKSoFVJ/43/1jdUzXY7wx8DtBOrPiRBd9KBEFAt4D3B+B3Bhp/B92jCg8CFArRc7sn4hJ0kNhUwCm/OG3XA5DsgSFv1D5SHajroLVFaMdg4zcOZTwFApfPAL7gBypwAcB8YYth2tQl5yHEZn62l+WZjcNigF029Jg1TSQbU3cyxQvg8wXRdm5NNa1h2iT2Y0M9YAR98TkpBMAYz8KZB3m5CQAFxttOJakxUHrzaJHGhsU8mlHSchQJgAmz5K+L5sL5WKOkGJKFrVngKOvQ/YjcIo3nFFQMpg/8OMMRwsE3ZcHZGlo9JQBDDGYLC52j+QdxsgkQkdCQ0XIx9YFKBPZ7ThPcDFXcC5/xWWmkgeDIxbCsQnB5xW3mhFdYsdMgmH8XmxvRd9X2CwuvDev65iy8Fr8PA3lHpUWiBnovD9pV3CJGQSgBJRNHGYhV6xysPC/fypwKifCgPn2nB7eOy5IGx9PCZH3+/WkxaDRiWDy8Ngc3pwrsbY/oTcKULngd0IXNhBVbQbUCKKFg0XgcPvCMV7mQIY+ROhYbqDcUHfXWlEk8WJeKUUkwqSRAi2/5FIOIzzljz3X2kMnAQLADIlMPxugJMAdeeAsj0iRNl39YlE9OabbyI/Px8qlQqTJ0/GwYMHxQ6p77AbgbNfCCOlfeODxhUBqbd0eLrF4cbJSgMA4M5haTG//XNfMiZbB41KBpPdjd3n69ufoBsA3DJH+L78u9bhFkT8RPThhx+iuLgYa9aswdGjRzFmzBjMnTsXdXV1YocmLmuT0BZ08C3g+hnhWM4kIQnd0B7UVrxShp9NyMG0ISnC9tAkYmRSCebcmgHOOyt/74V68De2F2UVAoNnCyXZqqPA4XeF0m4/nxjLMZF3OJw8eTImTpyIv/71rwAAnueRk5ODX//613j22We7/Fmj0QidTgeDwQCtNkpHDfM84LIKkyMt9YD5OtB0BbA0tJ6jyxY+vG2WeGWMwe7i0WJz4lqjFQzADwZ2nqBI5By91ow93hLR/FEZGJYhfDYZY61TbBouAee3C6VcQGjQTh4svNfqJKE9SaoEJKKXFXqlu7+jorZiOp1OHDlyBKtWrfIfk0gkmD17Nr777rt25zscDjgcrcstGI0dNAp2pKkMuPx1xw2EAcdY58cCjnd0rPX42RojbA43GAAOrM0pPORSCcZk64VzGcOFqgaYHYH7yTMGgONgjs/F9DvnA4kFAMdh+8kaVDRbwRjg4Xm4PK3PLZdyGJujp6pYHzAuNxFxCim+u9yIQamtqx3sPFOLsgYr5FIOEk4KBZuLdPMxJBvPQcq3YLTNAK7qKACgvMmCFqsLvEQBj0QGQALGScDAAeAwLj8ZMu/YpSsNFtR7N7xUy6UYPUAXmRc6+l4hgYaAqImooaEBHo8H6enpAcfT09Px/ffftzt/3bp1WLt2bc+fyO0AzB3U2cPEbTPC6eik7i/lAJfSf9fl4eFwM7glSljletjkSTAp09GsyoFUGYfpSQP95zo9nnbTCOKVUmTo1BiYEt/5oDoSccMytBiSpgl4T8wOD+wuD+z+oUYS1KvGQ6IYA529EqMGMHDWesDaCKfbBKvTA8DW0eUBC+8vLTGzGbxJSERMIQUsHYxlCgcWuupkVPXrrlq1CsXFxf77RqMROTnd2CVBnwOMuU/43t/L1OaXNqDniev8WDd/PmOoQ+g1aXOu71uphAO06tZzHRySJSqAk/jP4bz/cAhMLHcOS4fb2xsjk0gQr5RCJo3uonssu/EPw49GZ8LicMPDM3gYg4dvW1rOB5eo9n9mEo1WqG1WcG47wLsAxsAxHgAPMAaJRiEsEQkgyeKEyun9XEgBdGepl1C0yChCt+SwqIkoJSUFUqkU169fDzh+/fp1ZGRktDtfqVRCqVS2O35TivjWBc0joCdtxMk9eC91annPgyF9hm9Txu5I0cYB2rhunZuYCET7kFVR/5wqFAqMHz8eX3/9tf8Yz/P4+uuvMWXKFBEjI4REkuhVs+LiYixduhQTJkzApEmT8MYbb8BiseDhhx8WOzRCSISInojuvfde1NfXY/Xq1aitrcXYsWOxY8eOdg3YhJDYJfo4ot6IiXFEhMSw7v6OUpcLIUR0lIgIIaKjREQIEZ3ojdW94Wve6vZUD0JIRPl+N2/WFB3VichkMgFA90ZXE0JEYzKZoNN1PgcuqnvNeJ5HdXU1NBpNn9w40DcFpaKiIuZ69ei1RadIvzbGGEwmE7KysiDpYiWBqC4RSSQSZGdnix3GTWm12pj7QPvQa4tOkXxtXZWEfKixmhAiOkpEhBDRUSIKI6VSiTVr1gS3YkAfR68tOvXV1xbVjdWEkNhAJSJCiOgoERFCREeJiBAiOkpEhBDRUSLqoZ7uSvvxxx9j2LBhUKlUGDVqFP7v//4v4HHGGFavXo3MzEyo1WrMnj0bFy9eDOdL6FRPXtvbb7+NadOmITExEYmJiZg9e3a784uKisBxXMDXvHnzwv0yOtST11ZaWtoubpUqcEH6aH3fZsyY0e61cRyHBQsW+M8R5X1jpNu2bt3KFAoFe/fdd9mZM2fYo48+yvR6Pbt+/XqH5+/bt49JpVL28ssvs7Nnz7L/9//+H5PL5ezUqVP+c9avX890Oh377LPP2IkTJ9jdd9/NCgoKmM1mi9TLYoz1/LXdf//97M0332THjh1j586dY0VFRUyn07HKykr/OUuXLmXz5s1jNTU1/q+mpqZIvSS/nr62kpISptVqA+Kura0NOCda37fGxsaA13X69GkmlUpZSUmJ/xwx3jdKRD0wadIktmLFCv99j8fDsrKy2Lp16zo8/2c/+xlbsGBBwLHJkyezxx9/nDHGGM/zLCMjg/37v/+7//GWlhamVCrZli1bwvAKOtfT13Yjt9vNNBoNe++99/zHli5dyhYtWhTqUHusp6+tpKSE6XS6Tq8XS+/b66+/zjQaDTObzf5jYrxvVDXrJt+utLNnz/Yf62pXWgD47rvvAs4HgLlz5/rPLysrQ21tbcA5Op0OkydP7vSa4RDMa7uR1WqFy+VCUlJSwPHdu3cjLS0NQ4cOxb/927+hsbExpLHfTLCvzWw2Iy8vDzk5OVi0aBHOnDnjfyyW3rd33nkH9913H+LjA/e1ivT7Romom7ralba2trbDn6mtre3yfN9tT64ZDsG8ths988wzyMrKCvilmDdvHjZt2oSvv/4aL730Evbs2YP58+fD4+lkF9wwCOa1DR06FO+++y4+//xzvP/+++B5HrfddhsqKysBxM77dvDgQZw+fRrLly8POC7G+xbVs+9J37B+/Xps3boVu3fvDmjUve+++/zfjxo1CqNHj8agQYOwe/duzJo1S4xQu2XKlCkB++rddtttGD58ON566y288MILIkYWWu+88w5GjRqFSZMmBRwX432jElE39XRXWgDIyMjo8nzfbU+uGQ7BvDafV155BevXr8dXX32F0aNHd3nuwIEDkZKSgkuXLvU65u7qzWvzkcvlKCws9McdC++bxWLB1q1b8cgjj9z0eSLxvlEi6qZgdqWdMmVKwPkAsGvXLv/5BQUFyMjICDjHaDTiwIEDEd3pNtgdd19++WW88MIL2LFjByZMmHDT56msrERjYyMyMzNDEnd3hGI3YY/Hg1OnTvnjjvb3DRCGlTgcDvziF7+46fNE5H2LaNN4lNu6dStTKpWstLSUnT17lj322GNMr9f7u3YffPBB9uyzz/rP37dvH5PJZOyVV15h586dY2vWrOmw+16v17PPP/+cnTx5ki1atEi0buCevLb169czhULBPvnkk4BuXpPJxBhjzGQysZUrV7LvvvuOlZWVsb///e9s3LhxbMiQIcxut/fp17Z27Vq2c+dOdvnyZXbkyBF23333MZVKxc6cORPw+qPxffO5/fbb2b333tvuuFjvGyWiHvrLX/7CcnNzmUKhYJMmTWL79+/3PzZ9+nS2dOnSgPM/+ugjdssttzCFQsFGjBjBtm/fHvA4z/PsD3/4A0tPT2dKpZLNmjWLnT9/PhIvpZ2evLa8vDwGoN3XmjVrGGOMWa1WNmfOHJaamsrkcjnLy8tjjz76aLvxOJHSk9f29NNP+89NT09nd911Fzt69GjA9aL1fWOMse+//54BYF999VW7a4n1vtEyIIQQ0VEbESFEdJSICCGio0RECBEdJSJCiOgoERFCREeJiBAiOkpEhBDRUSIihIiOEhGJKqWlpdDr9f77zz33HMaOHeu/X1RUhMWLF0c8LtI7lIhIh3zrFv/yl79s99iKFSvAcRyKiooCzg91AsjPz8cbb7wRcOzee+/FhQsXOv2ZDRs2oLS01H9/xowZePrpp0MaFwk9SkSkUzk5Odi6dStsNpv/mN1ux+bNm5GbmytKTGq1GmlpaZ0+rtPpAkpMJDpQIiKdGjduHHJycrBt2zb/sW3btiE3NxeFhYW9unZHJZXFixf7S1kzZsxAeXk5fvOb3/h3kgDaV81u1LZkVlRUhD179mDDhg3+a5SVlWHw4MF45ZVXAn7u+PHj4DguomslkVaUiEiXli1bhpKSEv/9d999Fw8//HDYn3fbtm3Izs7G888/j5qaGtTU1PT4Ghs2bMCUKVPw6KOP+q+Rm5vb7jUBQElJCe644w4MHjw4VC+B9AAlItKlX/ziF/j2229RXl6O8vJy7Nu3r1uLafVWUlISpFIpNBoNMjIyglr5UKfTQaFQIC4uzn8NqVSKoqIinD9/3r//l8vlwubNm7Fs2bJQvwzSTbRmNelSamoqFixYgNLSUjDGsGDBAqSkpIgdVq9kZWVhwYIFePfddzFp0iT8z//8DxwOB5YsWSJ2aP0WlYjITS1btgylpaV47733QlZqkEgkuHEpLJfLFZJrd8fy5cv9DfElJSW49957ERcXF7HnJ4EoEZGbmjdvHpxOJ1wuF+bOnRuSa6ampga0+3g8Hpw+fTrgHIVC0estbDq7xl133YX4+Hhs3LgRO3bsoGqZyKhqRm5KKpXi3Llz/u87YzAYcPz48YBjycnJyMnJaXfunXfeieLiYmzfvh2DBg3Ca6+9hpaWloBz8vPzsXfvXtx3331QKpVBVQnz8/Nx4MABXL16FQkJCUhKSoJEIvG3Fa1atQpDhgyJ6KL3pD0qEZFu0Wq10Gq1XZ6ze/duFBYWBnytXbu2w3OXLVuGpUuX4qGHHsL06dMxcOBAzJw5M+Cc559/HlevXsWgQYOQmpoaVNwrV66EVCrFrbfeitTUVFy7ds3/2COPPAKn0xmRXkDSNVqzmvRb//znPzFr1ixUVFS02y2VRBYlItLvOBwO1NfXY+nSpcjIyMAHH3wgdkj9HlXNSL+zZcsW5OXloaWlBS+//LLY4RBQiYgQ0gdQiYgQIjpKRIQQ0VEiIoSIjhIRIUR0lIgIIaKjREQIER0lIkKI6CgREUJE9/8B2f0996LWQd4AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:07:11.113345Z", + "iopub.status.busy": "2024-03-26T07:07:11.112987Z", + "iopub.status.idle": "2024-03-26T07:07:11.438261Z", + "shell.execute_reply": "2024-03-26T07:07:11.437073Z" + }, + "papermill": { + "duration": 0.343203, + "end_time": "2024-03-26T07:07:11.440509", + "exception": false, + "start_time": "2024-03-26T07:07:11.097306", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABAjElEQVR4nO3deXxU9b34/9eZfZLMTBLIShIS9n2T5SLuckW0Fmvr0rqAqG2vtPdaftwq7S1WfViwVStt/dFNQK8i1dattRWpFfAiILIJyE4IARKykWQymcx2zvePMzPJkIUsM3Myyef5eIRkzpyceR8meeezfyRFURQEQRA0pNM6AEEQBJGIBEHQnEhEgiBoTiQiQRA0JxKRIAiaE4lIEATNiUQkCILmRCISBEFzBq0D6AlZljl37hw2mw1JkrQORxCEiyiKgtPpJDc3F52u/XJPQieic+fOkZ+fr3UYgiBcQmlpKXl5ee0+n9CJyGazAepN2u12jaMRBOFi9fX15Ofnh39X25PQiShUHbPb7SIRCUIvdqmmE9FYLQiC5kQiEgRBcyIRCYKguYRuI+oMRVHw+/0EAgGtQ+nXjEYjer1e6zCEXqpPJyKv10tZWRmNjY1ah9LvSZJEXl4eKSkpWoci9EJ9NhHJskxxcTF6vZ7c3FxMJpMY9KgRRVGorKzkzJkzDB8+XJSMhFb6bCLyer3Iskx+fj5JSUlah9PvZWRkcOrUKXw+n6aJSFEUPiuuoaS6kTG5dsYNcmgWi9CszyaikI6GlQvx01tKo3tKa/n0RDUAZ2vdpCYZyUsTf6i0Jn5LhX7D65f5rLgGAJNB/dHffbpWw4iEkD5fIhKEkOMVDbi9ARxWI1+bPIjTNY2Myul46oEQH6JE1Atdc801PPLII1qH0eccq3ACMDrHTlqyiYn5qZgNouG8N9A8EZ09e5Z77rmHAQMGYLVaGT9+PJ9//rnWYSWETZs2IUkStbW1WoeSEMbm2hmRZWNElhhC0NtoWjW7cOECs2bN4tprr+Uf//gHGRkZHDt2jLS0NC3DEvqoYZk2hmU2V8UavX72n6nD7QtwzchMDSMTNC0RPfPMM+Tn57NmzRqmT59OUVERN9xwA0OHDo3p63r9crsf/oDc6XN9nTi3pzweD48++ij5+fmYzWaGDRvGSy+9xKlTp7j22msBSEtLQ5IkFixYcMnrOZ1O7r77bpKTk8nJyeGXv/xlq6rg//7v/zJ16lRsNhvZ2dl861vfoqKiIvx8qCT20UcfMXXqVJKSkrj88ss5cuRIj+83ngKywqcnqtlXWtfqvRTiS9MS0XvvvcecOXO4/fbb2bx5M4MGDeLhhx/moYceavN8j8eDx+MJP66vr+/W67748fF2nysamMytkweFH/9+ywl8gbZ35c5Ls3L71OaF2VZvLcbtjZxK8oN/H9GtGEPuu+8+tm3bxq9+9SsmTpxIcXExVVVV5Ofn85e//IWvf/3rHDlyBLvdjtVqveT1Fi9ezNatW3nvvffIyspi2bJl7N69m0mTJoXP8fl8PPXUU4wcOZKKigoWL17MggUL+Pvf/x5xrR//+Mc899xzZGRk8N3vfpeFCxeydevWHt1vrBwuryc92URGijk8lCDFbCDZrMflCVDV4CHHcen/PyE2NE1EJ0+eZNWqVSxevJgf/ehH7Ny5k//8z//EZDIxf/78VucvX76cJ554QoNItXH06FHeeOMNNm7cyOzZswEYMmRI+Pn09HQAMjMzSU1NveT1nE4nL7/8MuvWreP6668HYM2aNeTm5kact3DhwvDXQ4YM4Ve/+hXTpk2joaEhYorG008/zdVXXw3AY489xs0330xTUxMWi6V7NxwjXr/MhgPnkRWFhVcU4bAaAXVs04BkMy5PI9UNXpGINKRpIpJlmalTp/Kzn/0MgMmTJ3PgwAF++9vftpmIli5dyuLFi8OPQ6u/ddWia4e1+5zuonF3376q/WrixWP0Fs4q6nIsHdm7dy96vT78y95TJ0+exOfzMX369PAxh8PByJEjI87btWsXP/3pT9m3bx8XLlxAltVqy+nTpxkzZkz4vAkTJoS/zsnJAaCiooKCgoKoxBst5+ubkBUFm8UQTkIh6ckmTtc0Utvo0yg6ATRORDk5ORE/2ACjR4/mL3/5S5vnm81mzGZzj183NJhNy3M7ozNVrWhzuVzMmTOHOXPm8Nprr5GRkcHp06eZM2cOXq834lyjsfmXOlTdCSWt3uR8fRMA2Y7WJbXUJPUeahq9rZ4T4kfTxupZs2a1auA8evQogwcP1iii3mX8+PHIsszmzZvbfN5kMgF0eomTIUOGYDQa2blzZ/hYXV0dR48eDT8+fPgw1dXVrFixgiuvvJJRo0ZFNFQnogqn2q6YaWudiNKT1f/DCy6RiLSkaSL6wQ9+wPbt2/nZz37G8ePHWbduHb///e9ZtGiRlmH1GoWFhcyfP5+FCxfyzjvvUFxczKZNm3jjjTcAGDx4MJIk8be//Y3KykoaGho6vJ7NZmP+/Pn893//Nx9//DEHDx7kgQceQKfThUs0BQUFmEwmfv3rX3Py5Enee+89nnrqqZjfayyFSkRZ9tal6bRgIqp3+5DltjslhNjTNBFNmzaNt99+m9dff51x48bx1FNP8cILL3D33XdrGVavsmrVKr7xjW/w8MMPM2rUKB566CFcLhcAgwYN4oknnuCxxx4jKyuL733ve5e83vPPP8/MmTP5yle+wuzZs5k1axajR48ONzBnZGSwdu1a3nzzTcaMGcOKFSt49tlnY3qPsdTkC4Tbf9oqEdnMBhbOKuLha4ehu7iBUIgbSVGUhP0zUF9fj8PhoK6urtUuHk1NTRQXF1NUVNTrenF6E5fLxaBBg3juued44IEHYvY6Wr0fpTWN/HnXGexWIw9cEd3OBOHSOvodbUlMeu1n9uzZw+HDh5k+fTp1dXU8+eSTAMybN0/jyGIjy27h61Py8Iqlgns1kYj6kIu71y/25ZdfAvDss89y5MgRTCYTl112GZ988gkDBw6MV5hxZTLoKBjQ8XpDxyucHK9wMXhAEqNzxP54WhCJqA/Jzc1l7969HT5fUFDArl274hdUAqhwejhUVo9BJ4lEpBGRiPoQg8HAsGHtD9bsbxRFYduJatKSTYzIsqFvpzHablHHEjk9YlCjVkQiEvqserefHcU1GHQSI7PaXwAtnIia/PEKTbiI5usRCUKsVLnUgYxpyaYOu+ZtFvXvsbPJTwJ3Iic0kYiEPqu6QR0tPTDF1OF5KcFE5PXLeKKwdIvQdSIRCX1WdYNaIkpP7nh+olGvC88TbPSKbn4tiEQk9FnVwfljAy5RIgJ1bSJQV20U4k80Vgt9kiwr1AQT0cBLlIgAbp+ah0mvw6AXf5u1IP7Xe6HetotHb4unM2rdPgKyglEvYbde+u9tkskgkpCGRImoj/J6veFlQvqjtCQjC2cV4fT4es0us0L7+tefAEUBvzf+H13oEl6wYAGbN29m5cqVSJKEJEmcOHGCBx54gKKiIqxWKyNHjmTlypWtvu/WW2/l6aefJjc3N7zq4qeffsqkSZOwWCxMnTqVd955B0mSIkZgHzhwgLlz55KSkkJWVhb33nsvVVVV7cZz6tSpHr8VsSZJEo4ubCddWtPIBwfK2VVSE+PIhLb0rxJRwAefPBf/173y/wND50onK1eu5OjRo4wbNy48ITUtLY28vDzefPNNBgwYwKeffsq3v/1tcnJyuOOOO8Lf+9FHH2G329m4cSOgzny+5ZZbuOmmm1i3bh0lJSWtqli1tbVcd911PPjgg/zyl7/E7Xbz6KOPcscdd/Cvf/2rzXgyMjKi8J/Su9Q3+ThUVo/bl8Rlg9O1Dqff6V+JKAE4HA5MJhNJSUlkZ2eHj7fcNKCoqIht27bxxhtvRCSi5ORk/vjHP4arZL/97W+RJIk//OEPWCwWxowZw9mzZyN2SfnNb37D5MmTw+uGA6xevZr8/HyOHj3KiBEj2oynt/v0RBUSEmMH2cMjpzuSbFJ/FVwe0X2vhf6ViPRGtXSixev20Isvvsjq1as5ffo0brcbr9cbsQUQqEvLtmwXOnLkCBMmTIhY/6flwvkA+/bt4+OPP47YnSPkxIkTjBjRs+2QtKAoCntLa/H4ZIZlpkAnlj9KMqtbT7s8ovteC/0rEUlSp6tIvcn69etZsmQJzz33HDNnzsRms/GLX/yCHTt2RJyXnJzc5Ws3NDRwyy238Mwzz7R6LrQzR6Jx+wJ4fOoI6dDi+JcSKhG5fQFkWRGrNcZZ/0pECcJkMkUsiL9161Yuv/xyHn744fCxEydOXPI6I0eO5NVXX8Xj8YR3P2m5cD7AlClT+Mtf/kJhYSEGQ9s/DhfH09tdCC4Na7MYMF7cJa8oUFsCkg4c+eE9oaxGPZKkPt3oC4QHOArx0b96zRJEYWEhO3bs4NSpU1RVVTF8+HA+//xzNmzYwNGjR/nJT37SKqG05Vvf+hayLPPtb3+bQ4cOsWHDhvD606Eu7UWLFlFTU8M3v/lNdu7cyYkTJ9iwYQP3339/OPlcHE9v3DKopdCOHKEdOsIUBQ6+BXtfhz2vwaG/hns0dToJq1Gtnl28W68QeyIR9UJLlixBr9czZswYMjIymDNnDrfddht33nknM2bMoLq6OqJ01B673c5f//pX9u7dy6RJk/jxj3/MsmXLAMLtRrm5uWzdupVAIMANN9zA+PHjeeSRR0hNTUWn07UZz+nTp2N381FwIbhHWVrSRYno7G6oPKqWgiQJzh+EikPhp60mNRE1+UQiijexeH4/89prr3H//fdTV1cX1w0c4/l+vLfvHCcqGrhmZAaTC9LUgwE/bP//weuC4TeA3w3Fn0BSOkz/NkgSbm8Ao14SI6yjSCyeLwDwyiuvMGTIEAYNGsS+ffvCY4S02EU2XurcahtRRImo6qiahMw2yJ0EcgBOb4fGGqgrhdSCcIlIiD9NU/9Pf/rT8Gjd0MeoUaO0DKnPKS8v55577mH06NH84Ac/4Pbbb+f3v/+91mHF1LemF7Dg8kJyU1sk2wp14wCyx4NOr/aeZo4OPneo9UWEuNK8RDR27Fj++c9/hh+313MjdM8Pf/hDfvjDH2odRlzpdVJ4B1cA/B6oOal+ndlil5OBI6DsC6g+AYpCcXUjR8rryXFYmZifGteY+zvNf+sNBkNCjdgVElDtabUqZk2DlBbTU1IHg84ATXXQWM2FRj2HypzICiIRxZnmrXLHjh0jNzeXIUOGcPfdd3fYI+PxeKivr4/4uJQEbovvU+L1Phw97+SDA+UcO+9sPnjhlPo5rTDyZIMJHHnq17UlovteQ5omohkzZrB27Vo++OADVq1aRXFxMVdeeSVOp7PN85cvX47D4Qh/5Ofnt3tto1EdUdvY2BiT2IWu8XrVLnW9PrYNwmcuNHKorJ4Kp6f5YHuJCJoTUd2Z5kQkuu/jTtOq2dy5c8NfT5gwgRkzZjB48GDeeOONNvdhX7p0KYsXLw4/rq+vbzcZ6fV6UlNTqaioACApKUmsS6MRWZaprKwkKSkp5m2AF1xqj1l4aoenAVxV6rihtMGtvyGciM5izRPjiLSieRtRS6mpqYwYMYLjx4+3+bzZbA5PVeiMUNtTKBkJ2tHpdBQUFMT8j0FoMGNqqOu+/pz6OXkgGNsYsmDPVZNUUx0WWS09u70BFEURf7jiqFclooaGBk6cOMG9994bletJkkROTg6ZmZn4fGIXTy2ZTKbwSO1Y8QdkGoKz51OtwRKRM5iIbLltf5PBDEkDwFWF1aMuBueXFXwBBZNBJKJ40TQRLVmyhFtuuYXBgwdz7tw5Hn/8cfR6Pd/85jej+jp6vT7mbROC9uqb/CgKmAw6kkKDE+vL1M/2DlYSSMkEVxVGdyUGXSZ+WcHtC4S3GBJiT9NEdObMGb75zW9SXV1NRkYGV1xxBdu3b++TKwAKsRcaUe2wGtVqlaKAM5iIbB0loiw4/yWSq4KFV8zAYtSjF8uAxJWmiWj9+vVavrzQx7g8fiRJTUQAuC+ogxl1Bkju4I9bSqb6uaGCZLH8hybE/7rQZ4wb5GBUtg1fIDhmKdRQbctSp3W0JzmYiNwX1M0OEnDxvEQnEpHQpxj0OgyhnNNwXv2ccomR++YUMCWD18WRkycp9joYnmVjaEbr5XOF2BCtcULf5VJ7wUgeeOlzg+fUVpVxqMxJRb3nEt8gRJNIREKfoCgKb35eyoaD5Xj8wQGJrkr1c6gNqCNJAwBIDqjThpr8YlBjPIlEJPQJTo+fMxfcHC5zYtTpwOcGT3CqUEcN1SFWdS8zq78OAI8YXR1XIhEJfUJdcMF8u9Wg7sARKg1Z7OqgxUtJCiYiXy0AHn/vXpe7rxGJSOgTQmOIwnPMQokouRPVMghXzcz+OlBkMd8szkQiEvqEcCKyBrveG0KJqBMN1QAWB+gMGFAw+500+USJKJ5EIhL6hNpw1eziElEnR+lLEiSlYdBLWP21zQ3eQlyIcURCn1DrDs26NwZ3SQx13XdhulDSAKzOCu4aloypaEgMohTaIxKR0CfIstI8vcPnBl+T+kSwEbpTrOnoJAlrwAlirllciUQk9An3zizEH5DRSRI4z6oHzTbQGzt/EWuq+tldG+3whEsQbURCn2HQ69Sue/cF9UBXSkOgNlgDJ0vP8MGBsvCQACH2RCIS+p7GGvWzNa1r32dJBaDuQhWHztVR3yQSUbyIqpmQ8L44U8vhciejs+2Mz3M0l4isXSwRme0g6TDqFEyBRtFzFkeiRCQkvEqnh7MX3Dg9wRKMO1gi6mrVTKcDiwODTsLirxdjieJIJCIh4YWqUHZLsOs+XCLqYtUMwJqKQSdh9jtFiSiORCISEp6zSV0w324xgtelLm4mSeE2ny6xpKLX60SJKM5EIhISmqIo1Aend9gshuZqmdkO+m40gbYoEYn5ZvEjEpGQ0Jp8cnhpWDURdbPrPsSSGm4jEjPw40f0mgkJLdQ+lGI2YNDrut91H2JNJcNm5up0I/qxl1hiVogakYiEhOb1y6SYDditwR/l7nbdh1gcGHQ6CDSB7AOdWEg/HkQiEhJafnoSD101BEUJ7tzh7mGJyGBRd/Hwe9UVHg0DohOo0KFe00a0YsUKJEnikUce0ToUIQGFN1TsaRuRJOE12CiudrH14MnoBSh0qFckop07d/K73/2OCRMmaB2KkMg8Tgj4QdKF5411h2yyUV7XxKlzZc0lLSGmNE9EDQ0N3H333fzhD38gLa2bxWmh33pnz1ne/LyUSqenuTRkcXS8oeIlGJPUJGb2OUXPWZxonogWLVrEzTffzOzZsy95rsfjob6+PuJD6N/O1bk5c8GtLh/U0/ahIH1SKnoJTIEGPGJQY1xo2li9fv16du/ezc6dOzt1/vLly3niiSdiHJWQKJp8gXCisFmMcL6H7UMhFgd6nQ6zv4EmfwAHXVjTSOgWzUpEpaWl/Nd//RevvfYaFoulU9+zdOlS6urqwh+lpaUxjlLozUJTO6wmPSZDyzFEPUxEZjsGvYRZlIjiRrMS0a5du6ioqGDKlCnhY4FAgC1btvCb3/wGj8eDXh9ZzzebzZjNndijSugXIia7QosxRKk9u7DFjkEnYfK58Pj8PbuW0CmaJaLrr7+e/fv3Rxy7//77GTVqFI8++mirJCQIF4uYY6YozUu89rRqZrKh0+uRFD/eRidg79n1hEvSLBHZbDbGjRsXcSw5OZkBAwa0Oi4IbQnPurcawVMPsl/tLTN3v+seAJ2OIYOykTz16NNF9308aN5rJgjdpddJ6vQOi6G5fciSqi5w1kPm5FRMeh16j+iZjYdeNcVj06ZNWocgJJBZwwYya1hwJ9ezwVHQPey6D7M4oO6MWtISYq5bfzpOnhRD34VeJjy1IzqJqDpgobjKxYkz56JyPaFj3UpEw4YN49prr+XVV1+lqakp2jEJQtc19mB52DbUyVbK65uorqqMyvWEjnUrEe3evZsJEyawePFisrOz+c53vsNnn30W7dgEoV01Li8v/V8x7+4NbqbY0+U/LmIITvPA44zK9YSOdSsRTZo0iZUrV3Lu3DlWr15NWVkZV1xxBePGjeP555+nslL8FRFiq97tC38gy9BUqz4RpRKRMTkVAJ1oI4qLHnUvGAwGbrvtNt58802eeeYZjh8/zpIlS8jPz+e+++6jrKwsWnEKQoTwYEarETx1IAdAZ+jRrPuWTEmpACg+N/g9Ubmm0L4eJaLPP/+chx9+mJycHJ5//nmWLFnCiRMn2LhxI+fOnWPevHnRilMQIkTs3BGe2pGq7t4RBWZrEgGdiYCsoDSJUlGsdav7/vnnn2fNmjUcOXKEm266iVdeeYWbbroJXXD8RlFREWvXrqWwsDCasQpCWOTOHbXqwWh13QMWgw6PPgW9XIPXVYs5JSNq1xZa61YiWrVqFQsXLmTBggXk5OS0eU5mZiYvvfRSj4IThPZEVM3qu7mzawcMeh1+ow18NXgbaxEzHGOrW4lo48aNFBQUhEtAIYqiUFpaSkFBASaTifnz50clSEG4WKhqZrMYmpf/iGKJCODysUWYK+oxyK6oXldorVttREOHDqWqqqrV8ZqaGoqKinoclCB0RFEU7BZjcHqHMXrLf1zE5hiASa9D522I6nWF1rpVImpvHd+GhoZOry0kCN0lSRJ3TMtXH8gBaKpTv45yiQizTf0suvBjrkuJaPHixYD6g7Bs2TKSkpLCzwUCAXbs2MGkSZOiGqAgdKipDhRZ3V46lDii5Fi9DkOVi1SlCrGaemx1KRHt2bMHUEtE+/fvx2Rq3nzOZDIxceJElixZEt0IBaEj7hbtQ1Hqug8padCTXN+EyXSBNEWJ+vWFZl1KRB9//DGgLmC2cuVK7HaxYJQQfztOVnPgXD2T8h1cpgv1mEV/I0S91Q5IBPx+8LrAnBL11xBU3WojWrNmTbTjEIROqw1O7QjIgCc2DdUAZpMRrz6JgOxX55yJRBQznU5Et912G2vXrsVut3Pbbbd1eO5bb73V48AEoT0RgxnrorOFUFssRj0ufTJ+uTbYYN32mDmh5zqdiBwOh7qtb/BrQdBKxBKx7ugPZgwxG3R4DCn45QsgpnnEVKcTUcvqmKiaCVqRZaV5MKNRaU4QsaiaGfR49Cn4vYo6sVaImW4NaHS73TQ2NoYfl5SU8MILL/Dhhx9GLTBBaEuD14+sKOgkiZRAcK0gowVMSR1/YzdYjDq8hhQCsixKRDHWrUQ0b948XnnlFQBqa2uZPn06zz33HPPmzWPVqlVRDVAQWmo5tUPXFN3F0C6WZbcwd+oIxuY6xAJpMdbtFRqvvPJKAP785z+TnZ1NSUkJr7zyCr/61a+iGqAgXCw31UKW3RLT9iEAo15HauoAjHqdGF0dY93qvm9sbMRmU0exfvjhh9x2223odDr+7d/+jZKSkqgGKAgtDUq1cue0AvXBoV3q5xiViACwBMfKeV3BxdfExp+x0O3F89955x1KS0vZsGEDN9xwAwAVFRVikKMQPzEuEQFsL3VzsqYJnz8gSkUx1K1EtGzZMpYsWUJhYSEzZsxg5syZgFo6mjx5cqevs2rVKiZMmIDdbsdutzNz5kz+8Y9/dCckoZ+ImHAdo1n3Le07U0dJgwFvQBbtRDHUrarZN77xDa644grKysqYOHFi+Pj111/P1772tU5fJy8vjxUrVjB8+HAUReHll19m3rx57Nmzh7Fjx3YnNKGPe3V7Cd6Awi2jU8n0udWDMRjMGGI26PAakvEHGkTPWQx1e6fX7OxssrOzI45Nnz69S9e45ZZbIh4//fTTrFq1iu3bt4tEJLSiKAq1jT78soLFH0wKZhsYTB1/Yw9YjOpYooDsFFWzGOpWInK5XKxYsYKPPvqIiooKZFmOeL47O8EGAgHefPNNXC5XuKp3MY/Hg8fTvKNCfb34wehPGr0B/LKCJEFyoFY9GMP2IQCzUV272i/GEsVUtxLRgw8+yObNm7n33nvJyckJT/3ojv379zNz5kyamppISUnh7bffZsyYMW2eu3z5cp544oluv5aQ2ELrVKeYDejd1erBpIExfU2LQU+NIQW/rIgSUQx1KxH94x//4P3332fWrFk9DmDkyJHs3buXuro6/vznPzN//nw2b97cZjJaunRpeHE2UEtE+fn5PY5BSAz17hZzzFzBRJQc/eU/WgqViAJ+kYhiqVuJKC0tjfT06BSJTSYTw4YNA+Cyyy5j586drFy5kt/97netzjWbzZjNYj+F/iq8c4fFCA3BNdPjUCLyGpLxexRRNYuhbnXfP/XUUyxbtixivlm0yLIc0Q4kCCGh5T8cZqV5nerk2CaiCfmp3HHFWPLTreqOr2LX15joVonoueee48SJE2RlZVFYWIjRaIx4fvfu3Z26ztKlS5k7dy4FBQU4nU7WrVvHpk2b2LBhQ3fCEvo4u9VIjsNChs4FigJGK5iSY/qaKWaDuiCaOQl8TWqpSGy2GHXdSkS33nprVF68oqKC++67j7KyMhwOBxMmTGDDhg38+7//e1SuL/Qt0wrTmVaYDuUH1AMxLg1FMNvVROQRiSgWupWIHn/88ai8uNgJVuiWxvi0DwE4m3zsP1tHRp3EcD2iwTpGutVGBOryH3/84x9ZunQpNTXqUPvdu3dz9uzZqAUnCCGyrCDLwekdrmAiikOJyO0LsONkDcfrg78qosE6JrpVIvriiy+YPXs2DoeDU6dO8dBDD5Gens5bb73F6dOnw2sVCUK0nHc28cbOM+Q4LNwhh7ruY5+IzAZ1tn0DySgoSKJEFBPdKhEtXryYBQsWcOzYsYidXW+66Sa2bNkSteAEIaTera7MKCm+5r3M4lA1sxjVXxG3LhlZQZSIYqRbiWjnzp185zvfaXV80KBBlJeX9zgoQbhYaAzRQMmp9piZkmLeYwZg0uvQSRJeQwr+gCzaiGKkW4nIbDa3Oc/r6NGjZGSIHgUh+kJjiNKV4NIfKVlx2XlVkiTMRh1efTIBWVGXAmm5FIkQFd1KRF/96ld58skn8fnUHw5Jkjh9+jSPPvooX//616MaoCAA1AUTUaq/RSKKE7NBh1efhE9GXaXR64rba/cX3UpEzz33HA0NDWRkZOB2u7n66qsZNmwYNpuNp59+OtoxCkI4Edk0SEQWox5F0uMzBHcKEdWzqOtWr5nD4WDjxo1s3bqVffv20dDQwJQpU5g9e3a04xMEArKiTnhVZJL9NSAR10Q0Z6y67pb90EBwnlMbrO25cXv9/qDLiUiWZdauXctbb73FqVOnkCSJoqIisrOzURSlR0uCCEJbfAGZEVkpeOsqMPkCoDfGdFXGi6UnBxdeszrURCSWjI26LlXNFEXhq1/9Kg8++CBnz55l/PjxjB07lpKSEhYsWNClZWIFobMsRj1zx+cwb7gBCQmSM0HX7bG43WdWd64Ru75GX5dKRGvXrmXLli189NFHXHvttRHP/etf/+LWW2/llVde4b777otqkIIAgLNM/WzL7vi8KDtb66ak2kVeo4ECEGOJYqBLf1Zef/11fvSjH7VKQgDXXXcdjz32GK+99lrUghMEgCZfQJ3eUX9OPRDn9pmyWjc7TtZQ0hD8uy0aq6OuS4noiy++4MYbb2z3+blz57Jv374eByUILX1woJwX/3WEyrLg5p32QXF9fYtRnebh0gUHUIoSUdR1KRHV1NSQldV+b0VWVhYXLlzocVCC0FJtoxdLUyUmKbgGURwbqkEdRwTgUqzqAa8LAv64xtDXdSkRBQIBDIb2m5X0ej1+v3iDhOiRZYX6Jj8p3gp13pd9UFxGVLcULhEpJtAFf/69oucsmrrUWK0oCgsWLGh33WixxKsQbc4mPwFZIdVbgcmg02T8TqhE5PErYLGrO8w21ce9ZNaXdSkRzZ8//5LniB4zIZqqXR5QFDKVCiRM4MiLewyhpUCafAGwBxORaLCOqi4lojVr1sQqDkFoU43Li9V3AbvOA/qkuDdUg7qlEIBfVgiYbOhBNFhHWbe3nBaEeKh2eXF4zmI16sFRAPr4/8iaDTrump6P2aBHKg+OZRKjq6NKJCKhVxuUaiXTXI1NMkJaoSYxSJJEjiPYY2Z1qJ9F1SyqRCISerVxOSlwoh4C2iWiCBa7+rlJTPOIJpGIhN7twikI+NS9xVIyNQvjcHk9NQ1eRtotDABRIooyDWYOCkLnNHj8uM4cREGBgSPiPn6opYNn69lRXEOFNzgT3+9V9zkTokLTRLR8+XKmTZuGzWYjMzOTW2+9lSNHjmgZktCL7C+9wO7dn1Fc6YKMkZrGEhrU2CTrwRjcMEKUiqJG00S0efNmFi1axPbt29m4cSM+n48bbrgBl0ssxSlAY9lhDHIT5mSb2mOmodCgxiafDJZgg7Xowo8aTduIPvjgg4jHa9euJTMzk127dnHVVVdpFJXQW0hl6gRqc94kbdYfasFqajGo0ZIKzvPQVKtpTH1Jr2qsrqtTeyLS09PbfN7j8URMI2lrJxGhb2iorcTqLEEC7EVTtQ4nnIjcvgAkpaoH3bWaxdPX9JrGalmWeeSRR5g1axbjxo1r85zly5fjcDjCH/n5+XGOUogX5+FNgII/tRCTPfYbKV6KNdhG5PYGS0QgSkRR1GsS0aJFizhw4ADr169v95ylS5dSV1cX/igtLY1jhELcuKpoOvsFAFLRlRoHowonIl8ArKnqQbdY8iZaekXV7Hvf+x5/+9vf2LJlC3l57U9qNJvN7c78F/oIWYbD7+Ns9FJjLWT4oCFaRwRAtsPCndPySTYZgOD0jqZadbNFsWFEj2maiBRF4fvf/z5vv/02mzZtoqioSMtwhN7g5L+g/hx5GanU59/EoFSr1hEBavd9bigW2aEmn4BfXSTNnKJtcH2Apolo0aJFrFu3jnfffRebzUZ5eTmg7ptmtfaOH0AhThQFTm6C0p0ApE+5lSsye0dpqBWdHsx2dZpHU61IRFGgaRvRqlWrqKur45prriEnJyf88ac//UnLsIR483vhy3fh9Hb18dDrIHO0tjG1YV9pLZ8er6LR62/RTlSrZUh9huZVM6Gfa6yBA38BVxVIOjxDb2C7O4+hFxoZlGrtVRt27jxVg7PJz5CMFJIsqUCJ6DmLkl7Tayb0Q7WlsGuNmoRMyTDpmxyQC9ldcoGPj1RqHV0robFEokQUfb2i10zoh+rOwBfr1QZfxyAY+zU8+iR2f6FuGTQpL7VXlYbgoi58MZYoqkQiEuKvqU6tjgX8kD4Ext2GLBn418FyGjx+UpOMjM6xaR1lK0ktp3mkpKoHRYkoKkQiEuJLUeDL98DbCCmZNAy/hS+K6zhR5aLK6UEnScwenYVB3/taDSzh0dVyc4nI41QTqgZL2PYl4n9PiK+zu9Vqmd4I425DkYzsKK4BwGTQccOYLPLTkzQOsm2hqlmj169u9GgwqT1+TXWQPEDj6BKbSERC/Hhd+E98TIPbS+rEG8Cahg24bHAaqUlGhmfawg3CvVHExFdJUktFDRXqVA+RiHpEJCIhbpRTWzlZVkOpz87AwBAmBo9fNSJD07g6a0hGCgNTzKRYgr821tTmRCT0iEhEQny4L1BzfAfVLi+lWTMYZbdoHVGXpZgNpJhb/MokBUtB7hptAupDel+LoNAn+U9to6TSSZ0ljxGjxveaOWQ9Yg2um9VYrW0cfYBIRELseV1UHvscj1+mNmMq04raXviut5NlhT2nL/B/x6rwB2RICiUiUSLqKVE1E2IucGYXZRcaaDBlMHrUWIy9sGu+MyQJPjlWRUBWGJ/nwBGqmnmcau+ZwaRtgAksMX8ihMQR8FF1ZJtaGhowmbGDHFpH1G2SJIUHNYa78I3BKqZoJ+oRkYiE2Crfj+RrBEsq+SMn98qBil2RHGysdnkC6oFQqUhUz3pEVM2E2JFlOLOTTJuFgZNmExiU+GNtQomo0etXDySlqwM0RYN1j4hEJMRO9TG1pGC0oMudiC7BS0MAycGqWbhEFOo5E1WzHkn8nwyh1wqUbKem0YucPbnPNOQmmUJVs1CJSFTNokEkIiE2akupKSvm8HkXb1dmaR1N1CSbgyWillUzUKtmYqG/bhNVMyE2SndQUd9EZdJwBmVlah1N1AzJSCHDZm4eYW1JDS6k7wNvA5h73/IliUCUiIToc1XTWHaE+iY/ZY6JjM21ax1R1KSYDeQ4rNgsRvWA3tC8JIhosO42kYiE6DvzGZVONxesg8nJGdT8S9tXJQd3onVVaRtHAhOJSIjkaQBnuVrV6A53LXLZF1Q6PZyzT2RsbuIOYGyLojRP8/D4gz1nycHVAxoqtAsswYk2IkHl98LxjVC+X210NZigYCbk/xvouvD3qmQrtQ1uqoy5yPZBFA1Mjl3MGpAkie0na2jyBRiVY8OcooeUYBuYSySi7tK0RLRlyxZuueUWcnNzkSSJd955R8tw+i85oK4hXfZFMAmZ1cR0cjN8+bb6fGe4qqB8P3VNfkodUxmT40Cv610L4EdDaD2ihqZgz1moROSqEj1n3aRpInK5XEycOJEXX3xRyzCE09tpPH+cgxVu1nlm8Rfr7VTn/7u6o2nlUTjw1qWTkaLAsQ9BUSgaOYl5V01jYn7fqpaF2II9Zg2hsUTWdNAZ1OqsWCStWzStms2dO5e5c+dqGYLQWEPjsc0cPFfP4bRrqNJnwwU36+ocfL3oK+SWvg/Vx9WdWMfc2n41rfwLuFCi9iINu55Ma+ItfNZZtmCJqL4p2I6m06njiRoq1FJRUmIuc6KlhGqs9ng81NfXR3wIPSMXf8Lx8jqqTIMw5ozj9ql5FA5Mwi8rvHPahHvkvGDJ6Agceb/tqkdDJRz7EFlR8ORdDta0+N9IHIV6AZ2hqhmIdqIeSqhEtHz5chwOR/gjPz9f65ASm6saKr4k22HBnTeLr0waRF5aErdMyCU/PYmrhmdgyRquloQkHZQfgCN/V7fPCV+jKrxR4lkpiz8UD2T7yb49niZUIopIROF2ot63Q20iSKhes6VLl7J48eLw4/r6epGMeuLMZ+iAzKLxfG3c1PDOqga9jq9PGdS802rGCBh9Cxx6T23QrjsLWWPB54ayPRDwIycNYIvvcnxuKWEXPuus5kTUYohDuAtfJKLuSKhEZDabMZvNWofRN/g9cP6g+nXetFbbO7d87PXLyOkjsYy/HQ7/TR1BXLyl+eS0wRwbcB0Vh+sxG3V9aiR1WzJsZu6clh9OSEBz1cxdI1Zr7IaESkRCFJ0/QHFFLSZ7BhkpebT3a1Na08iGg+XkOKzcPGEoTP8OnD8AzjLQGWHAUOS0oWzbru5Zf1lBWnhH1L7KbNCTe/Hi/2ab+uFxQkM5pBZoE1yC0jQRNTQ0cPz48fDj4uJi9u7dS3p6OgUF4o2MGUWh8dTnlNc1UaLPZ65fxtRO8jAbdLg8AY6edzKyIoVhmTbImxpxzp6SC1xo9GE16ZlUkBqHG+ilbNlqIqovE4moizStzH/++edMnjyZyZMnA7B48WImT57MsmXLtAyr76s7Q835M8iSAUveBBzW9ueCZdotTC1Ue8H+dbiieWXCoNpGL9tOqHOsrhg2ELOhb5eGQo6dd/LJsUrK6tzNB+256mdnmTZBJTBNS0TXXHMNihiJGn/n9lDj8lKVNIxhuZfeZXVGUTonKhuobvDy3t5z3DYlD5NB/RtW2+hDkiQGpVn6fNtQS8crGjhc7sRi1JPjCFbTbNnqZ5GIuqxvd28IrXkbcZ87iMsboNI2hmGZKZf8FoNex1cm5GIx6imra+L1z05TUu0CoHBgMndMzeem8TmtGrz7stQktVWttrFFz5ktR/3srgVvY/yDSmAiEfU35w9Q43TTaBxAenZBpxuW05NN3Do5l2SzXi1NNXjCz0UsFNZPpCap1dnaRm/zQaO1eTCnKBV1iUhE/YmiwLm9VLs8nE8ZzfCsrlWlchxW7ptZyFUjMlr3GvUzaW2ViKC5naj+XJwjSmwiEfUndaXIrioMRjMXbMMZmtn1JTosRj2XDU5rbhfpp0IlogaPH69fbn7Ckad+rivVIKrE1b/K0/3duT3oJIkxk2YybOjocIOz0HUWox6LUU+TL0Ct20umLTjJN3Ww+rnurDoVRi9+xTpD/CT2Fx6nOnEVIGeSSEJRkBYsFdW1rJ4lpYMpCWS/aCfqApGu+4szO/H7ffhSBmG152gdTZ9ww9hsTAZdeNNFQN3Rw5GvJv3a05Aq5kJ2hviz2B/4muDcHqobvPypPJcNB8u1jqhPSE82kWI2tB62EKqe1ZbEP6gEJRJRf3D2c/B7KfencME6ONzjI8RIepH6ubZUnVwsXJJIRH2dpwFOb8cvy+w3jgNJYmhG31rQXisBWWHr8Sre3XsWX6BFz1lSuvqhyFBTrF2ACUQkor7u5CYI+KjSDaTSMoS0JCPpyaJEFA06CQ6creNkpYsalzfyyfSh6ueaE/EPLAGJRNSXVR5VtweSJL5MmqaWhjJT+tVUjFiSJImBKer6WBX1F1XBBgQTUfUJkGWEjolE1Fc11qjLugK+nKkcalRHUXdmbpnQeVl2dfxQxCx8UJcBMVrA6xKN1p0gElFf1FQH+/+sLuVqy+Kk7TK8fhmH1Ui2ve/urqGF3NRQImqKfEKnh4xR6tcVX8Y5qsQjxhH1NbWlcPAtdfa3xQ7jb6dIn8xcnR5FQVTLoiw01aXG5cXtDWBtOaYoayyc2wuVh2H4HDHKugOiRNSXnN0N+15Xk1BKBky6G8w2TAYdo7LtjM7pP+sFxYvVpA83/p+7uHrmyFf/GPi9UHlIg+gSh0hEfYEcgCMfwNEN6teZo2DyfWBN1TqyfiE/3YrJoMPtvWg3XEmCXHX1Uc7sFNtRd0CUFROdpwEOvg11Z9Qf/KKroGAmSBKKovDW7rMMHpDEhLxUMb8sRi4fOpCrR2Si17VR7c2ZBCVbwXlenZEv1rJuk/jJTGT1ZbD7ZTUJGUww7hsw+HI1IQGnqhs5XdPIZ6dqNA60b7MY9W0nIVAnwGaNV78u3iJKRe0QiShRle+HPa9CU706infKAhg4LPy0oih8VqzuuDo21yFKQ3GgKAp1bl/rJwZfDjqD2pFQdSz+gSUA8dOZaOQAHNsIh/6mLjUxYBhMmQ/JAyJOK6lu5FxtEwadxGWD+/Ze9L1BXaOPlz89xeufnSYgX1Tqsdghf5r69fGN6iRkIYJIRInE06D2ip35XH1cOAvGf0MdONeCPyCz+ai69fHE/NR+t560FmwWA76Agtsb4FBZfesTCmaqnQdN9XD0A1FFu4hIRImi6hh8/pJavDeYYNzX1YbpNsYFbTtZTY3LS7JZz/SidA2C7X90OokpwZLn9pPVkZNgAQxmGP1VkHRQcQiKN2sQZe/VKxLRiy++SGFhIRaLhRkzZvDZZ59pHVLv0VQPX76njpQOjQ+asgAyRrR5usvj54szdQBcNyqzz2//3JtMzHNgsxhwNvnZdKSy9QmOQTDiBvXrkm3Nwy0E7RPRn/70JxYvXszjjz/O7t27mThxInPmzKGiokLr0LTVWKO2BX32Ozh/UD2WP11NQhe1B7WUbDZwx9R8rhw+UN0eWogbg17HDWOykYKz8rccrUS+uL0odzIMm62WZM/uhs9Xq6Xdfj4xVlI03mp1xowZTJs2jd/85jcAyLJMfn4+3//+93nsscc6/N76+nocDgd1dXXY7Qk6aliWwdeoTo50VULDeag5Ca6q5nMceeoPb4slXhVFocknU+v2crq6EQX4tyHtJyghfnafvsDmYIlo7vhsRmWrP5uKojRPsak6Dkfeb96I0WJXOx4ceWBNV9uT9GbQaV5W6JHO/o5q2orp9XrZtWsXS5cuDR/T6XTMnj2bbdu2tTrf4/Hg8TQvt1Bf30ajYFtqiuHER203EEYcU9o/FnG8rWPNx78sq8ft8aMAEkqLU2SMeh0T81LVcxWFo2eraPBE7ievKIAk0ZBcwNXXzYW0IpAk3v+ijNILjSgKBGQZX6D5tY16iUn5qaIq1gtMKUgjyaRn24lqhmY0r3aw4WA5xVWNGPUSOkmPSZlDVsMeBtQfQi/XMsFdh3R2NwAlNS5qG33IOhMBnQHQoUg6FCRAYkrhAAzBsUsnq1xUBje8tBr1TBjkiM+NTrhTTaBRoGkiqqqqIhAIkJWVFXE8KyuLw4cPtzp/+fLlPPHEE11/Ib8HGtqos8eI312P19NO3V8vgc8cfugLyHj8Cn6dmUZjKm5jOk5zFhcs+ejNSVydPiR8rjcQaDWNINmsJ9thZcjA5PYH1QlxNyrbzvBMW8R70uAJ0OQL0BQeaqSj0nIZOtNEHE1nGD9IQWqshMZqvH4njd4A4G7r8uCSw6UlpaEB2akmIsWkB1cbY5liQYledTKh+nWXLl3K4sWLw4/r6+vJz+/ELgmp+TDxLvXrcC9Ti1/aiJ4nqf1jnfz+7JEetdekxbmhL/U6CezW5nM9EgN0FpB04XOk4D8SkYnlulFZ+IO9MQadjmSzHoM+sYvufdnFfxi+MiEHl8dPQFYIKAoBuWVpuRApzRr+mUmrb8TqbkTyN4HsA0VBUmRABkVBZzOpS0QC6S4vFm/w50IPdGapl2i0yJiit+Swpolo4MCB6PV6zp8/H3H8/PnzZGdntzrfbDZjNptbHb8kU3LzguZx0JU24gFdeC8dVmPXgxF6jdCmjJ0x0J4E9qROnZuWBok+ZFXTP6cmk4nLLruMjz76KHxMlmU++ugjZs6cqWFkgiDEk+ZVs8WLFzN//nymTp3K9OnTeeGFF3C5XNx///1ahyYIQpxonojuvPNOKisrWbZsGeXl5UyaNIkPPvigVQO2IAh9l+bjiHqiT4wjEoQ+rLO/o6LLRRAEzYlEJAiC5kQiEgRBc5o3VvdEqHmr01M9BEGIq9Dv5qWaohM6ETmdToDOja4WBEEzTqcTh6P9OXAJ3WsmyzLnzp3DZrP1yo0DQ1NQSktL+1yvnri3xBTve1MUBafTSW5uLroOVhJI6BKRTqcjLy9P6zAuyW6397kf6BBxb4kpnvfWUUkoRDRWC4KgOZGIBEHQnEhEMWQ2m3n88ce7t2JALyfuLTH11ntL6MZqQRD6BlEiEgRBcyIRCYKgOZGIBEHQnEhEgiBoTiSiLurqrrRvvvkmo0aNwmKxMH78eP7+979HPK8oCsuWLSMnJwer1crs2bM5duxYLG+hXV25tz/84Q9ceeWVpKWlkZaWxuzZs1udv2DBAiRJivi48cYbY30bberKva1du7ZV3BZL5IL0ifq+XXPNNa3uTZIkbr755vA5mrxvitBp69evV0wmk7J69Wrl4MGDykMPPaSkpqYq58+fb/P8rVu3Knq9Xvn5z3+ufPnll8r//M//KEajUdm/f3/4nBUrVigOh0N55513lH379ilf/epXlaKiIsXtdsfrthRF6fq9fetb31JefPFFZc+ePcqhQ4eUBQsWKA6HQzlz5kz4nPnz5ys33nijUlZWFv6oqamJ1y2FdfXe1qxZo9jt9oi4y8vLI85J1Peturo64r4OHDig6PV6Zc2aNeFztHjfRCLqgunTpyuLFi0KPw4EAkpubq6yfPnyNs+/4447lJtvvjni2IwZM5TvfOc7iqIoiizLSnZ2tvKLX/wi/Hxtba1iNpuV119/PQZ30L6u3tvF/H6/YrPZlJdffjl8bP78+cq8efOiHWqXdfXe1qxZozgcjnav15fet1/+8peKzWZTGhoawse0eN9E1ayTQrvSzp49O3yso11pAbZt2xZxPsCcOXPC5xcXF1NeXh5xjsPhYMaMGe1eMxa6c28Xa2xsxOfzkZ6eHnF806ZNZGZmMnLkSP7jP/6D6urqqMZ+Kd29t4aGBgYPHkx+fj7z5s3j4MGD4ef60vv20ksvcdddd5GcHLmvVbzfN5GIOqmjXWnLy8vb/J7y8vIOzw997so1Y6E793axRx99lNzc3IhfihtvvJFXXnmFjz76iGeeeYbNmzczd+5cAoF2dsGNge7c28iRI1m9ejXvvvsur776KrIsc/nll3PmzBmg77xvn332GQcOHODBBx+MOK7F+5bQs++F3mHFihWsX7+eTZs2RTTq3nXXXeGvx48fz4QJExg6dCibNm3i+uuv1yLUTpk5c2bEvnqXX345o0eP5ne/+x1PPfWUhpFF10svvcT48eOZPn16xHEt3jdRIuqkru5KC5Cdnd3h+aHPXblmLHTn3kKeffZZVqxYwYcffsiECRM6PHfIkCEMHDiQ48eP9zjmzurJvYUYjUYmT54cjrsvvG8ul4v169fzwAMPXPJ14vG+iUTUSd3ZlXbmzJkR5wNs3LgxfH5RURHZ2dkR59TX17Njx4647nTb3R13f/7zn/PUU0/xwQcfMHXq1Eu+zpkzZ6iuriYnJycqcXdGNHYTDgQC7N+/Pxx3or9voA4r8Xg83HPPPZd8nbi8b3FtGk9w69evV8xms7J27Vrlyy+/VL797W8rqamp4a7de++9V3nsscfC52/dulUxGAzKs88+qxw6dEh5/PHH2+y+T01NVd59913liy++UObNm6dZN3BX7m3FihWKyWRS/vznP0d08zqdTkVRFMXpdCpLlixRtm3bphQXFyv//Oc/lSlTpijDhw9XmpqaevW9PfHEE8qGDRuUEydOKLt27VLuuusuxWKxKAcPHoy4/0R830KuuOIK5c4772x1XKv3TSSiLvr1r3+tFBQUKCaTSZk+fbqyffv28HNXX321Mn/+/Ijz33jjDWXEiBGKyWRSxo4dq7z//vsRz8uyrPzkJz9RsrKyFLPZrFx//fXKkSNH4nErrXTl3gYPHqwArT4ef/xxRVEUpbGxUbnhhhuUjIwMxWg0KoMHD1YeeuihVuNx4qUr9/bII4+Ez83KylJuuukmZffu3RHXS9T3TVEU5fDhwwqgfPjhh62updX7JpYBEQRBc6KNSBAEzYlEJAiC5kQiEgRBcyIRCYKgOZGIBEHQnEhEgiBoTiQiQRA0JxKRkFDWrl1Lampq+PFPf/pTJk2aFH68YMECbr311rjHJfSMSERCm0LLhX73u99t9dyiRYuQJIkFCxZEnB/tBFBYWMgLL7wQcezOO+/k6NGj7X7PypUrWbt2bfjxNddcwyOPPBLVuIToE4lIaFd+fj7r16/H7XaHjzU1NbFu3ToKCgo0iclqtZKZmdnu8w6HI6LEJCQGkYiEdk2ZMoX8/Hzeeuut8LG33nqLgoICJk+e3KNrt1VSufXWW8OlrGuuuYaSkhJ+8IMfhBdwh9ZVs4u1LJktWLCAzZs3s3LlyvA1iouLGTZsGM8++2zE9+3duxdJkuK6RInQTCQioUMLFy5kzZo14cerV6/m/vvvj/nrvvXWW+Tl5fHkk09SVlZGWVlZl6+xcuVKZs6cyUMPPRS+RkFBQat7AlizZg1XXXUVw4YNi9YtCF0gEpHQoXvuuYf/+7//o6SkhJKSErZu3dqpNWx6Kj09Hb1ej81mIzs7u1sLjjkcDkwmE0lJSeFr6PV6FixYwJEjR8Lb7vh8PtatW8fChQujfRtCJ4mlYoUOZWRkcPPNN7N27VoUReHmm29m4MCBWofVI7m5udx8882sXr2a6dOn89e//hWPx8Ptt9+udWj9ligRCZe0cOFC1q5dy8svvxy1UoNOp+PiFWh8Pl9Urt0ZDz74YLghfs2aNdx5550kJSXF7fWFSCIRCZd044034vV68fl8zJkzJyrXzMjIiGj3CQQCHDhwIOIck8nU450j2rvGTTfdRHJyMqtWreKDDz4Q1TKNiaqZcEl6vZ5Dhw6Fv25PXV0de/fujTg2YMAA8vPzW5173XXXsXjxYt5//32GDh3K888/T21tbcQ5hYWFbNmyhbvuuguz2dytKmFhYSE7duzg1KlTpKSkkJ6ejk6nC7cVLV26lOHDh8d1rWmhNVEiEjrFbrdjt9s7PGfTpk1Mnjw54uOJJ55o89yFCxcyf/587rvvPq6++mqGDBnCtddeG3HOk08+yalTpxg6dCgZGRndinvJkiXo9XrGjBlDRkYGp0+fDj/3wAMP4PV649ILKHRMLBUr9FuffPIJ119/PaWlpa02KRTiSyQiod/xeDxUVlYyf/58srOzee2117QOqd8TVTOh33n99dcZPHgwtbW1/PznP9c6HAFRIhIEoRcQJSJBEDQnEpEgCJoTiUgQBM2JRCQIguZEIhIEQXMiEQmCoDmRiARB0JxIRIIgaE4kIkEQNPf/AJevpNZz5baqAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:07:11.470252Z", + "iopub.status.busy": "2024-03-26T07:07:11.469886Z", + "iopub.status.idle": "2024-03-26T07:07:11.628752Z", + "shell.execute_reply": "2024-03-26T07:07:11.627634Z" + }, + "papermill": { + "duration": 0.176558, + "end_time": "2024-03-26T07:07:11.631582", + "exception": false, + "start_time": "2024-03-26T07:07:11.455024", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEnCAYAAAD4jcW1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAryElEQVR4nO3de1xU5b4/8M8MMMMdIXBAQlBxC1ZAwkGxOmmby9ataXShLEE09k6lbc0xjF+v4JAVmohamZw8oqbt5Lh1m1tNRZR2JaZCuNEQjyjihasKw0WHYeb5/eFh5cgAgw6sWWu+79eLF8xaz6z5LtbwYa1n1nqWhDHGQAghAiPluwBCCLkfFF6EEEGi8CKECBKFFyFEkCi8CCGCROFFCBEkCi9CiCBReBFCBInCixAiSNZ8FzDYdDodrl27BicnJ0gkEr7LIYTcgzGGlpYWDBs2DFJpL/tXjGeff/458/X1ZXK5nIWHh7Off/651/Y3b95kCxYsYJ6enkwmk7HRo0ezvXv3Gv16ly9fZgDoi77oy8y/Ll++3OvfMq97Xnl5eVAqlcjJycH48eOxevVqxMTEoKKiAkOHDu3WvqOjA1FRURg6dCj+9re/wdvbG5cuXcKQIUOMfk0nJycAwOXLl+Hs7GyqVTE7Go0GBw8eRHR0NGxsbPguh5iApWxTlUoFHx8f7m+1J7yGV3Z2NpKSkpCYmAgAyMnJwd69e5Gbm4t33323W/vc3FzcuHEDR48e5Taen59fv16z61DR2dlZ9OFlb28PZ2dnUb/RLYmlbdO+unV4C6+Ojg4UFxcjNTWVmyaVShEZGYmioiKDz9m9ezciIiKwcOFCfPvtt/Dw8MCsWbOwZMkSWFlZGXyOWq2GWq3mHqtUKgB33ggajcaEa2ReutZNzOtoaSxlmxq7fryFV2NjI7RaLRQKhd50hUKBs2fPGnzOhQsXcPjwYbz66qvYt28fzp8/jwULFkCj0SA9Pd3gczIzM5GRkdFt+sGDB2Fvb//gK2Lm8vPz+S6BmJjYt2l7e7tR7QT1aaNOp8PQoUPx5ZdfwsrKCqGhobh69SpWrFjRY3ilpqZCqVRyj7uOp6Ojo0V/2Jifn4+oqCiLOMSwBJayTbuOjvrCW3i5u7vDysoKdXV1etPr6urg6elp8DleXl6wsbHRO0QMDAxEbW0tOjo6IJPJuj1HLpdDLpd3m25jYyPqN0AXS1lPSyL2bWrsuvF2kqpMJkNoaCgKCgq4aTqdDgUFBYiIiDD4nCeeeALnz5+HTqfjpp07dw5eXl4Gg4sQIl68nmGvVCqxfv16bN68GeXl5Zg/fz7a2tq4Tx/j4+P1OvTnz5+PGzduYNGiRTh37hz27t2Ljz/+GAsXLuRrFQghPOG1zysuLg4NDQ1IS0tDbW0tQkJCsH//fq4Tv7q6Wu8MWx8fHxw4cABvv/02goKC4O3tjUWLFmHJkiV8rQIhA6K9vb3bB1ett9Q4WlYJV/eTcLT7rSskICDAIj58upeEMcu6AYdKpYKLiwuam5tF32G/b98+TJ06VdT9I2JVUlKC0NBQo9oWFxdj3LhxA1zR4DH2b1RQnzYSYikCAgJQXFysN62ipgnK7WXIfvExjPEaotfWElF4EWKG7O3tu+1NSS9dh/yHWwh8NBghvg/xVJn5oCFxCCGCROFFCBEkCi9CiCBReBFCBInCixAiSBRehBBBovAihAgShRchRJAovAghgkThRQgRJAovQoggUXgRQgSJwosQIkgUXoQQQaLwIoQIEoUXIUSQKLwIIYJE4UUIESQKL0KIIFF4EUIEicKLECJIdPcgkbj3JqU93aAUsNyblBJxofASibNnzxq8SeknBtqK7SalxDJReInEvTcp7ekGpV1tCRE6Ci+RuPcmpXSDUiJ21GFPCBEkCi9CiCBReBFCBInCixAiSBRehBBBovAihAiSWYTX2rVr4efnB1tbW4wfPx7Hjx/vse2mTZsgkUj0vmxtbQexWkKIOeA9vPLy8qBUKpGeno6SkhIEBwcjJiYG9fX1PT7H2dkZNTU13NelS5cGsWJCiDngPbyys7ORlJSExMREjB07Fjk5ObC3t0dubm6Pz5FIJPD09OS+FArFIFZMCDEHvJ5h39HRgeLiYqSmpnLTpFIpIiMjUVRU1OPzWltb4evrC51Oh3HjxuHjjz/GI488YrCtWq2GWq3mHqtUKgCARqOBRqMx0ZqYn87OTu67mNfTkljKNjV23XgNr8bGRmi12m57TgqFQm+EhLuNGTMGubm5CAoKQnNzM7KysjBx4kScOXMGDz/8cLf2mZmZyMjI6Db94MGDoh5Z4XIrAFjj2LFjuHqa72qIKVjKNm1vbzeqneCubYyIiEBERAT3eOLEiQgMDMR//dd/YenSpd3ap6amQqlUco9VKhV8fHwQHR0NZ2fnQamZD6eqbwBlJzFhwgQED3fjuxxiApayTbuOjvrCa3i5u7vDysoKdXV1etPr6urg6elp1DJsbGzw+OOP4/z58wbny+VyyOXybtNtbGxgY2PT/6IFwtramvsu5vW0JJayTY1dN1477GUyGUJDQ1FQUMBN0+l0KCgo0Nu76o1Wq0VZWRm8vLwGqkxCiBni/bBRqVQiISEBYWFhCA8Px+rVq9HW1obExEQAQHx8PLy9vZGZmQkA+OCDDzBhwgT4+/ujqakJK1aswKVLl/D666/zuRqEkEHGe3jFxcWhoaEBaWlpqK2tRUhICPbv38914ldXV0Mq/W0H8ebNm0hKSkJtbS1cXV0RGhqKo0ePYuzYsXytAiGEB7yHFwAkJycjOTnZ4LzCwkK9x6tWrcKqVasGoSpCiDkzi/AihAAXG9vQpu7scX5lQxv3vavz/l4OcmuMcHcYkPrMDYUXIWbgYmMbJmcVGtX2P/5W1uv8I4snWUSAUXgRYga69rhWx4XAf6ij4Ta31NhTWIRpkyLgYNf99J/z9a14K6+01703MaHwErDeDjOMOcQALOswQwj8hzriUW8Xg/M0Gg1qPYBxvq6iPs/LWBReAmXsYUZfhxiA5RxmEHGh8BKovg4z+jrEACzvMIOIC4WXwPV0mEGHGETseB/PixBC7geFFyFEkCi8CCGCROFFCBEkCi9CiCBReBFCBInCixAiSBRehBBBovAihAgShRchRJAovAghgkThRQgRJLowmxAzoNbehtT2Ki6qKiC1NTwYYWdnJ651XkP5jXKDY7RdVLVCansVau1tAIbHBBMTCi+B6uvN3tcbHbC8N7s5u9Z2CQ4jPsP/O9532y/2f9HjPIcRwLW2EIRCYcLqzBOFl0AZ+2bv7Y0OWNab3ZwNc/BF28U3sSYuBKN6GAa6s7MTP/34E5548gmD/5Aq61uxKK8Uwyb7DnS5ZoHCS6D6erP39UYHLO/Nbs7kVrbQ3fbGCOcxGPtQz8NAX7S+iEC3QINjtOluN0N3uwFyK9uBLtcsUHgJVF9v9r7e6IDlvdmJuNCnjYQQQaLwIoQIEoUXIUSQKLwIIYJE4UUIESQKL0KIIPU7vC5cuDAQdRBCSL/0O7z8/f0xefJkbN26Fbdv3x6ImgghpE/9Dq+SkhIEBQVBqVTC09MTf/7zn3H8uBEXZPVi7dq18PPzg62tLcaPH2/08rZt2waJRIKZM2c+0OsTQoSn3+EVEhKCNWvW4Nq1a8jNzUVNTQ2efPJJPProo8jOzkZDQ0O/lpeXlwelUon09HSUlJQgODgYMTExqK+v7/V5VVVVWLx4MZ566qn+rgIhRATuu8Pe2toasbGx2L59O5YvX47z589j8eLF8PHxQXx8PGpqaoxaTnZ2NpKSkpCYmIixY8ciJycH9vb2yM3N7fE5Wq0Wr776KjIyMjBy5Mj7XQVCiIDd97WNJ0+eRG5uLrZt2wYHBwcsXrwY8+bNw5UrV5CRkYEZM2b0efjX0dGB4uJipKamctOkUikiIyNRVFTU4/M++OADDB06FPPmzcMPP/zQ62uo1Wqo1WrusUqlAnDn2j+NRmPMqpqlzs5O7ruh9eia1ts69rUMMniM2RZ9bVOxbE9ja+93eGVnZ2Pjxo2oqKjA1KlT8dVXX2Hq1KmQSu/sxI0YMQKbNm2Cn59fn8tqbGyEVquFQqE/HItCocDZs2cNPufHH3/Ehg0bUFpaalS9mZmZyMjI6Db94MGDsLe3N2oZ5uhyKwBY48cff8QlwyOoAADy8/MfeBlk4PVnW/S0TcWyPdvb241q1+/wWrduHebOnYs5c+bAy8vLYJuhQ4diw4YN/V10n1paWjB79mysX78e7u7uRj0nNTUVSqWSe6xSqeDj44Po6Gg4OzubvMbBcuaaClllx/Dkk0/ikWHd10Oj0SA/Px9RUVE9jirR1zLI4DFmW/S1TcWyPbuOjvrS7/DKz8/H8OHDuT2tLowxXL58GcOHD4dMJkNCQkKfy3J3d4eVlRXq6ur0ptfV1cHT07Nb+8rKSlRVVWH69OncNJ1Od2dFrK1RUVGBUaNG6T1HLpdDLpd3W5aNjU2Pf9RC0DVGl7W1da/r0dt6GrsMMvD6sy162qZi2Z7G1t7vDvtRo0ahsbGx2/QbN25gxIgR/VqWTCZDaGgoCgoKuGk6nQ4FBQWIiIjo1j4gIABlZWUoLS3lvp599llMnjwZpaWl8PHx6e/qEEIEqt97Xowxg9NbW1tha9v/Qe2USiUSEhIQFhaG8PBwrF69Gm1tbUhMTAQAxMfHw9vbG5mZmbC1tcWjjz6q9/whQ4YAQLfphBBxMzq8uvqNJBIJ0tLS9Dq7tVotfv75Z4SEhPS7gLi4ODQ0NCAtLQ21tbUICQnB/v37uU786urqboeohBBidHj98ssvAO7seZWVlUEmk3HzZDIZgoODsXjx4vsqIjk5GcnJyQbnFRYW9vrcTZs23ddrEkKEzejwOnLkCAAgMTERa9asEfQndYQQ4et3n9fGjRsHog5CCOkXo8IrNjYWmzZtgrOzM2JjY3ttu3PnTpMURgghvTEqvFxcXCCRSLifCSGEb0aF192HinTYaB5uabQAgNNXmw3Ob7ulxskGwPPSTTjYdT9JFwDO17cOWH2EDDS66axAVf5f8Ly7s6yXVtbYcv5En8tykNPbgAiPUe/axx9/nDts7EtJSckDFUSME/3IncunRg11hJ2NVbf5FTXN+I+/lWHlC49hjFfPh/oOcmuMcHcYsDoJGShGhReNVGp+3BxkeDl8eI/zu4ZHGeXhgEe9qZ+SiI9R4ZWenj7QdRBCSL/QdTeEEEEyas/Lzc0N586dg7u7O1xdXXvt/7px44bJiiOEkJ4YFV6rVq2Ck5MT97OxnfeEEDJQjAqvuwcWnDNnzkDVQgghRut3n5eVlZXB25Jdv34dVlbdP7InhJCB0O/w6mkwQrVarTdMDiGEDCSjT63+9NNPAdwZjPC///u/4ej42+1JtFot/vnPfyIgIMD0FRJCiAFGh9eqVasA3NnzysnJ0TtElMlk8PPzQ05OjukrJMQC9HWtKtD39aqWdq2q0eF18eJFAMDkyZOxc+dOuLq6DlhRhFga465VBYy5XtVSrlXt91p2jahKCDGdvq5VBYy7XtWSrlXtd3jNnTu31/m5ubn3XQwhlqqva1UBul71Xv0Or5s3b+o91mg0OH36NJqamvDMM8+YrDBCCOlNv8Pr73//e7dpOp0O8+fP73a3akIIGSgmuTBbKpVCqVRyn0gSQshAM9moEpWVldwxOSGEDLR+HzZ23Tm7C2MMNTU12Lt3r941kIQQMpD6HV5dd87uIpVK4eHhgZUrV/b5SSQhhJgKnedFCBEkGkmVECJIFF6EEEGi8CKECBKFFyFEkEwWXleuXMGf/vQnUy2OEEJ6ZbLwun79OjZs2GCqxRFCSK/M4rBx7dq18PPzg62tLcaPH4/jx4/32Hbnzp0ICwvDkCFD4ODggJCQEGzZsmUQqyWEmAPewysvLw9KpRLp6ekoKSlBcHAwYmJiDN7kA7hzD8n33nsPRUVF+Ne//oXExEQkJibiwIEDg1w5IYRPvIdXdnY2kpKSkJiYiLFjxyInJwf29vY9jgs2adIkPPfccwgMDMSoUaOwaNEiBAUF4ccffxzkygkhfDL6DPvY2Nhe5zc1NfX7xTs6OlBcXIzU1FRumlQqRWRkJIqKivp8PmMMhw8fRkVFBZYvX26wjVqthlqt5h6rVCoAd8Yh02g0/a5ZKLouku/s7BT1eloSS9mmxq6b0eHl4tL7yI0uLi6Ij483dnEAgMbGRmi1WigUCr3pCoUCZ8+e7fF5zc3N8Pb2hlqthpWVFb744gtERUUZbJuZmYmMjIxu0w8ePAh7e/t+1Sskl1sBwBrHjh3D1dN8V0NMwVK2aXt7u1HtjA6vjRs33ncxpubk5ITS0lK0traioKAASqUSI0eOxKRJk7q1TU1N1RsJQ6VSwcfHB9HR0XB2dh7EqgfXqeobQNlJTJgwAcHD3fguh5iApWzTrqOjvvB6mxF3d3dYWVmhrq5Ob3pdXR08PT17fJ5UKoW/vz8AICQkBOXl5cjMzDQYXnK5HHJ599tE2djYwMbG5sFWwIxZW1tz38W8npbEUrapsetmdHgZO9xNf27AIZPJEBoaioKCAsycORPAnSGlCwoKkJycbPRydDqdXr8WIUT8jA6vTZs2wdfXF48//jgYYyYrQKlUIiEhAWFhYQgPD8fq1avR1taGxMREAEB8fDy8vb2RmZkJ4E4fVlhYGEaNGgW1Wo19+/Zhy5YtWLdunclqIoSYP6PDa/78+fjmm29w8eJFJCYm4rXXXoOb24Mfd8fFxaGhoQFpaWmora1FSEgI9u/fz3XiV1dXQyr97YyOtrY2LFiwAFeuXIGdnR0CAgKwdetWxMXFPXAthBDhkLB+7Eap1Wrs3LkTubm5OHr0KP74xz9i3rx5iI6OhkQiGcg6TUalUsHFxQXNzc2i7rAvvXQdM9cdw675ExDi+xDf5RATsJRtauzfaL9OUpXL5XjllVeQn5+PX3/9FY888ggWLFgAPz8/tLa2PnDRhBBirPs+w14qlUIikYAxBq1Wa8qaCCGkT/0KL7VajW+++QZRUVH43e9+h7KyMnz++eeorq6Go6PjQNVICCHdGN1hv2DBAmzbtg0+Pj6YO3cuvvnmG7i7uw9kbYQQ0iOjwysnJwfDhw/HyJEj8f333+P777832G7nzp0mK44QQnpidHjFx8cL5hNFQoj49eskVUIIMRe8j+dFCCH3g8KLECJIFF6EEEGi8CKECBKFFyFEkCi8CCGCROFFCBEkCi9CiCBReBFCBInCixAiSBRehBBBovAihAgShRchRJAovAghgkThJUIdHR34esM63MjPwdcb1qGjo4PvkggxOQovkUlJSYGDgwOylr6PlpI9yFr6PhwcHJCSksJ3aYSYlNGDERLzl5KSghUrVkChUCD62eext4rhj34SHNy9AytWrAAAfPLJJzxXSYhpUHiJRFNTE7Kzs7m7OG1Z/8Wd7wAUCgUcHR2RnZ2N2NhYBAUFwd7ensdqCXlwdNgoEkuXLoVWq0Vrayvq6ur05tXV1aG1tRVarRYRERE4e/YsT1USYjoUXiJx9x3L5XK53ry7H7/44osICAgYtLoIGSgUXiJhZWXF/axWq/Xm3f3Y3d2dDhmJKFB4iYSTk5NJ2xFi7ii8ROLy5csmbUeIuaPwEol7O+kftB0h5o7CSyQovIilofASCcaYSdsRYu7MIrzWrl0LPz8/2NraYvz48Th+/HiPbdevX4+nnnoKrq6ucHV1RWRkZK/tLcXdnzaaoh0h5o738MrLy4NSqUR6ejpKSkoQHByMmJgY1NfXG2xfWFiIV155BUeOHEFRURF8fHwQHR2Nq1evDnLl5qW6utqk7Qgxe4xn4eHhbOHChdxjrVbLhg0bxjIzM416fmdnJ3NycmKbN282qn1zczMDwJqbm++rXnNlY2PDAPT5ZWNjw3ep5D79UtXIfJfsYb9UNfJdyoAy9m+U12sbOzo6UFxcjNTUVG6aVCpFZGQkioqKjFpGe3s7NBoN3NzcDM5Xq9V6J2mqVCoAgEajgUajeYDqzYuVlZVR62NsO2J+Ojs7ue9i3obGrhuv4dXY2AitVguFQqE3XaFQGH393ZIlSzBs2DBERkYanJ+ZmYmMjIxu0w8ePCiqM80dHBxw+/Zto9rt27dvECoipna5FQCscezYMVw9zXc1A6e9vd2odoIeVWLZsmXYtm0bCgsLYWtra7BNamoqlEol91ilUnH9ZM7OzoNV6oDrT4f91KlTB7gaMhBOVd8Ayk5iwoQJCB5u+EhDDLqOjvrCa3i5u7vDysrK4CgInp6evT43KysLy5Ytw6FDhxAUFNRjO7lc3u1CZQCwsbGBjY3N/RVuhozd1dZoNKJab0tibW3NfRfzNjR23Xj9tFEmkyE0NBQFBQXcNJ1Oh4KCAkRERPT4vE8++QRLly7F/v37ERYWNhilmj1fX1+TtiPE3PF+qoRSqcT69euxefNmlJeXY/78+Whra0NiYiIAID4+Xq9Df/ny5Xj//feRm5sLPz8/1NbWora2Vm9IGEv03HPPmbQdIeaO9z6vuLg4NDQ0IC0tDbW1tQgJCcH+/fu5Tvzq6mpIpb9l7Lp1d24o8cILL+gtJz09Hf/5n/85mKWblfLycpO2I8Tc8R5eAJCcnIzk5GSD8woLC/UeV1VVDXxBAnTs2DGTtiPE3PF+2EhMgy4PIpbGLPa8yIO7O5Q8PDwQGBiIxsZGuLu7o7y8HA0NDd3aESJkFF4icfcHFg0NDVxY9daOECGjw0aRMPau2HT3bCIWFF4i8cgjj5i0HSHmjsJLJEaPHs39bG1tjeDgYAQEBCA4OJg7M/vedoQIGfV5icTdfVydnZ04depUn+0IETIKL5GgW5+JS3t7e7eRVSpqmqCuPY/y03bQXR/CTQ8ICBDVCCnGovASidmzZ2Pr1q1wdHSEm5ub3oipvr6+uH79OlpbWzF79mweqyTGOnv2LEJDQw3Om7VZ/3FxcTHGjRs3CFWZFwljlnVHBpVKBRcXFzQ3N4tqSBytVgs3NzeoVCoMHToUs2bNQnt7O+zt7fHXv/4V9fX1cHZ2xo0bN+hcLwEwtOfVekuNvUeK8MfJEXC0+22kFLHteRn7N0p7XiJhZWWFjRs34vnnn0dDQwNWr17NzZNIJACAjRs3UnAJhL29fbe9KY1Gg5uN9YgIDxP1kDjGok8bRSQ2NhY7duyAj4+P3vThw4djx44diI2N5akyQkyPDhtFSKvV4siRI/juu+8wZcoUTJ48mfa4BK6jowOfffYZDh8+jGeeeQZvvvkmZDIZ32UNCGP/Rim8REqj0WDfvn2YOnUqHWIIXEpKClatWsXdgAO4cy7f22+/jU8++YTHygYG9XkRIgIpKSlYsWIFFAoFMjIyIJfLoVarkZ6ejhUrVgCAKAPMGNTnRYiZ6ujowKpVq6BQKHDlyhXMnTsXrq6umDt3Lq5cuQKFQoFVq1ZZ7PWqFF6EmKkvvvgCnZ2d+PDDD/Uu8QLuHDZ+8MEH6OzsxBdffMFThfyi8CLETFVWVgIApk2bZnB+1/SudpaGwosQMzVq1CgAwJ49e9DQ0IDRo0fj5ZdfxujRo9HQ0IA9e/botbM09GmjSNGnjcLX0dEBBwcHvU8Z72VtbY22tjZRnTZh7N8o7XkRYqZkMpnenbM8PT3x5ptv6t2QWSqViiq4+oNOlSDETDU0NOh9klhbW4vPPvtMr01HRwcaGhrg4eEx2OXxjva8CDFT4eHhAO6MfqtWq5GVlYWpU6ciKysLarUagYGBeu0sDe15EWKmugaOXL58OWQyGf7yl7/A39+f68f86KOPEBsba7EDTNKeFyFmqutQcMmSJQbnv/fee3rtLA2FFyFm6vjx4wCAM2fOQKVS6c1TqVQoLy/Xa2dpKLwIMVMeHh5wcXEBALi4uCA4OBhFRUUIDg7Wm26pe17U50WIGWtqasKQIUPQ3NyM8vJybm8LuBNcTU1N/BXHM9rzIsTMNTU1ob6+Hr6+vrC1tYWvry/q6+stOrgA2vMiRBAcHR0xZcoUHDt2DBMmTICjoyPfJfGO9rwIMXMzZ86Evb09cnJyUFpaipycHNjb22PmzJl8l8YrCi9CzNjMmTPx7bffQiaTISUlBevWrUNKSgpkMhm+/fZbiw4w3sNr7dq18PPzg62tLcaPH9/rx75nzpzB888/Dz8/P0gkEr075BAiNrdu3eKCq6WlBR9++CG8vLzw4YcfoqWlhQuwW7du8V0qL3gNr7y8PCiVSqSnp6OkpATBwcGIiYlBfX29wfbt7e0YOXIkli1bpndxKiFi9M477wAAlEplt4uvZTIZ3nrrLb12lobX8MrOzkZSUhISExMxduxY7lg+NzfXYPt/+7d/w4oVK/Dyyy9DLpcbbEOIWPzv//4vAOD11183OH/evHl67SwNb582dnR0oLi4GKmpqdw0qVSKyMhIFBUVmex11Go11Go197jrTGWNRgONRmOy1zE3Xesm5nUUu5EjRwIAvvzyS3z44Yfdtun69eu5dmLazsauC2/h1djYCK1WC4VCoTddoVB0u835g8jMzERGRka36QcPHhTVLdJ7kp+fz3cJ5D5NmjQJOTk5yM7ORlhYGHfomJ+fj46ODq7Pd9KkSdi3bx+PlZpWe3u7Ue1Ef55XamoqlEol91ilUsHHxwfR0dGiH0k1Pz8fUVFRNJKqgE2fPh3/+Mc/8NprryE5ORmjRo1CZWUlPv/8c3R2dmL69OmiuxP6vddx9oS38HJ3d4eVlRXq6ur0ptfV1Zm0M14ulxvsH7OxsbGIP2pLWU+x2r17N3e6RHZ2tt68GTNmYNeuXfwUNoCMfb/y1mEvk8kQGhqKgoICbppOp0NBQQEiIiL4KosQs7Nr1y60t7fjjTfeQEhICN544w20t7eLMrj6g9fDRqVSiYSEBISFhSE8PByrV69GW1sbEhMTAQDx8fHw9vZGZmYmgDud/L/++iv389WrV1FaWgpHR0f4+/vzth6EDDQ7Ozt8+umndFOVu/AaXnFxcWhoaEBaWhpqa2sREhKC/fv3c5341dXVejcguHbtGh5//HHucVZWFrKysvD000+jsLBwsMsnhPCI9w775ORkJCcnG5x3byD5+fnBwu7URgjpAe+XBxFCyP3gfc9rsHXtuRn7caxQaTQatLe3Q6VSUf+ISFjKNu362+zrKMviwqulpQUA4OPjw3MlhJDetLS0cMNdGyJhFtaJpNPpcO3aNTg5OUEikfBdzoDpOhn38uXLoj4Z15JYyjZljKGlpQXDhg3T+8DuXha35yWVSvHwww/zXcagcXZ2FvUb3RJZwjbtbY+rC3XYE0IEicKLECJIFF4iJZfLkZ6eTuOeiQhtU30W12FPCBEH2vMihAgShRchRJAovAghgkThxbNJkyZxd4EhhBiPwksgCgsLIZFI0NTUxHcppB/M7Z+TudXzICi8CDFzHR0dfJdglii8zIharcaSJUvg4+MDuVwOf39/bNiwAVVVVZg8eTIAwNXVFRKJBHPmzOlzeS0tLXj11Vfh4OAALy8vrFq1qtt/3i1btiAsLAxOTk7w9PTErFmz9G7627XHV1BQgLCwMNjb22PixImoqKgw9eqLzpw5c/D9999jzZo1kEgkkEgkqKysxLx58zBixAjY2dlhzJgxWLNmTbfnzZw5Ex999BGGDRuGMWPGAACOHj2KkJAQ2NraIiwsDLt27YJEIkFpaSn33NOnT2PKlClwdHSEQqHA7Nmz0djY2GM9VVVVg/XrMD1GePX000+zRYsWMcYYe+mll5iPjw/buXMnq6ysZIcOHWLbtm1jnZ2dbMeOHQwAq6ioYDU1NaypqanPZb/++uvM19eXHTp0iJWVlbHnnnuOOTk5ca/HGGMbNmxg+/btY5WVlayoqIhFRESwKVOmcPOPHDnCALDx48ezwsJCdubMGfbUU0+xiRMnmvpXITpNTU0sIiKCJSUlsZqaGlZTU8Nu377N0tLS2IkTJ9iFCxfY1q1bmb29PcvLy+Oel5CQwBwdHdns2bPZ6dOn2enTp1lzczNzc3Njr732Gjtz5gzbt28f+93vfscAsF9++YUxxtjNmzeZh4cHS01NZeXl5aykpIRFRUWxyZMn91hPZ2cnH78ak6Dw4llXeFVUVDAALD8/32C7rhC5efOmUctVqVTMxsaGbd++nZvW1NTE7O3t9cLrXidOnGAAWEtLi97rHjp0iGuzd+9eBoDdunXLqFos2d3/nHqycOFC9vzzz3OPExISmEKhYGq1mpu2bt069tBDD+n9ztevX68XXkuXLmXR0dF6y758+TL3T8/YeoSCDhvNRGlpKaysrPD000+bZHkXLlyARqNBeHg4N83FxYU7BOlSXFyM6dOnY/jw4XBycuJev7q6Wq9dUFAQ97OXlxcA6B1eEuOtXbsWoaGh8PDwgKOjI7788stuv+/HHnuMu8ksAFRUVCAoKAi2trbctLu3LQCcOnUKR44cgaOjI/cVEBAAAKisrBzANeKHxQ2JY67s7OwG/TXb2toQExODmJgYfP311/Dw8EB1dTViYmK6dRLfPXJn1zhoOp1uUOsVg23btmHx4sVYuXIlIiIi4OTkhBUrVuDnn3/Wa+fg4NDvZbe2tmL69OlYvnx5t3ld/3DEhMLLTDz22GPQ6XT4/vvvERkZ2W1+139hrVZr1PJGjhwJGxsbnDhxAsOHDwcANDc349y5c/j3f/93AMDZs2dx/fp1LFu2jBtZ9uTJk6ZYHfJ/ZDKZ3jb76aefMHHiRCxYsICbZsxe0ZgxY7B161ao1WruwuwTJ07otRk3bhx27NgBPz8/WFsb/tO+tx4ho8NGM+Hn54eEhATMnTsXu3btwsWLF1FYWIj/+Z//AQD4+vpCIpFgz549aGhoQGtra6/Lc3JyQkJCAt555x0cOXIEZ86cwbx58yCVSrk9p+HDh0Mmk+Gzzz7DhQsXsHv3bixdunTA19WS+Pn54eeff0ZVVRUaGxsxevRonDx5EgcOHMC5c+fw/vvvdwshQ2bNmgWdToc//elPKC8vx4EDB5CVlQXgtz3hhQsX4saNG3jllVdw4sQJVFZW4sCBA0hMTOQC6956hLz3TOFlRtatW4cXXngBCxYsQEBAAJKSktDW1gYA8Pb2RkZGBt59910oFIoebxd3t+zsbERERGDatGmIjIzEE088gcDAQK7fxMPDA5s2bcL27dsxduxYLFu2jPuDIKaxePFiWFlZYezYsfDw8EBMTAxiY2MRFxeH8ePH4/r163p7YT1xdnbGP/7xD5SWliIkJATvvfce0tLSAIDbnsOGDcNPP/0ErVaL6OhoPPbYY3jrrbcwZMgQbjjle+u5t69NSGhIHAvS1tYGb29vrFy5EvPmzeO7HPKAvv76ayQmJqK5uZmXPlO+UZ+XiP3yyy84e/YswsPD0dzcjA8++AAAMGPGDJ4rI/fjq6++wsiRI+Ht7Y1Tp05hyZIleOmllywyuAAKL8Gqrq7G2LFje5z/66+/AgCysrJQUVEBmUyG0NBQ/PDDD3B3dx+sMokJ1dbWIi0tDbW1tfDy8sKLL76Ijz76iO+yeEOHjQLV2dnZ66UdvX3iRIgYUHgRQgSJPm0khAgShRchRJAovAghgkThRQgRJAovQoggUXiRATVnzhxu1E4bGxsoFApERUUhNze3X9fVbdq0CUOGDBm4QnvQNaopMT8UXmTA/eEPf0BNTQ2qqqrw3XffYfLkyVi0aBGmTZuGzs5OvssjQsXnSIhE/BISEtiMGTO6TS8oKGAA2Pr16xljjK1cuZI9+uijzN7enj388MNs/vz53UZzvfsrPT2dMcbYV199xUJDQ5mjoyNTKBTslVdeYXV1ddzr3Lhxg82aNYu5u7szW1tb5u/vz3Jzc7n51dXV7MUXX2QuLi7M1dWVPfvss+zixYuMMcbS09O7ve6RI0cG5PdE+o/2vAgvnnnmGQQHB2Pnzp0AAKlUik8//RRnzpzB5s2bcfjwYaSkpAAAJk6ciNWrV8PZ2Rk1NTWoqanB4sWLAQAajQZLly7FqVOnsGvXLlRVVendnOT999/Hr7/+iu+++w7l5eVYt24dd3mURqNBTEwMnJyc8MMPP+Cnn36Co6Mj/vCHP6CjowOLFy/GSy+9xO051tTUYOLEiYP7iyI94zs9ibj1tOfFGGNxcXEsMDDQ4Lzt27ezhx56iHu8ceNG5uLi0ufr3TsG//Tp01liYqLBtlu2bGFjxoxhOp2Om6ZWq5mdnR07cOBAn/UTftGeF+ENY4wbSO/QoUP4/e9/D29vbzg5OWH27Nm4fv062tvbe11GX2Pwz58/H9u2bUNISAhSUlJw9OhR7rmnTp3C+fPn4eTkxI357ubmhtu3b4tyzHexofAivCkvL8eIESNQVVWFadOmISgoCDt27EBxcTHWrl0LoPcbrnaNwe/s7Iyvv/4aJ06cwN///ne9502ZMgWXLl3C22+/jWvXruH3v/89d8jZ2tqK0NBQlJaW6n2dO3cOs2bNGuC1Jw+Khh0gvDh8+DDKysrw9ttvo7i4GDqdDitXruRG/Owa/rqLobHXjR2D38PDAwkJCUhISMBTTz2Fd955B1lZWRg3bhzy8vIwdOhQODs7G6xTTGO+iw3teZEBp1arUVtbi6tXr6KkpAQff/wxZsyYgWnTpiE+Ph7+/v7QaDTcWPpbtmxBTk6O3jL8/PzQ2tqKgoICNDY2or293agx+NPS0vDtt9/i/PnzOHPmDPbs2YPAwEAAwKuvvgp3d3fMmDEDP/zwA3ffgL/85S+4cuUK97r/+te/UFFRgcbGRmg0msH5pZG+8d3pRsQtISGBO83A2tqaeXh4sMjISJabm8u0Wi3XLjs7m3l5eTE7OzsWExPDvvrqq2432X3jjTfYQw89pHeqxF//+lfm5+fH5HI5i4iIYLt37+52I9bAwEBmZ2fH3Nzc2IwZM9iFCxe4ZdbU1LD4+Hjm7u7O5HI5GzlyJEtKSmLNzc2MMcbq6+tZVFQUc3R0pFMlzAyN50UIESQ6bCSECBKFFyFEkCi8CCGCROFFCBEkCi9CiCBReBFCBInCixAiSBRehBBBovAihAgShRchRJAovAghgvT/AZW21UtefbipAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:07:11.662693Z", + "iopub.status.busy": "2024-03-26T07:07:11.662372Z", + "iopub.status.idle": "2024-03-26T07:07:11.874079Z", + "shell.execute_reply": "2024-03-26T07:07:11.873190Z" + }, + "papermill": { + "duration": 0.229991, + "end_time": "2024-03-26T07:07:11.876254", + "exception": false, + "start_time": "2024-03-26T07:07:11.646263", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEnCAYAAAA9/18kAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABg40lEQVR4nO2dd3hTZdvAf5ndg9lSKEv2RpbgwFEtigMn8OEAURQZAqIC+oILQQVFtuiL4EBB5VVERKEKytZS9oay6QK6R9b5/kiTJs06KQlNy/O7rlxtznnOyZPk5D73/dxLIUmShEAgEFRDlJU9AYFAIPAXQsAJBIJqixBwAoGg2iIEnEAgqLYIAScQCKotQsAJBIJqixBwAoGg2iIEnEAgqLYIAScQCKot6sqeQCBiMpk4f/48ERERKBSKyp6OQCAohyRJ5OXlERcXh1LpRk+TAoC5c+dKjRo1koKCgqTu3btL27dvdzl20aJF0k033SRFR0dL0dHR0h133OEw3mQySf/5z3+k2NhYKTg4WLrjjjukI0eOyJ7PmTNnJEA8xEM8Avxx5swZt7/lStfgli9fzrhx41i4cCE9evRg1qxZJCYmcvjwYerWreswfsOGDQwcOJBevXoRHBzMe++9x1133cX+/fupX78+AO+//z6zZ89m6dKlNGnShP/85z8kJiZy4MABgoODPc4pIiICgDNnzhAZGenbNywQCK6Y3Nxc4uPjrb9Vl8hWa/xE9+7dpREjRlifG41GKS4uTpo2bZqs4w0GgxQRESEtXbpUkiSz9hYbGyt98MEH1jHZ2dlSUFCQ9M0338g6Z05OjgRIOTk5XrwTgUBwtZD7G61UJ4NOpyM5OZmEhATrNqVSSUJCAlu3bpV1jsLCQvR6PTVr1gQgNTWVtLQ0u3NGRUXRo0cPl+csKSkhNzfX7iEQCKo+lSrgsrKyMBqNxMTE2G2PiYkhLS1N1jleffVV4uLirALNcpw355w2bRpRUVHWR3x8vLdvRSAQBCBVOkxk+vTpfPvtt/zvf/+TtbbmiokTJ5KTk2N9nDlzxoezFAgElUWlOhlq166NSqUiPT3dbnt6ejqxsbFuj50xYwbTp09n/fr1dOjQwbrdclx6ejr16tWzO2enTp2cnisoKIigoKAKvovAxGg0otfrK3saggBFpVKhVqurfRhUpQo4rVZLly5dSEpKol+/foA5Bi0pKYmRI0e6PO79999n6tSp/Pbbb3Tt2tVuX5MmTYiNjSUpKckq0HJzc9m+fTvDhw/311sJKPLz8zl79iySKNYscENoaCj16tVDq9VW9lT8RqWHiYwbN46nnnqKrl270r17d2bNmkVBQQFDhgwB4Mknn6R+/fpMmzYNgPfee4/JkyezbNkyGjdubF1XCw8PJzw8HIVCwZgxY3jnnXdo3ry5NUwkLi7OKkSrM0ajkbNnzxIaGkqdOnWq/R1a4D2SJKHT6cjMzCQ1NZXmzZu7D5atwlS6gOvfvz+ZmZlMnjyZtLQ0OnXqxNq1a61OgtOnT9t9+AsWLECn0/HII4/YnWfKlCm88cYbALzyyisUFBQwbNgwsrOzuemmm1i7du0VrdNVFfR6PZIkUadOHUJCQip7OoIAJSQkBI1Gw6lTp9DpdJX+28gt1vPw/C1oVEpWjbwRtco3AlchCTvGgdzcXKKiosjJyalygb7FxcWkpqbSpEmTSr9oBYFNIF0rmXkldJu6HoUCTrx7j0fLQ+5vtHrqpQKBoEqhM5oA0KiUPl1WEQJOIBBUOnqDWcBpfWSaWhACThAQ3HrrrYwZM6ayp1Ft2LBhAwqFguzs7Mqeiiz0pRqcVi0EnOAapqr9cAXyKDFYTFTfev2FgBMIAhSdTlfZU7hq6G3W4HyJEHDVHEmSKNQZKuVRUQd9SUkJr776KvHx8QQFBdGsWTP++9//cvLkSW677TYAatSogUKhYPDgwR7Pl5eXx6BBgwgLC6NevXp89NFHDibxl19+SdeuXYmIiCA2Npb/+7//IyMjw7rfojkmJSXRtWtXQkND6dWrF4cPH5b9vt555x3q1q1LREQEzzzzDBMmTLDLrhk8eDD9+vVj6tSpxMXF0bJlS1lzA1izZg0tWrQgJCSE2267jZMnT8qeVyCgN5qvFV+bqJUeByfwL0V6I20m/1Ypr33grURCtd5fYk8++SRbt25l9uzZdOzYkdTUVLKysoiPj+eHH37g4Ycf5vDhw0RGRsqK9Rs3bhybN29m1apVxMTEMHnyZHbu3GknXPR6PW+//TYtW7YkIyODcePGMXjwYNasWWN3rtdee42ZM2dSp04dnn/+eZ5++mk2b97scQ5ff/01U6dOZf78+dx44418++23zJw5kyZNmtiNS0pKIjIyknXr1sme25kzZ3jooYcYMWIEw4YN499//+Wll17yOKdAQucnJ4MQcIKA4siRI6xYsYJ169ZZK8Q0bdrUut9SFqtu3bpER0d7PF9eXh5Lly5l2bJl3HHHHQB8/vnnxMXF2Y17+umnrf83bdqU2bNn061bN/Lz8wkPD7fumzp1Kr179wZgwoQJ9O3bl+LiYo9xZHPmzGHo0KHWDJ3Jkyfz+++/k5+fbzcuLCyMzz77zC59ytPcFixYwHXXXcfMmTMBaNmyJXv37uW9997z+PkECv4yUYWAq+aEaFQceCux0l7bW3bt2oVKpbIKkSvlxIkT6PV6unfvbt0WFRVlNf8sJCcn88Ybb7B7924uX76MyWT+wZ0+fZo2bdpYx9kWdrAUc8jIyKBhw4Zu53H48GFeeOEFu23du3fnjz/+sNvWvn17h9xQT3M7ePAgPXr0sDumZ8+ebucTaFicDMJEFXiFQqGokJlYWVRGellBQQGJiYkkJiby9ddfU6dOHU6fPk1iYqLDQr9Go7H+bwlItQgcXxAWFlbhuVVlyjQ44UUVVGPat2+PyWRi48aNTvdbtBuj0SjrfE2bNkWj0fDPP/9Yt+Xk5HDkyBHr80OHDnHx4kWmT5/OzTffTKtWrRwW8a+Uli1b2s0BcHjuDDlza926NTt27LDbtm3btiuf9FWkLA7Oe63fHULACQKKxo0b89RTT/H000/z448/kpqayoYNG1ixYgUAjRo1QqFQsHr1ajIzMx3WsMoTERHBU089xcsvv8yff/7J/v37GTp0KEplWUpQw4YN0Wq1zJkzhxMnTrBq1Srefvttn76vUaNG8d///pelS5dy9OhR3nnnHfbs2eMxLUnO3J5//nmOHj3Kyy+/zOHDh1m2bBlLlizx6fz9TZmTQWhwgmrOggULeOSRR3jhhRdo1aoVzz77LAUFBQDUr1+fN998kwkTJhATE+O2bqCFDz/8kJ49e3LvvfeSkJDAjTfeSOvWra2OgTp16rBkyRK+++472rRpw/Tp05kxY4ZP39OgQYOYOHEi48eP5/rrryc1NZXBgwd7dE7ImVvDhg354Ycf+PHHH+nYsSMLFy7k3Xff9en8/Y2/nAyimogTRDWR6k1BQQH169dn5syZDB06tNLmceeddxIbG8uXX35ZKa8fSNfKfzel8vbqAzzQKY6PB3T2OF7ub7TqrD4LBBUkJSWFQ4cO0b17d3JycnjrrbcAeOCBB67aHAoLC1m4cCGJiYmoVCq++eYb1q9fbxfvdi2jM4gwEYHAgfJhHOU5cOAAYO7hcfjwYWuZ/L///pvatWv7bB5t27bl1KlTTvd98sknPPTQQ6xZs4apU6dSXFxMy5Yt+eGHH+zaW17LiDg4gcAJcXFx7Nq1y+3+hg0bkpyc7Nd5rFmzxmWTn5iYGEJCQli/fr1f51CVsQi4IBEHJxCUoVaradasWWVPg0aNGlX2FKo0OlFNRCAQVFd0opqIQCCoruj8lKolBJxAIKh0RD04gUBQbbHUg/O1k0EIOIFAUOn4Kw5OCDhBtaBx48bMmjXL+lyhUPDjjz9W2nwE3uEvJ4MIExFUSy5cuECNGjUqexoCmfirq5YQcIJqSWxsbGVPAUmSMBqNqNXiZ+YJEQcnqBiSBLqCynl4WcdBTnMYudiaqCdPnkShULBy5Upuu+02QkND6dixI1u3brU7ZtOmTdx8882EhIQQHx/P6NGjrVVMQH5jml9//ZUuXboQFBTEpk2bvJ77tYhVgxMmqsAr9IXwbpzncf5g0nnQhnkeV4qc5jBXwmuvvcaMGTNo3rw5r732GgMHDuTYsWOo1WqOHz9Onz59eOedd1i8eDGZmZmMHDmSkSNH8vnnnwPyG9NMmDCBGTNm0LRpU2Emy8RfcXBCwAkCArnNYa6E8ePH07dvXwDefPNN2rZty7Fjx2jVqhXTpk1j0KBBVm2xefPmzJ49m969e7NgwQKCg4NlN6Z56623uPPOO30272sBXWmYiHAyCLxDE2rWpCrrtWUitznMleCqYUyrVq3YvXs3e/bs4euvv7aOkSQJk8lEamoqrVu3lt2YpmvXrj6b87WCqCYiqBgKhVdmYnXGXcOY/Px8nnvuOUaPHu1wXMOGDb1q/lK+cYzAM8JEFVRrbJvDWFrwWZrD3HLLLX5//euvv54DBw64rEyyd+9ea/OX+Ph4AP7991+/z+tawV9OBuFFraLsPH2Z6b8eokgnr7tUoCOnOYw/efXVV9myZQsjR45k165dHD16lJ9++sna8+FqNKbxNyV6I0ZTYHYo8FccnBBwVZSpvxxk4cbjbDySWdlT8RmemsP4kw4dOrBx40aOHDnCzTffTOfOnZk8ebLVyXE1GtP4kyKdgcPpeRxOy6vsqTilxE9xcMJErYJIkmS9UHOLnVeRrYpERETYLfIXFBTw5ptvMmzYMI/Hnjx50u65bS+lxo0bU763UnR0tMO2bt268fvvv7t8jYEDBzJw4ECXr3Prrbc6nDNQyC02AGDwYZNqXyKcDAGKySShN5ko0hk5mpFP10Y1/G5Snc8pJr/EfMEW66uHiQqB0RymuhKYYrcMf1UTEQLuCli48TjTfz3EY10b8NeRLNJyi/nkiS4ktvVvmtCR9DIzo7qswVlw1hzm4MGD3H333S6P8dT8WUBASzijSbKuDQoNLoCw3G0KdEbScosBWHcg3e8C7qitgKtGGlznzp2dNocpKipy21hGIIfAlXAW8xRAIzS4wCEsyPzxFZSai+D7O5AzjqSXaSzF+sBcU/ElISEhAdFYpioTuOKtzMEAIkwkoAjTmgVcYUmZFvXNjtPsO5fj19e11eBcrcEF6mK3IHAIlGvEToMT1UQCh7AgFYB1wd9C/0+2OhvuE0wmiaMZZRpc+TU4lco8p/LR9YJrHCeyrLCwELDP8KgMyjyoCp876ISJegVYTNRCnb2AK/Djwv+57CIKbc5ffg1OrVYTGhpKZmYmGo0GpVLcwwSg15UgGcw3vaKiIgoLC8nIyCA6Otp6U6wsrGlafljeEQLuCrCYqOU1OH9yNMM+ULO8gFMoFNSrV4/U1FROnTp11eYlCGyyC/XW61RbFAKYYwEDoTCoVYPzsYMBAkDAzZs3jw8++IC0tDQ6duzInDlz7CpK2LJ//34mT55McnIyp06d4qOPPnIohvjGG2/w5ptv2m1r2bIlhw4d8vncLSbq5cKrF2xrcTColQoMJsnpGpxWq6V58+bCTBVYmZN0hB93mYtzJr10KxqNptI1Nws6g9l+rnYa3PLlyxk3bhwLFy6kR48ezJo1i8TERA4fPkzdunUdxhcWFtK0aVMeffRRxo4d6/K8bdu2Zf369dbn/ioZbTFRr2Z+nyUGrmVsBPvP57qMg1MqlVclxUlQNcjVKzmXZ75WAu268FfDGahkJ8OHH37Is88+y5AhQ2jTpg0LFy4kNDSUxYsXOx3frVs3PvjgAwYMGEBQUJDL86rVamJjY62P2rVr+2X+FhP1anK0VIPr0CAaqF5xcAL/YQoQj6kz/JVoD5Uo4HQ6HcnJySQkJJRNRqkkISHBoVa+txw9epS4uDiaNm3KoEGDOH36tNvxJSUl5Obm2j3kEKxRovR/oQsrZg+qWYPr2CAKEAJOII8ALSIC+NfJUGkCLisrC6PRSExMjN32mJgY0tLSKnzeHj16sGTJEtauXcuCBQtITU3l5ptvJi/PdRWFadOmERUVZX1Y6n15QqFQXFUt7szlQor1JrRqJS1iIwAouQYCfQVXTqDEvDnDaqKqfa8tVLsYgrvvvptHH32UDh06kJiYyJo1a8jOzmbFihUuj5k4cSI5OTnWx5kzZ2S/XmjQ1VuotTgYrqsTTnjp+p/Q4ARyCGgT1U9d7aESnQy1a9dGpVKRnp5utz09Pd2nruvo6GhatGjBsWPHXI4JCgpyu6bnDrOjoaSCs/MOi4OhRUw4IRqzYK1uyfYC/xDQJqqfqvlCJWpwlmoRSUlJ1m0mk4mkpCR69uzps9fJz8/n+PHj1iYjvsaViaoz+N50PGoVcBEEWwSc3hjQ5ocgMDAFsITzp5OhUsNExo0bx1NPPUXXrl3p3r07s2bNoqCggCFDhgDw5JNPUr9+faZNmwaYHRMHDhyw/n/u3Dl27dpFeHi4NRl7/Pjx3HfffTRq1Ijz588zZcoUVCqVQ6FCXxHmwkQtKDGgVWt9+loWE7V53XBCtGWvW2IwWQWeQOCMwDZR/VMqCSpZwPXv35/MzEwmT55MWloanTp1Yu3atVbHw+nTp+1Sjc6fP0/nzp2tz2fMmMGMGTPo3bs3GzZsAODs2bMMHDiQixcvUqdOHW666Sa2bdtGnTp1/PIeXGlwBToDNcJ8J+CMJonjmWYB1zI2gmCbu12RzigEnMAtAazAUeJHE7XSMxks3cOdYRFaFpyVni7Pt99+66upycIS7FueghLfro2dvlRYqqkpia8RilKpQKtSojOaKNIbEf3TBe4IbA3Of6la1c6LerVxZaIevCAvlk4uFgdDs7rhKEuD74I15q9PeFIFnghoAReoTob8/PwKBchWJ1yZqKv3+LabvNXBUDfCus2yDic8qQJPBGivGcC26XMAxMGlpqbSt29fwsLCiIqKokaNGtSoUYPo6Ghq1Lj2DKVQFybqxiOZ5PgwCd/qYIgpE3CWdbcSgxBwAvdUBQ0uIJwMjz/+OJIksXjxYmJiYq5KU95AJtyFiao3Svy2P43HusnLivCEbQychbJYuAC+PQsCAuFkkMnu3btJTk6mZcuWPp9MVSTUiYnaMT6a3Wey+XnPeZ8IOIPRxInMAsAcA2fBNhZOIHBHQGtwljCRQHAydOvWzatUpupOuBMTtV8nczf0LccvkpV/5VkOpy4VojOaCNGoqB8dYt0eIgScQCYBLeACyUT97LPPeP755zl37hzt2rVzqOfeoUMHn02uKhCqdTRRm9QOo339KPaey+HXfWk8cUOjK3oNi4OheUyZBxXKnAzFwskg8EAgm6gWJ4Ovmz5DBQRcZmYmx48ft2YbgLmqhiRJKBQKjMZr68fmLA5OpVRwX8d67D2Xw+rd569YwJVlMETYbRcanEAugZzOZ9t0xtd4LeCefvppOnfuzDfffCOcDLgWcH07xPHumkPsOHmJtJxiYqMqXkXVmYMBxBqcQD6BbKL6s6Kv1wLu1KlTrFq1SjTiLSXMiYmqUiioHx1Cl0Y1SD51mV/2XmDoTU0q/BqWKr62DgaAEG1poK8wUQUeqBpxcAHgZLj99tvZvXu3zydSVXGlwQHc18FcweRKgn71RhMnsiwxcOU0OHXpGpzQ4AQeCGQNLqCcDPfddx9jx45l7969tG/f3sHJcP/99/tsclUBZ5kMFkfAPR3q8dbqA6SczubMpULia4Z6ff6TWQXojRJhWnsPKtg4GWQIuMy8EkZ8vZOBPeJ5sHMDt2MLdQa2HLvITc1riyT+akIAyzf0RvPkAsLJ8PzzzwPw1ltvOey7Fp0Mzir6qkrXJetGBNOjSS22nrjIL3sv8Hzv67w+v8XB0CwmwmG905s1uPfWmtcDd5y85FHAvbRiN7/uS+OhzvX5sH8nr+csCDyMASzhdH6s6Ov1GU0mk8vHtSbcwPylNKplr5mpbEI57utojon7eXfFzFSrg6FuuMO+Mi+q5wWWnCL5aWO/7jP3xFiZck72MYLAJpBN1IBpG6jX61Gr1ezbt8/nE6nK/DTiRhY+fr31udJG0+rTLhaVUsH+87mcKK3n5g2WLlrlHQzgXbJ9AF/fgqtAVYiDq3Qng0ajoWHDhtekpuaO6FAt9aLK1sdsNbiaYVpuambuy7p6zwWvz12WZO9ag5OzBucqDmrzsSwemr+Zw2muu44Jqj5VoWS5P+LgvBaZr732GpMmTeLSpUs+n0yVoygbvh8KWUdR23w55TVti5m6avd5rwIudQYTJ7PMOagtYx01OG/W4FyZKP9LOcfO09msP5judL+gehDIJqo/68F57WSYO3cux44dIy4ujkaNGhEWFma3f+fOnT6bXMCzdgLs+x7S96G5/yfrZmU5Z8BdbWPQrlRyLCOfw+l5tIqNlHX61KwCDCaJiCA1sZGOgcJemagutluEY4kfmuQIAocAVuD8aqJ6LeD69evn80lUWRLehON/QuYh6m18GegPKOxMVIDIYA29W9Zh3YF0Vu++IFvAHbHJQXWWMWLpyyDHRHV1gZeUHmswyhNwJpNklw8rqBoEcqqWzhhATWemTJni80lUWSJi4LGlsKQvEcdW8bQqisXGux00ODCbqesOpPPznvO8dFcLWSlutm0CnWHV4K5gDc6iuellCLglm1OZ8fsRlj3bgw4Noj2OFwQOxgBW4fwZ6FvhMyYnJ/PVV1/x1VdfkZKS4ss5VS0a3gCJ7wIwSf013RSHHDQ4gDta1SVYo+TUxUL2nsuRdWpnVXxt8cbJ4ArLsZZgS3e88fMB8ksMvPL9ngq/nqByCOQ1OIuJGlp4DvTFPj231wIuIyOD22+/nW7dujF69GhGjx5Nly5duOOOO8jMzPTp5KoM3YdR1PJB1AoT87Sz0RZmOAwJC1JzR2tzO0S53tQjGc6T7C34wslQXBpDp5Nporo7lyBwCeRvTG80EUkBdX/sD4sTIeesz87ttYAbNWoUeXl57N+/n0uXLnHp0iX27dtHbm4uo0eP9tnEqhQKBfmJH3LIFE9dRTZRvzwLRsfA2vs6mL2pq3ef9+i2LzEYOXWxEPBsohbrTR7PZ5tsbTJJPPHf7YxYtrNMg/PCySDkW9UjUL8zk0nCaDIyU7MQdXYqFF4Cjfcpja7wWsCtXbuW+fPn07p1a+u2Nm3aMG/ePH799VefTayqoQoKZ7h+DLlSCJpz22HdZIcxt7asQ3iQmvM5xaScuez2fCcyCzCaJCKD1dSNCHI6JsQmT9STF1SyuYdn5Zfw99EsftlzgUKdxUT1QsDJHikIFAJ1DU5nNDFc9TN3qpKRVEHmNe3Qmj47f4VStcon2IM5CNgUyDVZ/IwCSJXq8ZJ+uHnDtvmw93u7McEaFXe1MZupP+92b6YesXEwuHJI2CbCezJTba/vQpuwkiIv1uAsBLJHTuCcQF1WMB37g5fUKwAw9Hkf6l/v4QjvqFC5pBdffJHz58tyK8+dO8fYsWO54447fDq5qoRF2KwzdaWk5xjzxlWjIOOg3ThL0O8vey+4vase9eBgAHPGhCV2yJOAsxVKtgKuUGcAvFuDC8yfisAdASnfss8QvGoYKoXEN4bbUHV5yucv4bWAmzt3Lrm5uTRu3JjrrruO6667jiZNmpCbm8ucOXN8PsGqQohWxccDOvFR/44E3TkZmvQGfSEsfxyKyxpi39isNlEhGjLzStieetHl+VxV8XV4XY28YF/bC9xWGFqcDN6YqELCVT3caXCSJDFw0TaeXvLP1dPO9cWw4gmURZfYY2rC26bBfomv9DoOLj4+np07d7J+/XoOHToEQOvWrUlISPD55KoaD3SqX/bkkcXwyS1w8Rj8OBz6fwUKs8Z1d7tYvv3nDD/vvkCv62o7PdfRDOdVfMsTrFGSU+Q5VMT2snUmDMUaXPXGnbWQW2Rg6wnzzbZYb7I6r/zK2lfhfArG4BoMzx6DpKl4SX93VCgOTqFQcOeddzJq1ChGjRolhJszwmrDY1+CSguHVsPmj6277i31pv6674JTwVKsN3LyojkH1VmSvS1yG8+Y7ExUg8N+sQZXvXHnY7C9Nq5KdeiUryB5CaAgPWEu56jjlzQtqIAGB5CUlERSUhIZGRkOjoXFixf7ZGLVggZd4O73YPVYSHoT4jpD097c0LQmtcO1ZOXr2Hwsi1tb1rU77FhGPpIE0aEa6oQ796BaCJYZ7OvKRLUgNLjqjbubkp2AM/hZwJ3fBavHmf+/7TVy6t8C/O2XLAaogAb35ptvctddd5GUlERWVhaXL1+2ewjK0WUIdBoEkgm+fxpyzqFWKbmnvblfgzNvqrUGXF3XHlQLchPubS/wKzZRhYSrcrhdg7P5v1hG8dQKU3gJVjwBxhJo0Qdufsmmkoh/8pu91uAWLlzIkiVLeOKJJ/wxn+qHQgF9Z0LaHkjbCyuehCFruLdDHF9sPcXv+9Mo1rezC/lwVwOuPPJN1LL/C50JOIMXJqrQ4aoc7tbgTB5ufj7BZIKVz0L2aajRBB78BJRKv1YSgQpocDqdjl69evljLtUXTYh5PS44Cs79C79NomujGsRGBpNXYuCvI/Ypbp6S7G2Rm49qK5Su2EQV8q3K4e47s93nNxN143twbD2oQ6D/lxASDfi3XDlUQMA988wzLFu2zB9zqd7UbAIPfWb+/5/PUO75lntL2wr+XC431RsNLlimiWq7VOrMyeBVHJwQcFUOdyaq350MR343CziAez+C2PbWXf7W4Lw2UYuLi1m0aBHr16+nQ4cODlkNH374oc8mV+1ocRf0ngAbp8PqMTx670o+A9YfSKdQZyBUq6ZIZ+TMZXMOaksvNDhPjWdsL2+nJqpXAk5IuKqGOy+qnQbnawF3KRVWPgNI0O0Z6DTQbrfej7XgoAICbs+ePXTq1AnAofmMnBpn1zy9X4VzyXBsHS02vkDrGm9z8DL8cSiDezvEWT2otcK01PLgQQVzHBx4l8ng7CL2KkxE9khBoCBfg/Ohk0FfZHYqFOdAg26QOM1xiB/LlUMFBNyff/7pj3lcOyiV8NAiWNQbxeWTzIn+hDt5np93n+feDnF2VXzlIHsNzqOTQZio1RnZa3BXqMEdSc/j7OVCbmtRB8XqcWbHWmhteHQpqLUO4wPORBX4gNCaZqfDf++iWfYmRqjqsejww+QV621qwHk2T0F+qpZ9oK/jWO9yUYWEq2q4T9Uq+19ObcHyGIwm1h/MYOmWk9aMiFU3HKHD7mWgUJqzeqLqOz1W58eOWiAEXOUR1wnu/RB+GsE4zffs0TVl3YH2spLsbQnWytPgPIUCGLwopyM0uKqHu872FTVRLxXo+Paf03y97TTnsous2zsqjtF611TzkzumQNPeLs/hz3LlIARc5dL5cTj7D8rkJXysmcfU5HYcuWQ2TZ11sneG3Dg4u1xUJ2ONJgmjSXJabt3duQRVA3c3JbleVKNJ4ofksxxKy+NcdiEbDmda6xDWCNUwoHtDutU10fKnUWjQY2x5L6obX3Q7L2GiVnfufp/iM7uokbGLp85M5mfdFEDrtYnqce3EwxocmO+mKqXnRGuhwVUv7DMZXF9H209c5JUf7PtxtKsfyVM9G3NfxziCVSB9+SAKxUWOm+pxtt1b9PbgePS3k8Hrs/71118YDI5xVAaDgb/++ssnk7qmUAcR/H9fkaOIpL0ylTfUS6kdHkSNMMcFWWfI7axlb6I6fn/gTaiIkHDVCU8edguXCnUA1I8O4fW+rVn5Qi9+HnkTj3aNN2fi/PEOitSN6JTBPK8fy9qjBR5f26LBBUyg72233ea0q31OTg633XabTyZ1zREdz4Z20zBJCgaq/2RY+N+yDw2WWw/O5n/XGpw8wSVXg3vx2xSGXs0aY4IKYbLzorq+yVlugE3rhPHMzU25vmGNstCwQ7/AJnMM7PEbpnNUasD6gxkee4VYeqIGTKqWJElO490uXrzo0OVeDvPmzaNx48YEBwfTo0cPduzY4XLs/v37efjhh2ncuDEKhYJZs2Zd8TkDhc63PsQMw6MAPJ0zH87tlHVcsMxAXzn5hnI1ODniSmcw8dOu8yQdyrA2zxEEJnK9qJYboLr8Ou3F4/C/583/3/AC193+FBFBajLzSth1NtvtaweMk+Ghhx4CzMG8gwcPJiioLAjVaDSyZ88er3NUly9fzrhx41i4cCE9evRg1qxZJCYmcvjwYerWreswvrCwkKZNm/Loo48yduxYn5wzUGhYK5St9Z5kXfpx7lQlm5Pyh22EsFpuj5O7BmebquXqItbJjIWTo5HZ3gO9qhYsuOrIdTIYnGUd6ArMVatLcqFhT7jzLbQqJb1b1mH1ngusO5DO9Q1ruDxnwCTbR0VFERUVhSRJREREWJ9HRUURGxvLsGHD+Oqrr7x68Q8//JBnn32WIUOG0KZNGxYuXEhoaKjLmnLdunXjgw8+YMCAAXYC9krOGUh88FhnTt4yE6lGU8g5Y05xMbkXXHLj4GxxFRJyJRrcxfwSnln6D+sPpJvH2AzyJgRFcPWRGybioG1JEvz8ImQcgPAYeHQJqMypm3eWNldaV3o9eDpnpZdL+vzzzwFo3Lgx48ePr5A5aotOpyM5OZmJEydatymVShISEti6detVPWdJSQklJSXW57m5uS7H+pNmdcNpdmdnaP8lfJYAx/+ADdPh9tdcHhOilZeqJaer0pWswU379RDrD2aw/mAGJ6f3tdsXqC3rBGbkZjLoywfl7lgEe78Dhcos3CJirWNvbVkXtVLBsYx8TmTm07SO87Anf5uoXp91ypQpVyzcALKysjAajcTExNhtj4mJIS0t7aqec9q0aXYaaXx8fIVe32fEtoP7Skuc//U+HF7rcmhFKvq6Qq4G50xYZuWXOBnp3XkFlYN8AVe6BqdSwult8Nsk84673oFG9stTUSEael5nXl5xp8WVBIqJaiE9PZ0nnniCuLg41Go1KpXK7lEVmThxIjk5OdbHmTNnKntK0LE/dB9m/n/lMPNCrhMsJmqJwX13eznpVe4E0Y8p52xP5oCqnOPJ9vWEBhfYyC1Zbii9PmqYLsN3g8FkgLYPwQ3DnY6XY6YGXDWRwYMHc/r0af7zn/9Qr169ClcQqV27NiqVivR0+zefnp5ObGysi6P8c86goCCXa3qVyl1TzTXsz+4wOx2GrgNtqN0Q2w5IxQYjoVrnX6kcGePKRM0t1jNm+S7rc2ej3F0HYg3u6uAqwsETciv66o0m1BgYdGYK5F+AOq3g/jn2HiUbElrHMPmn/SSfvkxWfgm1nVTHsRR50ASKBrdp0ya+/vprhg8fTr9+/XjggQfsHnLRarV06dKFpKQk6zaTyURSUhI9e/b0dlp+O2elotbCY0shrA6k7zM3rylnHgarbbrbu7k45Xg+XWlw+cX2gcHOzlX+Bmw7RGhw/mfCD3tInPVXhaqByO3JoDdJvKJeTuP8XaCNMLfCDHKdUhgXHUK7+pFIEvxxMMPpGEuyfVCgrMHFx8f7LHBz3LhxfPrppyxdupSDBw8yfPhwCgoKGDJkCABPPvmkncNAp9Oxa9cudu3ahU6n49y5c+zatYtjx47JPmeVIzIOHvncvJC751v49792u5Uyu9vL+cpcVRQxlNPsnJ3KXQ6rWIPzP9/+c4Yj6fmsP+jea+kM299ziRsTtVnGOoapfzE/6Tcfajf3eO47W5stp99dmKlWJ4PaP15UrwXcrFmzmDBhAidPnrziF+/fvz8zZsxg8uTJdOrUiV27drF27Vqrk+D06dNcuFBWzvv8+fN07tyZzp07c+HCBWbMmEHnzp155plnZJ+zStLkZkh4w/z/rxPgzD92u+XEwrnzolosDFc14QzlWkM6O5XSjWkkNLirR0V0D9uvx6UVkHmY+06aK4TsiHsC2twv69yWdbhNxzKdnrvEz6laXq/B9e/fn8LCQq677jpCQ0MdSpY7S+Nyx8iRIxk5cqTTfRs2bLB73rhxY1nao7tzVll6jYKz/8DBVeb1uOf+gvA6gFnA5RTpKdK51pTcfWrhQWryig0u1+DKb3fmsHCnwYk1uMDGvumMk2uoJA+WP47WVMQWYxv+bTKC7jLP3bpeBPWjQziXXcTfRzO5q639WnjAVfR1lR4l8DMKhdksyDwEWUfg+yHwxI+gUstKuHd3X4gM1pQKOOcCsrzZ4uxc5b2otpQ3cQWBha12bzRJ6I0m+2Den0ZA1hGy1XUYVTyKoeWUGncoFArubBPDki0nWXcg3aWA85eTwWsB99RTT/ljHgI5BJUu7H56O5z8G/54C+58yyYftWImakSw+TJwtQZXPoXL2ZmUbjU4sQYXyJS/Nor1xjIBt3UuHPgJlBqWNHiTi4ei0Ci9E0Z3lQq4pEMZDjUHLddWwDgZAI4fP87rr7/OwIEDycgwe0d+/fVX9u/f79PJCZxQpyU8MNf8/+aP4cAqQkobz6zefZ5nlv7LySzHMjXuNDiLgHOtwZXbLicOTnhRqw7lvh7rjTL1b1g3xfx/n2kc07YGvC8v3q1JTSKD1Vwq0JF86rLdPmscXKCEiWzcuJH27duzfft2Vq5cSX6+ucT27t27mTJlis8nKHBC2wehZ+ka448v0FRxHoDvks+y/mA6a/c7Zm24W7sMDyoVcC6cDOVNVGfaoFsNTpioVw3bT/qNVfu5++O/ZZSzt39eojdB7nnzMohkhA4DoNsz1hug2kttS6NScnsrc6GLdQfsr82Aqwc3YcIE3nnnHdatW4dWW1aU8fbbb2fbtm0+nZzADQlvQKMbQZfH2EtvE0qxdZczjcmdEhUebF5TceUMKNF7NlEd4uBsRgknQ+WwZMtJDl7IZdXu827HOZioxUXmTIWCTIhpZ27WrFBYb1QVcQhY1t7WHUi3u9nqAq2i7969e3nwwQcdttetW5esrCyfTEogA5XGHB8XHkt9/Sk+Dv0vTWuFuhzuKlVLrVRYTVxXa3DlTVRn2qC7MBGxBnf1cPYtlA/ULk/5b7PGprfgzHYIioL+X1qzZ3RWDc77mLVbWtRBq1Jy8mIhxzLyrdutXtRAiYOLjo62i02zkJKSQv36zluDCfxERIw500Gp5k7TZsZGmjM4nAkgVxZqsEZlNQ/0BhcaXHkvqpMxbgWcMFGvGs4+aW8qzTyg3ETtA0vMTx5aBDWbWvc5rQcnk/AgNb2amZPvbYN+A85EHTBgAK+++ippaWkoFApMJhObN29m/PjxPPnkk/6Yo8AdDW8w56wC91yYTzfFISTJLOSeWryDIZ/vQJIkNwJOWSbgZGtwjmPKCzjhZAgcPJazL/2yWilOM13zmXnjLS9Dyz524xzKJXmJs+T7Mg0uQATcu+++S6tWrYiPjyc/P582bdpwyy230KtXL15//XV/zFHgiR7PQbtHUGFknnY2obosMvJK2Hgkkz8PZ5JbZHAZJhKkVlkvLpcCzk1+4q4z2Xy17ZTDGpwtemGiViquenBYkCSIpIAFmo8IUejIirkJbp3oME5vurLKHwmtzQJu15lsMnKLkSQp8KqJaLVaPv30U/7zn/+wb98+8vPz6dy5M82be85LE/gJhQLun03asZ3EFp+gz8EJpLX53ro7t1jvMpPBrMGZ78gu4+Dc5JL2m7cZgMbl1v9sX88oTNRKpdBFFzULJpOJGZqFNFGmc1aqzZ4u73OPk/aRFi+7t15UCzGRwXSMj2b3mWzWH8zg4S5lS1oBY6JaaNiwIffccw+PPfaYEG6BgDaMrxu9Q64UQv3cXURtetu6K6dI76DBWaI6gtQqzyaqjAoVJ900lrF4Uf8+mkn3qev585DzyhIC/+BJg2t0cBF3qZIpkTQM140hXxHpdJzFWaSR0RzcFXdZzdQ0uxTAoMrMZBg3bhxvv/02YWFhjBs3zu3YDz/80CcTE3jPxeCGvKQfzqfaD2l8dCn3KiNZbepJbpHeYd0sTKsmr8Rgvwbn0slwZSamZQ3uif+au5sNWfKPQ1lzgf9wK+CO/0HzfbMA+I9hMHulpi6dEr4Iyr2zTQwf/HaYzccucrlAZ91eqSZqSkoKer3e+r8rKlr8UuAbFMA6U1f+afAU3c4u5T3NIg7r4sktvt5hbFiQRcCprCaq7EwG4FhGPs3quq4FZuvJFWtw8tl8LItNx7IYd2cLn/3oi/QuTNTsM/D9UBSY+NZwKyuM5r7GrgKDrYG+V6DBNa8bTqNaoZy6WEhSaWknlVLhtljDlSBLwP35559O/xcEJpsbPk/UpX20KExmoeYjduXe7DAmLMi8xmIbJuI6Ds7xgv9tfxrN6jaTNR+xBiefQZ9tB6BBjRAG9Wjkk3PaanBWHcRQYq5KU3SJ7Oi2TEkbbB3jquilLxrEKBQK7mwdw2ebUvll74XS8/lPMfKPXiioFCwXr6RQMyPiFc5LNblOeYHrU16jfISUJT1LVpiIkwv+dyfpYK74bFMqz335r+zxAriQXex5UCmeSojZholYRcmvr8L5nRBSg3+7f0wJZVlJrkzUK4mDs8USLvJvaV6qv8xTkKnBWZo+y2HlypUVnozgylCUXr4ScKwgmOG6MazQvkWTzD94ThXDJ8b7rGPDLAJOrbKmybgKyHVmou4+m8O57CKXcyl/pt/2e19p9lrGmx+9pxKJthqcUqGAlK8g+XNAAQ9/RmFBfaDM8ePKRNVdYRychS6NalAzTMul0jU4fzkYQKYGZ9tSLzIykqSkJP79t+yOnJycTFJSElFRUX6bqMALJIkLOcXslprxpsFc3uoV9bf0VJZVe7EIuCCNylouWm6qlmW9ZNUu9zmOgorjTeCrpwUAW42srSIVVpc6Cm97DZolOGiArsqW+0qDU9sk3/vifO6QdebPP//c+oiJieGxxx4jNTWVlStXsnLlSk6cOMGAAQOoXbu23yYq8IzFRM0tNlgv6mXG29kRmYhKITFHM4dYLqJVl8W+yctksL/gG9QIAeC9tYecjjeZXGdO2JJTqOe5L//lNy/M3WsFb7QkuSZqFPnMVX8ExhJo0QdufglwTLZ3lfngyybNFjPVV+dzhddnXrx4MePHj7frgapSqRg3bhyLFy/26eQE3mH5SVzIKbLb+lnUKPabGlFbkcsC7cdEaUxWj7d9HJw8E9Vd3imAUWZjgJnrDvPb/nSe+zJZ1vhrCU8a3LGMfA6l5QL2lWKcCbsCnQEFJj7WzCNekQk1GsODC6G0cGX5Q5w5GSRJssYzViTZvjw3N69tNU39laYFFRBwBoOBQ4cc79yHDh3CJMIBAoK0HPsF6os6Fc/rx5AjhdJZeYwJii+sQso2k0FumIinaCA5uaczfjtMeq78hfRrDXdajcFoIuHDjfSZ9TcFJQaPTb0lCV5Ur+RW1W6KJY25KnRIDev+8l+Xs+bPtjc/X2hcoVo1Nzev7bPzucLrVK0hQ4YwdOhQjh8/Tvfu5tYT27dvZ/r06VW3NV81waKVXSgVcMEaJcV6EzlFes5IMbyoH8ES7Qc8bFpLUe71/Ew7+zARVwUvyy06u+u/AKUmjwcZN/fPY1ZPrsCMrfblrj7a5UK99f9CndFakRmcx6LeqkxhtOp/ALwhPcv02PZ2++WYqLYlr3wV1nFXm1jWH8wgTOuYFuYrvL7CZsyYQWxsLDNnzrSWTapXrx4vv/wyL730ks8nKPCezPwSAFrERLDnbA45ReYfxAZTZ2YZHmKMeiUDMz5kQ43p3Nz8JvJK64VZNLh6UcFWIQmOgs+jiSqzekh+ifscyWsNWyePO7Mtp6gsA8BTgGy8Ip2PNfNQKiS+MNzJKmVvppcf5KDBOd7obLNcfKVxPdA5juOZ+dzasq7nwRXE65kqlUpeeeUVzp07R3Z2NtnZ2Zw7d45XXnnFbl1OUHlYbsgtYiIA82K+hY8ND5Gi7YrKWMxnwbNoW8PksAYXFaKxOZfkuAbn4UclVioqhu2NxJ0Qybb5PiVJstPAbLXAIHQs1MwiSlFIiqkZ7xged1oQ02KSRpRq1M5yj22zUa4kk8GWILWKife0pud1tXxyPmdckSiOjIwkMtJ5Yq7g6lNesWoRY06lstUMJJR8WmciRDeEyyfhf8+jLb0K9E4qthpMTgScpzU4SfK4LiRwxFbAuatwa2uiSriIg5MkpmkX01Z5iiwpkuG6F9GhcWrCWtZCG9U2V4RxFuhrWwuuKqVkVmgR5Pvvv2fFihWcPn0anU5nt2/nzp0+mZjAexTl7s/N6oajUDj+AEzBNaDvl/Dfu+DIWmKj5gLX2+Qalt33DEbJIUxEjomqdLJO42wugjJsb0Tlv0tbbJPUyy93WoVP8uc8pPwLo6RglH4UadQqPa8jaTnmJY1GtcLYdy7XaaCvJQZO7WXLwMrG69nOnj2bIUOGEBMTQ0pKCt27d6dWrVqcOHGCu+++2x9zFFSQuOgQq9lhS6hWBXGd4F5z5Zfa/8zkFuVuq6Zmu4isN5m8NlGNLuLgPAnGax1X1VzKc7nQRsAh2ZmlkiTB2WRzKhbwvmEAW01tyw528hVYNDhLTT9nYSK+ymK42ngt4ObPn8+iRYuYM2cOWq2WV155hXXr1jF69GhycnL8MUeBTMrLj9jIYCJDHLuQh1i8Vp0fhy6DUSDxsWYeUSUX0BtNDo15HZ0M7udxw7QkTjjpzVq1fhpXH52xTHNyZ+LbmqhI9mEe2pJL5iR6o47fTd34xHiv3bHOvgNL3GTjWmGAcxPVV1kMVxuvZ3v69Gl69eoFQEhICHl5eQA88cQTfPPNN76dncArbC/eILWSqBCNncPAQqitW77Pe0hxnamhyGe+ZhaXcnLtzKPCEseL3VOYCMC7aw46bBPtA90jt+6erYlqkrDaqEpMdP13POSehVrNeNU0nPIizZn2nZ5rNlEb1zYLOJ3BhKncd+XLLIaridezjY2N5dKlS4C5qq+lF2pqaqrHlBHB1aNeVDAKhYLIYCcanMZGwGmCUTz2BdlE0EGZiua3V+20h7wSvcPxckxNoa15j6s4xPI4mKil39dL6hXUydwGmlDo/xUFkmMbyfLfS36JwRqu08im7Hx5YevMAVUV8FrA3X777axatQowB/2OHTuWO++8k/79+zvtlyq4etjKnZjIYAAiQxzX4EK05bZFN+T98FcwSQpqHv6W2wp+te4qcKLByVlKu9bW2zLzSlwmqcvFNlvAna5gHyZiftyp/JcRavPvkvvnQN3WTs3c8h5QS9ZLRJCaWmFB1u3lzVT9FTR9rky89qIuWrTImpI1YsQIatWqxZYtW7j//vt57rnnfD5BgXxsL956UWYB59FELeVszRuYcflRXtGsYEjOfH5W1GGf1JQCJ8G4cqqvXkvy7WRWAbfO2ECjWqFsfPm2Cp/HnQaXU6jnzOVC2tWP4pKdBgeKy8eZqVkAwPHrnuS69o+Y9zkRkuW/FouDISYqGJVSgUalQG+UHDyphmtBgzMYDLzzzjukpZVVfxgwYACzZ89m1KhRaLVaN0cLriYxpQLOqYnqRMDVDteywHg/qbV6o0XPQu0soslzmm0gRzvzVwnqQMTS5/OUm8Y7crB1MpTn2S/+5d45m/jrSCbZNgIOXQHhPw4mUlHEDlNL9rd1n03kSoOLLdX4g0uXL8oLON21sAanVqt5//33MRhEik0gYnvpWi5YuRpcnfAgJJSsaPAaF5T1aKDI4mPNPAqKSxzGGmSkKljSv64FvAlq/iH5LL2mJXHgfK7Dvqx8+/g2W3acNK97z/z9sI0XVSJ6/XjUWYfIkKIZoRuNSWGTheLk9cvfm9IsGlw5AVfeRLXGwVVnAQdwxx13sHHjRn/MRXCl2Fy8FhPVaZiIxomAizCvv5wt1jI1YhJFkpbeqj20ODjPYaynTukAe89VPGRo1e7zvPa/vVazqDrx0ne7OZ9TzLgVu+y2n7lUyCvf7/F4/O6zOdZc36dUvxN25H9IChUjdKPJpIadsHXm9HNlosZGmb//YI1ZJJSPhbN2oK9iJqrXa3B33303EyZMYO/evXTp0oWwsDC7/ffff7/PJieoOO6dDM5MVPMFnpVXwmVFYybqn2GWdj7Xn/yU25VR/GEq68zlqc/mlTL6G3Pnti6NavDQ9Q38+lq+wJVDQJIk3lp9gGZ1wx0ayJQvSLBqt3115PKnrBMRRGZemTbdRXGY19VfAbCr9Xj+2dnKYS6yNLhyJqrl5lc+H9XS1b6qZTJ4LeBeeOEFwHn/U4VCgdHNOoLAv9jGr8W6dTI4fu1WAZdfgkmS+NF0E50Mxxis/p1Zmvncq5vKaclchdXfAs7CpQKd50EBgCsDdeuJi3y++SSAg4Arv47pqQJLdIjGKuDqkM087Ww0CiPpDfuyqeYjwFHAsbZbecqngFk0uLoeTFRLV/sr6YlaGXg9W5PJ5PIhhFvlYvnNKBXmNTVw7mRwugZXaqJm5pdYf2xTDY9zKrQdkYpCFmpmEYz5B+aqKYnAHncCOq9Yz+G0POtzuUHQagzM1c4mVnGZI6b6JHd8k/T8Ms3OPm3L8fjyGlxGqdAsvwZX3kT1RVf7yqBqiWOBLOpEBFkXg51pcM7W4GqHmz3g2YV6691bj5pFMZPJlCJpozzFVM1/AUl2SfIrpapUrXD1cbjqUgZwPqeYxFl/sa90rdJYznFTfv3M8uwV9XJ6KA+RJ4UwXD+GHGOQNVnedpwrbD9RSZLIKhWOdSMsa3CuvKhVM1VLtolaVFREUlIS995rzm2bOHEiJSVlH6xKpeLtt98mODjY97MUyMJy8VrWU8DRydCoVqj1bm1LjVAtKqUCo0myW+s5b6rBKP1ovtK+y8OqTaSYmvOzdI9f5l+eqiHeXOOqS5ktW45n0a5+FO6GFuuNXMwv4R7lNoapfwFgvP45jkv1KSgxkJFXVpzUIhhdZRXZ3jRyivTWAN5apTe44FITtHzZ8mofB7d06VI++eQT6/O5c+eyZcsWUlJSSElJ4auvvmLBggV+maRAHpZr17L+BvYmaqvYCNaP6+20WqxSqaBWmPkit7WWCkqMbDO14VPtkwBMVn9BO+mwH2bvSBVR4FyGibjT4CxY1sTKa3C2VXXvmLmRmkUn+UBj/v0tNNzHbyZzu4C8YoNdbwtvlGvLjSwqREOQ2qy5WRxQ5T3lZV7UqqXByZ7t119/zbBhw+y2LVu2jD///JM///yTDz74gBUrVvh8ggL5WHqdNqpV5tm2uP3BnF/ozsSwOBpsySsN9P0t8lHWGLujVRiZKX1ILfxfOaaKyDc7JEmyChxXTXxssQjx8kNnrTti/T87+xKfaD4iTFHCFmMbPjA8Zt2XU6S307gt8k2OoLM6LSLKvvfgUkFnm4t6Mb+ErccvAlVPg5Ntoh47doz27cuaVQQHB6O0cRl3796dESNG+HZ2Aq8Y2K0hWpWSBzrVt26zNUk8xa/ViQiCC/bbLKla0WFaXtY/RwvFWZopzzNHM4cn9BMx4r8y9VVxDe6j9UeZnXSU1/u2lnWsxZtaXoM7kVXAyp1nWb7jNO9rPqGZ8jwXpJqM0o+y+8xPXSyw07gt5cvlmMeW3h2W9VewjYMzUqw3snhzKvP/PG7NaOkYHy3rfQUKsjW47OxsuzW3zMxMGjdubH1uMpns9guuPjXCtDxzc1O7O7Itzup82eJMg7MIuBqhWgoI4Tn9WPKlYHqpDjBe7TuNPSOvmDV7L9gF91YR+WbH7CRzuMY7vxx02WfWFotT0pkXddyK3bQ/8xV9VTvQSeZg3otE2Y05nmlfd88ibBdvTvX42mUaXNmSRnCpibrpWBa3z9jA+2sPk19ioF39SJY928Mh3CXQkS3gGjRowL59+1zu37NnDw0aVCwoc968eTRu3Jjg4GB69OjBjh073I7/7rvvaNWqFcHBwbRv3541a9bY7R88eDAKhcLu0adPnwrNrTrhKbyjdoRjLrHFRLV4Y49L9XlFb16qGK7+mUSl++9KLn1m/c0LX+9kyZaT1m1VUL7ZIScTQ2HV4BwF3A3KA0xQm2ssvmV4kp1SC4cxpy/Z579aznI0Pd/ja1s0uDrhjiZqyulszucUExcVzEf9O7JqxE30uq62x3MGGrIF3D333MPkyZMpLnZs1ltUVMSbb75J3759vZ7A8uXLGTduHFOmTGHnzp107NiRxMREMjIynI7fsmULAwcOZOjQoaSkpNCvXz/69evnIHz79OnDhQsXrA9RjNNzQcU6TjQ4S4WLYI2KOhFBKBSwI7Q3nxrMntQZmk9oqjjvcJy3WGLG/jhk871XRRXOBr2M2DaLBncuu8huewyXmKOZjVph4gfjTXxlTJD1mp68qLY4W4OzOKjCg9S80qclf4y/lQc7N/BYpj5Qkb0GN2nSJFasWEHLli0ZOXIkLVqY7yaHDx9m7ty5GAwGJk2a5PUEPvzwQ5599llr0+iFCxfyyy+/sHjxYiZMmOAw/uOPP6ZPnz68/PLLALz99tusW7eOuXPnsnDhQuu4oKAgYmNjvZ7PtYwr0xbMFYI3v3o7eqOJuz76i/cMA+igPEEP5SEWaj6in+5tCrnyEKEtpYvZUPU1OHlOBvO7/PtolnWbBgPztLOpo8jloKkhr+mHIvfTsMg1OXHDFgFnuwb3SJcG1I0IolN8NLWc3PCqGrI1uJiYGLZs2ULr1q2ZMGECDz74IA8++CATJ06kTZs2bNq0iZiYGK9eXKfTkZycTEJC2d1JqVSSkJDA1q1bnR6zdetWu/EAiYmJDuM3bNhA3bp1admyJcOHD+fixYu4oqSkhNzcXLvHtYizNTgLQRolWrWSsCA1JknCgJqRutGkS9G0UJ7jPc0iPIeZekdVKZrpSluSY6I6e4+T1F/TVXmEXCmU5/RjKUa+oLHMRU7zbWcanEal5I7WMdVCuIGXmQxNmjRh7dq1ZGZmsm3bNrZt20ZmZiZr166ladOmXr94VlYWRqPRQTDGxMTY1ZyzJS0tzeP4Pn368MUXX5CUlMR7773Hxo0bufvuu12mkk2bNo2oqCjrIz4+3uv3Esjc3srcOfzm5u7XUNxpcLbxT5YfTybRvKB7Eb2k4j7VNoao1vpgtmVUEfnmFKUCr5wMFh5QbmKI+jcAxuqHW/N/5WJ5RTklrSzlmdx971WdCvVFrVmzJt27d/f1XHzGgAEDrP+3b9+eDh06cN1117FhwwbuuOMOh/ETJ05k3Lhx1ue5ubnVSsh99Fgnft5znr7t67kd516DKwtNsO2kniy1ZKphEG9ovmCSehn7TE34R2p15ZOuQrhqkShHyNgK8VaK00zXfAbAbEM/kkxdZM8hRKOiSG+0mqaeNDijSeJSgaMGV92o1LDk2rVro1KpSE9Pt9uenp7ucv0sNjbWq/EATZs2pXbt2hw7dszp/qCgICIjI+0e1YmoUA2P39CIGmHuKy5Hh2hQu1hMDlI7anAWlhgT+cnYC43CaF474vKVT5qqvQanVCi8ymSIpIAFmo8IUej4y9ieWYZHvHq9oNL4NYuJ6il5/2JBCSbJLGBrhlbfStyVKuC0Wi1dunQhKSnJus1kMpGUlETPnj2dHtOzZ0+78QDr1q1zOR7g7NmzXLx4kXr13Gsw1zpKpcKak1geWwHneMdXMEH/DIdM8dRVZDNXOwc1V17Rt8qswTnZplTKC7ZVKEAyGZmhWUgTZTpnpdqM1o/E5OVPs3wKlScNzrL+VitMW+Wq9HpDpb+zcePG8emnn7J06VIOHjzI8OHDKSgosHpVn3zySSZOnGgd/+KLL7J27VpmzpzJoUOHeOONN/j3338ZOXIkAPn5+bz88sts27aNkydPkpSUxAMPPECzZs1ITEyslPdYlXBlpgbbmKjzB11PjyY17fYXEcxw/RhypRB6KA9Z47euhLPlQieqEkqFgpU7z1mfH8vIczpu24lL6Dd+yF2qZEokNcN1Y8gmQtZrhNmUvbLkF1uWDzxpj2cvmz9bd8sS1YFKF3D9+/dnxowZTJ48mU6dOrFr1y7Wrl1rdSScPn2aCxfK8od69erFsmXLWLRoER07duT777/nxx9/pF27doC5qsmePXu4//77adGiBUOHDqVLly78/fffBAVV7y/TFzhbj+kYH20n0JrVjWD5c44ac6pUj5f0wwF4Rv0r9yqde8LlYskKCHScrcGVLwqa8OFfTo9N3/Ur6o3vAjDZMIS9kntnna1SGx5ctoRuEXCSzDW49aWNcro1rul2XFWnQk4GXzNy5EirBlaeDRs2OGx79NFHefTRR52ODwkJ4bfffvPl9K4pnN3Rlw7pRrTMdZp1pq7MN9zPC+pVvKdZxGFdPEelwC87fjUoL3TiyGK2Zg5KTHxruJXlRs8tB7UqpTVg21xcocS6HeR5UfVGE7+XCrh7PDieqjqVrsEJAgtnAs7bpPeZhkfZZGxLmKKEhZqPCKfi7fSctS2sqtgKHS165mtnUVORzxFVM6YYBss6h22pq4igMv0kqJyJ6s5C3Xwsi5wiPbXDg+jepHprcELACexwZqJ6m6VjRMVo/SjOSzW5TnmhtI5ZxYKA3TVDDhTktg201eCmqL+gk/IEl6Vw3gx5lRLkaci2zoSwIHcmquvPbc1e85LP3e1iq33/WiHgBHbUduJFrUjZoktEMlw3hhJJzd2qfximWl2h+chJd6ps5BaZtAT+PqrawCB1EiZJwYv6EZyV6sh+LVsNLtyJgPt9fxqFOoNLJ8O57CJ+2WMWcNXdPAUh4ATlaFAjxGFbRW/yu6VmvGl4CoBX1d/SU7nf63NUDQ1OHkaTRFtFKu+oPwfgI8PD/GXqKCtezoKdgLN1MpRqdrvP5jD1l4Nu4+AKdEa6N67p4AmvjggBJ7Dj+oY1eOO+NjxxQ1ndL0/xaK1iXYc1LDPezvfGW1ApJOZo5hCL65xgZ1wq0HH7jA1M/eWAV8cFIsb8iyzUzCJIoSfJ2Jm5xn6AvHg5C7YVmcO0jhocwNfbT3Msw3W5pPrRIcx//PoqWyHEG4SAE9ihUCgYfGMTujauYbPN/THuBaCC1/RPs9/UiNqKXOZrP0bjRRDwF1tPcSKrgE//9lzAsdKQYaMqMBHx6wvEKzM5ZarLWP1wpNKfX6EXjhRbmWRbjl6rll9Z+fW+rat9/JsFIeAETrEVWq4E2Ct9WqJSKnjnwXZuz1WCluf1Y8iRQrleeYzX1V/Knkd16cE6WvU/gk/+QbGk4Xn9WHIJt+4r8KKRtq3paau1yW0Gc2OzWvRpd+2UERMCTuAU25xUVwLuhVubcejtPlzfsIbT/backWJ4UW/u2fGUeh0PKv+WNY/yRTqPpOcxYtlOjqY7zwyoDDzpb7cqU3hRvRKASfqhHJQqXvbb1hMbpHbMZHCG7de3dEj3KtPrwhcIASdwim34gLufgzeNgDeYOjPL8BAA72r+S2vFKY/HlF+fGrhoG7/sucD/fbZd9utWJvGKdD7WzEOpkPjSkMBK0y1XdD5bh4StieoOlY1Aq855p864tt6tQDa27eHk3PBdVSEpz8eGh/hHfT0hCh0LNR8RifveAbpyDYgvlpY2t22VV5nkFev5YqtzQR2EjoWaWUQpCkkxNeNtwxNX/Hr1bHre2pqltp/Hy4kt7Y5pXc9cHae6x7w5Qwg4gVNszVI5Jo2tiTTlvjYux0koeS90PGdMdWikzOBDzQIUuPYiVnaYSJHO6Davc9L/9pFTpHeyR+Id9WLaKk+RJUUyXPciOjROxsnntXtac0PTWtbntjX6bJs/j7itGQO7N7Q+nz/oevp1iuOnETde0etXRYSAEzhFrfTu0rAVcI90cZ97mq+M4Hn9GEokDQmqFF5QrXI51l2jHEtEvr/ILdbTevJa7puzyeWYdQecV54eqPqDR9V/YZQUjNKPIo1aTsfJ5ekbm/DsLU0psdFobUtY2Qo4gHvamx0J9aNDiK8ZyqwBnWlX377l4LWAEHACp3gp3+zW4uTUcdsvNeF1g7kk1kvq77hZucf5uPNl/THe+tk+Fu6Fr3fyz8lL3k3UC7YcMzeCOXDBux4dHRXHeEO9FIAPDP3Zamp7xXNpHmP2ut7YzFx2XqtScnuruigV0KVRDTLKmew3N6/DmtE389vYK1vzq+oIASdwijfOA7BfD/K01mMxeb8z3soyw+0oFRKzNXNpoMh0e5yzZsaH0/znTZXTmap8CFxNzLF+QQoDa43dWGi8r8Kvb1vvrXlds4Dr3aIOXz/Tg02v3kZ0qJb9b/ZhxXM9SWht7rthm53QJi7SLp3rWkQIOIFTOsdH075+lMc+DhZszSVvohDeNDzJblNTaijyma+ZRRA6r+bpz4gHOTmmtkOUmJitmUN9xUWOm+rxsv455BZe796kJm/eb6/p2d4ompUKOIVCwY3NalM30uxsCNGqUCkVvP9IR968vy2fPCG/j8O1gBBwAqeoVUpWjbyReYOulzXedg3Ok4lqu7cELS/oXuSSFE4HZarVtJOLP8uau6oSYjCamLX+CN8nn7VzgrykXsFNqv0USkE8rx9LHqGyX6teVDBP9Wpsty23uCzDwVM9vpphWp7q1Vh23b5rhWtbfxW4xZuAUG/W4LIL7bW0c9RhtH4UX2imM1D9JylSM1bIKP4I/m1M48xETTqYztCl/zpsv1P5LyPUZmfJq/pnvS7yeVMz9y0dBRVDaHACn3Bbae/V8CC1x+oj53OKHbZtMrVnhsFcpflt9RLaKU7Iel3/mqhlEi6nSI/OYHIq3BorLjBTswCAxYY+/Gzq5fVr3dXGMX3q9b6t6da4hjA7rwChwQl8wojbriMuKpibmteucCrQAuP9dFYe505VMgu1s7i3ZKrHBiyv/rCX/t0auh1TUWzX4Dq++bvTMSEUs1Azi0hFEf+YWvCu4f8q9FoRwY4/xSd6NuKZm71vqC4oQ2hwAp8QpFYxoHtDGtSQv+5UHgklL+mf56QphgaKLHOKk5sgYH/juVKvxDTNZ7RSniFDiuYF3YsYKqgzWEoXLR7clbvbxbLzP3fa5ZoKKoYQcAK/ckPTmnw+pJvs8bmE8bx+LEWSlt6qPbyo/sHjMZLckrpe4um0T6l+p59qCwZJyQjdaDLxXHTAE7e3imHB412o6aFJt0AeQsAJ/EpmXgm3tazr1TGHpIZM1D8DwIvq/3G7cqfb8Z5a5FUUdwLuesURXld/BcC7hkH8I7WSfV65pY0EV474pAV+pV6UYwl0OfxouoklhrsA+Egzn4aKdJdjs53mgl45JhcSrg7Z5sKdCiM/G29gsbGPV+eV26RGcOUIASfwC2tG38xD19dn2kPtK3yOqYbHSTY1J0pRyELNLIJxXkGk6zvrK/warsjILebl7x3Tx9QYmKudTaziMkdN9XlVPwxvg1XKy8129SOvYKYCdwgBJ/ALbeIi+fCxTsTXdHQ6jL6juaxz6FHzgu5FMqVI2ihPMVXzXyraftBbPvjtsNPtr6iX00N5iDwphOf1Yygk2Ok4d9i+g2kPtWfpkO4VnKXAE0LACa46tjmWnkinJqP0ozFISh5WbeJxle+1NWfsOZvjsO0e5TaGqX8BYLz+OY5L9St0blunyMDuDal1jfRHqAyEgBNcdbwNk9tmasN7hgEATFZ/QSfFMYcxI5ft5MylQl9Mj1MXCzhcriT6dYpzvK9ZBMBCw738ZqqY1hUbGSxW4K4iQsAJAgpXLQg/NfZljbE7WoWR+dpZ1MJew1q95wKjvknBaJLYfSb7ihpG7zx92e55GEV8ovmIcEUxW41t+MDQ36vzvfVAW9rGRdK3fT2+eqaH7EbRgitHCDhBwPD4DQ1ZO8ZV/TIFL+uf45gpjjjFJeZo5qDCvpz5rjPZzFp/hAfmbWbCD3tlvWbSwXTeWLXfKhBvnP4HY5fvthkh8b7mE5opz3NBqsko/SiMyDexp9zXhid7NuaX0Tczb9D11qoggquDEHCCq8KtLevY/S3P4F6Neaefo8d1/bje1v8LCOE5/VgKpCB6qQ4wXr3CYfzcP83m6w87z8qa19Cl/7Jky0m+/ecMGw5ncC67yG7/M6o19FXtQCepGKEbTRbyq+LOfLQjQ25sInu8wPcIASe4Kswe2Jn3H+nAxwM6O90/oHu89f/X7mkNmL2t5XM0j0v1eUX/HADD1T+TqPzHbr/G21LEpZzPLmLw5/bn6qE4yAT1NwC8ZXiSnVILr87prwBkgXxEsr3gqhAZrOGxrvEO238ccSOFJQZaxZbFgj17S1Pu6xhHTGSQ0+5Zv5huoJPhGM+q1zBDs5AjugakSubCnBqVAi/6KFtZuuWk3fMYLjFX+zFqhYmVxpv4ypjg9TkNQsBVOkKDE1QqneKj6eWkFlpsVDAKhcJlZZL3DAPYbmpFhKKIhZqPCMVcgsmbLvG2FNocp8HAPO1s6ihyOWhqyCT9UOQE8zasGWpX2shgcu7oiA69su5aAvkIAScIaFzVljOgZqRuNOlSNC2VZ3lPswhfBQFPUn9NV+URcqVQntePoRh5cWp/jr+VxLZldd1smzTb0i7u2utuVVkIASe46gRr5Hshw9w0TcnEXKJIL6m4T7WNIaq1dvvTnBTW9MQDyk0MUf8GwFj9cE5JjoUo7eZXGrQ8sHtDh2Y7rtbgrsX2fZWFEHCCq86jXeLp3DCaMQmeU7Y8CcNkqSVTDYMAmKReRlfFIeu+G6YleRX821JxmumazwCYbehHkslzJd33HunAV0N78Mb9js2u9S5M1Od7NyUuKphHPfSPFVw5QsAJrjohWhX/e+FGxiTI80q296DxLDEm8pOxFxqFkfna2dShLFD3u3/PyHqNCApZqPmIEIWOv4ztmWV4RN5xwRpual7baXFKowsTNTpUy6ZXb+f9RzrIeg1BxRECThDw/DDcU48DBRP0z3DIFE9dRTZztXNQY+5IpXSyiKc3mlh3IJ2txy+WHm1ipmYBTZTpnJVq86J+BCYXP4360WXln25sVosbmtZ0GKNRmV/zxuauG8kola4dKALfIQScIOCxbUnoiiKCGa4fQ64UQg/lIWv8WpHe6JC2Ne/PYzz7xb8M/HQbAMNVP3OXKpkSSc1w3Rgu47p80St9Wlr///qZG5xqbtsnJbBq5I1c3/DKK/wKrgwh4ARVgsdv8NxYplbDNozXPw/AM+pf6avcxicbT3D3x3/bjfs+uSzL4SblXl4qzYiYbBjCXsmxycsjXRqwY9IdnJzeV1Yeac0wLR0aRHseKPA7ItBXUCV4p197bmpWh++Tz7L+oPPqvkuf7k7bKZeZb7ifF9SreF/zCYd1DTiWYb+Yf/ayOR0rjixma+agUkh8a7iV5U56saZOu8fOlLyrbQyNaoXSo4mjaSoIPIQGJ6gy9GkXy2dPdbUzE22xhJTMNDzKJmNbwhQl5iogFDo0m9aiZ752FjUV+ew1NWaKYbDD+eJrhjisk4Vq1WwYfyvvP9LRN29K4FeEgBNUOV64tRm/j72FuhHOA3CNqBitH8V5qSbXKS/wgeYTOr1l39d0ivoLOilPcFkKZ7h+LCXYd7HSqpSsGX2z0/ML50DVISAE3Lx582jcuDHBwcH06NGDHTt2uB3/3Xff0apVK4KDg2nfvj1r1qyx2y9JEpMnT6ZevXqEhISQkJDA0aNH/fkWBFeZFjER7HgtgfmDrgdg+6Q77PZfIpLhujGUSGruVv3DMNVqckqb0zyq2sAgdRImScEY/QjOSvYVTlaNvJE9b9xFRLBIqarqVLqAW758OePGjWPKlCns3LmTjh07kpiYSEZGhtPxW7ZsYeDAgQwdOpSUlBT69etHv3792Ldvn3XM+++/z+zZs1m4cCHbt28nLCyMxMREiou9j2wXBDb3tK/Hyel9iYl07I2wW2rGW4YnAXhV/S2Lli7hrUXLeEf9OQAfGR5mo8ne1Fww6Ho6NIj2KttCELgoJH91zZVJjx496NatG3PnzgXAZDIRHx/PqFGjmDBhgsP4/v37U1BQwOrVq63bbrjhBjp16sTChQuRJIm4uDheeuklxo8fD0BOTg4xMTEsWbKEAQMGeJxTbm4uUVFR5OTkEBkpOh5VJYr1RgwmibScIhI+/AuQmKH5hEdUf6GXVEiAVmEkydiZZ/QvIaFk9+S7iAhWO42ZEwQmcn+jlarB6XQ6kpOTSUgoK0WjVCpJSEhg69atTo/ZunWr3XiAxMRE6/jU1FTS0tLsxkRFRdGjRw+X5ywpKSE3N9fuIaiaBGtUhAepaVY3otRsVfCa/mn2mxqhURjRKoycMtVlrH44EkoOvd2HqFCNEG7VlEoVcFlZWRiNRmJiYuy2x8TEkJaW5vSYtLQ0t+Mtf70557Rp04iKirI+4uMd65YJqh4xkcGcnN6X5SNu43n9GC5J4RRIQQzXj+HVB2/g6NS7hSlazRFxcMDEiRMZN26c9Xlubq4QctWITvHR/D3taXIu30+hrog1MeK7vVaoVAFXu3ZtVCoV6en2gZvp6enExjovUxMbG+t2vOVveno69erVsxvTqVMnp+cMCgoiKEj0pqzuRNVwnRsqqJ5Uqomq1Wrp0qULSUlJ1m0mk4mkpCR69uzp9JiePXvajQdYt26ddXyTJk2IjY21G5Obm8v27dtdnlMgEFRTpErm22+/lYKCgqQlS5ZIBw4ckIYNGyZFR0dLaWlpkiRJ0hNPPCFNmDDBOn7z5s2SWq2WZsyYIR08eFCaMmWKpNFopL1791rHTJ8+XYqOjpZ++uknac+ePdIDDzwgNWnSRCoqKpI1p5ycHAmQcnJyfPtmBQKBT5D7G630Nbj+/fuTmZnJ5MmTSUtLo1OnTqxdu9bqJDh9+jRKm05JvXr1YtmyZbz++utMmjSJ5s2b8+OPP9KuXTvrmFdeeYWCggKGDRtGdnY2N910E2vXriU42DFWSiAQVF8qPQ4uEBFxcAJBYFMl4uAEAoHAnwgBJxAIqi1CwAkEgmpLpTsZAhHLsqRI2RIIAhPLb9OTC0EIOCfk5eUBiGwGgSDAycvLIyrKddc14UV1gslk4vz580RERFSp4oaWFLMzZ84I768TxOfjnqr0+UiSRF5eHnFxcXZhZOURGpwTlEolDRpU3aa8kZGRAX+BVibi83FPVfl83GluFoSTQSAQVFuEgBMIBNUWIeCqEUFBQUyZMkVURnGB+HzcUx0/H+FkEAgE1RahwQkEgmqLEHACgaDaIgScQCCotggBJxAIqi1CwFVxLl26xKBBg4iMjCQ6OpqhQ4eSn5/v9phFixZx6623EhkZiUKhIDs7++pM9iowb948GjduTHBwMD169GDHjh1ux3/33Xe0atWK4OBg2rdvz5o1a67STCsHbz6f/fv38/DDD9O4cWMUCgWzZs26ehP1EULAVXEGDRrE/v37WbduHatXr+avv/5i2LBhbo8pLCykT58+TJo06SrN8uqwfPlyxo0bx5QpU9i5cycdO3YkMTGRjIwMp+O3bNnCwIEDGTp0KCkpKfTr149+/fqxb9++qzzzq4O3n09hYSFNmzZl+vTpLptABTz+rZwu8CcHDhyQAOmff/6xbvv1118lhUIhnTt3zuPxf/75pwRIly9f9uMsrx7du3eXRowYYX1uNBqluLg4adq0aU7HP/bYY1Lfvn3ttvXo0UN67rnn/DrPysLbz8eWRo0aSR999JEfZ+cfhAZXhdm6dSvR0dF07drVui0hIQGlUsn27dsrcWZXH51OR3JyMgkJCdZtSqWShIQEtm7d6vSYrVu32o0HSExMdDm+KlORz6c6IARcFSYtLY26devabVOr1dSsWZO0tLRKmlXlkJWVhdFotDYrshATE+Pys0hLS/NqfFWmIp9PdUAIuABkwoQJKBQKt49Dhw5V9jQFgoBHlEsKQF566SUGDx7sdkzTpk2JjY11WCA2GAxcunSp6i4KV5DatWujUqlIT0+3256enu7ys4iNjfVqfFWmIp9PdUBocAFInTp1aNWqlduHVqulZ8+eZGdnk5ycbD32jz/+wGQy0aNHj0p8B1cfrVZLly5dSEpKsm4zmUwkJSXRs2dPp8f07NnTbjzAunXrXI6vylTk86kWVLaXQ3Bl9OnTR+rcubO0fft2adOmTVLz5s2lgQMHWvefPXtWatmypbR9+3brtgsXLkgpKSnSp59+KgHSX3/9JaWkpEgXL16sjLfgM7799lspKChIWrJkiXTgwAFp2LBhUnR0tJSWliZJkiQ98cQT0oQJE6zjN2/eLKnVamnGjBnSwYMHpSlTpkgajUbau3dvZb0Fv+Lt51NSUiKlpKRIKSkpUr169aTx48dLKSkp0tGjRyvrLXiNEHBVnIsXL0oDBw6UwsPDpcjISGnIkCFSXl6edX9qaqoESH/++ad125QpUyTA4fH5559f/TfgY+bMmSM1bNhQ0mq1Uvfu3aVt27ZZ9/Xu3Vt66qmn7MavWLFCatGihaTVaqW2bdtKv/zyy1We8dXFm8/Hcu2Uf/Tu3fvqT7yCiHJJAoGg2iLW4AQCQbVFCDiBQFBtEQJOIBBUW4SAEwgE1RYh4AQCQbVFCDiBQFBtEQJOIBBUW4SAEwgE1RYh4ARVhsGDBzutrNKnT5/KnpogQBHVRARVij59+vD555/bbXPViV2v16PRaOy26XQ6tFqt169b0eMElYvQ4ARViqCgIGJjY+0eNWrUAEChULBgwQLuv/9+wsLCmDp1Km+88QadOnXis88+o0mTJgQHBwNw+vRpHnjgAcLDw4mMjOSxxx6zKyXk6jhB1UIIOEG14o033uDBBx9k7969PP300wAcO3aMH374gZUrV7Jr1y5MJhMPPPAAly5dYuPGjaxbt44TJ07Qv39/u3OVP05Q9RAmqqBKsXr1asLDw+22TZo0ydoh7P/+7/8YMmSI3X6dTscXX3xBnTp1AHPNt71795Kamkp8fDwAX3zxBW3btuWff/6hW7duTo8TVD2EgBNUKW677TYWLFhgt61mzZrW/20b8Fho1KiRnZA6ePAg8fHxVuEG0KZNG6Kjozl48KBVwJU/TlD1EAJOUKUICwujWbNmbvfL2Sb3tQRVG7EGJ7jmaN26NWfOnOHMmTPWbQcOHCA7O5s2bdpU4swEvkZocIIqRUlJiUObO7VaTe3atWWfIyEhgfbt2zNo0CBmzZqFwWDghRdeoHfv3k5NXEHVRWhwgirF2rVrqVevnt3jpptu8uocCoWCn376iRo1anDLLbeQkJBA06ZNWb58uZ9mLagsRMlygUBQbREanEAgqLYIAScQCKotQsAJBIJqixBwAoGg2iIEnEAgqLYIAScQCKotQsAJBIJqixBwAoGg2iIEnEAgqLYIAScQCKotQsAJBIJqixBwAoGg2vL/jZKbCt0d9MsAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.015014, + "end_time": "2024-03-26T07:07:11.906494", + "exception": false, + "start_time": "2024-03-26T07:07:11.891480", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 303.656298, + "end_time": "2024-03-26T07:07:14.641908", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/lct_gan/42/mlu-eval-load.ipynb", + "output_path": "eval/contraceptive/lct_gan/42/mlu-eval-load.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/contraceptive/lct_gan/42", + "path_prefix": "../../../../", + "random_seed": 42, + "single_model": "lct_gan" + }, + "start_time": "2024-03-26T07:02:10.985610", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/lct_gan/mlu-eval.ipynb b/contraceptive/lct_gan/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..2e7ad3f3331334e1c66d8e877973efc90ea2ed57 --- /dev/null +++ b/contraceptive/lct_gan/mlu-eval.ipynb @@ -0,0 +1,2269 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:48:55.756313Z", + "iopub.status.busy": "2024-03-24T16:48:55.755963Z", + "iopub.status.idle": "2024-03-24T16:48:55.791654Z", + "shell.execute_reply": "2024-03-24T16:48:55.790907Z" + }, + "papermill": { + "duration": 0.051893, + "end_time": "2024-03-24T16:48:55.793856", + "exception": false, + "start_time": "2024-03-24T16:48:55.741963", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:48:55.820237Z", + "iopub.status.busy": "2024-03-24T16:48:55.819887Z", + "iopub.status.idle": "2024-03-24T16:48:55.826763Z", + "shell.execute_reply": "2024-03-24T16:48:55.825852Z" + }, + "papermill": { + "duration": 0.022407, + "end_time": "2024-03-24T16:48:55.828668", + "exception": false, + "start_time": "2024-03-24T16:48:55.806261", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:48:55.852180Z", + "iopub.status.busy": "2024-03-24T16:48:55.851939Z", + "iopub.status.idle": "2024-03-24T16:48:55.856102Z", + "shell.execute_reply": "2024-03-24T16:48:55.855228Z" + }, + "papermill": { + "duration": 0.018137, + "end_time": "2024-03-24T16:48:55.858103", + "exception": false, + "start_time": "2024-03-24T16:48:55.839966", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:48:55.882533Z", + "iopub.status.busy": "2024-03-24T16:48:55.882167Z", + "iopub.status.idle": "2024-03-24T16:48:55.886502Z", + "shell.execute_reply": "2024-03-24T16:48:55.885612Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018935, + "end_time": "2024-03-24T16:48:55.888576", + "exception": false, + "start_time": "2024-03-24T16:48:55.869641", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:48:55.912595Z", + "iopub.status.busy": "2024-03-24T16:48:55.912264Z", + "iopub.status.idle": "2024-03-24T16:48:55.918703Z", + "shell.execute_reply": "2024-03-24T16:48:55.917860Z" + }, + "papermill": { + "duration": 0.021317, + "end_time": "2024-03-24T16:48:55.921283", + "exception": false, + "start_time": "2024-03-24T16:48:55.899966", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "af92ecea", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:48:55.947000Z", + "iopub.status.busy": "2024-03-24T16:48:55.946747Z", + "iopub.status.idle": "2024-03-24T16:48:55.951653Z", + "shell.execute_reply": "2024-03-24T16:48:55.950827Z" + }, + "papermill": { + "duration": 0.019815, + "end_time": "2024-03-24T16:48:55.953665", + "exception": false, + "start_time": "2024-03-24T16:48:55.933850", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"lct_gan\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 42\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/lct_gan/42\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011109, + "end_time": "2024-03-24T16:48:55.976177", + "exception": false, + "start_time": "2024-03-24T16:48:55.965068", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:48:56.000093Z", + "iopub.status.busy": "2024-03-24T16:48:55.999462Z", + "iopub.status.idle": "2024-03-24T16:48:56.008249Z", + "shell.execute_reply": "2024-03-24T16:48:56.007468Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.022732, + "end_time": "2024-03-24T16:48:56.010121", + "exception": false, + "start_time": "2024-03-24T16:48:55.987389", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/lct_gan/42\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:48:56.033962Z", + "iopub.status.busy": "2024-03-24T16:48:56.033701Z", + "iopub.status.idle": "2024-03-24T16:48:58.104299Z", + "shell.execute_reply": "2024-03-24T16:48:58.102687Z" + }, + "papermill": { + "duration": 2.08582, + "end_time": "2024-03-24T16:48:58.107485", + "exception": false, + "start_time": "2024-03-24T16:48:56.021665", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:48:58.147952Z", + "iopub.status.busy": "2024-03-24T16:48:58.147383Z", + "iopub.status.idle": "2024-03-24T16:48:58.160625Z", + "shell.execute_reply": "2024-03-24T16:48:58.159716Z" + }, + "papermill": { + "duration": 0.039124, + "end_time": "2024-03-24T16:48:58.162865", + "exception": false, + "start_time": "2024-03-24T16:48:58.123741", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:48:58.188688Z", + "iopub.status.busy": "2024-03-24T16:48:58.188092Z", + "iopub.status.idle": "2024-03-24T16:48:58.196834Z", + "shell.execute_reply": "2024-03-24T16:48:58.195931Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.023806, + "end_time": "2024-03-24T16:48:58.199192", + "exception": false, + "start_time": "2024-03-24T16:48:58.175386", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:48:58.232505Z", + "iopub.status.busy": "2024-03-24T16:48:58.231709Z", + "iopub.status.idle": "2024-03-24T16:48:58.338342Z", + "shell.execute_reply": "2024-03-24T16:48:58.337444Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.125269, + "end_time": "2024-03-24T16:48:58.340746", + "exception": false, + "start_time": "2024-03-24T16:48:58.215477", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:48:58.369454Z", + "iopub.status.busy": "2024-03-24T16:48:58.369151Z", + "iopub.status.idle": "2024-03-24T16:49:03.130885Z", + "shell.execute_reply": "2024-03-24T16:49:03.129963Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.779479, + "end_time": "2024-03-24T16:49:03.133283", + "exception": false, + "start_time": "2024-03-24T16:48:58.353804", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-24 16:49:00.662068: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-24 16:49:00.662158: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-24 16:49:00.664213: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:49:03.159806Z", + "iopub.status.busy": "2024-03-24T16:49:03.158703Z", + "iopub.status.idle": "2024-03-24T16:49:03.164356Z", + "shell.execute_reply": "2024-03-24T16:49:03.163663Z" + }, + "papermill": { + "duration": 0.020618, + "end_time": "2024-03-24T16:49:03.166282", + "exception": false, + "start_time": "2024-03-24T16:49:03.145664", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:49:03.192968Z", + "iopub.status.busy": "2024-03-24T16:49:03.192703Z", + "iopub.status.idle": "2024-03-24T16:49:11.591059Z", + "shell.execute_reply": "2024-03-24T16:49:11.589986Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.41478, + "end_time": "2024-03-24T16:49:11.593583", + "exception": false, + "start_time": "2024-03-24T16:49:03.178803", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-24T16:49:11.621678Z", + "iopub.status.busy": "2024-03-24T16:49:11.621156Z", + "iopub.status.idle": "2024-03-24T16:49:11.628321Z", + "shell.execute_reply": "2024-03-24T16:49:11.627434Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.023898, + "end_time": "2024-03-24T16:49:11.630234", + "exception": false, + "start_time": "2024-03-24T16:49:11.606336", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 46,\n", + " 'realtabformer': (24, 72, Embedding(72, 672), True),\n", + " 'lct_gan': 40,\n", + " 'tab_ddpm_concat': 10}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:49:11.654694Z", + "iopub.status.busy": "2024-03-24T16:49:11.654418Z", + "iopub.status.idle": "2024-03-24T16:49:11.659115Z", + "shell.execute_reply": "2024-03-24T16:49:11.658253Z" + }, + "papermill": { + "duration": 0.019272, + "end_time": "2024-03-24T16:49:11.661078", + "exception": false, + "start_time": "2024-03-24T16:49:11.641806", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:49:11.685585Z", + "iopub.status.busy": "2024-03-24T16:49:11.685304Z", + "iopub.status.idle": "2024-03-24T16:49:12.202349Z", + "shell.execute_reply": "2024-03-24T16:49:12.201381Z" + }, + "papermill": { + "duration": 0.532005, + "end_time": "2024-03-24T16:49:12.204786", + "exception": false, + "start_time": "2024-03-24T16:49:11.672781", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/aug_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_bs_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/bs_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_synth_test/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:49:12.236168Z", + "iopub.status.busy": "2024-03-24T16:49:12.235297Z", + "iopub.status.idle": "2024-03-24T16:49:12.563979Z", + "shell.execute_reply": "2024-03-24T16:49:12.563107Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.346363, + "end_time": "2024-03-24T16:49:12.566071", + "exception": false, + "start_time": "2024-03-24T16:49:12.219708", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'loss_balancer_beta': 0.7999999999999999,\n", + " 'loss_balancer_r': 0.95,\n", + " 'tf_pma_low': 8,\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7,\n", + " 'bias_weight_decay': 0.05,\n", + " 'head_activation': torch.nn.modules.activation.ReLU6,\n", + " 'tf_activation': torch.nn.modules.activation.Tanh,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.07,\n", + " 'n_warmup_steps': 200,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'fixed_role_model': 'lct_gan',\n", + " 'g_loss_mul': 0.1,\n", + " 'd_model': 256,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 512,\n", + " 'ada_n_layers': 9,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 512,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'single_model': True,\n", + " 'models': ['lct_gan'],\n", + " 'max_seconds': 3600,\n", + " 'Body': 'twin_encoder',\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 128,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.5, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "# params = {\n", + "# **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "# }\n", + "params = getattr(PARAMS, dataset_name).BEST_DICT[gp][gp_multiply][single_model]\n", + "if isinstance(params, (list, tuple)):\n", + " params = params[param_index]\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:49:12.594922Z", + "iopub.status.busy": "2024-03-24T16:49:12.594370Z", + "iopub.status.idle": "2024-03-24T16:49:12.703911Z", + "shell.execute_reply": "2024-03-24T16:49:12.703086Z" + }, + "papermill": { + "duration": 0.126462, + "end_time": "2024-03-24T16:49:12.706021", + "exception": false, + "start_time": "2024-03-24T16:49:12.579559", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_train/lct_gan/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/contraceptive [400, 0]\n", + "Caching in ../../../../contraceptive/_cache_aug_val/lct_gan/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/contraceptive [0, 200]\n", + "Caching in ../../../../contraceptive/_cache_bs_train/lct_gan/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/contraceptive [100, 0]\n", + "Caching in ../../../../contraceptive/_cache_bs_val/lct_gan/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/contraceptive [0, 50]\n", + "Caching in ../../../../contraceptive/_cache_synth/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/contraceptive [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-24T16:49:12.735014Z", + "iopub.status.busy": "2024-03-24T16:49:12.734314Z", + "iopub.status.idle": "2024-03-24T16:49:13.136389Z", + "shell.execute_reply": "2024-03-24T16:49:13.135431Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.418903, + "end_time": "2024-03-24T16:49:13.138768", + "exception": false, + "start_time": "2024-03-24T16:49:12.719865", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['lct_gan'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:49:13.168988Z", + "iopub.status.busy": "2024-03-24T16:49:13.168657Z", + "iopub.status.idle": "2024-03-24T16:49:13.173072Z", + "shell.execute_reply": "2024-03-24T16:49:13.172203Z" + }, + "papermill": { + "duration": 0.02214, + "end_time": "2024-03-24T16:49:13.175165", + "exception": false, + "start_time": "2024-03-24T16:49:13.153025", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:49:13.201165Z", + "iopub.status.busy": "2024-03-24T16:49:13.200908Z", + "iopub.status.idle": "2024-03-24T16:49:13.207683Z", + "shell.execute_reply": "2024-03-24T16:49:13.206823Z" + }, + "papermill": { + "duration": 0.021906, + "end_time": "2024-03-24T16:49:13.209657", + "exception": false, + "start_time": "2024-03-24T16:49:13.187751", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "7604993" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:49:13.235982Z", + "iopub.status.busy": "2024-03-24T16:49:13.235318Z", + "iopub.status.idle": "2024-03-24T16:49:13.317960Z", + "shell.execute_reply": "2024-03-24T16:49:13.317115Z" + }, + "papermill": { + "duration": 0.097864, + "end_time": "2024-03-24T16:49:13.319865", + "exception": false, + "start_time": "2024-03-24T16:49:13.222001", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 40] --\n", + "├─Adapter: 1-1 [2, 1179, 40] --\n", + "│ └─Sequential: 2-1 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 512] 20,992\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-16 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-9 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-17 [2, 1179, 256] 131,328\n", + "│ │ │ └─LeakyHardsigmoid: 4-18 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 40] (recursive)\n", + "│ └─Sequential: 2-2 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-10 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-32 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-17 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-33 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-34 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-18 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-36 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 2048] --\n", + "│ └─Encoder: 2-3 [2, 8, 256] --\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-6 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1179, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-12 [2, 1179, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-18 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1179, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-24 [2, 1179, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-39 [2, 8, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-30 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1179, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-36 [2, 1179, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 8, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 8, 256] 2,048\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 8, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 8, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 8, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 8, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 8, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-42 [2, 8, 256] --\n", + "│ └─Encoder: 2-4 [2, 8, 256] (recursive)\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-40 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-8 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-21 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-22 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-48 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-23 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 294, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-54 [2, 294, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-25 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-60 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 294, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-66 [2, 294, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-31 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-42 [2, 8, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-72 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 294, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-78 [2, 294, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-14 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-40 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 8, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 8, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-84 [2, 8, 256] --\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-21 [2, 512] --\n", + "│ │ │ └─Linear: 4-43 [2, 512] 1,049,088\n", + "│ │ │ └─ReLU6: 4-44 [2, 512] --\n", + "│ │ └─FeedForward: 3-22 [2, 512] --\n", + "│ │ │ └─Linear: 4-45 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-46 [2, 512] --\n", + "│ │ └─FeedForward: 3-23 [2, 512] --\n", + "│ │ │ └─Linear: 4-47 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-48 [2, 512] --\n", + "│ │ └─FeedForward: 3-24 [2, 512] --\n", + "│ │ │ └─Linear: 4-49 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-50 [2, 512] --\n", + "│ │ └─FeedForward: 3-25 [2, 512] --\n", + "│ │ │ └─Linear: 4-51 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-52 [2, 512] --\n", + "│ │ └─FeedForward: 3-26 [2, 512] --\n", + "│ │ │ └─Linear: 4-53 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-54 [2, 512] --\n", + "│ │ └─FeedForward: 3-27 [2, 512] --\n", + "│ │ │ └─Linear: 4-55 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-56 [2, 512] --\n", + "│ │ └─FeedForward: 3-28 [2, 512] --\n", + "│ │ │ └─Linear: 4-57 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-58 [2, 512] --\n", + "│ │ └─FeedForward: 3-29 [2, 1] --\n", + "│ │ │ └─Linear: 4-59 [2, 1] 513\n", + "│ │ │ └─LeakyHardsigmoid: 4-60 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 7,604,993\n", + "Trainable params: 7,604,993\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 24.24\n", + "========================================================================================================================\n", + "Input size (MB): 0.47\n", + "Forward/backward pass size (MB): 257.33\n", + "Params size (MB): 30.42\n", + "Estimated Total Size (MB): 288.22\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:49:13.349035Z", + "iopub.status.busy": "2024-03-24T16:49:13.348769Z", + "iopub.status.idle": "2024-03-24T17:54:43.774627Z", + "shell.execute_reply": "2024-03-24T17:54:43.773664Z" + }, + "papermill": { + "duration": 3930.457885, + "end_time": "2024-03-24T17:54:43.791813", + "exception": false, + "start_time": "2024-03-24T16:49:13.333928", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.019665115947168993, 'avg_role_model_std_loss': 1.3259430897919648, 'avg_role_model_mean_pred_loss': 0.0016351528701719095, 'avg_role_model_g_mag_loss': 0.12722583417470257, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0199518630333801, 'n_size': 900, 'n_batch': 225, 'duration': 270.090327501297, 'duration_batch': 1.20040145556132, 'duration_size': 0.30010036389033, 'avg_pred_std': 0.11211167593641827}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005123136696509189, 'avg_role_model_std_loss': 1.9587845133610828, 'avg_role_model_mean_pred_loss': 3.225599730537245e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005123136696509189, 'n_size': 450, 'n_batch': 113, 'duration': 87.16603398323059, 'duration_batch': 0.7713808317100053, 'duration_size': 0.19370229774051242, 'avg_pred_std': 0.048274144109964896}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004459613269524804, 'avg_role_model_std_loss': 0.5696247978427467, 'avg_role_model_mean_pred_loss': 4.493151174229456e-05, 'avg_role_model_g_mag_loss': 0.02243374695568087, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004538138056490829, 'n_size': 900, 'n_batch': 225, 'duration': 271.9333186149597, 'duration_batch': 1.2085925271775988, 'duration_size': 0.3021481317943997, 'avg_pred_std': 0.10111145882142915}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003131905284826644, 'avg_role_model_std_loss': 1.191799519885096, 'avg_role_model_mean_pred_loss': 1.3011600207322418e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003131905284826644, 'n_size': 450, 'n_batch': 113, 'duration': 92.20275020599365, 'duration_batch': 0.8159535416459616, 'duration_size': 0.20489500045776368, 'avg_pred_std': 0.05275093076289096}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0035123983815234777, 'avg_role_model_std_loss': 0.4977147144650333, 'avg_role_model_mean_pred_loss': 2.859574940254904e-05, 'avg_role_model_g_mag_loss': 0.018076600913837965, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003567821487797321, 'n_size': 900, 'n_batch': 225, 'duration': 275.1772153377533, 'duration_batch': 1.2230098459455703, 'duration_size': 0.30575246148639257, 'avg_pred_std': 0.1008471870360275}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0031233079556032962, 'avg_role_model_std_loss': 1.3221473698086115, 'avg_role_model_mean_pred_loss': 4.6298641943619236e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0031233079556032962, 'n_size': 450, 'n_batch': 113, 'duration': 88.90858387947083, 'duration_batch': 0.7868016272519542, 'duration_size': 0.1975746308432685, 'avg_pred_std': 0.05913279911882318}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003161510648399902, 'avg_role_model_std_loss': 0.5058521567848904, 'avg_role_model_mean_pred_loss': 2.4743936033731546e-05, 'avg_role_model_g_mag_loss': 0.014727871190666014, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00322171315592843, 'n_size': 900, 'n_batch': 225, 'duration': 272.91378474235535, 'duration_batch': 1.2129501544104682, 'duration_size': 0.30323753860261704, 'avg_pred_std': 0.09905971649413307}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004386183474238755, 'avg_role_model_std_loss': 2.076401953705954, 'avg_role_model_mean_pred_loss': 2.7025729084248062e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004386183474238755, 'n_size': 450, 'n_batch': 113, 'duration': 88.29646849632263, 'duration_batch': 0.7813846769586074, 'duration_size': 0.19621437443627252, 'avg_pred_std': 0.03421469124248334}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0026987572804662505, 'avg_role_model_std_loss': 0.35000921227432324, 'avg_role_model_mean_pred_loss': 1.5105901600198398e-05, 'avg_role_model_g_mag_loss': 0.015139500791944253, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0027399578163426162, 'n_size': 900, 'n_batch': 225, 'duration': 272.80935049057007, 'duration_batch': 1.2124860021803114, 'duration_size': 0.30312150054507786, 'avg_pred_std': 0.09980787922110823}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002216681500544332, 'avg_role_model_std_loss': 1.5294503328664861, 'avg_role_model_mean_pred_loss': 5.906305432745928e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002216681500544332, 'n_size': 450, 'n_batch': 113, 'duration': 88.42259788513184, 'duration_batch': 0.7825008662401047, 'duration_size': 0.19649466196695964, 'avg_pred_std': 0.05325051893447155}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002033243282720731, 'avg_role_model_std_loss': 0.23602186178530385, 'avg_role_model_mean_pred_loss': 7.030283134612784e-06, 'avg_role_model_g_mag_loss': 0.015872358748973865, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0020601135703246756, 'n_size': 900, 'n_batch': 225, 'duration': 275.2858557701111, 'duration_batch': 1.2234926923116047, 'duration_size': 0.3058731730779012, 'avg_pred_std': 0.10312151636100478}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002322478660401733, 'avg_role_model_std_loss': 1.320404739686366, 'avg_role_model_mean_pred_loss': 9.749587738599938e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002322478660401733, 'n_size': 450, 'n_batch': 113, 'duration': 90.18929839134216, 'duration_batch': 0.7981353839941784, 'duration_size': 0.20042066309187148, 'avg_pred_std': 0.0515262810760631}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001614498706185259, 'avg_role_model_std_loss': 0.1809647554011328, 'avg_role_model_mean_pred_loss': 2.9324602819437286e-06, 'avg_role_model_g_mag_loss': 0.01615508396980456, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016328314457981225, 'n_size': 900, 'n_batch': 225, 'duration': 275.734769821167, 'duration_batch': 1.2254878658718533, 'duration_size': 0.30637196646796333, 'avg_pred_std': 0.10563728055192365}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0019716585887686556, 'avg_role_model_std_loss': 1.9351633055870725, 'avg_role_model_mean_pred_loss': 4.450378878835354e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019716585887686556, 'n_size': 450, 'n_batch': 113, 'duration': 88.78998851776123, 'duration_batch': 0.7857521107766481, 'duration_size': 0.19731108559502494, 'avg_pred_std': 0.04906956392414538}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0014726841067183867, 'avg_role_model_std_loss': 0.21683113766619005, 'avg_role_model_mean_pred_loss': 2.8254440730661005e-06, 'avg_role_model_g_mag_loss': 0.016018104005423892, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0014894471139349562, 'n_size': 900, 'n_batch': 225, 'duration': 272.83155059814453, 'duration_batch': 1.2125846693250868, 'duration_size': 0.3031461673312717, 'avg_pred_std': 0.10693864568240112}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0019270910127347127, 'avg_role_model_std_loss': 1.2538913027706378, 'avg_role_model_mean_pred_loss': 3.4612182754591007e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019270910127347127, 'n_size': 450, 'n_batch': 113, 'duration': 89.2393867969513, 'duration_batch': 0.789729086698684, 'duration_size': 0.19830974843766955, 'avg_pred_std': 0.050700269499381556}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001313657746925148, 'avg_role_model_std_loss': 0.1492658925630795, 'avg_role_model_mean_pred_loss': 3.4148349597811518e-06, 'avg_role_model_g_mag_loss': 0.016415953670317927, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013283096294455592, 'n_size': 900, 'n_batch': 225, 'duration': 286.74962306022644, 'duration_batch': 1.2744427691565618, 'duration_size': 0.31861069228914046, 'avg_pred_std': 0.1061259970565637}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0018737697858078818, 'avg_role_model_std_loss': 1.1538138667777444, 'avg_role_model_mean_pred_loss': 4.4807563205433855e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0018737697858078818, 'n_size': 450, 'n_batch': 113, 'duration': 99.40478444099426, 'duration_batch': 0.879688357884905, 'duration_size': 0.22089952097998725, 'avg_pred_std': 0.05413557932530818}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001177909866010446, 'avg_role_model_std_loss': 0.1193157453701771, 'avg_role_model_mean_pred_loss': 2.5867055472029534e-06, 'avg_role_model_g_mag_loss': 0.01608747973675943, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001191412492368929, 'n_size': 900, 'n_batch': 225, 'duration': 287.26546144485474, 'duration_batch': 1.2767353841993543, 'duration_size': 0.31918384604983857, 'avg_pred_std': 0.10842714142054319}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0018042887282919966, 'avg_role_model_std_loss': 0.9737242848529073, 'avg_role_model_mean_pred_loss': 4.131605657280651e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0018042887282919966, 'n_size': 450, 'n_batch': 113, 'duration': 97.79816961288452, 'duration_batch': 0.8654705275476506, 'duration_size': 0.21732926580641004, 'avg_pred_std': 0.05228815563806061}\n", + "Time out: 3681.44966673851/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'lct_gan', 'n_size': 1050, 'n_batch': 263, 'role_model_metrics': {'avg_loss': 0.0014803996638372717, 'avg_g_mag_loss': 0.005388011875739759, 'avg_g_cos_loss': 0.0038071341092082937, 'pred_duration': 4.85994815826416, 'grad_duration': 2.964001417160034, 'total_duration': 7.823949575424194, 'pred_std': 0.10189060121774673, 'std_loss': 0.00683511421084404, 'mean_pred_loss': 2.3611642063769978e-06, 'pred_rmse': 0.03847596049308777, 'pred_mae': 0.028478950262069702, 'pred_mape': 0.09100271761417389, 'grad_rmse': 0.022572096437215805, 'grad_mae': 0.011139169335365295, 'grad_mape': 0.344256192445755}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0014803996638372717, 'avg_g_mag_loss': 0.005388011875739759, 'avg_g_cos_loss': 0.0038071341092082937, 'avg_pred_duration': 4.85994815826416, 'avg_grad_duration': 2.964001417160034, 'avg_total_duration': 7.823949575424194, 'avg_pred_std': 0.10189060121774673, 'avg_std_loss': 0.00683511421084404, 'avg_mean_pred_loss': 2.3611642063769978e-06}, 'min_metrics': {'avg_loss': 0.0014803996638372717, 'avg_g_mag_loss': 0.005388011875739759, 'avg_g_cos_loss': 0.0038071341092082937, 'pred_duration': 4.85994815826416, 'grad_duration': 2.964001417160034, 'total_duration': 7.823949575424194, 'pred_std': 0.10189060121774673, 'std_loss': 0.00683511421084404, 'mean_pred_loss': 2.3611642063769978e-06, 'pred_rmse': 0.03847596049308777, 'pred_mae': 0.028478950262069702, 'pred_mape': 0.09100271761417389, 'grad_rmse': 0.022572096437215805, 'grad_mae': 0.011139169335365295, 'grad_mape': 0.344256192445755}, 'model_metrics': {'lct_gan': {'avg_loss': 0.0014803996638372717, 'avg_g_mag_loss': 0.005388011875739759, 'avg_g_cos_loss': 0.0038071341092082937, 'pred_duration': 4.85994815826416, 'grad_duration': 2.964001417160034, 'total_duration': 7.823949575424194, 'pred_std': 0.10189060121774673, 'std_loss': 0.00683511421084404, 'mean_pred_loss': 2.3611642063769978e-06, 'pred_rmse': 0.03847596049308777, 'pred_mae': 0.028478950262069702, 'pred_mape': 0.09100271761417389, 'grad_rmse': 0.022572096437215805, 'grad_mae': 0.011139169335365295, 'grad_mape': 0.344256192445755}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T17:54:43.825332Z", + "iopub.status.busy": "2024-03-24T17:54:43.824674Z", + "iopub.status.idle": "2024-03-24T17:54:43.828973Z", + "shell.execute_reply": "2024-03-24T17:54:43.828164Z" + }, + "papermill": { + "duration": 0.02359, + "end_time": "2024-03-24T17:54:43.830986", + "exception": false, + "start_time": "2024-03-24T17:54:43.807396", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T17:54:43.862693Z", + "iopub.status.busy": "2024-03-24T17:54:43.862393Z", + "iopub.status.idle": "2024-03-24T17:54:43.933774Z", + "shell.execute_reply": "2024-03-24T17:54:43.932612Z" + }, + "papermill": { + "duration": 0.09002, + "end_time": "2024-03-24T17:54:43.936193", + "exception": false, + "start_time": "2024-03-24T17:54:43.846173", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T17:54:43.971892Z", + "iopub.status.busy": "2024-03-24T17:54:43.971478Z", + "iopub.status.idle": "2024-03-24T17:54:44.297234Z", + "shell.execute_reply": "2024-03-24T17:54:44.296063Z" + }, + "papermill": { + "duration": 0.347246, + "end_time": "2024-03-24T17:54:44.300426", + "exception": false, + "start_time": "2024-03-24T17:54:43.953180", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA1HUlEQVR4nO3de1xUdf4/8NfMMBduw3C/KFcVMbyuBmGmlWxUVItZlktqrqm1WCpbmZXa1iZq+ctSN7t8V7eSUHe1LXUtw0ur4A1FMZG8IKAyICAz3Of2+f0xcHS4Ofdh4P18PObhcOYz53yGmFfnfM7nwmOMMRBCiJPgO7oChBBiCgotQohTodAihDgVCi1CiFOh0CKEOBUKLUKIU6HQIoQ4FRdHV8BedDodrl+/Dk9PT/B4PEdXhxDSDmMMdXV1CAkJAZ/f9flUnwmt69evIzQ01NHVIITcQVlZGfr379/l630mtDw9PQHofyFSqdTBtSGEtKdUKhEaGsp9V7vSZ0Kr7ZJQKpVSaBHSg92p+YYa4gkhToVCixDiVCi0CCFOxaw2rfXr1+ODDz6AXC7HiBEjsHbtWsTFxXVZftu2bViyZAmuXLmCQYMGYeXKlXj00UcBAGq1Gm+//TZ2796Ny5cvw8vLC4mJiVixYgVCQkK4fdTU1ODll1/GDz/8AD6fj8mTJ+Pjjz+Gh4eHOR+BmEir1UKtVju6GsSJCYVCCAQCi/djcmht2bIF6enp2LBhA+Lj47FmzRokJSWhqKgIAQEBHcrn5ORg6tSpyMjIwGOPPYbMzEykpKTg5MmTGDp0KBobG3Hy5EksWbIEI0aMwM2bNzF//nw88cQTOHHiBLef1NRUlJeXY+/evVCr1Zg5cybmzJmDzMxMy34DpFuMMcjlctTW1jq6KqQXkMlkCAoKsqivJM/USQDj4+Nx9913Y926dQD0nTZDQ0Px8ssv44033uhQ/plnnkFDQwN27tzJbbvnnnswcuRIbNiwodNjHD9+HHFxcSgpKUFYWBgKCwtx11134fjx4xgzZgwAYM+ePXj00Udx9epVgzOyriiVSnh5eUGhUNDdQxOUl5ejtrYWAQEBcHNzo465xCyMMTQ2NqKyshIymQzBwcEdyhj7HTXpTEulUiEvLw+LFy/mtvH5fCQmJiI3N7fT9+Tm5iI9Pd1gW1JSEr777rsuj6NQKMDj8SCTybh9yGQyLrAAIDExEXw+H0ePHsWkSZM67KOlpQUtLS3cz0ql0piPiPoWDQrLlVBrdRg7wM+o9/RWWq2WCyxfX19HV4c4OVdXVwBAZWUlAgICzL5UNKkhvqqqClqtFoGBgQbbAwMDIZfLO32PXC43qXxzczMWLVqEqVOncmkrl8s7XHq6uLjAx8eny/1kZGTAy8uLexjbGz7nYhWe3pCL5bsLjSrfm7W1Ybm5uTm4JqS3aPtbsqR9tEfdPVSr1ZgyZQoYY/j0008t2tfixYuhUCi4R1lZmVHvi/LXN+wX32gATZ+vR5eExFqs8bdk0uWhn58fBAIBKioqDLZXVFQgKCio0/cEBQUZVb4tsEpKSrBv3z6Da9qgoCBUVlYalNdoNKipqenyuGKxGGKx2OjP1ibMxw0CPg8NKi0qlC0I8pKYvA9CiO2YdKYlEokwevRoZGdnc9t0Oh2ys7ORkJDQ6XsSEhIMygPA3r17Dcq3BdaFCxfw888/d2g/SUhIQG1tLfLy8rht+/btg06nQ3x8vCkf4Y5ELnyE+ehPYS/fqLfqvgkhljP58jA9PR1ffPEF/vnPf6KwsBAvvfQSGhoaMHPmTADA9OnTDRrq58+fjz179mD16tU4f/483nnnHZw4cQLz5s0DoA+sp556CidOnMDmzZuh1Wohl8shl8uhUqkAAEOGDMHDDz+M2bNn49ixYzh8+DDmzZuHZ5991qg7h6aK8nMHAFyqarD6vknfwOPxur3Z5IzeeecdjBw50tHVAJgZ1q5dy8LCwphIJGJxcXHsyJEj3GsTJkxgM2bMMCi/detWFh0dzUQiEYuNjWW7du3iXisuLmYAOn3s37+fK1ddXc2mTp3KPDw8mFQqZTNnzmR1dXVG11mhUDAATKFQ3LHs33b+ysIX7WTvfH/W6P33Rk1NTezcuXOsqanJ0VVxOgDYjh07HFqHjRs3Mi8vL6vtr66ujlVVVVm0j+7+poz9jprVI37evHncmVJ7Bw4c6LDt6aefxtNPP91p+YiICKMavH18fOzWkbStMf7yDTrTIr2fSqWCSCS6YzkPD48eMQKlR9097CnaLg8vV1Gb1u0YY2hUaRzyMOZ/bLfbs2cPxo0bB5lMBl9fXzz22GO4dOkSAGDs2LFYtGiRQfkbN25AKBTil19+AaDvVJucnAxXV1dERkYiMzMTERERWLNmjVm/u4KCAjz44INwdXWFr68v5syZg/r6W39fBw4cQFxcHNzd3SGTyXDvvfeipKQEAHD69Gk88MAD8PT0hFQqxejRow1Gi3TmwIEDmDlzJtfnkcfj4Z133gGgP1F47733MH36dEilUsyZMwcAsGjRIkRHR8PNzQ1RUVFYsmSJQdeE9peHzz//PFJSUvDhhx8iODgYvr6+SEtLs/lwrz4zn5YpIv31oXX1ZhOa1VpIhJaPl+oNmtRa3LX0R4cc+9y7SXATGf/n2tDQgPT0dAwfPhz19fVYunQpJk2ahPz8fKSmpmLVqlVYsWIFdwt+y5YtCAkJwX333QdA3zZbVVWFAwcOQCgUIj09vcMdbFPqkpSUhISEBBw/fhyVlZV44YUXMG/ePGzatAkajQYpKSmYPXs2vv32W6hUKhw7doyrW2pqKkaNGoVPP/0UAoEA+fn5EAqF3R5z7NixWLNmDZYuXYqioiIAMDhL+vDDD7F06VIsW7aM2+bp6YlNmzYhJCQEBQUFmD17Njw9PfH66693eZz9+/cjODgY+/fvx8WLF/HMM89g5MiRmD17tlm/K2NQaHXC30MMT7EL6lo0KKluxOCg7mdSJD3P5MmTDX7+xz/+AX9/f5w7dw5TpkzBggULcOjQIS6kMjMzMXXqVPB4PJw/fx4///yzwbCxL7/8EoMGDTKrLpmZmWhubsZXX30Fd3f9/xDXrVuHxx9/HCtXroRQKIRCocBjjz2GAQMGANDffGpTWlqK1157DTExMQBgVD1EIhG8vLzA4/E67Rb04IMP4i9/+YvBtrfffpt7HhERgVdffRVZWVndhpa3tzfWrVsHgUCAmJgYJCcnIzs7m0LL3ng8HqL83XH6qgKXb9RTaLVyFQpw7t0khx3bFBcuXMDSpUtx9OhRVFVVQafTAdAHwNChQ/HQQw9h8+bNuO+++1BcXIzc3Fx89tlnAICioiK4uLjgd7/7Hbe/gQMHwtvb26y6FxYWYsSIEVxgAcC9994LnU6HoqIijB8/Hs8//zySkpLw+9//HomJiZgyZQo3Pi89PR0vvPACvv76ayQmJuLpp5/mws1ctw+Ja7NlyxZ88sknuHTpEurr66HRaO44Tjc2NtZgOE5wcDAKCgosqtudUJtWF7jGeOr2wOHxeHATuTjkYWpP6scffxw1NTX44osvcPToURw9ehQAuG40qamp+Ne//gW1Wo3MzEwMGzYMw4YNs/rvzFgbN25Ebm4uxo4diy1btiA6OhpHjhwBoG9L+vXXX5GcnIx9+/bhrrvuwo4dOyw63u0BCujH96ampuLRRx/Fzp07cerUKbz11lvc76sr7S9TeTwe9z8IW6HQ6gLXGE93EJ1OdXU1ioqK8Pbbb2PixIkYMmQIbt68aVDmD3/4A5qbm7Fnzx5kZmYiNTWVe23w4MHQaDQ4deoUt+3ixYsd9mGsIUOG4PTp02houPW3dPjwYfD5fAwePJjbNmrUKCxevBg5OTkYOnSowd3y6OhoLFy4ED/99BOefPJJbNy48Y7HFYlE0Gq1RtUxJycH4eHheOuttzBmzBgMGjSIuxHQ01BodeHWmRbdQXQ23t7e8PX1xeeff46LFy9i3759HWYacXd3R0pKCpYsWYLCwkJMnTqVey0mJgaJiYmYM2cOjh07hlOnTmHOnDlwdXU1a+xcamoqJBIJZsyYgbNnz2L//v14+eWXMW3aNAQGBqK4uBiLFy9Gbm4uSkpK8NNPP+HChQsYMmQImpqaMG/ePBw4cAAlJSU4fPgwjh8/btDm1ZWIiAjU19cjOzsbVVVVaGxs7LLsoEGDUFpaiqysLFy6dAmffPKJxWdztkKh1YUo/1tnWqbebieOxefzkZWVhby8PAwdOhQLFy7EBx980KFcamoqTp8+jfvuuw9hYWEGr3311VcIDAzE+PHjMWnSJO5OmkRi+lhUNzc3/Pjjj6ipqcHdd9+Np556ChMnTuTmpHNzc8P58+cxefJkREdHY86cOUhLS8PcuXMhEAhQXV2N6dOnIzo6GlOmTMEjjzyCv/71r3c87tixY/Hiiy/imWeegb+/P1atWtVl2SeeeAILFy7EvHnzMHLkSOTk5GDJkiUmf1Z7MHkSQGdl6iSAzWothizdA8aAvLcT4eth+uBrZ9fc3Izi4mJERkaa9WXtTa5evYrQ0FD8/PPPmDhxoqOr47S6+5uyySSAfYlEKECIlyuu1TbhclVDnwytvmzfvn2or6/HsGHDUF5ejtdffx0REREYP368o6vW59HlYTduXSJSu1Zfo1ar8eabbyI2NhaTJk2Cv78/19F08+bN3JCW9o/Y2Fi71fGRRx7psh7Lly+3Wz3sjc60ujHA3wP/u1BFdxD7oKSkJCQldd4n7YknnuhySqQ79VS3pi+//BJNTU2dvubj42O3etgbhVY3ItumqKHQIrfx9PSEp6fjOxz369fP0VVwCLo87AZ3eUjdHgjpMSi0utHWV6u0uhFqrW17+RJCjEOh1Y1gqQQSIR8aHcPVm523HRBC7ItCqxt8Pg+Rfm0TAtIlIiE9AYXWHdzeM54Q4ngUWncwgGYxJWbojQtb9BQUWnfQ1hhP3R6Is9m0aRNkMplV93ngwAHweDzU1tZadb+moNC6A7o8JKRnodC6g7YOplX1LVA223bC/h6PMUDV4JgHLWxhtYUtWlpa8Oqrr6Jfv35wd3dHfHy8wSpaJSUlePzxx+Ht7Q13d3fExsZi9+7duHLlCh544AEA+ul/eDwenn/+ebN+H5agHvF34CkRIsBTjMq6Fly+0YCRoTJHV8lx1I3AcusvjmuUN68DIvc7l2tFC1t0vbDFvHnzcO7cOWRlZSEkJAQ7duzAww8/jIKCAgwaNAhpaWlQqVT45Zdf4O7ujnPnzsHDwwOhoaH497//jcmTJ6OoqAhSqRSurq5m/U4sQaFlhEg/99bQqu/boeVEaGGLzhe2KC0txcaNG1FaWsqtzv7qq69iz5492LhxI5YvX47S0lJMnjyZm346KiqKe3/bmMaAgACrt5cZi0LLCFH+HjhaXEPtWkI3/RmPo45tAlrYonMFBQXQarWIjo422N7S0gJfX18AwCuvvIKXXnoJP/30ExITEzF58mQMHz7crOPZArVpGWEAjUHU4/H0l2iOeNDCFlZZ2KK+vh4CgQB5eXnIz8/nHoWFhfj4448BAC+88AIuX76MadOmoaCgAGPGjMHatWut9lktRaFlBLqD6FxoYQu9zha2GDVqFLRaLSorKzFw4ECDx+2XkaGhoXjxxRexfft2/OUvf8EXX3zB7ROA0Qtm2AKFlhGiWofyFFc1QKfrE7NTOzVa2EKvs4UtoqOjkZqaiunTp2P79u0oLi7GsWPHkJGRgV27dgEAFixYgB9//BHFxcU4efIk9u/fzx0vPDwcPB4PO3fuxI0bNwzugNoN6yMUCgUDwBQKhcnvVWu0bOCbu1j4op2srKbBBrXrmZqamti5c+dYU1OTo6tisr1797IhQ4YwsVjMhg8fzg4cOMAAsB07dnBldu/ezQCw8ePHd3j/9evX2SOPPMLEYjELDw9nmZmZLCAggG3YsMGo47c/1pkzZ9gDDzzAJBIJ8/HxYbNnz2Z1dXWMMcbkcjlLSUlhwcHBTCQSsfDwcLZ06VKm1WpZS0sLe/bZZ1loaCgTiUQsJCSEzZs3z+j/Ji+++CLz9fVlANiyZcsYY4ypVCq2dOlSFhERwYRCIQsODmaTJk1iZ86cYYwxNm/ePDZgwAAmFouZv78/mzZtGquqquL2+e6777KgoCDG4/HYjBkzjKpHm+7+poz9jtLCFkZK/H8HcbGyHl/9KQ7jo/1tUMOehxa2uIUWtrAOayxsQZeHRrq1eGsfb4zvI/bt24fvv/8excXFyMnJwbPPPksLW/QQFFpGurV4KzXG9wW0sEXPRf20jER3EPsWWtii56LQMtIAWk6MtKKFLRyLLg+N1Nbt4bqiGY0qjYNrY1995F4NsQNr/C1RaBnJ210EmZv+1L+4j7RrtV3qNDY2OrgmpLdo+1uy5DKaLg9NEOXnjpOltbh8owGxIV6Oro7NCQQCyGQybnYDNzc3szpXEsIYQ2NjIyorKyGTySAQCMzeF4WWCaL8PbjQ6ivahnaYOy0LIbeTyWQGw4XMQaFlgrY7iMV9aOA0j8dDcHAwAgICoFb38UkQiUWEQqFFZ1htKLRM0NYY3xf7agkEAqv8wRFiKWqIN8GA2/pq0R01QhyDQssEYb5u4POA+hYNbtS1OLo6hPRJFFomELsIEOqjn0GTlhQjxDEotEwURYu3EuJQFFom4gZO05kWIQ5BoWWiKBqDSIhDmRVa69evR0REBCQSCeLj43Hs2LFuy2/btg0xMTGQSCQYNmwYdu/ebfD69u3b8dBDD8HX1xc8Hg/5+fkd9nH//fdzi062PV588UVzqm+RSO7ykM60CHEEk0Nry5YtSE9Px7Jly3Dy5EmMGDECSUlJXfaYzsnJwdSpUzFr1iycOnUKKSkpSElJwdmzZ7kyDQ0NGDduHFauXNntsWfPno3y8nLusWrVKlOrb7EBrZeHZTWNaNE4bnJ/QvoskyZ4ZozFxcWxtLQ07metVstCQkJYRkZGp+WnTJnCkpOTDbbFx8ezuXPndihbXFzMALBTp051eG3ChAls/vz5plaXY8kc8bfT6XTsriX/ZeGLdrLf5EqL9kUIucXY76hJZ1oqlQp5eXlITEzktvH5fCQmJiI3N7fT9+Tm5hqUB/QTrHVVvjubN2+Gn58fhg4disWLF3c7+0BLSwuUSqXBwxp4PB7XGE/dHgixP5OG8VRVVUGr1SIwMNBge2BgIM6fP9/pe+Ryeafl5XK5SRX94x//iPDwcISEhODMmTNYtGgRioqKsH379k7LZ2Rk4K9//atJxzBWlL87Cq4p+swUNYT0JE4z9nDOnDnc82HDhiE4OBgTJ07EpUuXOl0ifPHixQZr3SmVSoSGhlqlLtwYRLqDSIjdmRRafn5+EAgEqKioMNheUVHR5XQTQUFBJpU3Vtsc3RcvXuw0tMRiMcRisUXH6ArX7YHOtAixO5PatEQiEUaPHo3s7Gxum06nQ3Z2NhISEjp9T0JCgkF5ANi7d2+X5Y3V1i0iODjYov2Yg/pqEeI4Jl8epqenY8aMGRgzZgzi4uKwZs0aNDQ0YObMmQCA6dOno1+/fsjIyAAAzJ8/HxMmTMDq1auRnJyMrKwsnDhxAp9//jm3z5qaGpSWluL69esAgKKiIgD6s7SgoCBcunQJmZmZePTRR+Hr64szZ85g4cKFGD9+PIYPH27xL8FUbX21bjaqcbNBBW93kd3rQEifZc6tybVr17KwsDAmEolYXFwcO3LkCPfahAkTOiyVvXXrVhYdHc1EIhGLjY1lu3btMnh948aNDECHR9sy3qWlpWz8+PHMx8eHicViNnDgQPbaa6+Z1H3BWl0e2iQs/5mFL9rJTlyptsr+COnrjP2O8hjrGxNDGbvktrGe+/IoDl2swqqnhmPKGOs08BPSlxn7HaWxh2bihvNQXy1C7IpCy0zUGE+IY1BomYmbooa6PRBiVxRaZmqbDLCkugEarc7BtSGk76DQMlM/mSvELnyotQzXapscXR1C+gwKLTPx+TxqjCfEASi0LNDWGH+JGuMJsRsKLQv05cVbCXEUCi0LULcHQuyPQssCtDIPIfZHoWWBtjOtyroW1DWrHVwbQvoGCi0LSCVC+Hno5+yiWUwJsQ8KLQtFUbcHQuyKQstC1BhPiH1RaFmI66tFl4eE2AWFloVuLXJBoUWIPVBoWajtTOtKVQN0uj4xnyIhDkWhZaFQHze48HloUmshVzY7ujqE9HoUWhYSCvgI83UDQJeIhNgDhZYV3BqDSHcQCbE1Ci0rGOBPfbUIsRcKLSugKWoIsR8KLSuggdOE2A+FlhW0zWB6XdGEZrXWwbUhpHej0LICX3cRpBIXMEYDpwmxNQotK+DxeHSJSIidUGhZCQ2cJsQ+KLSsZAAt3kqIXVBoWQk3rxaFFiE2RaFlJbfatOrBGA2cJsRWKLSsJNzXDTweUNesQVW9ytHVIaTXotCyEolQgP7ergCoMZ4QW6LQsiJavJUQ26PQsiLq9kCI7VFoWRGtzEOI7VFoWVEU9dUixOYotKyo7fKwtKYRKo3OwbUhpHei0LKiIKkEbiIBtDqG0ppGR1eHkF6JQsuKeDweN00NNcYTYhsUWlbW1q5FU9QQYhsUWlZGdxAJsS0KLSvj+mrRyjyE2ASFlpUNoMkACbEpCi0ra2uIr25QQdGodnBtCOl9KLSszF3sgiCpBABwiS4RCbE6s0Jr/fr1iIiIgEQiQXx8PI4dO9Zt+W3btiEmJgYSiQTDhg3D7t27DV7fvn07HnroIfj6+oLH4yE/P7/DPpqbm5GWlgZfX194eHhg8uTJqKioMKf6NhdFi7cSYjMmh9aWLVuQnp6OZcuW4eTJkxgxYgSSkpJQWVnZafmcnBxMnToVs2bNwqlTp5CSkoKUlBScPXuWK9PQ0IBx48Zh5cqVXR534cKF+OGHH7Bt2zYcPHgQ169fx5NPPmlq9e2C+moRYkPMRHFxcSwtLY37WavVspCQEJaRkdFp+SlTprDk5GSDbfHx8Wzu3LkdyhYXFzMA7NSpUwbba2trmVAoZNu2beO2FRYWMgAsNzfXqHorFAoGgCkUCqPKW+LL/11m4Yt2srlfnbD5sQjpLYz9jpp0pqVSqZCXl4fExERuG5/PR2JiInJzczt9T25urkF5AEhKSuqyfGfy8vKgVqsN9hMTE4OwsLAu99PS0gKlUmnwsBfq9kCI7ZgUWlVVVdBqtQgMDDTYHhgYCLlc3ul75HK5SeW72odIJIJMJjN6PxkZGfDy8uIeoaGhRh/PUgNaJwO8Ut0IrY7miyfEmnrt3cPFixdDoVBwj7KyMrsdu5+3K0QufKg0Oly72WS34xLSF5gUWn5+fhAIBB3u2lVUVCAoKKjT9wQFBZlUvqt9qFQq1NbWGr0fsVgMqVRq8LAXAZ+HCF83AHSJSIi1mRRaIpEIo0ePRnZ2NrdNp9MhOzsbCQkJnb4nISHBoDwA7N27t8vynRk9ejSEQqHBfoqKilBaWmrSfuyJmy+euj0QYlUupr4hPT0dM2bMwJgxYxAXF4c1a9agoaEBM2fOBABMnz4d/fr1Q0ZGBgBg/vz5mDBhAlavXo3k5GRkZWXhxIkT+Pzzz7l91tTUoLS0FNevXwegDyRAf4YVFBQELy8vzJo1C+np6fDx8YFUKsXLL7+MhIQE3HPPPRb/EmyBGuMJsRFzbk2uXbuWhYWFMZFIxOLi4tiRI0e41yZMmMBmzJhhUH7r1q0sOjqaiUQiFhsby3bt2mXw+saNGxmADo9ly5ZxZZqamtif//xn5u3tzdzc3NikSZNYeXm50XW2Z5cHxhjbdqKMhS/ayaZ+blyXDEL6OmO/ozzG+sZyyEqlEl5eXlAoFHZp3zpZehNP/j0HQVIJjrw50ebHI8TZGfsd7bV3Dx2tbV4tubIZDS0aB9eGkN6DQstGZG4i+LiLANAspoRYE4WWDbWdbV2iMYiEWA2Flg3RbA+EWB+Flg3R4q2EWB+Flg1F0RQ1hFgdhZYN3b6cWB/pWUKIzVFo2VCYjxsEfB4aVVpUKFscXR1CegUKLRsSufAR5tM6cJouEQmxCgotG+O6PVBjPCFWQaFlY7e6PdCZFiHWQKFlY1G0eCshVkWhZWPcyjw0RQ0hVkGhZWNtl4dXbzahWa11cG0IcX4UWjbm7yGGp9gFjAEl1Y2Org4hTo9Cy8Z4PB41xhNiRRRadkBjEAmxHgotO6ApagixHgotO7h9DCIhxDIUWnZw+7xaNHCaEMtQaNlBpJ87eDxA0aRGTYPK0dUhxKlRaNmBRChAiJcrAGqMJ8RSFFp2Qt0eCLEOCi07GUBjEAmxCgotO4nkuj1QaBFiCQotO+EuD2ngNCEWodCyk7a+WqXVjVBrdQ6uDSHOi0LLToKlEkiEfGh0DGU1NHCaEHNRaNkJn89DpB81xhNiKQotO6J2LUIsR6FlRwNa7yDSGERCzEehZUdtjfHU7YEQ81Fo2dHtA6cJIeah0LKjtg6mVfUtUDarHVwbQpwThZYdeUqECPAUA6CzLULMRaFlZ9ySYjRwmhCzUGjZGS3eSohlKLTsbAD11SLEIhRadkZ3EAmxDIWWnUX53VrkQqej+eIJMRWFlp3193aFUMBDi0aHa7VNjq4OIU6HQsvOXAR8hPvScB5CzEWh5QBR1O2BELNRaDkA1+2BzrQIMRmFlgPQHURCzEeh5QADaDkxQsxmVmitX78eERERkEgkiI+Px7Fjx7otv23bNsTExEAikWDYsGHYvXu3weuMMSxduhTBwcFwdXVFYmIiLly4YFAmIiICPB7P4LFixQpzqn9nV/MATYtt9g1wM5heVzSjUaWx2XEI6Y1MDq0tW7YgPT0dy5Ytw8mTJzFixAgkJSWhsrKy0/I5OTmYOnUqZs2ahVOnTiElJQUpKSk4e/YsV2bVqlX45JNPsGHDBhw9ehTu7u5ISkpCc3Ozwb7effddlJeXc4+XX37Z1Orf2bU8YFMy8PUkoLHG+vsH4OMugsxNCIDuIBJiMmaiuLg4lpaWxv2s1WpZSEgIy8jI6LT8lClTWHJyssG2+Ph4NnfuXMYYYzqdjgUFBbEPPviAe722tpaJxWL27bffctvCw8PZRx99ZHQ9m5ubmUKh4B5lZWUMAFMoFN2/8eI+xpb3Z2yZlLGPRzJ244LRxzTFpPWHWPiinez7/Gs22T8hzkahUBj1HTXpTEulUiEvLw+JiYncNj6fj8TEROTm5nb6ntzcXIPyAJCUlMSVLy4uhlwuNyjj5eWF+Pj4DvtcsWIFfH19MWrUKHzwwQfQaLq+tMrIyICXlxf3CA0NNe5DDngAmPUT4BUG1FwGvpwIFP/PuPeagAZOE2Iek0KrqqoKWq0WgYGBBtsDAwMhl8s7fY9cLu+2fNu/d9rnK6+8gqysLOzfvx9z587F8uXL8frrr3dZ18WLF0OhUHCPsrIy4z9owBBgdjbQ/26guRb4OgU49Y3x7zcCLXJBiHlcHF0BY6Wnp3PPhw8fDpFIhLlz5yIjIwNisbhDebFY3Ol2o3kEADN+AL77M/DrduA/aUD1ReDBpQDf8puuUbScGCFmMenb5+fnB4FAgIqKCoPtFRUVCAoK6vQ9QUFB3ZZv+9eUfQJAfHw8NBoNrly5YspHMI3QFZj8f8D41jO6Qx8B22YAKssXW7292wNjNHCaEGOZFFoikQijR49GdnY2t02n0yE7OxsJCQmdvichIcGgPADs3buXKx8ZGYmgoCCDMkqlEkePHu1ynwCQn58PPp+PgIAAUz6C6fh84MG3gEmfAQIRUPg9sOlRoK7zy2Fjhfm6gc8DGlRaXL1JA6cJMZqpLfxZWVlMLBazTZs2sXPnzrE5c+YwmUzG5HI5Y4yxadOmsTfeeIMrf/jwYebi4sI+/PBDVlhYyJYtW8aEQiErKCjgyqxYsYLJZDL2n//8h505c4b94Q9/YJGRkaypqYkxxlhOTg776KOPWH5+Prt06RL75ptvmL+/P5s+fbrR9Tb2zkS3rhxmbEWE/s7i6rsYKz9j/r4YY/d/sJ+FL9rJhiz5L3vj32fYr9csqBshTs7Y76jJocUYY2vXrmVhYWFMJBKxuLg4duTIEe61CRMmsBkzZhiU37p1K4uOjmYikYjFxsayXbt2Gbyu0+nYkiVLWGBgIBOLxWzixImsqKiIez0vL4/Fx8czLy8vJpFI2JAhQ9jy5ctZc3Oz0XW2SmgxxljVRcY+Ga0PrvdDGCvaY/au9p2vYImrD7DwRTu5x+S/H2bfnbrKmtUay+rpCMpyxv77BmO7X2dM1ejo2hAnY+x3lMdY32hQUSqV8PLygkKhgFQqtWxnTTeBrdOB4l8AHh9IygDi5wI8nsm7YozhWHENvjpSgh/PyqFpnRjQ112EKXeH4o9xYQj1cbOsvrbWrARyPgFy1wPq1va+8HHA1ExA4uXYuhGnYex3lELLXFo1sHMhcOpr/c93vwA8vBIQmH9DtlLZjKzjZcg8Wgq5Uj8agMcDHhwcgOcSwjFhkD/4fNOD0WY0LcDx/wN++QBoah090G80UHUBaFECQcOB57YDHv6OrSdxChRa7Vg9tACAMf0Zxt5lABgwMBF4aiMgsWz/Gq0OPxdW4psjJTh0sYrbHubjhtT4MEwZEwpvd5GFlbeATgcUbAP2/w2oLdVv84sGJi4DYpIB+Rng6yeBxirAdyAwbQcgC3NcfYlToNBqxyah1abwB+DfswFNExBwF/DHLVb7kl66UY/NR0qxLa8Mdc36EQAiFz4eGx6MafeEY2SoDDwzLkvNwhhwMRv4+R2gokC/zTMYuH8xMDLV8Cyz6qK+U66iDJD20weX/2D71JM4JQqtdmwaWgBw7STw7VSgXg64+wPPfguE3m213TeqNPjh9HV8lVuCX68rue1D+0kx7Z5wPDGiH1xFAqsdr4NrefozyiutQ5rEXsC4BUD8i4CoizY3xTX9wPOqIsDVB3juX/rLR0I6QaHVjs1DC9B/STOf0Z+FCMTApE+BoZOtegjGGPLLavH1kRLsPFMOlUYHAJBKXPDU6FA8d08YN67RKqovAdnvAue+0/8sEAFxc4D7/gK4+dz5/Q3VwOangOsnAZEH8GwmEDXBevUjvQaFVjt2CS0AaKkH/j0L+G2P/ucH3wbue9WsO4t3UtOgwrYTZfjmaAnKam51UB030A/P3ROOxCEBcBGYOeSorgI4uBI4+U9ApwHAA0ZMBR5YbPqlb0sdkJUKFB/Uh95TG4Ehj5lXL9JrUWi1Y7fQAgCdFvjpbeDI3/U/j5gKPP4x4GLBWMjuDqdjOHjhBr7JLcG+okq0/RcNkkrwx/gwPHt3KAKkEuN21qwEctYCuetudV8YlAQkLgMCY82vpKZFH+aFP+i7iTyxFhj1nPn7I70OhVY7dg2tNsf/D9j9GsC0QNhY4JlvAHdfmx6yrKYRmcdKseV4GWoaVAAAFz4PSbFBGDvQF5F+7ojy80CgVGzYgK9RASf+AfyyCmis1m/rNwb4/V+BiHHWqZxWA+xccKubyEN/A8baYCJH4pQotNpxSGgB+rtt257X91vyjgRStwF+g2x+2BaNFv8tkOPrIyXIK7nZ4XVXoQARfu4Y4OuKh3k5eODaZ3BvvKp/0XegvvvCkMetf1nLGLB3qb6rCACMSwcmLrXJ5TNxLhRa7TgstACgshDInKLv0yTx0p9xRY632+F/va7A9/nXcaGyHsVVDSitaYRWxzCOX4A3XL7FUP4VAEAFk2GNZjJ+Ev0eYf7S1rMyd0T6eSDCzw2Rfu5wE1lpNqNDH+m7TgDA6JlA8mqAb8O7n6THo9Bqx6GhBQD1N4CsqcDV4wDfBXhsDfC7afavBwD11ZNQ71kCt6uHAADNfHf8x+NpbGh+CMXK7t8bJJUg0s8dkf76QIvw1T8P9XaDyMXERv+8TcAPCwAwIHYSMOlzwMWBnWaJQ1FotePw0AIAddOtSQUB4N4F+sswK0wqaJTqS8C+v906vkAE3D1b332hta2tSaXFleoGFFfpH5dvNKC4Sn+GdrNR3eWuBXweQr1dEennjkGBnhjWzwvD+3shzMet+86vv34H/PsFQKcGBjyoPwsVuVvxQxNnQaHVTo8ILUA/BOZAhr7BGwAi7gN8olrbdHgm/tvKmLJ15cDprFvdF4Y/AzzwJuAdbnTVaxtVXJgVVzXgclUDrrQ+b1RpO32PzE3IBdjw/jKM6C/reBPg0j4g6zlA3aCf4vqPW43rA0Z6FQqtdnpMaLU5vQX4fh6gVdn3uAN/r+++EDTMartkjKGyrgWXbzTgclU9zpfX4czVWhSW10Gl1XUo7+8pxojWEBvW3wsj+svgU3Na3wm1uVY/FOq57YA02Gp1JD0fhVY7PS60AEBeAFz4SX/2Baa/s2b0v+h8W1fv4Qn0g5kj77Pbx1NpdCiS1+H01VoUXFXg9NVaXKish1bX8U+un8wVDwfUIF3+BtxVN6DzCgd/+g7Ad4Dd6ksci0KrnR4ZWn1Qk0qLc+UKnC5ToOCaPshuX9yjP68SXwszEMmvQA1Phi/CV8N/4GgM7++F2BAv246vJA5FodUOhVbPpWxW4+w1Bc5cVaDgqgKlZVewsvEd3MUvgZK5YabqNeSxwRDweRgU4GHQPnZXiBSCnjTHGDEbhVY7FFrOpaaqEvwtUyG7cQItPDFe57+K/zR0HEbk4y7ChGh/3D/YHxOi/SFzoy4TzopCqx0KLSekatQv2XbhJ4DvgtqktTjq8SDXPpZfWou6llurjPN5wKgwbzwYE4D7B/vjrmCp/eYaIxaj0GqHQstJadXAdy/pZ0oFD3j0AyBuNgBArdXhZMlN7C+6gQNFlTgvrzN4a6BUjAcGB+D+wQEYN8gPHmKnWZu4T6LQaodCy4npdMCeRcCxz/U/P/AWMP61DuMVr9U24UBRJfafv4HDF6vQpL7Vd0wo4CEu0ocLsQH+7nQW1sNQaLVDoeXkGAMOrAAOrtD/HP8SkLS8y9EEzWotjhXXYH9RJfafr8SVasNVwUN9XPFgtC8mDvRAXIgEEtYMqBpuPdQNhj+7yvQDyX0HAh6BNMDbBii02qHQ6iWObNCfdQHAsClAzKOtwdIIqOr1c4AZhI9+e3NjHZrqldA010OgaYQbmiHhdT0sqVsiT33/Md+B+hk7fAfe+lnsab3P2sdQaLVDodWLnN6ib+dinQ8dMpWW8dAACZogRgOTQOfiCrGbFB5SKbyk3uCL3PRzjFVfBGpLANaxlz/HI6g1zAbeOjPzHaQfLiUQWqW+vZWx31FqmSTOZ8QzgJsvcHiNPkCEbvpB1m0PoZt+PnpR23aPLsswoSsuVKuxv6gK+89XIq/0JrQqBjQCqAI8xC4YN9AP90T5YMhYKWL8xfBqvqYPsKoL+n+rLwHVF4CGG/qFTerlQMkhwzrzBIB3RLszs9bnnkF0uWkCOtMi5DaKRjX+d/EG9p+/gYO/VaKqvuPY0H4yVwwJ9sSQYCligqQYEuyJcF93CFoUQM0l/fJp1W2PC/pQUzd2crRWIo9bl5eyMP2iKAIXgC/Un53xhbd+5ru0bnOxwmvCHhWWdHnYDoUWMZVOx1BwTYEDRTdQcE0/APxabVOnZV2FAgwO8uTCbEiwFIODPCGVCPU3EerKO56ZVV8EbpZY7TLXLG3hJRDqpyoSiG4953ex3aC8kWWH/OGOK41TaLVDoUWsQdGkxvlyJQrLlTgvr0NhuRJFFXVoVnfeztXf21UfYkG3wizMxw38tqFHGhVw88qtMzPlNf3MH1q1foEUnbr1uab1X3X3r+k0+rn4O3utu7Y4W5tzEAgZ2W0RCq12KLSIrWh1DFeqG1DYFmbl+jC7rmjutLybqO2sTMoFWkyw1PadX3U6fYgZBJrqVkhq1bc9v227rovtppR/6P07zt1GodUOhRaxt9pGFXc21nZmViSvQ4um8zOeMB83DAzwgMxNCKlECKnEBVJX/XPPTp57SlwgNHddyx6IQqsdCi3SE2i0utazMsMwK+/irOxOXIUCSF1dugw2w+cu8JQI4dVaXuoqhETYc6b6odBqh0KL9GQ3G1QolCtRUt0IZZMaymY1lE0a1DWroWzWQNmkRl2zpnW7Gg1dTG9tKrELHzI3IbxchZC5iuDFPW/9100fbjI3kcE2T4nQ6lMCUT8tQpyIt7sIYwf4YayRE7VqtDrUt2igbGoNsjuEHPe8ufV5kxo6BrRodKhQtqBC2WJSfXk8wFPsApmbyDDcWp+3haC09efYECk8JdbpXEuhRYgTchHw9Wc/Zs4fxhhDfYsGtY1qKJr0j7bntU0qKBrbb1ND0aiCovUsjzHow7FZc+eDAdjx57EYFeZtVl3bo9AipA/i8XjwlOgv80JNfK9Ko4OyWX1b4KluhZtBCKq4wPN1F1ut7hRahBCTiFz48PMQw8/DekFkit5zv5QQ0idQaBFCnAqFFiHEqVBoEUKcCoUWIcSpUGgRQpwKhRYhxKn0mX5abUMslUqlg2tCCOlM23fzTsOh+0xo1dXpF/IMDTW1/y8hxJ7q6urg5eXV5et9ZpYHnU6H69evw9PT846LdCqVSoSGhqKsrKzXzAjR2z5Tb/s8AH0mxhjq6uoQEhICfhfrWQJ96EyLz+ejf//+Jr1HKpX2mj+eNr3tM/W2zwP07c/U3RlWG2qIJ4Q4FQotQohTodDqhFgsxrJlyyAWO2YUuy30ts/U2z4PQJ/JWH2mIZ4Q0jvQmRYhxKlQaBFCnAqFFiHEqVBoEUKcCoUWIcSpUGi1s379ekREREAikSA+Ph7Hjh1zdJXMlpGRgbvvvhuenp4ICAhASkoKioqKHF0tq1qxYgV4PB4WLFjg6KpY5Nq1a3juuefg6+sLV1dXDBs2DCdOnHB0tcym1WqxZMkSREZGwtXVFQMGDMB77713x8HQxqDQus2WLVuQnp6OZcuW4eTJkxgxYgSSkpJQWVnp6KqZ5eDBg0hLS8ORI0ewd+9eqNVqPPTQQ2hoaHB01azi+PHj+OyzzzB8+HBHV8UiN2/exL333guhUIj//ve/OHfuHFavXg1vb+usE+gIK1euxKeffop169ahsLAQK1euxKpVq7B27VrLd84IJy4ujqWlpXE/a7VaFhISwjIyMhxYK+uprKxkANjBgwcdXRWL1dXVsUGDBrG9e/eyCRMmsPnz5zu6SmZbtGgRGzdunKOrYVXJycnsT3/6k8G2J598kqWmplq8bzrTaqVSqZCXl4fExERuG5/PR2JiInJzcx1YM+tRKBQAAB8fHwfXxHJpaWlITk42+O/lrL7//nuMGTMGTz/9NAICAjBq1Ch88cUXjq6WRcaOHYvs7Gz89ttvAIDTp0/j0KFDeOSRRyzed5+Z5eFOqqqqoNVqERgYaLA9MDAQ58+fd1CtrEen02HBggW49957MXToUEdXxyJZWVk4efIkjh8/7uiqWMXly5fx6aefIj09HW+++SaOHz+OV155BSKRCDNmzHB09czyxhtvQKlUIiYmBgKBAFqtFu+//z5SU1Mt3jeFVh+RlpaGs2fP4tChQ46uikXKysowf/587N27FxKJxNHVsQqdTocxY8Zg+fLlAIBRo0bh7Nmz2LBhg9OG1tatW7F582ZkZmYiNjYW+fn5WLBgAUJCQiz/TBZfYPYSLS0tTCAQsB07dhhsnz59OnviiSccUykrSUtLY/3792eXL192dFUstmPHDgaACQQC7gGA8Xg8JhAImEajcXQVTRYWFsZmzZplsO3vf/87CwkJcVCNLNe/f3+2bt06g23vvfceGzx4sMX7pjatViKRCKNHj0Z2dja3TafTITs7GwkJCQ6smfkYY5g3bx527NiBffv2ITIy0tFVstjEiRNRUFCA/Px87jFmzBikpqYiPz8fAoHA0VU02b333tuhK8pvv/2G8PBwB9XIco2NjR1mHxUIBNDpdJbv3OLY60WysrKYWCxmmzZtYufOnWNz5sxhMpmMyeVyR1fNLC+99BLz8vJiBw4cYOXl5dyjsbHR0VWzKme/e3js2DHm4uLC3n//fXbhwgW2efNm5ubmxr755htHV81sM2bMYP369WM7d+5kxcXFbPv27czPz4+9/vrrFu+bQqudtWvXsrCwMCYSiVhcXBw7cuSIo6tkNgCdPjZu3OjoqlmVs4cWY4z98MMPbOjQoUwsFrOYmBj2+eefO7pKFlEqlWz+/PksLCyMSSQSFhUVxd566y3W0tJi8b5pPi1CiFOhNi1CiFOh0CKEOBUKLUKIU6HQIoQ4FQotQohTodAihDgVCi1CiFOh0CKEOBUKLUKIU6HQIoQ4FQotQohT+f+xmxDKXTzqXgAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T17:54:44.345308Z", + "iopub.status.busy": "2024-03-24T17:54:44.344309Z", + "iopub.status.idle": "2024-03-24T17:58:38.026994Z", + "shell.execute_reply": "2024-03-24T17:58:38.025930Z" + }, + "papermill": { + "duration": 233.706406, + "end_time": "2024-03-24T17:58:38.029725", + "exception": false, + "start_time": "2024-03-24T17:54:44.323319", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T17:58:38.066343Z", + "iopub.status.busy": "2024-03-24T17:58:38.065514Z", + "iopub.status.idle": "2024-03-24T17:58:38.087460Z", + "shell.execute_reply": "2024-03-24T17:58:38.086416Z" + }, + "papermill": { + "duration": 0.042144, + "end_time": "2024-03-24T17:58:38.089709", + "exception": false, + "start_time": "2024-03-24T17:58:38.047565", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
lct_gan0.004060.0198720.001482.9478840.0111390.3442560.0225720.0000024.8571980.0284790.0910030.0384760.1018910.0068357.805082
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "lct_gan 0.00406 0.019872 0.00148 2.947884 0.011139 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "lct_gan 0.344256 0.022572 0.000002 4.857198 0.028479 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "lct_gan 0.091003 0.038476 0.101891 0.006835 7.805082 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T17:58:38.122299Z", + "iopub.status.busy": "2024-03-24T17:58:38.122027Z", + "iopub.status.idle": "2024-03-24T17:58:38.505706Z", + "shell.execute_reply": "2024-03-24T17:58:38.504742Z" + }, + "papermill": { + "duration": 0.402502, + "end_time": "2024-03-24T17:58:38.508059", + "exception": false, + "start_time": "2024-03-24T17:58:38.105557", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T17:58:38.544114Z", + "iopub.status.busy": "2024-03-24T17:58:38.543339Z", + "iopub.status.idle": "2024-03-24T18:03:02.462418Z", + "shell.execute_reply": "2024-03-24T18:03:02.461477Z" + }, + "papermill": { + "duration": 263.939983, + "end_time": "2024-03-24T18:03:02.464987", + "exception": false, + "start_time": "2024-03-24T17:58:38.525004", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/lct_gan/all inf False\n", + "Caching in ../../../../contraceptive/_cache_bs_test/lct_gan/all inf False\n", + "Caching in ../../../../contraceptive/_cache_synth_test/lct_gan/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T18:03:02.501612Z", + "iopub.status.busy": "2024-03-24T18:03:02.501249Z", + "iopub.status.idle": "2024-03-24T18:03:02.529244Z", + "shell.execute_reply": "2024-03-24T18:03:02.528445Z" + }, + "papermill": { + "duration": 0.048639, + "end_time": "2024-03-24T18:03:02.531339", + "exception": false, + "start_time": "2024-03-24T18:03:02.482700", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T18:03:02.566128Z", + "iopub.status.busy": "2024-03-24T18:03:02.565498Z", + "iopub.status.idle": "2024-03-24T18:03:02.570909Z", + "shell.execute_reply": "2024-03-24T18:03:02.569993Z" + }, + "papermill": { + "duration": 0.025176, + "end_time": "2024-03-24T18:03:02.573016", + "exception": false, + "start_time": "2024-03-24T18:03:02.547840", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'lct_gan': 0.38817292616480875}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T18:03:02.609131Z", + "iopub.status.busy": "2024-03-24T18:03:02.608503Z", + "iopub.status.idle": "2024-03-24T18:03:02.996812Z", + "shell.execute_reply": "2024-03-24T18:03:02.995877Z" + }, + "papermill": { + "duration": 0.408606, + "end_time": "2024-03-24T18:03:02.999055", + "exception": false, + "start_time": "2024-03-24T18:03:02.590449", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABDC0lEQVR4nO3deZxT9b0//tfJSXKyJ7MvMBsMgrKDgKgICEWQqpTWarEKWtS2WLWUx0Npb3FrL9TrRpfL118rM3JdsPqw2lurYr0OVERUEGRfZ2P2LdtkTz6/P04mM2EWMpkkZ5J5Px+PPJKcfObkcyYz73z2D8cYYyCEEAnJpM4AIYRQICKESI4CESFEchSICCGSo0BECJEcBSJCiOQoEBFCJEeBiBAiOQpEhBDJUSAiAyovLwfHcaiqqpI6KySFUSAiMffaa6/hhRdekDobJIlQICIxR4GIDBYFIkKI5CgQkUF7//33MX/+fOj1ehgMBsyaNQuvvfYaAGDBggV47733UF1dDY7jwHEciouLIz53IBDA448/jvz8fGg0GixcuBDHjx9HcXEx1qxZE0rX3t6ODRs2YPLkydDpdDAYDFi2bBkOHz4cdr6KigpwHIe//vWv+O1vf4vRo0dDpVJh0aJFOHv2bCx+HSQG5FJngCSX8vJy3HPPPZg4cSI2btwIk8mEr7/+Gh988AFWrVqFX/3qV7BYLLhw4QKef/55AIBOp4v4/Bs3bsTTTz+Nm266CTfccAMOHz6MG264AS6XKyzd+fPn8c477+DWW29FSUkJmpqa8OKLL2L+/Pk4fvw48vPzw9Jv2bIFMpkMGzZsgMViwdNPP4077rgD+/fvH/ovhQwdI2QAZWVlDACrrKxkZrOZ6fV6NmfOHOZ0OsPSBQKB0OPly5ezoqKiQb9XY2Mjk8vlbMWKFWHHH3/8cQaArV69OnTM5XIxv98flq6yspIJgsCefPLJ0LFPPvmEAWCXX345c7vdoeNbt25lANiRI0cGnU8Se1Q1IxH76KOPYLPZ8Oijj0KlUoW9xnHckM//8ccfw+fz4ac//WnY8Z/97Ge90gqCAJlM/PP1+/1oa2uDTqfD+PHjcfDgwV7p7777biiVytDzefPmARBLVkR6FIhIxM6dOwcAmDRpUlzOX11dDQAoLS0NO56eno60tLSwY4FAAM8//zzGjRsHQRCQmZmJrKwsfPPNN7BYLL3OXVhYGPa863wdHR2xvAQSJQpEJCn953/+J9avX4/rrrsOr7zyCj788EN89NFHmDhxIgKBQK/0PM/3eR5GKyUPC9RYTSI2duxYAMDRo0d7lVp6iraaVlRUBAA4e/YsSkpKQsfb2tp6lVzeeustLFy4EC+99FLYcbPZjMzMzKjen0iHSkQkYkuWLIFer8fmzZt79WL1LFlotdo+q0eXsmjRIsjlcmzbti3s+B//+MdeaXme71WaefPNN1FXVzfo9yXSoxIRiZjBYMDzzz+PtWvXYtasWVi1ahXS0tJw+PBhOBwOvPzyywCAmTNn4o033sD69esxa9Ys6HQ63HTTTZc8f05ODh566CE8++yzuPnmm7F06VIcPnwY77//PjIzM8NKWt/+9rfx5JNP4u6778bVV1+NI0eO4NVXX8WYMWPidv0kjiTutSPDXM/u+y5///vf2dVXX83UajUzGAxs9uzZ7PXXXw+9brfb2apVq5jJZGIABtWV7/P52K9//WuWm5vL1Go1u/7669mJEydYRkYG+/GPfxxK53K52C9+8QuWl5fH1Go1u+aaa9i+ffvY/Pnz2fz580Ppurrv33zzzbD3qaysZABYWVnZYH8lJA44xqi1jgxvZrMZaWlp+M1vfoNf/epXUmeHxAG1EZFhxel09jrWNYF2wYIFic0MSRhqIyIJ0dLSAr/f3+/rSqUS6enpeOONN1BeXo4bb7wROp0On376KV5//XUsWbIE11xzTQJzTBKJAhFJiFmzZoUGLPZl/vz5qKiowJQpUyCXy/H000/DarWGGrB/85vfJDC3JNGojYgkxN69e/usdnVJS0vDzJkzE5gjMpxQICKESI4aqwkhkkvqNqJAIID6+nro9fqYzP4mhMQWYww2mw35+fmh1RL6ktSBqL6+HgUFBVJngxByCbW1tRg9enS/ryd1INLr9QDEizQYDBLnhhByMavVioKCgtD/an+SOhB1VccMBgMFIkKGsUs1nVBjNSFEchSICCGSo0BECJFcUrcRRYIxBp/PN+A8J9I3nuchl8tpaASJu5QORB6PBw0NDXA4HFJnJWlpNBrk5eWF7YBBSKylbCAKBAKorKwEz/PIz8+HUqmkb/ZBYIzB4/GgpaUFlZWVGDdu3IAD0ggZipQNRB6PB4FAAAUFBdBoNFJnJymp1WooFApUV1fD4/H02suMkFhJ+a84+hYfmpHy+6O539JK2RIRIf3x+gPYdawJ43P18PoDOFxrxrRCEybk0qBYqYyMrztCejjf0onTTTbsOd2Cjk4PGiwunG6yS52tEY0C0QhXXFwcWhN6pKhq6wQAlGbrMCZLBwCo63BS9UxCFIjIiMIYQ227OJyjKEODLL0ABc/B5fWj1e6ROHcjFwWiFODx0D9QpKxOH2wuH3gZh3yTGryMQ7ZB7A1ssbklzt3INSIDkccX6Pfm8wciTuuNIG00FixYgAceeAAPPPAAjEYjMjMz8etf/zpUdSguLsZTTz2Fu+66CwaDAffddx8A4NNPP8W8efOgVqtRUFCABx98EJ2dnaHzNjc346abboJarUZJSQleffXVqPKXzJps4lbZmToBCl7888/SCQCAtk4KRFIZkb1mf/rkbL+vlWRqsWL6qNDz/2/POXj9fbcdjE5T49Yruxdm2763Ek5P+FSSn3/rsqjy+PLLL+NHP/oRvvjiC3z11Ve47777UFhYiHvvvRcA8Mwzz2DTpk147LHHAADnzp3D0qVL8Zvf/Abbt29HS0tLKJiVlZUBANasWYP6+np88sknUCgUePDBB9Hc3BxV/pJVs1UMNtl6IXQsQyeOGm+jqplkRmQgSgYFBQV4/vnnwXEcxo8fjyNHjuD5558PBaLrr78ev/jFL0Lp165dizvuuAMPP/wwAGDcuHH4/e9/j/nz52Pbtm2oqanB+++/jy+++AKzZs0CALz00ku4/PLLE35tUrK7fQCAHEP34MwMnQCtwEOlGJEVhGFhRAaidQtL+31NdtEskPuuG9tv2otnjNxzTclQshXmqquuCpuSMnfuXDz77LOhybtXXnllWPrDhw/jm2++CatuMcZCU11Onz4NuVwetmXPhAkTYDKZYpbnZLB0Ui4WjM8K++zyjaoBP2cSfyMyECnlkX/zxSvtUGm12rDndrsd999/Px588MFeaQsLC3H69OlEZW3YUyn4sOc0B1F6IzIQJYP9+/eHPf/8888xbtw48DzfZ/oZM2bg+PHjKC3tu7Q3YcIE+Hw+HDhwIFQ1O3XqFMxmc0zzTUg0qFI8TNXU1GD9+vU4deoUXn/9dfzhD3/AQw891G/6Rx55BJ999hkeeOABHDp0CGfOnMG7776LBx54AAAwfvx4LF26FPfffz/279+PAwcOYO3atVCr1Ym6JMkdrbPgrQMXcLTO0uu1z8614i//Po9DtebEZ4xQIBqu7rrrLjidTsyePRvr1q3DQw89FOqm78uUKVOwe/dunD59GvPmzcP06dOxadMm5Ofnh9KUlZUhPz8f8+fPx8qVK3HfffchOzs7EZczLDRaXKhtd8Dq8vZ6zR9gsLl8MDuo50wKVDUbphQKBV544QVs27at12tVVVV9/sysWbOwa9eufs+Zm5uLf/zjH2HH7rzzziHlM5m0B4NMurb3Im8GlQIAYHH2DlIk/qhEREaMjs5gINL0DkQ6lfid3OmmJYWlQIGIjAhOjx+O4GBTUx+BSC+IgcjuphKRFKhqNgxVVFRInYWU01Ut06vkfQ6z0AYDkcPjhz/AwF88oIzEFZWIyIgQqpb10T4EABolD17GgTGg0+NLZNYIqERERgivPwCtwCOtn0DEcRxyg9M+/P3MLSTxQ4GIjAjTC9MwvTBtwMXPvj+roN/XSHxJWjV7/PHHwXFc2G3ChAlSZomkOJrOMTxJXiKaOHEi/vWvf4Wey+WSZ4kQkmCS/9fL5XLk5uZKnQ2SwlxeP17bXwOTRoEV00ZB1k+P2LF6C/ada0NxhhaLr8hJcC5HNsl7zc6cOYP8/HyMGTMGd9xxB2pqavpN63a7YbVaw26EXIrF6YXF6UWr3d1vEAIAxgCbyxdas4gkjqSBaM6cOSgvL8cHH3yAbdu2obKyEvPmzYPNZusz/ebNm2E0GkO3ggJqXCSX1jVtw6hWDJhOoxRXNnB4aHR1okkaiJYtW4Zbb70VU6ZMwQ033IB//vOfMJvN+Otf/9pn+o0bN8JisYRutbW1Cc4xSUaRB6KuQY1UIko0yduIejKZTLjssstw9mzfa0oLggBBEPp8LSKMAX4JhvDzit7LOfZjx44d+PnPf476+vqwa12xYgX0ej3+53/+J165TFkWh/iZGy4RiNTBEpHT4wdjjHrYEmhYBSK73Y5z587Fb0a43wv8+9n4nHsg834ByPseSHexW2+9FQ8++CD+/ve/49ZbbwUg7r7x3nvvDTiznvSva9mPSKtmvgCDxx+AIO97EToSe5JWzTZs2IDdu3ejqqoKn332Gb7zne+A53n84Ac/kDJbklKr1Vi1alVo5w0AeOWVV1BYWIgFCxZIl7Ek1lU161rqoz8KXhaah3bxbiwkviQtEV24cAE/+MEP0NbWhqysLFx77bX4/PPPkZWVFZ835BVi6STR+IH/AS527733YtasWairq8OoUaNQXl6ONWvWUFUhCowxaAU5PL4AjJpLfw45BhUCAYYAzfJIKEkD0c6dOxP7hhwXcRVJStOnT8fUqVOxY8cOLFmyBMeOHcN7770ndbaSEsdx+P6Vkfeufm/m6DjmhvRnWLURkW5r167FCy+8gLq6OixevJiGKpCUJvmARtK3VatW4cKFC/jzn/+Me+65R+rsEBJXFIiGKaPRiO9+97vQ6XRYsWKF1NlJWp+fb0PZ3kocrOmIKP2hWjP+8u/z2HO6Jc45Iz1RIBrG6urqcMcddwxt7NQI19HpgdnhRSDC1ucAYzTNQwLURjQMdXR0oKKiAhUVFfjv//5vqbOT1HqNqmYMaD0NuKxA9gRA0IelVwXHDrm81H2fSBSIhqHp06ejo6MDv/vd7zB+/Hips5PUeg1mPPU+0HBYfFyzD5i5BlAZQulVCrGS4PIGEpnNEY8C0TDU375lZHA8vkBoeyCDWgG0nBKDEMcBCg3g6QROfwhMuTX0MyoFlYikQG1EJGV1lYYEhQwquQyo+lR8ofAqYNodYkBqOwvYmkI/ow4GIicFooRK+UA00BrF5NKS+fcX1j5krgHszeIo99GzAW0GkBWs9nZV1dBdIvL4AvDT8OqESdlApFCIbQIOh0PinCS3rt9f1+8zmcg4DrlGFbL1KqD5hHgw+3JAqREf50wW71tPiY3YAAS5DOlaJfJNKvgC1E6UKCnbRsTzPEwmE5qbmwEAGo2G5moNAmMMDocDzc3NMJlM4Pnkm4lekqlFSaYWCPiBz06KB7Mv706QVixO+XHbAcsFwFQAmYzD6quLpcjuiJaygQhAaC3srmBEBs9kMiX/muLWesDrBBQqwFTcfZyXA+ljgOaTgLkaMNE0GqmkdCDiOA55eXnIzs6G10t7mg+WQqFIypJQl9DiZh1V4oG0YkB2UWuEqSgYiPpfK53EX0oHoi48zyf1PxQZPMYY/t/u81ApZLhdfg5qQAxEFzMVifeWOsDvA3g5/u9kE863dOKa0kxcnmfo/TMk5lK2sZqMbE6vHy6vH/bOTigdjeLBvgKRJl1svA74ALuYzu0NwOby0SL6CUSBiKSkrq77bNYGHgxQGQF1Wu+EHAfo88XHwfFEKiUNakw0CkQkJXUFoiy0iwcM+f0n1gc3U7Q1AKD5ZlKgQERSUtfOHZmBVvGAYVT/iXXBXsFg1YzmmyUeBSKSkixOL8AYjN6uQJTXf2J9MBB1tgF+L803kwAFIpKSrC4fBL8NGrgBGd9d6umLoBcbrFkAsDeH5pu5fBSIEoUCEUlJGTolihQWcdNEXbY4eLE/HAdos8XHjlZolDzStcpL7oNGYmdEjCMiI8/C8dmAnANq5IB+gGpZF02GOPDR0YbsPBVN80gwKhGR1GUPrjutjWCfPG2GeN/ZFr/8kH5RICIpxx9g4vIlncE5hrrsS/+QJlO8d7TGL2OkX1Q1IynneL0V/z5eg6XmRozJ0kVYIgoGIpcF8HvxxsEG2Fw+fHfGaKRph/+mnMmOSkQk5VicXgjuVnHCq9oEyCPYBUWhEWfnMwY42mB3+2Fz+ajnLEGGTSDasmULOI7Dww8/LHVWSJKzOL3QeNshKGSRlYYAsecsVD1rgyAX/zXcNKgxIYZFIPryyy/x4osvYsqUKVJnhaQAi9MLjaddnKoRSftQl67qWWdrdyDyUSBKBMkDkd1uxx133IE///nPSEvrY1IiIYNkcXqh9bYFS0SDCERdk2Jd5tDoajdVzRJC8kC0bt06LF++HIsXL75kWrfbDavVGnYjpCeX1w+Xxwu1t0Ms1QymRKQyiffOjlCJiOabJYakvWY7d+7EwYMH8eWXX0aUfvPmzXjiiSfinCuSzKwuL1Q+KwRZAHKFsju4RKKrROQ0Q9BTiSiRJCsR1dbW4qGHHsKrr74KlUoV0c9s3LgRFosldKutrY1zLkmykXEcLjd6YdIoAXV676VhB6I2ifdeJwxyHzJ0ylAVjcSXZCWiAwcOoLm5GTNmzAgd8/v92LNnD/74xz/C7Xb3Wt5VEAQIQgRdsWTEytQJWFAgBzw6cdrGYMgFcfKrx4HpWRymjymOSx5Jb5IFokWLFuHIkSNhx+6++25MmDABjzzyCK0xTaLnCE7T0KQP/mdVJsDjAJzm7uVBSNxJFoj0ej0mTZoUdkyr1SIjI6PXcUIi5fL6IThawQGDLxEBYjuRtR5wdsQ6a2QANMWDpJQ3vqzF2DNnMDVbAb06ihJRsJ3I0tGCv9dXQVDw+P6VtN9ZvA2rQFRRUSF1FkgSY4zBbreB9zmhkAvRl4gAyFwdaLV7oFFSE0EiSD6OiJBYsbt9UHrM4AAoNUZxO+nBCnb3K7ziGDWXNyDO5CdxRYGIpAyL0wu11wxBIYNMG0VpCAiViOReOzjmR4AxeP0UiOKNAhFJGRanFyqfBYKcj65aBgBKLSCTg+cYVAEHABrUmAgUiEjK6CoRqeQycTBjNDgOEPTgwEGPrkBE0zzijQIRSRlWpxdqnxmCgo9uDFEXlREAoEUnAApEiTCses0IGYo8gwCV4IJWUAwxEBkAABm8C1Ydrc6YCBSISMqYmgkgW2zjgWCM/kTBEtG8AiXmTSiOSd7IwKhqRlJH12hoTdrgJrteTBBLRHBZhp4nEhEKRCQleHwBOMzNYGDRN1R3CZaI4Kb1rhKFAhFJCTXtDnz01TEcr7dG33XfJdhG1NDUhP/5rBL7z9NeZ/FGgYikBIvTA5XXAqVcNrSGakCsmnEcAn4fLFYLLE5vbDJJ+kWBiKQEs0PsulcNZTBjFxkPKHXgZRyUPjt13ycABSKSEqy2Tij8TnHB/KG2EQGAygC5jIPKb6NAlABRBaLz58/HOh+EDInLKm4vLWgN4kaJQ6UyQh4sEbm8NMUj3qIKRKWlpVi4cCFeeeUVuFyuWOeJkEHx+QPwd4oNyipDhBsqXopgAM9zEPxUNUuEqALRwYMHMWXKFKxfvx65ubm4//778cUXX8Q6b4RExOryQeUxi206+szYnFRlhFwmg+Cz06TXBIgqEE2bNg1bt25FfX09tm/fjoaGBlx77bWYNGkSnnvuObS0tMQ6n4T0S85zuMLkRbZeAKeNZSDioOcc0AtyWpMozobUWC2Xy7Fy5Uq8+eab+N3vfoezZ89iw4YNKCgowF133YWGhoZY5ZOQfhlUClxh8qE4QxubhmoAEAxQ8DLMG63AnXOLwXFcbM5L+jSkQPTVV1/hpz/9KfLy8vDcc89hw4YNOHfuHD766CPU19fjlltuiVU+CekfY4Cja3pHjAJRcFAjfG7A54nNOUm/opr0+txzz6GsrAynTp3CjTfeiB07duDGG2+ELDi/p6SkBOXl5SguLo5lXgnpU1tbC/ReDxRyObjB7Ow6ELkg3nxucaqHPEZVPtKnqEpE27Ztw6pVq1BdXY133nkH3/72t0NBqEt2djZeeumlmGSSkIF88vUJHKjugJnphjbZ9WIqA8422/HOvuOoNztjd17SS1Qloo8++giFhYW9gg9jDLW1tSgsLIRSqcTq1atjkklC+uMPMPhtwa57U4y67rsIRrh8fnTaOuDw+GJ7bhImqq+PsWPHorW1tdfx9vZ2lJSUDDlThETK5vJC8Joh42I4hqhLcHS14LPD5aWxRPEUVSDqryvTbrdDpYrBqFZCIhSaY6bgwQ11jtnFBDEQKf2dNKgxzgZVNVu/fj0AgOM4bNq0CRqNJvSa3+/H/v37MW3atJhmkJCBmJ1eqLxmqFQxmOx6MUEPXiaD4LHDTdM84mpQgejrr78GIJaIjhw5AqWyez1fpVKJqVOnYsOGDbHNISEDsNhsUPodUMlVseu67xKsmtEM/PgbVCD65JNPAAB33303tm7dCoPBMKQ337ZtG7Zt24aqqioAwMSJE7Fp0yYsW7ZsSOclI4fL0gIlAIVGDyjUsT15j/lmVi81VsdTVL1mZWVlMXnz0aNHY8uWLRg3bhwYY3j55Zdxyy234Ouvv8bEiRNj8h4ktY03eAGjCrq0nNifXNBDzvMQZAwqRpO74yniQLRy5UqUl5fDYDBg5cqVA6Z9++23IzrnTTfdFPb8t7/9LbZt24bPP/+cAhGJSLHaBWRogfTc2J9cxiM7IwPZOiVQTJ0w8RRxIDIajaH5NkbjELZq6Yff78ebb76Jzs5OzJ07t880brcbbrc79NxqpcXNRzxHcD3pWLcPdVEZALcNcFkBQ3583oNEHoh6VsdiVTUDgCNHjmDu3LlwuVzQ6XT429/+hiuuuKLPtJs3b8YTTzwRs/cmyc3u9sHX1gitPwBFrHvMuggGAHW0o0ecRTWOyOl0wuFwhJ5XV1fjhRdewK5duwZ9rvHjx+PQoUPYv38/fvKTn2D16tU4fvx4n2k3btwIi8USutXW1kaTfZIizjdbcfxcFc612ONWInLyWhyrt+CTw+ficn4iiqqx+pZbbsHKlSvx4x//GGazGbNnz4ZSqURrayuee+45/OQnP4n4XEqlEqWlpQCAmTNn4ssvv8TWrVvx4osv9korCAIEQYgmyyQF2cxtkLEABKV6aDu7DkCmMsDq8sEha4fPH4Ccp2Xe4yHqFRrnzZsHAHjrrbeQm5uL6upq7NixA7///e+HlKFAIBDWDkRIf5wWcZ1qhT4ztpNde1BqTeAAGksUZ1GViBwOB/R6PQBg165dWLlyJWQyGa666ipUV1dHfJ6NGzdi2bJlKCwshM1mw2uvvYaKigp8+OGH0WSLjDBuSwu0AFTG7Li9B6cygpd1r12tpQJ5XES9eP4777yD2tpafPjhh1iyZAkAoLm5eVCDHJubm3HXXXdh/PjxWLRoEb788kt8+OGH+Na3vhVNtsgI4g8w+O1ij5km1rPuexIM4GUcFH4n3B4qqcdLVCWiTZs2YdWqVfj5z3+ORYsWhbrbd+3ahenTp0d8HlqviETL6vRC8HaA5wB1HEtEUKghkysAnxveTguQpo/fe41gUQWi733ve7j22mvR0NCAqVOnho4vWrQI3/nOd2KWOUL60+7wQO21xGfWfU8cByYYAFcLvA4zgNHxe68RLKpABAC5ubnIzQ0fzTp79uwhZ4iQSGQqfdCmy8BxaiBWO3f0Q6YyQmlvhcxji+v7jGRRBaLOzk5s2bIFH3/8MZqbmxEIhPcm0E6wJN6MATOMBhWgTgN4RVzfa2ppIaDtAHTUaxYvUQWitWvXYvfu3bjzzjuRl5dHW62QxOsMrhAa59IQAEAItgvR6Oq4iSoQvf/++3jvvfdwzTXXxDo/hETkwoUaGN1eaDRZ4OP9Zl1bC7koEMVLVN33aWlpSE+P0yRDQi7B7fPjxJmzOFpnhU8dx4bqoJpOOY7VW3Cmti7u7zVSRRWInnrqKWzatClsvhkhiWLp9EDt7YCC5yAY47D8x0VcvA5Wlw9uu1nczJHEXFRVs2effRbnzp1DTk4OiouLoVCENxYePHgwJpkjpC9mcyt45oVKKYiN1XEmV4tVM+bzAD5X7FeCJNEFohUrVsQ4G4RErrO9EQDA6zIBWdxbiKBSqeCVqeHzu8V2IgpEMRdVIHrsscdinQ9CIuY2NwEAFIY4jqjuQZDL4JFr4fO5xJ4zfRyWpR3hop6ybDab8Ze//AUbN25Ee3s7ALFKVldHDXokvrxWMRCpTPFvHwIAQcHDzevgDzAw6jmLi6hKRN988w0WL14Mo9GIqqoq3HvvvUhPT8fbb7+Nmpoa7NixI9b5JASAuJUVs4lVM21GYpZuVQVLRAyA12GG8pI/QQYrqhLR+vXrsWbNGpw5cyZsZ9cbb7wRe/bsiVnmCOnF58bkdD+KMjTQZyZm3pecl4EJBih5Dn6nJSHvOdJEVSL68ssv+1xBcdSoUWhsbBxypgjpD9fZjDS1AmmmdECduJnwK+deARw7Dfg7E/aeI0lUJSJBEPrcQeP06dPIyorj2jCE2MT2IegT0z4UIgRHV9M0j7iIKhDdfPPNePLJJ+H1egEAHMehpqYGjzzyCL773e/GNIOE9NRYX4VWuxsOVYK/8LqmebjtQIAmv8ZaVIHo2Wefhd1uR1ZWFpxOJ+bPn4/S0lLo9Xr89re/jXUeCQmpq63CmWY7GnzxWSy/P/tq3TjaYEe73QXQciAxF1UbkdFoxEcffYS9e/fi8OHDsNvtmDFjBhYvXhzr/BESwnxuBOwtAABDVmIXKDM7vXB7lUj3+cUNF1WJDYSpbtCBKBAIoLy8HG+//TaqqqrAcRxKSkqQm5sLxhgtCULixtVeB58/AI9cC1NaYiddCwoZ3LwW/oBFHF1NcSimBlU1Y4zh5ptvxtq1a1FXV4fJkydj4sSJqK6uxpo1a2iZWBJXtqZKAIBflw9FgvcXE+Q8PHIdfAFGDdZxMKgSUXl5Ofbs2YOPP/4YCxcuDHvt//7v/7BixQrs2LEDd911V0wzSRKs9SxgawAyL0O7LA2tdjdGmdTQClGvLBwTztYaAIA8rSDh7y3IZeLoaj+jdYniYFB/Wa+//jp++ctf9gpCAHD99dfj0UcfxauvvkqBKJnVfgGc/RgA0HzsE/wD82ARRkEpl2HljFHIM0o04ZMx+DouAABUGUUJf3uVgoeH18HnC1CJKA4GVb795ptvsHTp0n5fX7ZsGQ4fPjzkTBGJODuA8xUAAIdMi8pmK8a2VkDN+1GcoUW6VsLJDY42uF0OBDg5dAma2tGTIJfBLRfnm8FFo6tjbVAlovb2duTk9D/zOCcnBx0dHUPOFJHIha+AgB9IK8an/LVQVr2EbIUb14wxgy+aIG1HhLkaY7O0MAv50GTqEv72gpyHX0FrV8fLoEpEfr8fcnn/sYvnefh8viFnikjA7wOajgIAnLkzcbLZhTrDDBSkqyFvPARO6pUJ2yshyHnkFE2AXhXfXTv6UpCuxtpvTcPEfCPgdQE+T8LzkMoGVSJijGHNmjUQhL43AHe7aUvepNVRKf6DCTqc9WbBH2gBl3sF9P5zgNMMmKvQyOfjQHUHRqWpMa3AlLi8BQKAuVp8nFaSuPftgeM4QC4AcqUYhNw2QB7/9bJHikGViFavXo3s7GwYjcY+b9nZ2YNqqN68eTNmzZoFvV6P7OxsrFixAqdOnRr0RZAYaA/uRZd5GUxaAWOzdRifnwFkXS4ebz6JZpsLp5tsOFTTAZbIEpKtHu0WG2psATQxiQfwhOacUTtRLA2qRFRWVhbTN9+9ezfWrVuHWbNmwefz4Ze//CWWLFmC48ePQ6vVxvS9yCV0BaL0MShI16AgXSM+75gA1H8NtJ7C+DGLsec0hw6HF01WN3KNqv7PF9O8VaLV7sFplo08sws5Rk1i3rcHxhj+8U0Dsup9mKkLQEFd+DEl6cCQDz74IOx5eXk5srOzceDAAVx33XUS5WoEcnaI1S9OBpgKw18zFgJKDeBxQLDXYUyWDqcabTjdZEtcIGo9DYfHB7O+AJN1fTcLxBvHcagzO+F3K+FVB6CgBuuYSuzw1EuwWMTibn97prndblit1rAbiQFLcHlffS4aOgPo6PR0V71kMiCjVHzcfg6X5Yg9VqebbImpnjna4bM2wuljaFcXIUsvTSACgis18jp4aVBjzA2bQBQIBPDwww/jmmuuwaRJk/pMs3nz5rA2qYKCxI+wTUnWevHeMAqfnGxB+WdVON1k7349fYx4316J4gwtlHIZbC4f6i2u+Oet5SQcHj8swiioNTpolNIV4tVKPjiWiAY1xtqwCUTr1q3D0aNHsXPnzn7TbNy4ERaLJXSrra1NYA5TmFUsEXm0OWi2icEl39Sj2pVWDHAc0NkKuceKsVliqehMU5yXw2AMaDwKu9uHNk0Jsg3SlYYAcXS1m0pEcSHt5KGgBx54AP/4xz+wZ88ejB7d//IOgiD0O3SARMnvBezNAIBmLhOM2aBXycPH6ijUgGEUYLkAdFRiXM44tNrdSNPEeaS1uQZwtMHq4dBmHIvLTNLuJ6ZSBCe+upjYfc+YGKDJkEkaiBhj+NnPfoa//e1vqKioQEmJNGNERjR7E8ACgFKLJrcYWHIMfTRCp48RA1HbOYyZNC1UKoqrugPinbIEfpkSuX3lK4HUCh4eXgNvgAEBH+B1AErq3Y0FSQPRunXr8Nprr+Hdd9+FXq8PLbxvNBqhVtNumgkRLA1Bn4tGqzhauO9AVAJU7gHM1eIo63iXBOzNQIs4pmzh9cswhTPBpEn8iOqe1EoePC+Hjw8OH3BZKRDFiKRtRNu2bYPFYsGCBQuQl5cXur3xxhtSZmtk6RRXPIQ2C01WsX2oz5KHLlesovk8oTYlt8+PM/HoPWMMOL9bfJw9ATJDDrL0QsLXILrYzMI0PHB9KUoLgpNuqcE6ZiSvmhGJBUtELiETFqe4GUKfjcIymdho3XwC6KhEwDAaL39WhU63H7fNKkB+LNtvmo4BbWfFfe2L58XuvEMkkwVLgSqD2NNIDdYxM2x6zYgEGAM6xUDEG7Jx09R8zBuXCZWC7zt9erANr70SMhmHwuDo69Ox7D2zNgCngwNdi67B26dc2HWsEVaXN3bvMVQ0zSPmKBCNZC6LWNWS8VDoMlGarcOVxQOsBZ1WLN7bGgCvE6XZ4rIYZ5vtQy/dMgY0fAMcelXsyUsrhi17JqrbHDjeYIVS4moZANjdPrx7qA67q4OTu6lEFDPSf7pEOl3tQ5oMsRp0KSojoM0Ug0ZHNYozNKHBjQ1DGdzodQLH3wFOvicGIVMhMGklasziP3yOQdV/KS2BOADnWzpR1SkXAy+1EcUMBaKRrKvHTJeNo3UWnGuxw+3zD/wzXctwdFRCzsswJlPsNToVbfXMXAt8tR1oPinOdRszH5j6A0AuoLJV3N65KD3xk1z70hUM3TKtuIg+lYhihgLRSBZsHwposlBxqhl/P1QPh/sSgahHOxEYw/hcsXp2ssEGr3+QO6DWHQQOvSb+Q6vTgBl3AkVXAzIZPL4AqoKBqDQ78Ssy9oWXcVAGl4z1BRjg6RQXlCNDRoFoJAtuVmiXm+D1M8hlHIzqS4zVMRaI1TiXBXB2oDhDC4NaAbfPj3qzM/L3vnAAOP2hOJgy+3LgynsAQ/da1Oda7PD6GdI0Ckknul5MreDhk6ngZcF/Hdr1NSaGxRQPIgG/D3C2AwBamBGADek6ZXcXdX/kSsA4GuioFnvPRqdjyRU5MKgVlw5iXVrPAGd2iY+L5gIl83sNkDxcawYAjM81DKtNO1UKHhaOg5vXAegUA7I6TepsJT0qEY1Uznax0VkuoNUtBpAMbYQljx7tRABQkK6JPAg5zcDJf4iPR83oMwgFAgxjsnQwqhWYMnp4bamqVor/MmIgAu3oESMUiEaqzlbxXpuJ1k5xjE6mLsJJrF3tRB1V4q4fPVzocMA3UFvRmY/EtbENeUDp4j6nishkHGaXpOPua4ol39TxYmoFD7mMg0cRDJBO2rUmFobXp0wSxxEMRJpMtHWI3eQZka5+qMsJrdoIc00oMH12thX7K9sxsygN112W1fvn2s4BbWdR1e7EvwOT4d9Xg3SdgBy9gDyjGkq5DH7GMCo4Sns4Vcm6LLkiF0sncUBtB3D2NAWiGKFANFIFS0Q+dTo6LoglooxIS0QcB2ReBtQfAlpOhgJR19SQA9UdkMs4zCpJBwegtsOJQpMA/uy/xLfOmo5Wtx5weNHh8OJcc/cibEq5DHfMKYQp3kuMRCnUhtbVLkSBKCYoEI1UjjYAgEybhVVzRqOt0w39YKpBWeODgegUMO4GQCZDabYec8d6sO9cG/ZXtuPLqg5wHOAPMKzMqkORox1QalEw9Vu43c/D52dotrnRZHWh0eKC1x9ArlElLjw23PUMRLQu0ZBRIBqJAv7QN7lMm4EstTD4LnJTsTgb3+sU9xwLloquGpMBk0aBvWfbYHV6AQYYZS4oL+wFtDwwZgEMOh2Cs7W6dwtJEs1WF/adb4NWDizmOHGKDK1LNGQUiEYip1kMRrxCnLYRDZmsu1TU+E13AzaACbkGjM/Rw+YWB/vpKz8A18iLDdS5k4eefwl5AwznWzrFXkJBLw7GdHZQIBoi6jUbiUIN1Rn4ps6CA9UdsDiimN2eN028bzkljjLugeM4GFQKGFyN4BrFrawxbknSV2E0wWkeTq+f2oliiALRSNSj6/5QrRl7TregwxHFXu6GPECfK5au6r/u/XogAJz5UHycNyVs5HSy0ghiIPL4AvAqqQs/VigQjUTBEpFfnQFzsCSUHmmP2cUKZov3tfvF7vyeaveL00gUKmDMwmhzO6woeRnkwZ4zt5wCUaxQIBqJgiUiG2+CP8CglMsG12PWU/YVgC5bbLQ997HYgwSIU0Aq94iPx14vjjtKARzHQRP8XTnk4oRfCkRDR43VI00gADjEOWbtTAfAhTSNMvrBgxwHjPuWOIu+8SjAC4AmHThfIU5ozbkCyJ0Ss+wPBxolD6vTi05ZsO/PaZY0P6mASkQjjcssboUjk6PFK45gTtcOcfCgqRAYu0h8XHdAnMbh94o9aZctS/oG6otplDwUPAdXV4nI6xRvJGpUIhppggMZoUlHu0PsXo94RPVACmaJvUj1BwG/B8gcL05qjWTlxyTz7Sn5kHHBKSg1OsBtF6tnCtoCK1oUiEaaHj1m7TaxpyxmO7Zmloq3FMf3XCpFkyEGos7WlOgVlAoFopGmx2TX7102Gh2dXsk3LkxqmgyxYb6rpEmiQoFopOlRIhLkPHKNqVd1ircmqwufn2+DTpBjkT5TPEiBaEgoEI0kgUBYiYhEx+ML4HxLJ9I0CiAnuP0SBaIhoV6zkcRlFpeI5eU4bZPjk5PNqG7rvOSPkXC64DiiTo8fTJMhHnR20EL6QyBpINqzZw9uuukm5Ofng+M4vPPOO1JmJ/XZm8R7bRaq2pw4VGtGvXkI+5GNUDqVGIg8vgDcMg0gF8SBnME1wMngSRqIOjs7MXXqVPzpT3+SMhsjR9c+ZtpstHeKPWYx6bofYRS8LLTHmd3jFxusAaqeDYGkbUTLli3DsmXLpMzCyBLc2ZVps9DWIAaiIQ9mHKF0KjlcXj/sLh8ytZmAtb67I4AMWlI1Vrvdbrjd7tBzq5V22hyUYImoU5kBj88HGcfBFOnuGySMXpCj1eaG3e3rUSKiQBStpGqs3rx5M4xGY+hWUFAgdZaSh9cV2vqmPbg+olEth5xPqj+BYUMnyKHgOXF3Ww114Q9VUv0Vbty4ERaLJXSrra2VOkvJI7i9NFQGtLjEjz1zGO2gmmwWTsjGuoWlmF6YBmi7AlF7r+2VSGSSqmomCAIEgf55omIL9pjpcmB1id3MmZFuH0R6CZvmoTKKPWc+t9hOpM+RLmNJKqkCERkCW714r8/DwuJsXFWSIW1+UgnHiXu9mWvEIRIUiAZN0kBkt9tx9uzZ0PPKykocOnQI6enpKCwslDBnKcgaDESGPACAWklTO4ai0+3Dv040weML4NYrC7oDUVcVmAyKpIHoq6++wsKF3UuIrl+/HgCwevVqlJeXS5SrFOTp7F68S08zxGOBl3E43yKOSvf6A1DossUX7BSIoiFpIFqwYAEYS4LN9JKdrVG812TgvNmHQ7UXMCZLh2kFJkmzlcwEuQxKuQweXwA2lw/poUDURBsuRiGpes1IlKx14r0hDw0WF6rbHGi1uQf+GTIgjuPEvc0AWJxesQufk4nDJNw0vm2wKBCNBJYL4r0hH612MQBR1/3QdQUis8MD8HJAG+wAoOrZoFEgSnV+H2AJlohMxWgJloQyaY7ZkIWViABAL3YEhDoGSMQoEKU6a524WL5Si07eAJvLB47D4Pe6J730CkRdS8VSIBo0CkSpzlwt3qcVoSlYGkrXKiHIqft+qIxqBRQ8170Vk2GUeG+rFxehIxGjAY2prv28eG8qQpNVDETZepWEGUodhekarFtY2h2INJkArxA3m3S0ihtPkohQiSiVuW2AtUF8nDEWjDEIChlyDFQtiwWZjAvfmFIm61E9q5MmU0mKSkSprPWMeG/IBwQ9ri7VY+7YDARo6Fb8GEaJu3pY64H86VLnJmlQIEplLafE+8zLQoc4jgNPY+1i5kB1O4432DBttAmTRxu724ksVCIaDKqapSqnubuhOnsC/FQMigunJ4BWmxvNtuDa38ZR4qhqRxvgooGNkaJAlKoaj4hTDdKKAHUa9pxuwUufVuJEA/1zxFLXmt9twTXAoVAD+lzxcdcXAbkkCkSpyO8DGg6Jj3OnAAAudDhgdXohl1G9LJZCgcju6Z43aSoS7zuqpMlUEqJAlIqajoj7sQt6IGsCnB4/Wu3iN/aoNLXEmUst6RolOA5wef1weIKrM6YVi/cdVWKplFwSBaJU4/cC1Z+JjwvmALwcdWYHAPHbW6Ok/olYkvOy0AYEXdNnYBwNyOTil0Fw5xQyMApEqaZ6r9hIqjIA+dMAALXtTgDAaCoNxUWuURwg2mAJNljzCiC9RHzcelqiXCUXCkSpxNYI1H4hPi5dDPAKMMZQ2Sou4FWUoZUwc6krz6hGhk4JpbzHv1PmOPG+awgFGRCV01OF1wUc+5u4i0TmuNDYofZODyzBRuqCNI3EmUxNU0YbMfXiReYyxond+PZmwNkBqNMkyVuyoBJRKgj4gRN/F8cOqYzAhOWhFQLlMhmmFZpweZ4h/BubxAzX12qMSg1gCq673nwisRlKQvSXmewYA079E2g7Jy7ONXGFOJYlyKhRYOH4bCy+gnaWiDd/gKHT7es+kDNJvG/4hnrPLoECUbI7/wnQeFRcpvSK73RPuiQJdarRhv+3+xz+72SP1RmzJgBypVg1M9dIl7kkQIEomVV/BtTsFx+PXwZkloa9fKbJhtp2BwI0vSPujGoFPL4Aatod3dNp5Eoge6L4uO6AdJlLAhSIklX1Z8D53eLjsQuBvClhL3v9AfzrRDPeOnAB54O9ZiR+cgwCNEoeHl8Ate2O7hdGzRTvW08DdhpT1B8KRMmoel93EBozHyi8qleS4/VWuLx+GNUKjMmkbvt44zgOl+XoASB8Pp8uC8gaL7YRVX8qUe6GPwpEyYQxoOpT4HyF+LzkOqDo6l7JPL4AvqhsBwDMKEqDjOaXJcSEPDEQnWuxw+X1d79QfK3Yi9l8UlyriPRCgShZBALA6Q+Byn+Lz0vmAcXX9Jl0f2Ub7G4fjGoFJuUbEpjJkS3XoEKWXoDXz/B1jbn7BV02kDdNfHzqfcBHe8pdjAJRMvB0Akf+CtR/LX6zjlsifsv2oaq1EweqOwAA88dnQc7TR5woHMdhdkk6AOB4gzW8k2DMAnESsrMDOPG/4tgvEjIs/kr/9Kc/obi4GCqVCnPmzMEXX3whdZaGB8bEKQJfbQfaK8VxQlesAEbP7DO5xxfAh8cawZg42ndsli6x+SUozdJhVnE6bptVEF4lVqiASSsBGS8u4Xvsb+Ii+wTAMAhEb7zxBtavX4/HHnsMBw8exNSpU3HDDTeguXkE75bJmDhA8fDrwNG3xVncmgxgxhoge0K/P6aUy7BkYi5KMrWYf1lW4vJLQmQyDteOy4ROEGdPMcbg8we3FjLkAxNXijPzW88AB8rEz5kGO4JjTNrfwpw5czBr1iz88Y9/BAAEAgEUFBTgZz/7GR599NEBf9ZqtcJoNMJiscBgSNK2kIAf8DrEHTfszYCtAWg7KwYfQPwGLZgDFF0NJpPD7QvA6fHD6vKi1e5BndmJTK0SV5dmhk7JGOt72gFJuK9rOnCgugNTRptQlKFBmkYJpf0CcPzv4mcOAJp0cX6gPh/QZgFKLSAXQtN0klmk/6OSTnr1eDw4cOAANm7cGDomk8mwePFi7Nu3r1d6t9sNt7u7oc9qjXDZ0/ZK4Oy/+n4tFIdZH8cAxgL4qrrjom8tFkpn1ChwWbY+dOyr6vbutoGLfsagUmBCbnd16evqdgS87tDP9kytVmswccY8YPSVgNqE/9lXhbZOT59fnq1qBeaOzQgFHwpCw4PXH8DXNWbYXD7sPduKvWfF4wqeg5pbiFLnUczX1QKOdqBmP84020KLqzGZHAFODsbJEOB48LwcM4oyQuc+0WCF1eULf8Pgxy7jOMwq6p5ke6rJjmy9gDRtjLcZn/x9cbmZGJA0ELW2tsLv9yMnJ3weVE5ODk6ePNkr/ebNm/HEE08M/o18bqCzNcpcMvg6O/p9NcApgB6dIH6nrd/tegIyBeDt/pX7PC74AgwMHLy8Gk5FGjoVGbCoRkGTXYKJ48aE0rp9gVAQUspl0AlyZOiUyNIJKMmicULDkYKX4a65RTjZaMPZZjuarC44PH54/QxeKHDWMBvzr1ohVs86qtDeegJeXyv4gBeAHz3/sDieAzq7/7ACdgsCFweirrQcgM7uxnBmt4JxKoDFOBCx2O1mm1TLgGzcuBHr168PPbdarSgoKLj0D5oKgKm3i49DpYUepYawEgQXfowxFI5xd78SPM7AgeMAlYIHtN0bFhaPd4OxrmRc2M8oeFlY2pKJXgTkAiBXg+O6m+s4DlDIwpvvvjdzNOS8DCq5jHrCkoicl2HSKCMmjTICEJeUdXsDcPv9YhFYLgA5VwA5V6A4/3p4fAEwnxeczwku4AUCfnDMDxn8QI+NMXNK3TD5+ip5B//cTKrQ8cxSN9RKOaCI8Tbjyth9AUoaiDIzM8HzPJqamsKONzU1ITc3t1d6QRAgCFHsUqrUdq+YN0gcgFGmyNMHB9dGZDADnk2aGH+bEUmoFLz45QVFr9dyDD23AjcOeJ7MQSxvlJkeeVqpSPrVqlQqMXPmTHz88cehY4FAAB9//DHmzp0rYc4IIYkkedVs/fr1WL16Na688krMnj0bL7zwAjo7O3H33XdLnTVCSIJIHohuu+02tLS0YNOmTWhsbMS0adPwwQcf9GrAJoSkLsnHEQ1FSowjIiSFRfo/St0vhBDJUSAihEiOAhEhRHKSN1YPRVfzVsRTPQghCdX1v3mppuikDkQ2mzhpMKLR1YQQydhsNhiN/Q/STOpes0AggPr6euj1+mE50bNrCkptbW1K9uql8vWl8rUBibs+xhhsNhvy8/Mhk/XfEpTUJSKZTIbRo0dLnY1LMhgMKfnH3CWVry+Vrw1IzPUNVBLqQo3VhBDJUSAihEiOAlEcCYKAxx57LLoVA5JAKl9fKl8bMPyuL6kbqwkhqYFKRIQQyVEgIoRIjgIRIURyFIgIIZKjQDRIg92V9s0338SECROgUqkwefJk/POf/wx7nTGGTZs2IS8vD2q1GosXL8aZM2fieQn9Gsy1/fnPf8a8efOQlpaGtLQ0LF68uFf6NWvWgOO4sNvSpUvjfRn9Gsz1lZeX98q7SqUKS5Osn92CBQt6XRvHcVi+fHkoTcI/O0YitnPnTqZUKtn27dvZsWPH2L333stMJhNramrqM/3evXsZz/Ps6aefZsePH2f/8R//wRQKBTty5EgozZYtW5jRaGTvvPMOO3z4MLv55ptZSUkJczqdibosxtjgr23VqlXsT3/6E/v666/ZiRMn2Jo1a5jRaGQXLlwIpVm9ejVbunQpa2hoCN3a29sTdUlhBnt9ZWVlzGAwhOW9sbExLE2yfnZtbW1h13X06FHG8zwrKysLpUn0Z0eBaBBmz57N1q1bF3ru9/tZfn4+27x5c5/pv//977Ply5eHHZszZw67//77GWOMBQIBlpuby/7rv/4r9LrZbGaCILDXX389DlfQv8Fe28V8Ph/T6/Xs5ZdfDh1bvXo1u+WWW2Kd1agM9vrKysqY0Wjs93yp9Nk9//zzTK/XM7vdHjqW6M+OqmYR6tqVdvHixaFjA+1KCwD79u0LSw8AN9xwQyh9ZWUlGhsbw9IYjUbMmTOn33PGQzTXdjGHwwGv14v09PC9ayoqKpCdnY3x48fjJz/5Cdra2mKa90hEe312ux1FRUUoKCjALbfcgmPHjoVeS6XP7qWXXsLtt98OrTZ8f6tEfnYUiCI00K60jY2Nff5MY2PjgOm77gdzzniI5tou9sgjjyA/Pz/sH2Lp0qXYsWMHPv74Y/zud7/D7t27sWzZMvj9/gHOFHvRXN/48eOxfft2vPvuu3jllVcQCARw9dVX48KFCwBS57P74osvcPToUaxduzbseKI/u6SefU+Ghy1btmDnzp2oqKgIa9C9/fbbQ48nT56MKVOmYOzYsaioqMCiRYukyGrE5s6dG7a33tVXX43LL78cL774Ip566ikJcxZbL730EiZPnozZs2eHHU/0Z0cloggNdldaAMjNzR0wfdf9YM4ZD9FcW5dnnnkGW7Zswa5duzBlypQB044ZMwaZmZk4e/bskPM8GEO5vi4KhQLTp08P5T0VPrvOzk7s3LkTP/rRjy75PvH+7CgQRSiaXWnnzp0blh4APvroo1D6kpIS5ObmhqWxWq3Yv39/Qne6jXbH3aeffhpPPfUUPvjgA1x55ZWXfJ8LFy6gra0NeXl5Mcl3pGKxo7Df78eRI0dCeU/2zw4Qh5a43W788Ic/vOT7xP2zS1izeArYuXMnEwSBlZeXs+PHj7P77ruPmUymULfunXfeyR599NFQ+r179zK5XM6eeeYZduLECfbYY4/12X1vMpnYu+++y7755ht2yy23SNYFPJhr27JlC1Mqleytt94K6+K12WyMMcZsNhvbsGED27dvH6usrGT/+te/2IwZM9i4ceOYy+VK6LVFc31PPPEE+/DDD9m5c+fYgQMH2O23385UKhU7duxYKE2yfnZdrr32Wnbbbbf1Oi7FZ0eBaJD+8Ic/sMLCQqZUKtns2bPZ559/Hnpt/vz5bPXq1WHp//rXv7LLLruMKZVKNnHiRPbee++FvR4IBNivf/1rlpOTwwRBYIsWLWKnTp1KxKX0MphrKyoqYgB63R577DHGGGMOh4MtWbKEZWVlMYVCwYqKiti9997bayxOIg3m+h5++OFQ2pycHHbjjTeygwcPhp0vWT87xhg7efIkA8B27drV61xSfHa0DAghRHLURkQIkRwFIkKI5CgQEUIkR4GIECI5CkSEEMlRICKESI4CESFEchSICCGSo0BEkkp5eTlMJlPo+eOPP45p06aFnq9ZswYrVqxIeL7I0FAgIn3qWrP4xz/+ca/X1q1bB47jsGbNmrD0sQ4AxcXFeOGFF8KO3XbbbTh9+nS/P7N161aUl5eHni9YsAAPP/xwTPNFYo8CEelXQUEBdu7cCafTGTrmcrnw2muvobCwUJI8qdVqZGdn9/u60WgMKzGR5ECBiPRrxowZKCgowNtvvx069vbbb6OwsBDTp08f0rn7KqmsWLEiVMpasGABqqur8fOf/zy0iwTQu2p2sZ4lszVr1mD37t3YunVr6ByVlZUoLS3FM888E/Zzhw4dAsdxCV8riYgoEJEB3XPPPSgrKws93759O+6+++64v+/bb7+N0aNH48knn0RDQwMaGhoGfY6tW7di7ty5uPfee0PnKCws7HVNAFBWVobrrrsOpaWlsboEMggUiMiAfvjDH+LTTz9FdXU1qqursXfv3ogW0hqq9PR08DwPvV6P3NzcqFY9NBqNUCqV0Gg0oXPwPI81a9bg1KlTob2/vF4vXnvtNdxzzz2xvgwSIVqzmgwoKysLy5cvR3l5ORhjWL58OTIzM6XO1pDk5+dj+fLl2L59O2bPno3//d//hdvtxq233ip11kYsKhGRS7rnnntQXl6Ol19+OWalBplMhouXwvJ6vTE5dyTWrl0baogvKyvDbbfdBo1Gk7D3J+EoEJFLWrp0KTweD7xeL2644YaYnDMrKyus3cfv9+Po0aNhaZRK5ZC3r+nvHDfeeCO0Wi22bduGDz74gKplEqOqGbkknudx4sSJ0OP+WCwWHDp0KOxYRkYGCgoKeqW9/vrrsX79erz33nsYO3YsnnvuOZjN5rA0xcXF2LNnD26//XYIghBVlbC4uBj79+9HVVUVdDod0tPTIZPJQm1FGzduxLhx4xK64D3pjUpEJCIGgwEGg2HANBUVFZg+fXrY7Yknnugz7T333IPVq1fjrrvuwvz58zFmzBgsXLgwLM2TTz6JqqoqjB07FllZWVHle8OGDeB5HldccQWysrJQU1MTeu1HP/oRPB5PQnoBycBozWoyYv373//GokWLUFtb22unVJJYFIjIiON2u9HS0oLVq1cjNzcXr776qtRZGvGoakZGnNdffx1FRUUwm814+umnpc4OAZWICCHDAJWICCGSo0BECJEcBSJCiOQoEBFCJEeBiBAiOQpEhBDJUSAihEiOAhEhRHL/P0isGI8XMmDyAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T18:03:03.033745Z", + "iopub.status.busy": "2024-03-24T18:03:03.033448Z", + "iopub.status.idle": "2024-03-24T18:03:03.399582Z", + "shell.execute_reply": "2024-03-24T18:03:03.398574Z" + }, + "papermill": { + "duration": 0.385969, + "end_time": "2024-03-24T18:03:03.401750", + "exception": false, + "start_time": "2024-03-24T18:03:03.015781", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABBa0lEQVR4nO3deZxT9b34/9fJPskkmX2DWVkFWUTE4lJBqIjW6qV1qbaCqN1oH9fy41Zpb7Hqwwtt1RZbH95+WwG91VJtrd1FSwUsUlQQlHVYZgNmmDWZZCZ7zu+Pk8kQZgZmSXJmMp/n45FHMicnJ+9DyDuf81klWZZlBEEQVKRROwBBEASRiARBUJ1IRIIgqE4kIkEQVCcSkSAIqhOJSBAE1YlEJAiC6kQiEgRBdTq1AxiKcDjMmTNnsFqtSJKkdjiCIJxHlmVcLhdFRUVoNH2Xe0Z0Ijpz5gzFxcVqhyEIwkXU1dUxduzYPp8f0YnIarUCyknabDaVoxEE4Xzt7e0UFxdHv6t9GdGJqOtyzGaziUQkCMPYxapORGW1IAiqE4lIEATViUQkCILqRnQdUX/IskwwGCQUCqkdyqim1+vRarVqhyEMUymdiPx+P/X19XR2dqodyqgnSRJjx44lPT1d7VCEYShlE1E4HKaqqgqtVktRUREGg0F0elSJLMs0NTVx6tQpJkyYIEpGQg8pm4j8fj/hcJji4mLMZrPa4Yx6ubm5VFdXEwgERCISekj5yuoLdSsXkme4l0bF1O3qStkSkSD0JRAK89bBs0wqsBIIhdlf52BmSQaTC0SnWLWI4oIw6pxs6qDyrIsdlU20dfipd3qpPOtWO6xRTSSiYWjevHk89NBDaoeRsqpbOgAYn5dORa7Sine6zSMuz1QkEtEItm3bNiRJwuFwqB3KiCHLMnWtSneO0mwzuVYjeq2ENxCi2e1XObrRSyQiYVRp9wRxeYNoNRJFGWloNRJ5NhMATS6fytGNXqMyEfmD4T5vwVC43/sG+rHvUPl8Ph5++GGKi4sxGo2MHz+eF154gerqaubPnw9AZmYmkiSxbNmyix7P5XJxzz33YLFYKCws5Cc/+UmPS8H/+7//Y/bs2VitVgoKCrj77rtpbGyMPt9VEtu6dSuzZ8/GbDZz1VVXcfTo0SGfb6KddXkByEk3otcq//1z040AtHSIRKSWUdlq9tw7x/t8rjzHwm2XjYn+/f92nCAQ6r3uYGxmGrfP7p6YbcPOKjz+2KEk3/7MxCHFeu+997Jr1y6effZZZsyYQVVVFc3NzRQXF/P73/+ez3/+8xw9ehSbzUZaWtpFj7dy5Up27tzJn/70J/Lz81mzZg179+5l5syZ0X0CgQBPPPEEkyZNorGxkZUrV7Js2TL+9re/xRzre9/7Hk8//TS5ubl87WtfY/ny5ezcuXNI55toje1KssmzGqPbstMNALSISzPVjMpENFJUVlby6quv8vbbb7Nw4UIAKioqos9nZWUBkJeXR0ZGxkWP53K5ePHFF3nllVdYsGABABs3bqSoqChmv+XLl0cfV1RU8Oyzz3LFFVfgdrtjhmg8+eSTXHfddQA88sgj3HzzzXi9Xkwm0+BOOAncviAA+bbuGLPTjViMWkz6UXmBMCyMykS0Yv74Pp/TnNfv7iufHtfnvuf30Vt+dflQwuph3759aLXa6Jd9qE6ePEkgEGDOnDnRbXa7nUmTJsXst2fPHn7wgx+wf/9+2traCIeVS8za2lqmTJkS3W/69OnRx4WFhQA0NjZSUlISl3gT4cZLC5g3KTfmsyuymy74OQuJNyoTkUHX/1++RO3bH/251Iq3jo4OFi1axKJFi3j55ZfJzc2ltraWRYsW4ffHXrro9fro466e011Jazgz6WOHmAz3Xt+jgSiLDmPTpk0jHA6zffv2Xp83GJS6jf5OcVJRUYFer+eDDz6IbnM6nVRWVkb/PnLkCC0tLaxbt45rr72WyZMnx1RUC0IiiEQ0jJWVlbF06VKWL1/OG2+8QVVVFdu2bePVV18FoLS0FEmS+Mtf/kJTUxNu94V7B1utVpYuXcp//dd/8c4773Dw4EHuv/9+NBpNtFRQUlKCwWDgZz/7GSdPnuRPf/oTTzzxRMLPNRkOnHbyuz2nOHDa2eO5904086t3T7KvzpH8wASRiIa7559/ni984Qt84xvfYPLkyTz44IN0dCg9g8eMGcNjjz3GI488Qn5+Pt/85jcverxnnnmGuXPn8tnPfpaFCxdy9dVXc8kll0QrmHNzc9m0aROvvfYaU6ZMYd26dTz11FMJPcdkaXB6qWvtpN0b6PFcKCzj8gZxdIqWMzVI8gju197e3o7dbsfpdPZYxcPr9VJVVUV5efmwbsVRW0dHB2PGjOHpp5/m/vvvT9j7DIfP49UP6zjd5mHxtIIeA1z31zn455FGKnIt3DpzTB9HEAbqQt/Rc43KyurR7KOPPuLIkSPMmTMHp9PJ448/DsCtt96qcmSJ19ahlHayzIYez6WblK9Ch09MKawGkYhSyPnN6+c7dOgQAE899RRHjx7FYDBw+eWX8+6775KTk5OsMFXh8YfojHQ2zeglEVmNylfB7et52SYknkhEKaSoqIh9+/Zd8PmSkhL27NmTvKCGidZI3Y/VpOu1m4Ulkog6/SFCYRnt+R3KhIQSiSiF6HQ6xo/vu7PmaBa9LLP0LA0BmA1atBqJUFimwx/EZtL3up+QGCIRCaNCIBTGYtSS2UcikiSJgsiwj1AfYwuFxFG1+f4HP/gBkiTF3CZPnqxmSEKKuqwkk698ehzzJub2uc8dVxRzxxXFfSYrIXFULxFNnTqVf/zjH9G/dTrVQxJSmBjOMTyp/q3X6XQUFBSoHYYgCCpSvWf1sWPHKCoqoqKignvuuYfa2lq1QxJSjDcQYsO/qnh97ynC4b7rfw6ecfKrd0/yj0NnkxidACqXiK688ko2bdrEpEmTqK+v57HHHuPaa6/lwIEDWK3WHvv7fD58vu5Z9Nrb25MZrjBCOT0BnJ4AwXAYzQWa5WUZXN5gdM4iIXlULREtXryY22+/nenTp7No0SL+9re/4XA4ooM6z7d27Vrsdnv0Vlxc3Ot+I91wW8VjuMUzUE6P0knRnnbhJnmzQZkepNMvelcnm+qXZufKyMhg4sSJHD/e+1Suq1evxul0Rm91dXVJjnDkOH/uoNGs/4moq1OjKBEl27BKRG63mxMnTkRn+zuf0WjEZrPF3AZEliHoT/5tAOOKly1bxvbt21m/fn20S8OJEye4//77KS8vJy0tjUmTJrF+/foer7vtttt48sknKSoqis66+N577zFz5kxMJhOzZ8/mjTfeQJKkmB7YBw4cYPHixaSnp5Ofn8+Xv/xlmpub+4ynurp6YP/uKnN2KonIdpFElBYpEXn8IbHGWZKpWke0atUqbrnlFkpLSzlz5gyPPvooWq2WL37xi4l5w1AA3n06Mce+kGv/P9D1r2/K+vXrqays5NJLL40OSM3MzGTs2LG89tprZGdn89577/GVr3yFwsJC7rjjjuhrt27dis1m4+233waUOrRbbrmFm266iVdeeYWampoel1gOh4Prr7+eBx54gJ/85Cd4PB4efvhh7rjjDv75z3/2Gk9ubt99cYajrmk/+ntpFgzL+ENhjDrtBfcX4kfVRHTq1Cm++MUv0tLSQm5uLtdccw3//ve/R9x/9Hiy2+0YDAbMZnNMt4bHHnss+ri8vJxdu3bx6quvxiQii8XCr371q+jMjf/7v/+LJEn88pe/xGQyMWXKFE6fPs2DDz4Yfc3Pf/5zLrvsMv7nf/4num3Dhg0UFxdTWVnJxIkTe41nJOm6NLvYsA29VoNBp8EfDOPxh0QiSiJVE9HmzZuT+4ZavVI6STbt0MctPffcc2zYsIHa2lo8Hg9+vz9mCSBQppbtSkIAR48eZfr06THz/5w7cT7A/v37eeedd2JW5+hy4sQJJk4c2nJIapNlGYtRhz8Yxm6++OeQbzMRDstcoJVfSADVOzQmlST1+xJpONm8eTOrVq3i6aefZu7cuVitVn784x+ze/fumP0sFsuAj+12u7nlllv44Q9/2OO5vurqRhJJkrhjdv9bV79w+dgERiP0ZXQlohHCYDDETIi/c+dOrrrqKr7xjW9Et504ceKix5k0aRK//vWv8fl8GI3KgoLnTpwPMGvWLH7/+99TVlbW5/Ca8+MRhHgbVq1mgqKsrIzdu3dTXV1Nc3MzEyZM4MMPP2TLli1UVlby/e9/v0dC6c3dd99NOBzmK1/5CocPH2bLli3R+ae7xlytWLGC1tZWvvjFL/LBBx9w4sQJtmzZwn333RdNPufHMxKWDBJGFpGIhqFVq1ah1WqZMmUKubm5LFq0iCVLlnDnnXdy5ZVX0tLSElM66ovNZuPPf/4z+/btY+bMmXzve99jzZo1ANF6o6KiInbu3EkoFOKGG25g2rRpPPTQQ2RkZKDRaHqNZyQNw/n3yRY27qxib21bv/bfV+fgV++eZEdlU4IjE84lJs8fZV5++WXuu+8+nE5nUhdwVOvz+Psn9RxpcHHthBxml2VddP+9tW1sP9rEpAIrN00b+XVkahOT5wsAvPTSS1RUVDBmzBj2798f7SOkxiqyaujRq1qWobkSvO2QNxmMsWMaTZEme29A1Iklk0hEKa6hoYE1a9bQ0NBAYWEht99+O08++aTaYSVNj86MR/8O9fuVx7W74PJlYOr+pTbplctRb0DUgyWTSEQp7jvf+Q7f+c531A5DFf5gOLo8kC1ND01HlSQkSaA3g78DKrfA9NujrzHpRYlIDaKyWkhZXaUho16DSaeB6n8pT5R8CmbeoySkluPg6p5/KC2SiDwiESVVyieiEVwXn1LU+Bxi6occteBuVHq5j50DlmzIVQYGRy/V6C4R+YNhQqJ7ddKkbCLS65U6gc7OTpUjEaB7WhKtNnnjtzSSRIHdRJ7VBI2HlY15l4DBrDzOn6bcNx+NzpBg1GnIshgoyjARFP2lkiZl64i0Wi0ZGRk0NjYCYDabxcTpKgmHwzQ1NWE2m5O6OEJ5joXyHAuEQ/DeEWVj3iXdO2SWKUN+fG5wnoKMYjQaiaVXlSUtRkGRsokIiI4W70pGgno0Gg0lJSXq/Bi0n4GAB/QmyCjr3q7VQVYFNB4BRw1kpOaMnyNBSiciSZIoLCwkLy+PQECsaa4mg8EQ7amdLLIsK4mvrVrZkFkG58eQURpJRCOnt3gqSulE1EWr1Sa1bkJQnyzL/O/2k5j0Gu7SnSANlER0voxS5d55GkJB0Or455GznGzq4OrxOVxSOMBZQIVBSdnKamF08wRCeAMh3B0dGDoblI29JSJzllJ5HQ6CW9nPFwjj8gbFJPpJJBKRkJK6mu7z5Ba0yGCyQ1pmzx0lCaxFyuNIfyKTQXRqTDaRiISU1JWIcmlVNtiK+t7Zmq/cu+oBMd5MDSIRCSmpa+WOnLCyGgm2MX3vnB6ZiztyaSbGmyWfSERCSnJ6AiDL2ANdiegCU3pYI4moowVCATHeTAUiEQkpqd0bxBhyYcYHGm13qac3RqtSYS2Hwd0YHW/mDYpElCwiEQkpKTvdQKneqSyamJ6ndF7siySBJU953NmM2aAly2K46DpoQvyMin5Ewugzf1Ie6CSo1YG1HzMtmrOVjo+dLeQVmsQwjyQTJSIhdbkj805b+rFgpyVbue9oSVw8Qp9EIhJSTigsK9OOdETGGKbnXfxF5hzlvrM5cYEJfRKXZkLKOXSmnXcP1XKjo4GK3PR+logiicjrhFCA3+6tx+UN8vlZY8m0jLxFOUeaYVMiWrduHZIk8dBDD6kdijDCOT0BjL5mZcBrWgbojBd/kd6sjM6XZehswe0L4fIGRctZkgyLRPTBBx/wi1/8gunTp6sdipACnJ4A5kArRr2mf6UhUFrOopdnLRh1ylfDJzo1JoXqicjtdnPPPffwy1/+kszMXsYCCcIAOT0BzP5WZahGf+qHunRdnnU0dyeioEhEyaB6IlqxYgU333wzCxcuVDsUIUU4PQEsgZZIiWgAiahrUKzXEe1d7ROXZkmhamX15s2b2bt3b7/WcQfw+Xz4fL7o3+3t7YkKTRihvIEQXn+AtEAbRp1tYCUiU4Zy72nDaBLjzZJJtRJRXV0d//mf/8nLL7/c7yWI165di91uj96Ki8XUnkKsdm8AU7AdoyaMTm/oTi790VUi8jgwihJRUqmWiPbs2UNjYyOzZs1Cp9Oh0+nYvn07zz77LDqdjlCo53+A1atX43Q6o7e6ujoVIheGM40kcYk9QIbZAGlZPaeGvZC0DOU+4MGmC5KdboheogmJpdql2YIFC/jkk09itt13331MnjyZhx9+uNepXY1GI0ZjP5pihVErJ93IvGId+NOVYRsDoTMqg1/9nVyWK3FZRVlCYhR6Ui0RWa1WLr300phtFouF7OzsHtsFYUA6I8M0zFkDf60pA/yd4HF0Tw8iJJzqrWaCEE/eQAi5a5jGQEtEcE49UVv8ghIualgN8di2bZvaIQgj3G8/qGPcsWPMyNNjTRtEiShST+Rsa+JPZ6ox6rXcMVs0iiTasEpEgjAUsizjdrvQBj3odcYhlYg03jaa3X7MBlFZnQzi0kxIGW5fEIPfgQQYzHZlOemBijT36wNKHzVvIKyM5BcSSiQiIWU4PQHSAg6Meg0ayyBKQxAtEekCbiQ5RFiWCYREIko0kYiElOH0BDAFnRh12sFdlgEYLKDRoZVkTOFOQHRqTAaRiISU0VUiMuk0SmfGwZAkMFqRkLDSlYjEMI9EE4lISBntngBpwcjwjMH0IepisgNgoQMQiSgZRKuZkDIKbUZMRi8Wo36IicgGQLbWS3u6mJ0xGUQiElLGjBwgT6njwWgf/IEiJaJriw1cO7ksLrEJFyYuzYTU0dUb2pw5sMGu5zMqJSK8zqHHJPSLSERCSvAHw3Q6GpGRB19R3SVSIsIn5rtKFpGIhJRQ29rJ2x8e5NCZ9sE33XeJ1BHVnz3L/71Xxe6TYq2zRBOJSEgJTo8fU8CJQacZWkU1KJdmkkQ4FMTZ7sTpCcQnSKFPIhEJKcHRqTTdm4bSmbGLRguGdLQaCUPQLZrvk0AkIiEltLs60Ic8yoT5Q60jAjDZ0GkkTCGXSERJMKhEdPLkyXjHIQhD4m1Xlpc2WmzKQolDZbKji5SIvAExxCPRBpWIxo8fz/z58/n1r3+N1+uNd0yCMCDBUJhQh1KhbLL1c0HFizHa0GoljCFxaZYMg0pEe/fuZfr06axcuZKCggK++tWv8v7778c7NkHol3ZvEJPfodTpWHPic1CTHZ1GgzHoFoNek2BQiWjmzJmsX7+eM2fOsGHDBurr67nmmmu49NJLeeaZZ2hqaop3nILQJ51WYkpGgDyrEckSz0QkYZU6sRp1Yk6iBBtSZbVOp2PJkiW89tpr/PCHP+T48eOsWrWK4uJi7r33Xurr6+MVpyD0yWbSMyUjSFm2JT4V1QBGG3qthmvH6vny3DIkSYrPcYVeDSkRffjhh3zjG9+gsLCQZ555hlWrVnHixAnefvttzpw5w6233hqvOAWhb7IMnV3DO+KUiCKdGgn6IOiPzzGFPg1q0OszzzzDxo0bOXr0KDfddBMvvfQSN910E5rI+J7y8nI2bdpEWVlZPGMVhF61tDRhDfjR63RIA1nZ9UJ0RuUW9ClDPXRxuuQTejWoEtHzzz/P3XffTU1NDW+88Qaf/exno0moS15eHi+88EJcghSEC3nno8PsqWnDIacPbbDr+Uw2jje6eWPXIc44PPE7rtDDoEpEb7/9NiUlJT2SjyzL1NXVUVJSgsFgYOnSpXEJUhD6EgrLhFyRpvuMODXddzHa8QZDdLja6PQH43tsIcagfj7GjRtHc3Nzj+2tra2Ul5cPOShB6C+XN4Ax4EAjxbEPUZdI72pj0I03IPoSJdKgElFfTZlutxuTKQ69WgWhn6JjzPRapKGOMTufUUlEhlCH6NSYYAO6NFu5ciUAkiSxZs0azGZz9LlQKMTu3buZOXNmXAMUhAtxeAKYAg5MpjgMdj2f0YpWo8Hod+MTwzwSakCJ6KOPPgKUEtEnn3yCwdA9n6/BYGDGjBmsWrUqvhEKwgU4XS4MoU5MOlP8mu67RC7NxAj8xBtQInrnnXcAuO+++1i/fj02m21Ib/7888/z/PPPU11dDcDUqVNZs2YNixcvHtJxhdHD62zCAOjNVtCnxffg54w3aw+IyupEGlSr2caNG+Py5mPHjmXdunVMmDABWZZ58cUXufXWW/noo4+YOnVqXN5DSG2TbAGwm0jPzI//wY1WdFotRo2MSRaDuxOp34loyZIlbNq0CZvNxpIlSy647+uvv96vY95yyy0xfz/55JM8//zz/Pvf/xaJSOiXsjQvZFsgqyD+B9doycvOJi/dAGWiESaR+p2I7HZ7dLyN3T6EpVr6EAqFeO211+jo6GDu3Lm97uPz+fD5fNG/29vF5OajXmdkPul41w91MdnA5wJvO9iKEvMeQv8T0bmXY/G6NAP45JNPmDt3Ll6vl/T0dP7whz8wZcqUXvddu3Ytjz32WNzeWxjZ3L4gwZYGLKEw+ni3mHUx2oDTYkWPBBtUPyKPx0NnZ2f075qaGn7605/y1ltvDfhYkyZNYt++fezevZuvf/3rLF26lEOHDvW67+rVq3E6ndFbXV3dYMIXUsTJxnYOnajmRJM7YSUij9bCwTNO3tl/IiHHFxSDqqy+9dZbWbJkCV/72tdwOBzMmTMHg8FAc3MzzzzzDF//+tf7fSyDwcD48eMBuPzyy/nggw9Yv349v/jFL3rsazQaMRqNgwlZSEEuRwsaOYzRkDa0lV0vQGOy0e4N0qlpJRgKo9OKad4TYdAzNF577bUA/O53v6OgoICamhpeeuklnn322SEFFA6HY+qBBKEvHqcyT7XemhPfwa7nMFgykED0JUqwQZWIOjs7sVqtALz11lssWbIEjUbDpz71KWpqavp9nNWrV7N48WJKSkpwuVy88sorbNu2jS1btgwmLGGU8TmbsAAme17C3kMy2dFquueutogCeUIMevL8N954g7q6OrZs2cINN9wAQGNj44A6OTY2NnLvvfcyadIkFixYwAcffMCWLVv4zGc+M5iwhFEkFJYJuZUWM3O8R92fy2hDq5HQhzz4/KKkniiDKhGtWbOGu+++m29/+9ssWLAg2tz+1ltvcdlll/X7OGK+ImGw2j0BjIE2tBKkJbBEhD4NjU4PQR+BDidkWhP3XqPYoBLRF77wBa655hrq6+uZMWNGdPuCBQv4j//4j7gFJwh9ae30kxZwJmbU/bkkCdloA28TgU4HMDZx7zWKDSoRARQUFFBQENubdc6cOUMOSBD6I8cQxJKlQZLSIF4rd/RBY7JjcDej8bsS+j6j2aASUUdHB+vWrWPr1q00NjYSDse2JoiVYIVEs4cd2G0mSMsErT6h7zVjfAlY2iBdtJolyqAS0QMPPMD27dv58pe/TGFhoVhqRUi+jsgMoQkuDQFgjNQLid7VCTOoRPT3v/+dv/71r1x99dXxjkcQ+uXUqVrsvgBmcy7aRL9Z19JCXpGIEmVQzfeZmZlkZSVokKEgXIQvGOLwseMcON1OMC2BFdURtR06Dp5xcqzudMLfa7QaVCJ64oknWLNmTcx4M0FIFmeHn7RAG3qthNGegOk/zuPVptPuDeJzO5TFHIW4G9Sl2dNPP82JEyfIz8+nrKwMvT62snDv3r1xCU4QeuNwNKOVA5gMRqWyOsF0acqlmRz0Q9Ab/5kghcElottuuy3OYQhC/3W0NgCgTc8BTcJriDCZTAQ0aQRDPqWeSCSiuBtUInr00UfjHYcg9JvPcRYAvS2BParPYdRp8OssBINepeXMmoBpaUe5QQ9Zdjgc/OpXv2L16tW0trYCyiXZ6dOiQk9IrEC7kohMGYmvHwIw6rX4tOmEwjKyaDlLiEGViD7++GMWLlyI3W6nurqaBx98kKysLF5//XVqa2t56aWX4h2nIADKUlayS7k0s2QnZ+pWU6REJAOBTgeGi75CGKhBlYhWrlzJsmXLOHbsWMzKrjfddBM7duyIW3CC0EPQx7SsEKXZZqw5yRn3pdNqkI02DFqJkMeZlPccbQZVIvrggw96nUFxzJgxNDQ0DDkoQeiL1NFIZpqezIwsSEveSPglc6fAwUoIdSTtPUeTQZWIjEZjrytoVFZWkpubwLlhBMGl1A9hTU79UJQx0rtaDPNIiEElos997nM8/vjjBAIBACRJora2locffpjPf/7zcQ1QEM7VcKaaZrePTlOSf/C6hnn43BAWg1/jbVCJ6Omnn8btdpObm4vH4+G6665j/PjxWK1WnnzyyXjHKAhRp+uqOdbopj6YmMny+7KrzseBejetbi+I6UDiblB1RHa7nbfffpudO3eyf/9+3G43s2bNYuHChfGOTxCi5KCPsLsJAFtucicoc3gC+AIGsoIhZcFFU3ITYaobcCIKh8Ns2rSJ119/nerqaiRJory8nIKCAmRZFlOCCAnjbT1NMBTGr7OQkZncQddGvQaf1kIo7FR6V4s8FFcDujSTZZnPfe5zPPDAA5w+fZpp06YxdepUampqWLZsmZgmVkgo19kqAELpReiTvL6YUafFr0snGJZFhXUCDKhEtGnTJnbs2MHWrVuZP39+zHP//Oc/ue2223jppZe499574xqkkGTNx8FVDzkTadVk0uz2MSYjDYtx0DMLx4WnuRYAXWZx0t/bqNMovatDspiXKAEG9D/rN7/5Dd/97nd7JCGA66+/nkceeYSXX35ZJKKRrO59OL4VgMaD7/AXrsVpHINBp2HJrDEU2lUa8CnLBNtOAWDKLk3625v0WvzadILBsCgRJcCAyrcff/wxN954Y5/PL168mP379w85KEElnjY4uQ2ATo2FqsZ2xjVvI00boizbQpZFxcENnS34vJ2EJR3pSRracS6jToNPp4w3wyt6V8fbgEpEra2t5Of3PfI4Pz+ftra2IQclqOTUhxAOQWYZ/9Jeg6H6BfL0Pq6ucKAtnaxuQ4SjhnG5FhzGIsw56Ul/e6NOS0gv5q5OlAGViEKhEDpd37lLq9USDAaHHJSgglAQzh4AwFNwOUcavZy2zaI4Kw1dwz4ktWcmbK3CqNOSXzoZqymxq3b0pjgrjQc+M5OpRXYIeCHoT3oMqWxAJSJZllm2bBlGY+8LgPt8YkneEautSvmCGdM5HsglFG5CKpiCNXQCPA5wVNOgLWJPTRtjMtOYWZyRvNjCYXDUKI8zy5P3vueQJAl0RtAZlCTkc4Eu8fNljxYDKhEtXbqUvLw87HZ7r7e8vLwBVVSvXbuWK664AqvVSl5eHrfddhtHjx4d8EkIcdAaWYsuZyIZFiPj8tKZVJQNuZco2xuP0OjyUnnWxb7aNuRklpBcZ2h1uqh1hTkrq9yBJzrmTNQTxdOASkQbN26M65tv376dFStWcMUVVxAMBvnud7/LDTfcwKFDh7BYLHF9L+EiuhJRVgXFWWaKs8zK322T4cxH0HyUSRUL2VEp0dYZ4Gy7jwK7qe/jxTW2KprdfirlPAodXvLt5uS87zlkWeYvH9eTeybI5elh9KIJP65U7Rjy5ptvxvy9adMm8vLy2LNnD5/+9KdVimoU8rQpl1+SBjJKYp+zl4DBDP5OjO7TVOSmc7TBReVZV/ISUXMlnf4gDmsx09J7rxZINEmSOO3wEPIZCKSF0YsK67hKbvfUi3A6leJuX2um+Xw+2tvbY25CHDgj0/taC6jvCNPW4e++9NJoIHu88rj1BBPzlRaryrOu5FyedbYSbG/AE5RpTSsl16pOIoLITI3adAKiU2PcDZtEFA6Heeihh7j66qu59NJLe91n7dq1MXVSxcXJ72GbktrPKPe2MbxzpIlN71VTedbd/XxWhXLfWkVZtgWDToPLG+SM05v42JqO0OkP4TSOIc2cjtmgXiE+zaCN9CUSnRrjbdgkohUrVnDgwAE2b97c5z6rV6/G6XRGb3V1dUmMMIW1KyUivyWfRpeSXIoyzrnsyiwDSYKOZnT+dsblKqWiY2cTPB2GLEPDAdy+IC3mcvJs6pWGQOld7RMlooRQd/BQxDe/+U3+8pe/sGPHDsaO7Xt6B6PR2GfXAWGQQgFwNwLQKOUgyy6sJl1sXx19GtjGgPMUtFUxIX8CzW4fmeYE97R21EJnC+1+iRb7OCZmqLuemEkfGfjqlZXme1lWErQwZKomIlmW+da3vsUf/vAHtm3bRnm5On1ERjX3WZDDYLBw1qcklnxbL5XQWRVKImo5QcWlM6OlooQ6vUe5M5QT0hgo6C2uJErTa/FrzQTCMoSDEOgEg2jdjQdVE9GKFSt45ZVX+OMf/4jVao1OvG+320lLE6tpJkWkNIS1gIZ2pbdw74moHKp2gKNG6WWd6JKAuxGalD5l869fzHQpgwxz8ntUnyvNoEWr1RHURroPeNtFIooTVeuInn/+eZxOJ/PmzaOwsDB6++1vf6tmWKNLhzLjIZZczrYr9UO9ljzSC5RLtKA/WqfkC4Y4lojWM1mGk9uVx3mT0djyybUakz4H0fkuL8nkm9ePZ3xxZNCtqLCOG9UvzQSVRUpEXmMOTo+yGEKvlcIajVJp3XgY2qoI28by4nvVdPhC3HlFMUXxrL85exBajivr2pddG7/jDpFGEykFmmxKS6OosI6bYdNqJqhAlqFDSURaWx63zCji2gk5mPTa3vfPitThtVah0UiURHpfV8az9ay9HiojHV1Lr+b1o17eOthAuzcQv/cYKjHMI+5EIhrNvE7lUkujRZ+ew/i8dGaXXWAu6Mwy5d5VDwEP4/OUaTGON7qHXrqVZaj/GPa9rLTkZZbhyrucmpZODtW3Y1D5sgzA7Qvyx32n2V4TGdwtSkRxo/6nK6inq37InK1cBl2MyQ6WHCVptNVQlm2Odm6sH0rnxoAHDr0BR/6qJKGMErh0CbUO5QufbzP1XUpLIgk42dRBdYdOSbyijihuRCIazbpazNLzOHDayYkmN75g6MKv6ZqGo60KnVZDRY7SanR0sJdnjjr4cAM0HlHGulVcBzO+CDojVc3K8s6lWckf5NqbrmTo01iUSfRFiShuRCIazSL1Q2FzLtuONvKnfWfo9F0kEZ1TT4QsM6lAuTw7Uu8iEBrgCqin98K+V5QvdFomzPoylF4FGg3+YJjqSCIan5f8GRl7o9VIGCJTxgbDMvg7lAnlhCETiWg0iyxW6NZlEAjJ6DQS9rSL9NWxFyuXcV4neNooy7ZgS9PjC4Y44/D0/71P7YHKLUpnyrxLYPZysHXPRX2iyU0gJJNp1qs60PV8aXotQY2JgBz56ohVX+NiWAzxEFQQCoKnFYAm2Q64yEo3dDdR90VnAPtYaKtRWs/GZnHDlHxsafqLJ7Euzcfg2FvK49K5UH5djw6S++scAEwqsA2rRTtNei1OScKnTQc6lISclql2WCOeKBGNVp5WpdJZZ6TZpySQbEs/Sx7n1BMBFGeZ+5+EPA448hfl8ZhZvSahcFimIjcde5qe6WOH15KqaQblK6MkIsSKHnEiEtFo1dGs3FtyaO5Q+ujkpPdzEGtXPVFbtbLqxzlOtXUSvFBd0bG3lbmxbYUwfmGvQ0U0Gok55Vncd3WZ6os6ni9Nr0WnkfDrIwnSI1atiYfh9SkLydMZSUTmHFralGby7P7OfpieH521EUdtNDG9d7yZ3VWtXF6ayacn5vZ8XcsJaDlOdauHd8PTCO2qJSvdSL7VSKE9DYNOQ0iWGRPppT2cLsm63DClgBsvlaCuDY5XikQUJyIRjVaRElEwLYu2U0qJKLu/JSJJgpyJcGYfNB2JJqKuoSF7atrQaSSuKM9CAuraPJRkGNEe/4fy1rmX0eyzQmeAts4AJxq7J2Ez6DTcc2UJGYmeYmSQonVoXfVCIhHFhUhEo1VnCwAaSy53XzmWlg4f1oFcBuVOiiSiozBhEWg0jM+zMnecn10nWthd1coH1W1IEoTCMktyT1Pa2QoGC8UzPsNdIS3BkEyjy8fZdi8NTi+BUJgCu0mZeGy4OzcRiXmJhkwkotEoHIr+kmss2eSmGQfeRJ5RpozGD3iUNccipaJPVWSTYdaz83gL7Z4AyGDXeDGc2gkWLVTMw5aeTmS0VvdqISNEY7uXXSdbsOhgoSQpQ2TEvERDJhLRaORxKMlIq1eGbQyGRtNdKmr4uLsCG5hcYGNSvhWXT+nsZ616E6lBq1RQF0wbevwqCoRlTjZ1KK2ERqvSGdPTJhLREIlWs9EoWlGdzcenneypacPZOYjR7YUzlfumo0ov43NIkoTNpMfmbUBqUJayZsINI/4SxhwZ5uEJhEQ9URyJRDQandN0v6/OwY7KJto6B7GWu60QrAVK6erMRz2fD4fh2BblceH0mJ7TI5XZqCQifzBMwCCa8ONFJKLRKFIiCqVl44iUhLL622J2vuI5yn3dbqU5/1x1u5VhJHoTVMwfbLTDikGrQRdpOfPpRCKKF5GIRqNIicilzSAUljHoNANrMTtX3hRIz1MqbU9sVVqQQBkCUrVDeTzueqXfUQqQJAlz5N+qU6cM+BWJaOhEZfVoEw5DpzLGrFVOB7xkmg2D7zwoSTDhM8oo+oYDoDWCOQtOblMGtOZPgYLpcQt/ODAbtLR7AnRoIm1/Hoeq8aQCUSIabbwOZSkcjY6mgNKDOcsyxM6DGSUwboHy+PQeZRhHKKC0pE1cPOIrqM9nNmjRayW8XSWigEe5CYMmSkSjTaQjI+YsWjuV5vV+96i+kOIrlFakM3sh5IecScqg1v7M/DjCfHZ6ERopMgSlNh18buXyTC+WwBoskYhGm3NazFpdSktZ3FZszRmv3FKc9typUszZSiLqaE6JVkG1iEQ02pwz2PULE8fS1hFQfeHCEc2crVTMd5U0hUERiWi0OadEZNRpKbCn3qVTop1t9/Lvky2kG3UssOYoG0UiGhKRiEaTcDimRCQMjj8Y5mRTB5lmPeRHll8SiWhIRKvZaOJ1KFPEanVUunS8c6SRmpaOi75MiJUe6UfU4Q8hm7OVjZ42MZH+EKiaiHbs2MEtt9xCUVERkiTxxhtvqBlO6nOfVe4tuVS3eNhX5+CMYwjrkY1S6SYlEfmDYXwaM+iMSkfOyBzgwsCpmog6OjqYMWMGzz33nJphjB5d65hZ8mjtUFrM4tJ0P8rotZroGmduf0ipsAZxeTYEqtYRLV68mMWLF6sZwugSWdlVtuTSUq8koiF3Zhyl0k06vIEQbm+QHEsOtJ/pbggQBmxEVVb7fD58Pl/07/Z2sdLmgERKRB2GbPzBIBpJIqO/q28IMaxGHc0uH25f8JwSkUhEgzWiKqvXrl2L3W6P3oqLi9UOaeQIeKNL37RG5ke0p+nQaUfUf4FhI92oQ6+VlNVtzaIJf6hG1P/C1atX43Q6o7e6ujq1Qxo5IstLY7LR5FU+9pxhtILqSDN/ch4r5o/nspJMsHQlotYeyysJ/TOiLs2MRiNGo/jyDIor0mKWnk+7V2lmzunv8kFCDzHDPEx2peUs6FPqiaz56gU2Qo2oRCQMgeuMcm8tZH5ZHp8qz1Y3nlQiScpab45apYuESEQDpmoicrvdHD9+PPp3VVUV+/btIysri5KSEhUjS0HtkURkKwQgzSCGdgxFhy/IPw6fxR8Mc/vs4u5E1HUJLAyIqonoww8/ZP787ilEV65cCcDSpUvZtGmTSlGlIH9H9+RdVjFCPB60GomTTUqv9EAojD49T3nCLRLRYKiaiObNm4csj4DF9EY6V4Nyb87mpCPIvrpTVOSmM7M4Q9WwRjKjToNBp8EfDOPyBsmKJqKzYsHFQRhRrWbCILWfVu5thdQ7vdS0dNLs8l34NcIFSZKkrG0GOD0BpQlf0ijdJHyif9tAiUQ0GjhPKfe2IprdSgISTfdD15WIHJ1+0OrAEmkAEJdnAyYSUaoLBcEZKRFllNEUKQnliDFmQxZTIgKwKg0B0YYBod9EIkp17aeVyfINFjq0NlzeIJLEwNe6F3rokYi6pooViWjARCJKdY4a5T6zlLOR0lCWxYBRJ5rvh8qepkevlbqXYrKNUe5dZ5RJ6IR+Ex0aU13rSeU+o5Sz7UoiyrOaVAwodZRkmVkxf3x3IjLngFavLDbZ2awsPCn0iygRpTKfC9rrlcfZ45BlGaNeQ75NXJbFg0YjxS5MqdGcc3l2Wp2gRihRIkplzceUe1sRGK1cNd7K3HHZhEXXrcSxjVFW9Wg/A0WXqR3NiCESUSprOqrc50yMbpIkCa3oaxc3e2paOVTvYubYDKaNtXfXEzlFiWggxKVZqvI4uiuq8yYTEsWghPD4wzS7fDS6InN/28covao7W8ArOjb2l0hEqarhE2WoQWYppGWyo7KJF/5VxeF68eWIp645v1sic4CjTwNrgfK464dAuCiRiFJRKAj1+5THBdMBONXWSbsngE4jrsviKZqI3P7ucZMZpcp9W7U6QY1AIhGlorOfKOuxG62QOxmPP0SzW/nFHpOZpnJwqSXLbECSwBsI0emPzM6YWabct1UrpVLhokQiSjWhANS8pzwuvhK0Ok47OgHl19tsEO0T8aTTaqILEHQNn8E+FjQ65ccgsnKKcGEiEaWamp1KJanJBkUzAahr9QAwVpSGEqLArnQQrXdGKqy1esgqVx43V6oU1cgiElEqcTVA3fvK4/ELQatHlmWqmpUJvEqzLSoGl7oK7Wlkpxsw6M75OuVMUO67ulAIFyTK6aki4IWDf1BWkciZEO071NrhxxmppC7ONKscZGqaPtbOjPMnmcueoDTjuxvB0wZpmarENlKIElEqCIfg8J+UvkMmO0y+OTpDoE6jYWZJBpcU2mJ/sYW4kXqbjdFghozIvOuNh5Mb0Agk/meOdLIMR/8GLSeUybmm3qb0ZYmwm/XMn5THwiliZYlEC4VlOnzB7g35lyr39R+L1rOLEIlopDv5DjQcUKYpnfIf3YMuhaQ62uDif7ef4J9HzpmdMXcy6AzKpZmjVr3gRgCRiEaymvegdrfyeNJiyBkf8/Sxsy7qWjsJi+EdCWdP0+MPhqlt7eweTqMzQN5U5fHpPeoFNwKIRDRS1bwHJ7crj8fNh8LpMU8HQmH+cbiR3+05xclIq5mQOPk2I2aDFn8wTF1rZ/cTYy5X7psrwS36FPVFJKKRqGZXdxKquA5KPtVjl0Nn2vEGQtjT9FTkiGb7RJMkiYn5VoDY8XzpuZA7SakjqvmXStENfyIRjSSyDNX/gpPblL/LPw2lV/XYzR8M835VKwCzSjPRiPFlSTG5UElEJ5rceAOh7ifKrlFaMRuPKHMVCT2IRDRShMNQuQWq3lX+Lr8Wyq7uddfdVS24fUHsaXouLbIlMcjRrcBmItdqJBCS+ajW0f1Eeh4UzlQeH/07BMWacucTiWgk8HfAJ6/CmY+UX9YJNyi/sr2obu5gT00bANdNykWnFR9xskiSxJzyLAAO1bfHNhJUzFMGIXva4PCflb5fQtSw+F/63HPPUVZWhslk4sorr+T9999XO6ThQZaVIQIfboDWKqWf0JTbYOzlve7uD4bZcrABWVZ6+47LTU9uvALjc9O5oiyLO68ojr0k1pvg0iWg0SpT+B78gzLJvgAMg0T029/+lpUrV/Loo4+yd+9eZsyYwaJFi2hsHMWrZcqy0kFx/2/gwOvKKG5zNsxaBnmT+3yZQafhhqkFlOdYuG5ibvLiFaI0GolrJuSQblRGT8myTDAUWVrIVgRTlygj85uPwZ6NyucsOjsiybK6/wpXXnklV1xxBT//+c8BCIfDFBcX861vfYtHHnnkgq9tb2/HbrfjdDqx2UZoXUg4BIFOZcUNdyO46qHluJJ8QPkFLb4SSq9C1ujwBcN4/CHavQGa3X5OOzzkWAxcNT4nekhZlnsfdiAk3Ue1beypaWP62AxKs81kmg0Y3Kfg0J+UzxzAnKWMD7QWgSUXDBbQGaPDdEay/n5HVR306vf72bNnD6tXr45u02g0LFy4kF27dvXY3+fz4fN1V/S1t/dz2tPWKjj+j96fi+ZhuZdtIMthPqxpO+9XS47uZzfrmZhnjW77sKa1u27gvNfYTHomF3RfLn1U00o44Iu+9ty909LMTJ11LYydDWkZ/N+ualo6/L3+eDan6Zk7LjuafEQSGh4CoTAf1TpweYPsPN7MzuPKdr1WIk2az3jPAa5Lr4POVqjdzbFGV3RyNVmjIyzpkCUNYUmLVqtjVml29NiH69tp9wZj3zDysWskiStKuwfZHj3rJs9qJNMS52XGp92hTDcTB6omoubmZkKhEPn5seOg8vPzOXLkSI/9165dy2OPPTbwNwr6oKN5kFHKBDva+nw2LOnhnEaQkMfV53I9YY0eAt3/5EG/l2BYRkYioE3Do8+kQ5+N0zQGc145UydURPf1BcPRJGTQaUg36shON5CbbqQ8V/QTGo70Wg33zi3lSIOL441uzrZ76fSHCIRkAug5bpvDdZ+6Tbk8a6umtfkwgWAz2nAACHHufyxJK0FH93+ssNtJ+PxE1LWvBHR0V4bL7nZkyQRynBORHL/VbEfUNCCrV69m5cqV0b/b29spLi6++AszimHGXcrjaGnhnFJDTAlCit0my5RU+LqfiWyXkZAkMOm1YOlesLBskg9Z7tpNinmNXquJ2bd8aoCwzgi6NCSpu7pOkkCvia2++8LlY9FpNZh0GtESNoLotBouHWPn0jF2QJlS1hcI4wuFlCKwzgj5UyB/CmVF1+MPhpGDAaSgBykcgHAISQ6hIQTnLIyZP95HRrC3knfkv1uGKbo9Z7yPNIMO9HFeZtwQvx9AVRNRTk4OWq2Ws2fPxmw/e/YsBQUFPfY3Go0YjYNYpdRg6Z4xb4AkYExG//ePdK7tl4F0eM4wx/nXTFCFSa9VfrzQ93gu33buUuD2Cx4nZwDTG+Vk9X9ftaj602owGLj88svZunVrdFs4HGbr1q3MnTtXxcgEQUgm1S/NVq5cydKlS5k9ezZz5szhpz/9KR0dHdx3331qhyYIQpKonojuvPNOmpqaWLNmDQ0NDcycOZM333yzRwW2IAipS/V+REOREv2IBCGF9fc7KppfBEFQnUhEgiCoTiQiQRBUp3pl9VB0VW/1e6iHIAhJ1fXdvFhV9IhORC6XMmiwX72rBUFQjcvlwm7vu5PmiG41C4fDnDlzBqvVOiwHenYNQamrq0vJVr1UPr9UPjdI3vnJsozL5aKoqAiNpu+aoBFdItJoNIwdO1btMC7KZrOl5H/mLql8fql8bpCc87tQSaiLqKwWBEF1IhEJgqA6kYgSyGg08uijjw5uxoARIJXPL5XPDYbf+Y3oympBEFKDKBEJgqA6kYgEQVCdSESCIKhOJCJBEFQnEtEADXRV2tdee43JkydjMpmYNm0af/vb32Kel2WZNWvWUFhYSFpaGgsXLuTYsWOJPIU+DeTcfvnLX3LttdeSmZlJZmYmCxcu7LH/smXLkCQp5nbjjTcm+jT6NJDz27RpU4/YTSZTzD4j9bObN29ej3OTJImbb745uk/SPztZ6LfNmzfLBoNB3rBhg3zw4EH5wQcflDMyMuSzZ8/2uv/OnTtlrVYr/+hHP5IPHTok//d//7es1+vlTz75JLrPunXrZLvdLr/xxhvy/v375c997nNyeXm57PF4knVasiwP/Nzuvvtu+bnnnpM/+ugj+fDhw/KyZctku90unzp1KrrP0qVL5RtvvFGur6+P3lpbW5N1SjEGen4bN26UbTZbTOwNDQ0x+4zUz66lpSXmvA4cOCBrtVp548aN0X2S/dmJRDQAc+bMkVesWBH9OxQKyUVFRfLatWt73f+OO+6Qb7755phtV155pfzVr35VlmVZDofDckFBgfzjH/84+rzD4ZCNRqP8m9/8JgFn0LeBntv5gsGgbLVa5RdffDG6benSpfKtt94a71AHZaDnt3HjRtlut/d5vFT67H7yk5/IVqtVdrvd0W3J/uzEpVk/da1Ku3Dhwui2C61KC7Br166Y/QEWLVoU3b+qqoqGhoaYfex2O1deeWWfx0yEwZzb+To7OwkEAmRlxa5ds23bNvLy8pg0aRJf//rXaWlpiWvs/THY83O73ZSWllJcXMytt97KwYMHo8+l0mf3wgsvcNddd2GxxK5vlczPTiSifrrQqrQNDQ29vqahoeGC+3fdD+SYiTCYczvfww8/TFFRUcwX4sYbb+Sll15i69at/PCHP2T79u0sXryYUCh0gSPF32DOb9KkSWzYsIE//vGP/PrXvyYcDnPVVVdx6tQpIHU+u/fff58DBw7wwAMPxGxP9mc3okffC8PDunXr2Lx5M9u2bYup0L3rrruij6dNm8b06dMZN24c27ZtY8GCBWqE2m9z586NWVvvqquu4pJLLuEXv/gFTzzxhIqRxdcLL7zAtGnTmDNnTsz2ZH92okTUTwNdlRagoKDggvt33Q/kmIkwmHPr8tRTT7Fu3Treeustpk+ffsF9KyoqyMnJ4fjx40OOeSCGcn5d9Ho9l112WTT2VPjsOjo62Lx5M/fff/9F3yfRn51IRP00mFVp586dG7M/wNtvvx3dv7y8nIKCgph92tvb2b17d1JXuh3sirs/+tGPeOKJJ3jzzTeZPXv2Rd/n1KlTtLS0UFhYGJe4+yseKwqHQiE++eSTaOwj/bMDpWuJz+fjS1/60kXfJ+GfXdKqxVPA5s2bZaPRKG/atEk+dOiQ/JWvfEXOyMiINut++ctflh955JHo/jt37pR1Op381FNPyYcPH5YfffTRXpvvMzIy5D/+8Y/yxx9/LN96662qNQEP5NzWrVsnGwwG+Xe/+11ME6/L5ZJlWZZdLpe8atUqedeuXXJVVZX8j3/8Q541a5Y8YcIE2ev1JvXcBnN+jz32mLxlyxb5xIkT8p49e+S77rpLNplM8sGDB6P7jNTPrss111wj33nnnT22q/HZiUQ0QD/72c/kkpIS2WAwyHPmzJH//e9/R5+77rrr5KVLl8bs/+qrr8oTJ06UDQaDPHXqVPmvf/1rzPPhcFj+/ve/L+fn58tGo1FesGCBfPTo0WScSg8DObfS0lIZ6HF79NFHZVmW5c7OTvmGG26Qc3NzZb1eL5eWlsoPPvhgj744yTSQ83vooYei++bn58s33XSTvHfv3pjjjdTPTpZl+ciRIzIgv/XWWz2OpcZnJ6YBEQRBdaKOSBAE1YlEJAiC6kQiEgRBdSIRCYKgOpGIBEFQnUhEgiCoTiQiQRBUJxKRMKJs2rSJjIyM6N8/+MEPmDlzZvTvZcuWcdtttyU9LmFoRCISetU1VejXvva1Hs+tWLECSZJYtmxZzP7xTgBlZWX89Kc/jdl25513UllZ2edr1q9fz6ZNm6J/z5s3j4ceeiiucQnxJxKR0Kfi4mI2b96Mx+OJbvN6vbzyyiuUlJSoElNaWhp5eXl9Pm+322NKTMLIIBKR0KdZs2ZRXFzM66+/Ht32+uuvU1JSwmWXXTakY/dWUrntttuipax58+ZRU1PDt7/97ejk7dDz0ux855bMli1bxvbt21m/fn30GFVVVYwfP56nnnoq5nX79u1DkqSkT1EiKEQiEi5o+fLlbNy4Mfr3hg0buO+++xL+vq+//jpjx47l8ccfp76+nvr6+gEfY/369cydO5cHH3wweoySkpIe5wSwceNGPv3pTzN+/Ph4nYIwACIRCRf0pS99iX/961/U1NRQU1PDzp07+zV/zVBlZWWh1WqxWq0UFBQMarIxu92OwWDAbDZHj6HValm2bBlHjx6NLrkTCAR45ZVXWL58ebxPQ+gnMVWscEG5ubncfPPNbNq0CVmWufnmm8nJyVE7rCEpKiri5ptvZsOGDcyZM4c///nP+Hw+br/9drVDG7VEiUi4qOXLl7Np0yZefPHFuJUaNBoN589AEwgE4nLs/njggQeiFfEbN27kzjvvxGw2J+39hVgiEQkXdeONN+L3+wkEAixatCgux8zNzY2p9wmFQhw4cCBmH4PBMORVI/o6xk033YTFYuH555/nzTffFJdlKhOXZsJFabVaDh8+HH3cF6fTyb59+2K2ZWdnU1xc3GPf66+/npUrV/LXv/6VcePG8cwzz+BwOGL2KSsrY8eOHdx1110YjcZBXRKWlZWxe/duqqurSU9PJysrC41GE60rWr16NRMmTEjqPNNCT6JEJPSLzWbDZrNdcJ9t27Zx2WWXxdwee+yxXvddvnw5S5cu5d577+W6666joqKC+fPnx+zz+OOPU11dzbhx48jNzR1U3KtWrUKr1TJlyhRyc3Opra2NPnf//ffj9/uT0gooXJiYKlYYtd59910WLFhAXV1djwUKheQSiUgYdXw+H01NTSxdupSCggJefvlltUMa9cSlmTDq/OY3v6G0tBSHw8GPfvQjtcMRECUiQRCGAVEiEgRBdSIRCYKgOpGIBEFQnUhEgiCoTiQiQRBUJxKRIAiqE4lIEATViUQkCILqRCISBEF1/z9ya8uK7xi4gwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T18:03:03.442011Z", + "iopub.status.busy": "2024-03-24T18:03:03.441185Z", + "iopub.status.idle": "2024-03-24T18:03:03.611855Z", + "shell.execute_reply": "2024-03-24T18:03:03.610864Z" + }, + "papermill": { + "duration": 0.192798, + "end_time": "2024-03-24T18:03:03.614119", + "exception": false, + "start_time": "2024-03-24T18:03:03.421321", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEpCAYAAADCh6TFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAnDUlEQVR4nO3de1hU17038O8MDIMjFyXAgATFSyOaRInwQjDxJDZcmtTU1CbyxlSQWM+pyntMpjSG9hQOsQ1Jo4jNIdJYiYlJI01ikzZaFYmkmuANo8UL+IgiXgBB5R6HYWa9f+QwyTgDDDqwZw/fz/Pw6F77Mr/Nhi97r9mztkIIIUBEJDNKqQsgIroVDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiXJwys/Px9hYWHw9PRETEwMDh482Ofyzc3NWL58OYKDg6FWq3HXXXdh+/btQ1QtETkLdylfvKioCDqdDgUFBYiJiUFeXh4SExNRVVWFwMBAq+W7uroQHx+PwMBAfPjhhwgJCcH58+cxatQou1/TZDLh8uXL8Pb2hkKhcODeEJEjCCHQ1taGMWPGQKns4/xKSCg6OlosX77cPG00GsWYMWNETk6OzeXXr18vJkyYILq6um75NS9cuCAA8Itf/HLyrwsXLvT5u6wQQppRJbq6uqDRaPDhhx/iiSeeMLenpKSgubkZn3zyidU6jz32GPz8/KDRaPDJJ58gICAACxYswMqVK+Hm5mbzdfR6PfR6vXm6paUFY8eOxblz5+Dt7e3w/XIWBoMBe/bswezZs6FSqaQuhxxguBzTtrY2jB8/Hs3NzfD19e11OckuG5uammA0GqHVai3atVotKisrba5z9uxZfPbZZ3jmmWewfft2nDlzBsuWLYPBYEBWVpbNdXJycpCdnW3VXlZWBo1Gc/s74sQ0Gg0OHDggdRnkQMPhmHZ2dgJAv906kvZ5DZTJZEJgYCDefPNNuLm5ITIyEpcuXcJrr73Wa3hlZGRAp9OZp1tbWxEaGoqEhAT4+PgMVelDzmAwoLi4GPHx8S79V3o4GS7HtLW11a7lJAsvf39/uLm5oaGhwaK9oaEBQUFBNtcJDg6GSqWyuEScMmUK6uvr0dXVBQ8PD6t11Go11Gq1VbtKpXLpH4Aew2U/hxNXP6b27ptkt0p4eHggMjISJSUl5jaTyYSSkhLExsbaXOeBBx7AmTNnYDKZzG2nT59GcHCwzeAiItcl6X1eOp0OGzZswNtvv41Tp05h6dKl6OjoQGpqKgAgOTkZGRkZ5uWXLl2Ka9euYcWKFTh9+jS2bduGl19+GcuXL5dqF4hIIpL2eSUlJaGxsRGZmZmor69HREQEduzYYe7Er62ttbjPIzQ0FDt37sTzzz+PadOmISQkBCtWrMDKlSul2gUikojkHfZpaWlIS0uzOa+0tNSqLTY2Fvv37x/kqojI2UkeXkRkrbOz0+qWofav9fiyohqj/Q/Da8S3b0KFh4e7/G0/tjC8iJxQZWUlIiMjbc77/U3T5eXlmDFjxuAX5WQYXkROKDw8HOXl5RZtVXXN0H1Qgdyn7sXk4FEWyw5HDC8iJ6TRaKzOppTnr0K992tMuWc6IsbdIVFlzkPyIXGIiG4Fw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInDQLuIm58209uTZoDh+7QZci0MLxfR29Nmbn7SDDB8nzZDroXh5SJuftpMb0+a6VmWSO4YXi7i5qfN8Ekz5OrYYU9EssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJklOEV35+PsLCwuDp6YmYmBgcPHiw12U3bdoEhUJh8eXp6TmE1RKRM5A8vIqKiqDT6ZCVlYUjR45g+vTpSExMxJUrV3pdx8fHB3V1deav8+fPD2HFROQMJA+v3NxcLFmyBKmpqZg6dSoKCgqg0WhQWFjY6zoKhQJBQUHmL61WO4QVE5EzkPTjQV1dXSgvL0dGRoa5TalUIi4uDmVlZb2u197ejnHjxsFkMmHGjBl4+eWXcffdd9tcVq/XQ6/Xm6dbW1sBAAaDAQaDwUF74ny6u7vN/7ryfg4nw+WY2rtvkoZXU1MTjEaj1ZmTVqu1GN7luyZPnozCwkJMmzYNLS0tWL16NWbOnIkTJ07gzjvvtFo+JycH2dnZVu27du1y6WFhLrQDgDv279+PS8elroYcYbgc087OTruWk90Hs2NjYxEbG2uenjlzJqZMmYI//vGPWLVqldXyGRkZ0Ol05unW1laEhoYiISEBPj4+Q1KzFI7VXgMqDuP+++/H9LF+UpdDDjBcjmnP1VF/JA0vf39/uLm5oaGhwaK9oaEBQUFBdm1DpVLhvvvuw5kzZ2zOV6vVUKvVVu0qlQoqlWrgRcuEu7u7+V9X3s/hZLgcU3v3TdIOew8PD0RGRqKkpMTcZjKZUFJSYnF21Rej0YiKigoEBwcPVplE5IQkv2zU6XRISUlBVFQUoqOjkZeXh46ODqSmpgIAkpOTERISgpycHADASy+9hPvvvx+TJk1Cc3MzXnvtNZw/fx4/+9nPpNwNIhpikodXUlISGhsbkZmZifr6ekRERGDHjh3mTvza2loold+eIF6/fh1LlixBfX09Ro8ejcjISHz55ZeYOnWqVLtARBKQPLwAIC0tDWlpaTbnlZaWWkyvXbsWa9euHYKqiMiZSX6TKhHRrWB4EZEsMbyISJYYXkQkS07RYU+35lxTBzr03TbnVTd2mP/tubnRlpFqd4z3Hzko9dHA9HU8AfuO6XA6ngohhJC6iKHU2toKX19ftLS0yPrjQeeaOjB7dalDtrUn/eFh8wPvrHg8v2Xv7yjPvGSq5y90XlIEJgV6Wc//Wo9PS8sw5+FYjBxh/fEoADhzpR3PFR3t8689DY3+jifQ/zEdbseT4SVzkwK9cE+Ir1W7wWBAfQAwY9xol/4cnKvp7XgCPKY3Y4c9EckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSR1KVKb3xBpSel3CutQpKT+thg7u7u3G5+zJOXTvV68MazrW2Q+l5CXrjDQC2R+8kclYML5m63HEeI8e/jl8d7Hu5N3a80ef8keOByx0RiITWgdURDT6Gl0yNGTkOHef+H9YlRWCijQc2dHd344t9X+CBBx/o9cyr+ko7VhQdxZjZ4wa7XCKHY3jJlNrNE6YbIRjvMxlT77D9AI5z7ucwxW9Krw9rMN1ogelGI9RunoNdLvWjv24AoP+ugOHWDcDwInIC9nYDAH13BQynbgCGF5ET6K8bAOi/K2C4dQMwvIicQH/dAED/XQHDrRtgwPd5nT17djDqICIakAGH16RJkzB79my8++67uHHjhkOKyM/PR1hYGDw9PRETE4ODB+248AewZcsWKBQKPPHEEw6pg4jkY8DhdeTIEUybNg06nQ5BQUH4j//4D7vDxpaioiLodDpkZWXhyJEjmD59OhITE3HlypU+16upqUF6ejpmzZp1y69NRPI14PCKiIjAunXrcPnyZRQWFqKurg4PPvgg7rnnHuTm5qKxsXFA28vNzcWSJUuQmpqKqVOnoqCgABqNBoWFhb2uYzQa8cwzzyA7OxsTJkwY6C4QkQu45Q57d3d3zJs3Dz/84Q/xxhtvICMjA+np6fjVr36F+fPn49VXX0VwcHCf2+jq6kJ5eTkyMjLMbUqlEnFxcSgrK+t1vZdeegmBgYFYvHgx9u7d2+dr6PV66PV683RrayuAbzo/DQaDPbvqlLq7u83/2tqPnra+9rG/bdDQsedY9HdMXeV42lv7LYfX4cOHUVhYiC1btmDkyJFIT0/H4sWLcfHiRWRnZ2Pu3Ln9Xk42NTXBaDRCq7W8J0Wr1aKystLmOvv27cPGjRtx9OhRu+rMyclBdna2VfuuXbug0Wjs2oYzutAOAO7Yt28fztt+Zx0AUFxcfNvboME3kGPR2zF1lePZ2dlp13IDDq/c3Fy89dZbqKqqwmOPPYZ33nkHjz32GJTKb65Ax48fj02bNiEsLGygm+5XW1sbFi5ciA0bNsDf39+udTIyMqDT6czTra2tCA0NRUJCAnx8fBxe41A5cbkVqyv248EHH8TdY6z3w2AwoLi4GPHx8b3eYd/fNmjo2HMs+jumrnI8e66O+jPg8Fq/fj2effZZLFq0qNfLwsDAQGzcuLHfbfn7+8PNzQ0NDQ0W7Q0NDQgKCrJavrq6GjU1NXj88cfNbSaTCcA3l7FVVVWYOHGixTpqtRpqtdpqWyqVqtdfajnouUnR3d29z/3oaz/t3QYNvoEci96OqascT3trH3B4FRcXY+zYseYzrR5CCFy4cAFjx46Fh4cHUlJS+t2Wh4cHIiMjUVJSYr7dwWQyoaSkBGlpaVbLh4eHo6KiwqLtv/7rv9DW1oZ169YhNDR0oLtDRDI14PCaOHEi6urqEBgYaNF+7do1jB8/HkajcUDb0+l0SElJQVRUFKKjo5GXl4eOjg6kpqYCAJKTkxESEoKcnBx4enrinnvusVh/1KhRAGDVTkSubcDhJYSw2d7e3g5Pz4F/LCEpKQmNjY3IzMxEfX09IiIisGPHDnMnfm1trdVZHhGR3eHV0+mtUCiQmZlp8U6d0WjEgQMHEBERcUtFpKWl2bxMBIDS0tI+1920adMtvSYRyZvd4fXVV18B+ObMq6KiAh4eHuZ5Hh4emD59OtLT0x1fIRGRDXaH1549ewAAqampWLdunaxvMyAi+Rtwn9dbb701GHUQEQ2IXeE1b948bNq0CT4+Ppg3b16fy27dutUhhRER9cWu8PL19YVCoTD/n4hIanaF13cvFXnZSETOgDdQEZEs2XXmdd9995kvG/tz5MiR2yqIiMgedoUXh1kmImdjV3hlZWUNdh1ERAPCPi8ikiW7zrz8/Pxw+vRp+Pv7Y/To0X32f127ds1hxRER9cau8Fq7di28vb3N/7e3856IaLDYFV7fHVhw0aJFg1ULEZHdBtzn5ebmZvOZilevXoWbm5tDiiIi6s+Aw6u3wQj1er3FMDlERIPJ7lEl/vCHPwD4ZjDCP/3pT/Dy+vbZSkajEf/85z8RHh7u+AqJiGywO7zWrl0L4Jszr4KCAotLRA8PD4SFhaGgoMDxFRIR2WB3eJ07dw4AMHv2bGzduhWjR48etKKIiPoz4MEIe0ZUJSKS0oDD69lnn+1zfmFh4S0XQ0RkrwGH1/Xr1y2mDQYDjh8/jubmZnz/+993WGFERH0ZcHj99a9/tWozmUxYunQpJk6c6JCiiIj645APZiuVSuh0OvM7kkREg81ho0pUV1eju7vbUZsjIurTgC8be56c3UMIgbq6Omzbts3iM5BERINpwOHV8+TsHkqlEgEBAVizZk2/70QSETkK7/MiIlniSKpEJEsMLyKSJYYXEckSw4uIZGnAHfa9uXjxIl566SW8+eabjtok9eFrgxEAcPxSi835HV/rcbgRCDp/HSNHqG0uc+ZK+6DVRzTYHBZeV69excaNG28pvPLz8/Haa6+hvr4e06dPx+uvv47o6Giby27duhUvv/wyzpw5A4PBgO9973v4xS9+gYULF97uLshK9f8Gz4tbK/pYyh2bzxzqd1sj1Q77MSAaMpL/1BYVFUGn06GgoAAxMTHIy8tDYmIiqqqqEBgYaLW8n58ffv3rXyM8PBweHh749NNPkZqaisDAQCQmJkqwB9JIuDsIADAx0AsjVNbPDqiqa8EvPqzAmifvxeRg3163M1LtjvH+IwetTqLBInl45ebmYsmSJUhNTQUAFBQUYNu2bSgsLMSLL75otfzDDz9sMb1ixQq8/fbb2Ldv37AKL7+RHvi/0WN7nd/zUa2JASNxT0jv4UUkV5KGV1dXF8rLy5GRkWFuUyqViIuLQ1lZWb/rCyHw2WefoaqqCq+++qrNZfR6PfR6vXm6tbUVwDdD+RgMhtvcA+fVE17d3d0uvZ+uwp7j1dPe23xXOeb21m53eM2bN6/P+c3NzfZuyqypqQlGoxFardaiXavVorKystf1WlpaEBISAr1eDzc3N7zxxhuIj4+3uWxOTg6ys7Ot2nft2gWNRjPgmuXiQjsAuGP//v24dFzqaqg/1a0A4I4tO7/AnSNtP6HLYAKu6YGzHxVDZeM+gYavFQDcsG/fPpz3sp4vF52dnXYtZ3d4+fr2fenh6+uL5ORkezd3W7y9vXH06FG0t7ejpKQEOp0OEyZMsLqkBICMjAyLD5O3trYiNDQUCQkJ8PHxGZJ6pXCs9hpQcRj3338/po/1k7oc6sdfDl8ETpzElrO3/+zTxEceQtgd8u3H7Lk66o/d4fXWW2/dcjG98ff3h5ubGxoaGizaGxoaEBQU1Ot6SqUSkyZNAgBERETg1KlTyMnJsRlearUaarX1rQIqlQoqler2dsCJubu7m/915f10FY9OC4Gbm1uvb8AA9r0J4wpvwNj78yppn5eHhwciIyNRUlKCJ554AsA3o7KWlJQgLS3N7u2YTCaLfi0iuenvDRiAb8LczO7wsne4m4E+gEOn0yElJQVRUVGIjo5GXl4eOjo6zO8+JicnIyQkBDk5OQC+6cOKiorCxIkTodfrsX37dmzevBnr168f0OsSkbzZHV6bNm3CuHHjcN9990EI2x2KtyIpKQmNjY3IzMxEfX09IiIisGPHDnMnfm1tLZTKb3snOzo6sGzZMly8eBEjRoxAeHg43n33XSQlJTmsJiJyfnaH19KlS/H+++/j3LlzSE1NxU9/+lP4+TmmIzgtLa3Xy8TS0lKL6d/+9rf47W9/65DXJSL5svuD2fn5+airq8MLL7yAv//97wgNDcX8+fOxc+dOh56JERHZY0CjSqjVajz99NMoLi7GyZMncffdd2PZsmUICwtDezs/5EtEQ+eWh8RRKpVQKBQQQsBoNDqyJiKifg0ovPR6Pd5//33Ex8fjrrvuQkVFBf7nf/4HtbW18PKS8S29RCQ7dnfYL1u2DFu2bEFoaCieffZZvP/++/D39x/M2oiIemV3eBUUFGDs2LGYMGECPv/8c3z++ec2l9u6davDiiMi6o3d4ZWcnAyFQjGYtRAR2W1AN6kSETkLPoCDiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGTJKcIrPz8fYWFh8PT0RExMDA4ePNjrshs2bMCsWbMwevRojB49GnFxcX0uT0SuSfLwKioqgk6nQ1ZWFo4cOYLp06cjMTERV65csbl8aWkpnn76aezZswdlZWUIDQ1FQkICLl26NMSVE5GUJA+v3NxcLFmyBKmpqZg6dSoKCgqg0WhQWFhoc/n33nsPy5YtQ0REBMLDw/GnP/0JJpMJJSUlQ1w5EUnJXcoX7+rqQnl5OTIyMsxtSqUScXFxKCsrs2sbnZ2dMBgM8PPzszlfr9dDr9ebp1tbWwEABoMBBoPhNqp3bt3d3eZ/XXk/h5Phckzt3TdJw6upqQlGoxFardaiXavVorKy0q5trFy5EmPGjEFcXJzN+Tk5OcjOzrZq37VrFzQazcCLlokL7QDgjv379+PScamrIUcYLse0s7PTruUkDa/b9corr2DLli0oLS2Fp6enzWUyMjKg0+nM062treZ+Mh8fn6Eqdcgdq70GVBzG/fffj+ljbZ+VkrwMl2Pac3XUH0nDy9/fH25ubmhoaLBob2hoQFBQUJ/rrl69Gq+88gp2796NadOm9bqcWq2GWq22alepVFCpVLdWuAy4u7ub/3Xl/RxOhssxtXffJO2w9/DwQGRkpEVne0/ne2xsbK/r/f73v8eqVauwY8cOREVFDUWpRORkJL9s1Ol0SElJQVRUFKKjo5GXl4eOjg6kpqYCAJKTkxESEoKcnBwAwKuvvorMzEz8+c9/RlhYGOrr6wEAXl5e8PLykmw/iGhoSR5eSUlJaGxsRGZmJurr6xEREYEdO3aYO/Fra2uhVH57grh+/Xp0dXXhySeftNhOVlYW/vu//3soSyciCUkeXgCQlpaGtLQ0m/NKS0stpmtqaga/ICJyepLfpEpEdCsYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWXKK5zbS7evs7ERlZaV5uqquGfr6Mzh1fARMV0dZLBseHg6NRjPEFRI5FsPLRVRWViIyMtKqfcHb1suWl5djxowZQ1AV0eBheLmI8PBwlJeXm6fbv9Zj254y/HB2LLxGqK2WJZI7hpeL0Gg0FmdTBoMB15uuIDY6CiqVSsLKiAYHO+yJSJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLIkeXjl5+cjLCwMnp6eiImJwcGDB3td9sSJE/jJT36CsLAwKBQK5OXlDV2hRORUJL3Pq6ioCDqdDgUFBYiJiUFeXh4SExNRVVWFwMBAq+U7OzsxYcIEPPXUU3j++eclqJhoaNz8cS+g9498DduPewkJRUdHi+XLl5unjUajGDNmjMjJyel33XHjxom1a9cO+DVbWloEANHS0jLgdeWkq6tLfPzxx6Krq0vqUugWlJeXCwB2fZWXl0tdrkPZ+zsq2ZlXV1cXysvLkZGRYW5TKpWIi4tDWVmZw15Hr9dDr9ebp1tbWwF8cwe6wWBw2Os4m559c+V9dGUTJ07EgQMHLNrav9Zj595DSJz1fyw+8jVx4kSXOs727otk4dXU1ASj0QitVmvRrtVqrU6Xb0dOTg6ys7Ot2nft2jUsTrWLi4ulLoEcaOa9E9HWfA1tzd+21dXVSVbPYOjs7LRrOZf/bGNGRgZ0Op15urW1FaGhoUhISICPj4+ElQ0ug8GA4uJixMfH87ONLmK4HNOeq6P+SBZe/v7+cHNzQ0NDg0V7Q0MDgoKCHPY6arUaarXaql2lUrn0D0CP4bKfw4mrH1N7902yWyU8PDwQGRmJkpISc5vJZEJJSQliY2OlKouIZELSy0adToeUlBRERUUhOjoaeXl56OjoQGpqKgAgOTkZISEhyMnJAfBNJ//JkyfN/7906RKOHj0KLy8vTJo0SbL9IKKhJ2l4JSUlobGxEZmZmaivr0dERAR27Nhh7sSvra2FUvntyeHly5dx3333madXr16N1atX46GHHkJpaelQl09EEpK8wz4tLQ1paWk2590cSGFhYRBCDEFVROTsJP94EBHRrZD8zGuo9Zy52ft2rFwZDAZ0dnaitbXVpd+ZGk6GyzHt+d3s7ypr2IVXW1sbACA0NFTiSoioL21tbfD19e11vkIMs04kk8mEy5cvw9vbGwqFQupyBk3PzbgXLlxw6Ztxh5PhckyFEGhra8OYMWMs3rC72bA781IqlbjzzjulLmPI+Pj4uPQP+nA0HI5pX2dcPdhhT0SyxPAiIllieLkotVqNrKwsm5/rJHniMbU07Drsicg18MyLiGSJ4UVEssTwIiJZYnhJ7OGHH8Zzzz0ndRlEssPwkonS0lIoFAo0NzdLXQoNgLP9cXK2em4Hw4vIyXV1dUldglNieDkRvV6PlStXIjQ0FGq1GpMmTcLGjRtRU1OD2bNnAwBGjx4NhUKBRYsW9bu9trY2PPPMMxg5ciSCg4Oxdu1aq7+8mzdvRlRUFLy9vREUFIQFCxbgypUr5vk9Z3wlJSWIioqCRqPBzJkzUVVV5ejddzmLFi3C559/jnXr1kGhUEChUKC6uhqLFy/G+PHjMWLECEyePBnr1q2zWu+JJ57A7373O4wZMwaTJ08GAHz55ZeIiIiAp6cnoqKi8PHHH0OhUODo0aPmdY8fP45HH30UXl5e0Gq1WLhwIZqamnqtp6amZqi+HY43yM+PpH489NBDYsWKFUIIIebPny9CQ0PF1q1bRXV1tdi9e7fYsmWL6O7uFh999JEAIKqqqkRdXZ1obm7ud9s/+9nPxLhx48Tu3btFRUWF+PGPfyy8vb3NryeEEBs3bhTbt28X1dXVoqysTMTGxopHH33UPH/Pnj0CgIiJiRGlpaXixIkTYtasWWLmzJmO/la4nObmZhEbGyuWLFki6urqRF1dnbhx44bIzMwUhw4dEmfPnhXvvvuu0Gg0oqioyLxeSkqK8PLyEgsXLhTHjx8Xx48fFy0tLcLPz0/89Kc/FSdOnBDbt28Xd911lwAgvvrqKyGEENevXxcBAQEiIyNDnDp1Shw5ckTEx8eL2bNn91pPd3e3FN8ah2B4SawnvKqqqgQAUVxcbHO5nhC5fv26XdttbW0VKpVKfPDBB+a25uZmodFoLMLrZocOHRIARFtbm8Xr7t6927zMtm3bBADx9ddf21XLcPbdP069Wb58ufjJT35ink5JSRFarVbo9Xpz2/r168Udd9xh8T3fsGGDRXitWrVKJCQkWGz7woUL5j969tYjF7xsdBJHjx6Fm5sbHnroIYds7+zZszAYDIiOjja3+fr6mi9BepSXl+Pxxx/H2LFj4e3tbX792tpai+WmTZtm/n9wcDAAWFxekv3y8/MRGRmJgIAAeHl54c0337T6ft97773w8PAwT1dVVWHatGnw9PQ0t3332ALAsWPHsGfPHnh5eZm/wsPDAQDV1dWDuEfSGHZD4jirESNGDPlrdnR0IDExEYmJiXjvvfcQEBCA2tpaJCYmWnUSf3fkzp5x0Ewm05DW6wq2bNmC9PR0rFmzBrGxsfD29sZrr72GAwcOWCw3cuTIAW+7vb0djz/+OF599VWreT1/cFwJw8tJ3HvvvTCZTPj8888RFxdnNb/nr7DRaLRrexMmTIBKpcKhQ4cwduxYAEBLSwtOnz6Nf/u3fwMAVFZW4urVq3jllVfMI8sePnzYEbtD/8vDw8PimH3xxReYOXMmli1bZm6z56xo8uTJePfdd6HX680fzD506JDFMjNmzMBHH32EsLAwuLvb/tW+uR4542WjkwgLC0NKSgqeffZZfPzxxzh37hxKS0vxl7/8BQAwbtw4KBQKfPrpp2hsbER7e3uf2/P29kZKSgp++ctfYs+ePThx4gQWL14MpVJpPnMaO3YsPDw88Prrr+Ps2bP429/+hlWrVg36vg4nYWFhOHDgAGpqatDU1ITvfe97OHz4MHbu3InTp0/jN7/5jVUI2bJgwQKYTCb8+7//O06dOoWdO3di9erVAL49E16+fDmuXbuGp59+GocOHUJ1dTV27tyJ1NRUc2DdXI+cz54ZXk5k/fr1ePLJJ7Fs2TKEh4djyZIl6OjoAACEhIQgOzsbL774IrRaba+Pi/uu3NxcxMbGYs6cOYiLi8MDDzyAKVOmmPtNAgICsGnTJnzwwQeYOnUqXnnlFfMvBDlGeno63NzcMHXqVAQEBCAxMRHz5s1DUlISYmJicPXqVYuzsN74+Pjg73//O44ePYqIiAj8+te/RmZmJgCYj+eYMWPwxRdfwGg0IiEhAffeey+ee+45jBo1yjyc8s313NzXJiccEmcY6ejoQEhICNasWYPFixdLXQ7dpvfeew+pqaloaWmRpM9UauzzcmFfffUVKisrER0djZaWFrz00ksAgLlz50pcGd2Kd955BxMmTEBISAiOHTuGlStXYv78+cMyuACGl2zV1tZi6tSpvc4/efIkAGD16tWoqqqCh4cHIiMjsXfvXvj7+w9VmeRA9fX1yMzMRH19PYKDg/HUU0/hd7/7ndRlSYaXjTLV3d3d50c7+nrHicgVMLyISJb4biMRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbxoUC1atMg8aqdKpYJWq0V8fDwKCwsH9Lm6TZs2YdSoUYNXaC96RjUl58PwokH3gx/8AHV1daipqcE//vEPzJ49GytWrMCcOXPQ3d0tdXkkV1KOhEiuLyUlRcydO9eqvaSkRAAQGzZsEEIIsWbNGnHPPfcIjUYj7rzzTrF06VKr0Vy/+5WVlSWEEOKdd94RkZGRwsvLS2i1WvH000+LhoYG8+tcu3ZNLFiwQPj7+wtPT08xadIkUVhYaJ5fW1srnnrqKeHr6ytGjx4tfvSjH4lz584JIYTIysqyet09e/YMyveJBo5nXiSJ73//+5g+fTq2bt0KAFAqlfjDH/6AEydO4O2338Znn32GF154AQAwc+ZM5OXlwcfHB3V1dairq0N6ejoAwGAwYNWqVTh27Bg+/vhj1NTUWDyc5De/+Q1OnjyJf/zjHzh16hTWr19v/niUwWBAYmIivL29sXfvXnzxxRfw8vLCD37wA3R1dSE9PR3z5883nznW1dVh5syZQ/uNot5JnZ7k2no78xJCiKSkJDFlyhSb8z744ANxxx13mKffeust4evr2+/r3TwG/+OPPy5SU1NtLrt582YxefJkYTKZzG16vV6MGDFC7Ny5s9/6SVo88yLJCCHMA+nt3r0bjzzyCEJCQuDt7Y2FCxfi6tWr6Ozs7HMb/Y3Bv3TpUmzZsgURERF44YUX8OWXX5rXPXbsGM6cOQNvb2/zmO9+fn64ceOGS4757moYXiSZU6dOYfz48aipqcGcOXMwbdo0fPTRRygvL0d+fj6Avh+42jMGv4+PD9577z0cOnQIf/3rXy3We/TRR3H+/Hk8//zzuHz5Mh555BHzJWd7ezsiIyNx9OhRi6/Tp09jwYIFg7z3dLs47ABJ4rPPPkNFRQWef/55lJeXw2QyYc2aNeYRP3uGv+5ha+x1e8fgDwgIQEpKClJSUjBr1iz88pe/xOrVqzFjxgwUFRUhMDAQPj4+Nut0pTHfXQ3PvGjQ6fV61NfX49KlSzhy5AhefvllzJ07F3PmzEFycjImTZoEg8FgHkt/8+bNKCgosNhGWFgY2tvbUVJSgqamJnR2dto1Bn9mZiY++eQTnDlzBidOnMCnn36KKVOmAACeeeYZ+Pv7Y+7cudi7d6/5uQH/+Z//iYsXL5pf91//+heqqqrQ1NQEg8EwNN806p/UnW7k2lJSUsy3Gbi7u4uAgAARFxcnCgsLhdFoNC+Xm5srgoODxYgRI0RiYqJ45513rB6y+/Of/1zccccdFrdK/PnPfxZhYWFCrVaL2NhY8be//c3qQaxTpkwRI0aMEH5+fmLu3Lni7Nmz5m3W1dWJ5ORk4e/vL9RqtZgwYYJYsmSJaGlpEUIIceXKFREfHy+8vLx4q4ST4XheRCRLvGwkIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiW/j+eoJUu3ZYQ6wAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T18:03:03.653383Z", + "iopub.status.busy": "2024-03-24T18:03:03.653088Z", + "iopub.status.idle": "2024-03-24T18:03:03.880388Z", + "shell.execute_reply": "2024-03-24T18:03:03.879469Z" + }, + "papermill": { + "duration": 0.249442, + "end_time": "2024-03-24T18:03:03.882455", + "exception": false, + "start_time": "2024-03-24T18:03:03.633013", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAErCAYAAACPTwXOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABW+0lEQVR4nO2dd3hU1daH3+lphBBKQgkEEGkCQUIQr4ooCDaaBbDRRBFBrxEVVECaoIKigCD2qyKgIvBxFS9GEKSpNOm9Q0goIX3q+f4YZjK9JJPMTLJfnzwyp83O5MzvrLX22mvJJEmSEAgEgjBDHuwBCAQCQWkQ4iUQCMISIV4CgSAsEeIlEAjCEiFeAoEgLBHiJRAIwhIhXgKBICwR4iUQCMISIV4CgSAsEeIlEAjCkpAQr3nz5pGcnExERASdOnXizz//9Om8xYsXI5PJ6NOnT/kOUCAQhBxBF68lS5aQnp7OxIkT2b59O+3ataNHjx5kZWV5PO/EiROMGTOGW2+9tYJGKhAIQglZsBdmd+rUiY4dOzJ37lwATCYTSUlJjB49mrFjx7o8x2g0cttttzF06FA2bNhATk4Oy5cvr8BRCwSCYKMM5pvrdDq2bdvGuHHjrNvkcjndunVj8+bNbs+bPHkyderUYdiwYWzYsMHje2i1WrRarfW1yWTi8uXL1KxZE5lMVvZfQiAQBBRJksjLy6NevXrI5e6dw6CK18WLFzEajSQkJNhtT0hI4MCBAy7P+eOPP/j000/ZuXOnT+8xffp0Jk2aVNahCgSCCub06dM0aNDA7f6gipe/5OXl8fjjj/Pxxx9Tq1Ytn84ZN24c6enp1tdXr16lYcOGnD59mtjY2PIaqkAgKCW5ubkkJSVRrVo1j8cFVbxq1aqFQqHgwoULdtsvXLhAYmKi0/FHjx7lxIkT3H///dZtJpMJAKVSycGDB2natKndORqNBo1G43St2NhYIV4CQQjjLawT1NlGtVpNhw4dyMjIsG4zmUxkZGTQuXNnp+NbtGjB7t272blzp/WnV69edO3alZ07d5KUlFSRwxcIBEEk6G5jeno6gwYNIjU1lbS0NGbPnk1BQQFDhgwB4IknnqB+/fpMnz6diIgIbrjhBrvz4+LiAJy2CwSCyk3Qxat///5kZ2czYcIEMjMzSUlJYfXq1dYg/qlTpzzOOAgEgqpJ0PO8Kprc3FyqV6/O1atXRcyrHJAkCYPBgNFoDPZQBCGKQqFAqVS6jWn5+h0NuuUlqDzodDrOnz9PYWFhsIciCHGioqKoW7cuarW61NcQ4iUICCaTiePHj6NQKKhXrx5qtVokAQuckCQJnU5HdnY2x48fp1mzZqUOCwnxcsOBzFxeWLKLmtFqvn6yU7CHE/LodDrr0q6oqKhgD0cQwkRGRqJSqTh58iQ6nY6IiIhSXUeIlxuMJon953OpU805R0zgHjG5IvCFQNwn4k5zQ6RKAUCRTgSeBYJQRIiXG6LUZqO0SC/ESyAIRYR4ucFieRlMEjqDKcijEZQnt99+O//+97+DPYxKw7p165DJZOTk5JTr+wjxckOkWmH9t7C+BFBxX0qBbwjxcoNKIUMhN0/1i7iXoKqh0+mCPQSvCPFyg0wmI8oStBeWV6mQJIlCnSEoP6VdOKLVannllVdISkpCo9Fw3XXX8emnn3LixAm6du0KQI0aNZDJZAwePNjr9fLy8nj00UeJjo6mbt26vPfee05u6ldffUVqairVqlUjMTGRRx55xK4MusXiy8jIIDU1laioKG6++WYOHjzo8+81depU6tSpQ7Vq1XjyyScZO3YsKSkp1v2DBw+mT58+TJs2jXr16tG8eXOfxgbw008/cf311xMZGUnXrl05ceKEz+MqCyJVwgORagV5WvOXQeA/RXojrSb8EpT33je5h3XSxR+eeOIJNm/ezAcffEC7du04fvw4Fy9eJCkpiR9++IEHHniAgwcPEhsbS2RkpNfrpaens3HjRlauXElCQgITJkxg+/btdsKh1+uZMmUKzZs3Jysri/T0dAYPHsxPP/1kd63XXnuNWbNmUbt2bUaMGMHQoUPZuHGj1zF88803TJs2jQ8//JB//etfLF68mFmzZtG4cWO74zIyMoiNjWXNmjU+j+306dP069ePZ599lqeeeoq///6bF1980euYAoEQLw9Y4l7CbawaHDp0iKVLl7JmzRq6desGQJMmTaz74+PjAahTp461mokn8vLy+PLLL1m0aBF33nknAJ9//jn16tWzO27o0KHWfzdp0oQPPviAjh07kp+fT0xMjHXftGnT6NKlCwBjx47l3nvvpbi42GuS55w5cxg2bJi1UsuECRP43//+R35+vt1x0dHRfPLJJ3ZLdryNbf78+TRt2pRZs2YB0Lx5c3bv3s1bb73l9fMpK0K8PBAp3MYyEalSsG9yj6C9t7/s3LkThUJhFYiycuzYMfR6PWlpadZt1atXt7pkFrZt28Ybb7zBrl27uHLlirXA5qlTp2jVqpX1uLZt21r/XbduXQCysrJo2LChx3EcPHiQkSNH2m1LS0vjt99+s9vWpk0bp7WG3sa2f/9+OnWyX4HiqhZfeSDEywMWy6tQWF6lQiaTlcp1Cxa+uIGBpqCggB49etCjRw+++eYbateuzalTp+jRo4dT0FylUln/bVk3ahGTQBAdHV3qsQUDEbD3QNQ18SoWlleVoE2bNphMJn7//XeX+y1Wia/lfpo0aYJKpeKvv/6ybrt69SqHDh2yvj5w4ACXLl1ixowZ3HrrrbRo0cJrz1J/ad68ud0YAKfXrvBlbC1btnRqEr1ly5ayD9oHhHh5wOJ6CMurapCcnMygQYMYOnQoy5cv5/jx46xbt46lS5cC0KhRI2QyGatWrSI7O9spZuRItWrVGDRoEC+99BJr165l7969DBs2DLlcbrWcGjZsiFqtZs6cORw7doyVK1cyZcqUgP5eo0eP5tNPP+XLL7/k8OHDTJ06lX/++cdr1Q9fxjZixAgOHz7MSy+9xMGDB1m0aBFffPFFQMfvDiFeHoi0LBES4lVlmD9/Pg8++CAjR46kRYsWDB8+nIKCAgDq16/PpEmTGDt2LAkJCYwaNcrr9d599106d+7MfffdR7du3fjXv/5Fy5YtrUH22rVr88UXX/Ddd9/RqlUrZsyYwcyZMwP6Oz366KOMGzeOMWPGcOONN3L8+HEGDx7sNdDvy9gaNmzIDz/8wPLly2nXrh0LFizgzTffDOj43SEqqXrg5e93sfTvM7zUoznPdr2ugkYYnhQXF3P8+HEaN25c6hInVYGCggLq16/PrFmzGDZsWNDG0b17dxITE/nqq6+C8v6e7hdRSTUARAnLS1BGduzYwYEDB0hLS+Pq1atMnjwZgN69e1fYGAoLC1mwYAE9evRAoVDw7bff8uuvv9rlc4UjQrw8ECFiXgIPOKYyOLJv3z4AZs6cycGDB62t/jZs2OBz02RfaN26NSdPnnS576OPPqJfv3789NNPTJs2jeLiYpo3b84PP/xgzWULV4R4ecAy21ikFxn2Amfq1avHzp07Pe5v2LAh27ZtK9dx/PTTT+j1epf7EhISiIyM5Ndffy3XMQQDIV4eiBIZ9gIPKJVKrrsu+LHQRo0aBXsIQUHMNnpAuI0CQegixMsDJW6jEC+BINQQ4uUBUcdeIAhdhHh5IFJYXgJByCLEywPC8hIIQhchXh4QHYQEvpCcnMzs2bOtr2UyGcuXLw/aeKoKIlXCA5Fqs7aL2UaBP5w/f54aNWoEexiVHiFeHhALswWlITExMdhDQJIkjEYjSmXl/YoLt9EDlgYcOqMJg1H0bvQbSQJdQXB+/Kw34EujDF+xdRtPnDiBTCZj2bJldO3alaioKNq1a8fmzZvtzvnjjz+49dZbiYyMJCkpieeee85azQJ8b9Lx888/06FDBzQaDX/88YffYw8nKq8sBwDH3o3VFELr/UJfCG/W835cefDqOVBHez/uGr40yigLr732GjNnzqRZs2a89tprDBw4kCNHjqBUKjl69Cg9e/Zk6tSpfPbZZ2RnZzNq1ChGjRrF559/DvjepGPs2LHMnDmTJk2aVHrXVYiXBzRKOTKZ+SFepDdSLULl/SRB2OFro4yyMGbMGO69914AJk2aROvWrTly5AgtWrRg+vTpPProo1Yrr1mzZnzwwQd06dKF+fPnExER4XOTjsmTJ9O9e/eAjTuUEeLlAZlMRqRKQaHOKOJepUEVZbaAgvXePuJro4yy4K55RosWLdi1axf//PMP33zzjfUYSZIwmUwcP36cli1b+tykIzU1NWBjDnWEeHkhSn1NvES6hP/IZH65bpUZT80z8vPzefrpp3nuueeczmvYsKFfjTAcm2hUZoR4eUEszq782DbKsLQRszTKuO2228r9/W+88Ub27dvntkLF7t27rY0wkpKSAPj777/LfVyhjohAe8HaQUiIV6XFl0YZ5ckrr7zCpk2bGDVqFDt37uTw4cOsWLHCWiO/Ipp0hCNCvLwgOghVDbw1yihP2rZty++//86hQ4e49dZbad++PRMmTLBOGFREk45wRDTg8MKAhZvZcuwyHwxsT692QZr2DwMqWwOOUGmUUVkRDTgqAIvlJdzGyk0oNMoQ+IcQLy9YFmcX6kQd+8qOq0YZ+/fv5+6773Z7jrfGs4LyQ4iXF0pqeonlQZWZ9u3bu2yUUVRU5LHJhiB4CPHyQklNL2F5VUUiIyNDosmGwBkx2+gFUcfeP6rY/I+glATiPgkJ8Zo3bx7JyclERETQqVMn/vzzT7fHLlu2jNTUVOLi4oiOjiYlJaVcW5aLJFXfsGSQFxYWBnkkgnDAcp/Yrjzwl6C7jUuWLCE9PZ0FCxbQqVMnZs+eTY8ePTh48CB16tRxOj4+Pp7XXnuNFi1aoFarWbVqFUOGDKFOnTr06NEj4OMTlpdvKBQK4uLirGVaoqKiKiTBUxBeSJJEYWEhWVlZxMXFoVAovJ/khqDneXXq1ImOHTsyd+5cwLzeKykpidGjRzN27FifrnHjjTdy7733+pR17G+e1382n2DCir3cfUMi8x/r4NN4qiqSJJGZmUlOTk6whyIIceLi4khMTHT5gAuLPC+dTse2bdsYN26cdZtcLqdbt25OxdpcIUkSv/32GwcPHuStt95yeYxWq0Wr1Vpf5+bm+jVGkWHvOzKZjLp161KnTh237ecFApVKVSaLy0JQxevixYsYjUYSEhLstickJHDgwAG35129epX69euj1WpRKBR8+OGHbmsYTZ8+nUmTJpV6jKL9mf8oFIqA3JwCgSdCImDvL9WqVWPnzp389ddfTJs2jfT0dNatW+fy2HHjxnH16lXrz+nTp/16L2vMS1heAkFIEVTLq1atWigUCi5cuGC3/cKFCx6bGMjlcmvuTUpKCvv372f69OncfvvtTsdqNBo0Gk2pxxipEu3PBIJQJKiWl2UZRkZGhnWbyWQiIyODzp07+3wdk8lkF9cKJJHC8hIIQpKgp0qkp6czaNAgUlNTSUtLY/bs2RQUFDBkyBAAnnjiCerXr8/06dMBcwwrNTWVpk2botVq+emnn/jqq6+YP39+uYxPpEoIBKFJ0MWrf//+ZGdnM2HCBDIzM0lJSWH16tXWIP6pU6eQy0sMxIKCAkaOHMmZM2eIjIykRYsWfP311/Tv379cxlcy2yiWBwkEoUTQ87wqGn/zvC7ma0md+isAx968B7lcJF4KQgOdwcR/d5/j5qa1SIgN/xpqFnz9joblbGNFYrG8AIoNwnUUhA7z1x3lhSW7uPv9DcEeSlAQ4uUFW/ESQXtBKPHbAfMs/eUCnZcjKydCvLwgl8vQKM0fk8iyF4QSVSre4wIhXj4gZhwFoYipaoWrnSjTbGN+fr61caYFX4Lg4UakSsEV9MJtFIQUVVy7/Le8jh8/zr333kt0dDTVq1enRo0a1KhRg7i4OGrUqFEeYww6lkRV4TYKQomqLl5+W16PPfYYkiTx2WefkZCQUCVqNlmacBQLt1EQQlRx7fJfvHbt2sW2bdto3rx5eYwnJBFlcQShSBVL0XTCb7exY8eOfldmCHdEWRyBIPTw2/L65JNPGDFiBGfPnuWGG25wqkHdtm3bgA0uVBAdhAShSBU3vPwXr+zsbI4ePWpdOA3mCpqSJCGTyTAaK591IlIlBKGISJXwk6FDh9K+fXu+/fbbKhOwjxCzjYIQpGpLVynE6+TJk6xcubJKNeKMUomaXoLQQwTs/eSOO+5g165d5TGWkEUE7AWhSNWWrlJYXvfffz8vvPACu3fvpk2bNk4B+169egVscKGCSFIVhCRVXL38Fq8RI0YAMHnyZKd9lTZgrxKWlyD0EAF7P3Fcy1gVEHXsBYLQw6+Yl16vR6lUsmfPnvIaT0gSeW15kBAvQShRte0uP8VLpVLRsGHDSukaesK6PEi4jYIQoqq7jX7PNr722mu8+uqrXL58uTzGE5JYklSLheUlCCGquHb5H/OaO3cuR44coV69ejRq1Ijo6Gi7/du3bw/Y4EKFCKvlFV7LgzYduciM1Qd4s28bbqhfPdjDEQQYIV5+0qdPn3IYRmgTFaYB+//uPs8/Z66yZt8FIV6CSoff4jVx4sTyGEdIExmmGfZGk/nRbKiCM8RVgaqeYV/qMtDbtm1j//79ALRu3Zr27dsHbFChhsXyKtQbrQvQw4ES8araN3llpar/Wf0Wr6ysLAYMGMC6deuIi4sDICcnh65du7J48WJq164d6DEGHcvCbEkCrcFkjYGFOpab22is4ne5oFLi92zj6NGjycvLY+/evVy+fJnLly+zZ88ecnNzee6558pjjEEnKkx7N1qm0oXlVTmRqniml9+W1+rVq/n1119p2bKldVurVq2YN28ed911V0AHFyooFXLUCjk6o4kivZFwaTMiYl6Vm6r+TPLb8jKZTE6LscGcwFqZlw5FqMKv8azxmuVlrOp3eSWlisfrS1cS5/nnn+fcuXPWbWfPnuWFF17gzjvvDOjgQolw7CBkslhebmJeBVpDWP0+Akfcq9fag1mM+Goblwt0FTieisVv8Zo7dy65ubkkJyfTtGlTmjZtSuPGjcnNzWXOnDnlMcaQIBzL4niabTQYTXR/93d6zF4vLLMwxZPl9fnGE6zem8n6Q9kVN6AKxu+YV1JSEtu3b+fXX3/lwIEDALRs2ZJu3boFfHChREn7s/DJsrdokivxytcaOHe1GICsvGLqVo+syKEJAoCnR06h1nyfVubJmlLleclkMrp370737t0DPZ6QxWJ5hZObZbLGvJxjkbZP7bNXioR4hSGeFmYXG4xejwl3SiVeGRkZZGRkkJWV5RSk/+yzzwIysFAjKozdRr2LmJftljNXikhNrpgxCSoGrf7a97Lyapf/4jVp0iQmT55MamoqdevWDZts87ISEYbVVE0eZhttl5aczSmqsDEJAocno0pYXi5YsGABX3zxBY8//nh5jCdkCcfF2Z4C9vaWV2EFjUgQSDy6jdcsr0oc8vJ/tlGn03HzzTeXx1hCmnBcnG0RL28xrzNXhOUVlngQJq2+8ltefovXk08+yaJFi8pjLCFNpDr8qqlablzXMS8bt1GIV1jiSZaKDSavx4Q7fruNxcXFLFy4kF9//ZW2bds6Zdu/++67ARtcKBGObqN1YbbLmFfJv8/kFGEyScjlVSN+WVlwVxLHZJLQWcSrEltefovXP//8Q0pKCoBTI47KHLwPZ7fRZczLZpPOYOJigZY61SIqamiCAOBOlnTGkjCBqRIHvfwWr7Vr15bHOEIeSwehcHQbDUYXMS+HW//MlSIhXmGGu3iWbS5iJdYu/2NeVZVgW14/7z7Pgcxcv84pCdh7trxAxL3CEXceoWWmESp3zEuIl49YY15BaMJx6EIez3yznae/2ubXeb6mSoCYcQxH3AmT1lDygK3MMS8hXj4SEUTLy5JEevJSIef8SCj16DY63NRnc0SuV9jhg+UlUiUEQV0elFNYUtbk75NXkCSJC7nFXs/ztDDb8Z4Wllf44a6Sqoh5uWH9+vUYDM6uk8FgYP369aUaxLx580hOTiYiIoJOnTrx559/uj32448/5tZbb6VGjRrUqFGDbt26eTw+UARzYfaVAr3139tOXOaNlXvp9GYGq/ec93ieScS8KjXuY15Gr8dUBvwWr65du7rsln316lW6du3q9wCWLFlCeno6EydOZPv27bRr144ePXqQlZXl8vh169YxcOBA1q5dy+bNm0lKSuKuu+7i7Nmzfr+3P5SUxAmC5VVUIl5/n7zCl5tPAvDOLwc9nmf0MUkVzJZXZY6PhCK/H8pm7A//lLrMkvuYl3AbXeKu9delS5ecumf7wrvvvsvw4cMZMmQIrVq1YsGCBURFRbmtTvHNN98wcuRIUlJSaNGiBZ988gkmk4mMjAy/39sfSgL2wXUb958vmXGM0XjOdHFcHvT7oWz6zNvIoQt51ieyWmG+BYr0Rq7aiKSg/Bn02Z8s/us089cdLdX5vqRKVOYHks95Xv369QPMiaiDBw9Go9FY9xmNRv755x+/1zzqdDq2bdvGuHHjrNvkcjndunVj8+bNPl2jsLAQvV5PfHy8y/1arRatVmt9nZvrX7qBhcggZthfKSwRFVsP0FKa2h0mh9nGQZ+Z3esRX2/j00EdAVAr5ehNJiQJa1a2oGI5l+M9fukKt26jneVVqkuHBT6LV/Xq5nbxkiRRrVo1IiNLitep1Wpuuukmhg8f7tebX7x4EaPRSEJCgt32hIQEa5VWb7zyyivUq1fPbSXX6dOnM2nSJL/G5YoolfmjMlxbeqFWVtxch8XyUivldgIT7c3yclMSJztXa30iy2Qgo3LnA4U6gV6Yoq0iMS+fxevzzz8HIDk5mTFjxpTKRQw0M2bMYPHixaxbt46ICNfZ4ePGjSM9Pd36Ojc3l6SkJL/fK0JdIlZFemMFi5fZ8vpX05qsPVhSk9ziyrrDOtvoEPOSKBErmcN2QcUT6CWlxVUk5uX38qCJEycG7M1r1aqFQqHgwoULdtsvXLhAYmKix3NnzpzJjBkzrAvE3aHRaOxc3NKiVshRyGUYTRJFOiPVI53bv5UXOUVmy+vOlgl24uXtxixxG+3dQZMkWZ/IMpnMHMO02SaoWGQEVr20VSTm5bf5cOHCBR5//HHq1auHUqlEoVDY/fiDWq2mQ4cOdsF2S/C9c+fObs97++23mTJlCqtXryY1NdXfX6FUyGSykiVCFRy0z7mWKnFTk3iq2biK3tI2LG6jSbJfoGsWPXu3EUQH5mAhD7ARrxUxL9cMHjyYU6dOMX78+ICUgU5PT2fQoEGkpqaSlpbG7NmzKSgoYMiQIQA88cQT1K9fn+nTpwPw1ltvMWHCBBYtWkRycjKZmZkAxMTEEBMTU6axeCNSrSBfa6jQDkJ6o4m8a51g4qM1pDSMY8Phi4B3EbWNdRnsxKvkppZREnOpxA/pECewlpfdbGMlfiD5LV5//PEHGzZssJbFKSv9+/cnOzubCRMmkJmZSUpKCqtXr7YG8U+dOoXc5tE0f/58dDodDz74oN11Jk6cyBtvvBGQMbnDYnlVZKKqbfpC9UgVHZPjS8TLy8ynrbVlK2QKmczObRQEl4DHvKpIhn2p+jYG2o8eNWoUo0aNcrlv3bp1dq9PnDgR0Pf2h2AsEbLMNMZGKFHIZTzaqSFL/jrN2ZwiivSeUxuMkq3lVXKsUiGzPpFlWGIulfkZHdoEfLaxigTs/fa2Z8+ezdixY4MqIsEiGIuzLTONNaLVANSM0fD+gJRr4/Dsvto+dW1nHJVyB8vL6jZW3hs9lAl0wL6qLA/y2/Lq378/hYWFNG3alKioKKcy0K6WDlUWgpFlb0lQjbOZ3fS1DZvJTcxLqZDbiJdNwL4S3+ihTODdRpt6XpX4j+q3eM2ePbschhEeBKOO/ZVrbmNclNq6zddsf1u30TbmpZQ7uI0i7BVUAh13tK3nJWJeNgwaNKg8xhEWRARhcfZVi9sYVWJ5lUwcuI95SQ55W3qjQ8zLxvISBJcvNp2gQY1Inry1SUCuJ+p5eeDo0aO8/vrrDBw40Fr94eeff2bv3r0BHVyoERy30YXldU28dEaTy0KD4LwkqMAmPqaS27iN1/4D4TYGk6n/3R+wa1WVmJff4vX777/Tpk0btm7dyrJly8jPzwdg165dAc2+D0WCUcfeUg4nztbyslkWVOxmMbXR4a4t0JaMWWHrNspKxHjf+auBGbQgqNjONlbmmJff4jV27FimTp3KmjVrUKtLrIE77riDLVu2BHRwoYa1g1AQUiVsA/Yam3WV7oTU8Z4t0JZYXtdWA5n/bXPMiK+3l22wgpCgquR5+S1eu3fvpm/fvk7b69Spw8WLFwMyqFAlGMuDLFVULakSYL9UyV3CrJPbaCNeBlNJTpdIUq18iDwvN8TFxXH+vHP54R07dlC/fv2ADCpUKZltrLjlQSVuo9pue6SX+Juj25hvI15Gk2RXEkdQuSir5fXbgQt89PvRkHc5/RavAQMG8Morr5CZmYlMJsNkMrFx40bGjBnDE088UR5jDBkighCwd+U2gvf4m2OnZDvLy2hreQVooIKQoawxr9d/3MP0nw+w73zpCndWFH6L15tvvkmLFi1ISkoiPz+fVq1acdttt3HzzTfz+uuvl8cYQ4aoCkiV2H7qCluOXbK+tmbYO1heEaqS8s2ucJ5ttH0aS3azjYLKRVlnGy9fe2CeuBja7fD8zvNSq9V8/PHHjB8/nj179pCfn0/79u1p1qxZeYwvpIgqxw5CWoORGT8f4PONJ1ApZGwf3x2VQm4Vp+pRDpaXl0RVT26jOdteuI2hhskkIS9jur0kSQ5uo3/qZTCarHliZ65UMvGy0LBhQxo2bBjIsYQ8EeW0MPtYdj6jv93B3nNmM11vlMgtNqC4piwKuYzYCPs/laUstTvLy/GeLXSIeZlczDYKgovBJKEuo3gZbP624H/Myzal5nRlEK/09HSmTJlCdHS0XUllV7z77rsBGVhI8PvbcN2dUL8DUOI2BjLP64dtZxi/Yg+FOiNxUSoKtUZ0RhNavfn/YI53Oc4KRnizvEyOllfJcQajSZTECUFc9df0F0evwN+YV562pATT6cuh3cvTJ/HasWMHer3e+m93VKovwu7vYe002DAL+nwINzzgdYbPH/K1BsYv38OPO8z9Jjs1juf9Ae25b84fXMzXojWYrPEuR5cRINLfmJeYbQx5zGWL/KtG7IjjkjF/5dA2vBBQyyvvAhxZA+0fC9glfRKvtWvXuvx3pabZXeafw/+D74dC1gGiWptrjpXVbdx95iqjv93OiUuFyGXw/J3XM+qO61DIZdYEVK3BxNVrtesdg/XgvTCiY6zDdnmQXZ5XmX4Tgb9sOXaJ6pEqWtaNddpXFstr6Bd/8VCHBrRpUN1uu78xL9uH3NlrjYjLbJSc3wXfDoTcs6CpBq16l+1616i4FjjhRkQsDFwMna8VSVz/NvXWPEMkxaW2vCRJ4pMNx+g3fyMnLhVSt3oEi5/qzPPdmqG4FuvQXLOodAaTy3I4FrwG7J3cRkfLy/zvSmUthzhnrhQyYOEW7n5/g8v9hjKI128Hsnjmm+1Olpe/l8wrLrlPtAYT2XlaD0f7wL6V8FlPs3DVuh4Sbijb9WzwyfKyNJz1hWXLlpV6MCGHXAE9pkGdlvB//ybqyH/5Tr2XJ3UvYjRJVsHxhUv5WsZ8t8va/eeuVgm8/WBbp+RTjdIsSlqD0eWibAveano5WV5OGfYlJXEEFcOpy57dsPKIeflredk+5MDsOtaJdd1W0COSBOtnwtqpAByMSWN2xDhGFtWijf9Xc4lPllf16tWtP7GxsWRkZPD3339b92/bto2MjAxrY9pKR/vHYND/IUXW5Ab5CVZqxqM9sdXn0zcdvcjd729g7cFs1Eo5U3q35qPHO7gUJavbqDe5LIdjwdtSJcdiE7azSOb9IuZV0XjLqSuL5WVB67hQ3+/ZRnvxOnOlFEF7fRH88KRVuOg0gmelsfx8pCigzWt8srwsDWfB3KH64YcfZsGCBdZWZ0ajkZEjRxIb6+zHVxoadYbhGRyYfT8t5KeRvukFvedB24fcnmIwmpj962HmrTuCJMF1dWKYM7C9y3iHBduYV4nl5V68ShPzgpL6XiJJteLw9qAwGgMgXmW0vGzdRoDTXqxF5wtkwuJH4Ow2kCvhnpmQOoScbb8CUC0icP1O/Y55ffbZZ4wZM8auR6NCoSA9PZ3PPvssYAMLRWTxjXlUmsIa443IjFpY9iRkTAaTc1maM1cK6b9wC3PXmoWrf2oSK0f9y6NwAWhUJW6jZbbRlYXmb8zL8YmqNwrLK9RwbA5cGooNAXYb/UmXOLcTFnY1C1dkDXj8R0gdcu265nu5WkSpU0ud8Fu8DAYDBw4ccNp+4MABTAH48EMedQxP69O5lPKM+fWGWbD0cdDmWw9Zvec897y/gW0nr1BNo2TOwPa89WBbotTe/3C2lpe7pUHgf8xL7/BUN1jFS6hXReHtkw5EzEtbxoC95SFXK8bcZd7ndIl9K8yB+bxz5sD8kxnQ+DbAbOVbJhICKV5+X2nIkCEMGzaMo0ePkpaWBsDWrVuZMWOGtVFsZSZSreBSgZyTN75CzeS28H/Pw4FV8FlPih/6hikbcvlm6ykA2iXFMWdAexrWjPL5+mprzMvok9voLm3D2xfB8pQX0hU6OC7pKg2Olpe/l7RYXi3rVmPDYa33mJckwfp3zDmRAE3vhIc+h4iS+LetKxqjCaJ4zZw5k8TERGbNmmUtjVO3bl1eeuklXnzxxYANLFSxxpp0Rkh5BOKbwpJH4cJuCufdxv7ifwPXM6JLU16863pUCv+MWzvLy0UVVes4vKyz9OYuCLex4vFm5RoCEPNySlL12200308t68ay4fBFzuUUuZ9Z1xfBimdhzw/m152egbumgsJeVvKviVeUWoHSz++DJ/wWL7lczssvv8zLL79Mbq55LV6lDtQ74Nh4VkpKY2Xa1zT/bTgtOMVizVSOd55O8x73lur6JakSppJyOJ5iXj7ONjpiqX0vxKvi8BqwD4jbWMaYV7H5gdmkVjQqhQy9UeL81SIa1HDwHnLPmwPz57abA/P3zoIOg11eM7c48PEuKGOSamxsbJUSLrCPNV0t0jNq0Q6e//kS/bRv8Hfkzagx0HzzS7BmostAvjcslteVQp3VOvKYKuGunpc3y8uSKiEcx6DgyiIKRKqEpaeBRSj9vaTFbYyNVFE/LhJwkS5xbid8fIdZuCJrwOPL3QoXlLiNgXQZoZRVJb7//nuWLl3KqVOn0Ol0dvu2b6/cddAtlteWY5d4a/UBzlwpQimX8XyPFG68ZZU5t+WPd2HjbLh4CPotNC+J8BFLhv2F3GLAHAOzCJUt3tqfORYjdERYXhWP7Uft6tkSyCTVSJWCQp3RpzQvk0ni8c+2opDLrUITrVGSFB/FiUuFnL5cyE1NapoP3rscfhwBhiKo1RweWQzxnlu25Vktr8ClSUApLK8PPviAIUOGkJCQwI4dO0hLS6NmzZocO3aMu+++O6CDC0Us7to3W09x5koRDeOj+P6Zm3m6S1PkCgV0mwj9PgaFBg7+BJ/2gJxTPl/f4jZmXjWLl6uKErbj8LUMtCPW2UafRyYoK7Z/RleWcSBSJSxJqpaHmy8xr9NXCtl45BLrD2WTee2hGaNR0qBG5LX9RWa1XfcWfDfILFzXdYMn13gVLiix5oLuNn744YcsXLiQOXPmoFarefnll1mzZg3PPfccV69W/tZZkaqSP0CvdvVY9dwtpCTF2R/U9mEY/F+IrgNZe825L6d866xkcRsv5JrXlLlKkzCPw788L0f0ltlGYXqVOxuPXKTfhxs5dKEkncbVXyeglte1h5svMa+j2SXjsqTnVItQWuNcFy5egR+Gwbo3zQfdNBIGLrGbUfSExZoLunidOnWKm2++GYDIyEjy8vIAePzxx/n2228DOrhQ5L52dWmRWI23H2jL+wNSiHVnCid1hKfWQmIbKLwIX9wHO77xen2LeGXlmZ+ArsrhgH3szdXT1dtNaxCzjRXGo59sZfupHMYt223d5urPE5CYl97e8vLFmDuaVeC0zeI21uEKw4+NMs8oypVw/wfQc7rTjKInrG6jJshuY2JiIpcvXwbM1VQtvRqPHz8e8t1GAkHX5nVY/e/beLhjknerpXoDGPoLtLwfTHpYMRL+Nx5M7qtSWMTLU7Ae7BvPOq1nw4/ZRs+HCcoJyYXtFYjlQRbLyxKbdfU+jthaXhZiNEquNx5hpeZ1rtMfgsh4eGIFdBjk95hCxvK64447WLlyJWBOWH3hhRfo3r07/fv3d9nPscqjjoaH/gO3vWR+vekD8xSzNs/l4ZaYlwV3bmOEl8azPs82CtMrKJSX5WWJL1ksc18u6VK8jq6i+U8Pkyi7wiFTfbRDf4XkW0o1pjxrzCuwlpffUrhw4ULrMqBnn32WmjVrsmnTJnr16sXTTz8d0MFVGuRyuON1qN0Clo+EQ6vh07tg4LdQI9nuUMtsowV3bqNSIUetkKMzmijSG6nhsN/dbKPlHGF5VQzuvJHymm20LO+xWOa+eENHsmzFS+JF9Y8ovv8egPVSCs/qRrFSlkjjUo7JmioRTMvLYDAwdepUMjMzrdsGDBjABx98wOjRo1GrXVsJgmu0eRCG/AwxCZC1z5wrc3KT3SEapf2fxJ3lBSXtz1wtEXI32xitMd/UIsO+Ypj/+1GX28trttEiXlFq3yyvywU6a9FLDTrmqOYwWm4WLm56lmmxE8kjqkydhPJCIUlVqVTy9ttvYzBUXMfoSkeDDjB8LdRtB4WX4MtesP0r625Ht9FVFVULnpYIuXuKWxaHl7UkzsHMPEZ+s43DF1y7vwIzb68+6HK7q4dLICyvPAe30ZvlZXEZ63CFperJ3K/YggEF9JoDPd+kfk1zjmJZmnFYLC/HDlhlxe+Y15133snvv/8e0EFUOarXhyGroVUfcyB/5Sj45TUwGZ0sL1dLgyx4KkjoLuZlyXIu62zjgIWb+Wl3Jv0X+pYCIrDHlVsfiJiX1W30MeZ1NCufNrJjrNS8Tjv5MS5LMYyPnQY3PgFgk+tVessr35phH+SY1913383YsWPZvXs3HTp0IDo62m5/r169Aja4So06Ch78HH5vAb/PgM1zIfsgkZ1m2h3mbrYRIPKaFeUYsF++4yyf/XHC6XilXGaNqZXkeZVu+BZX43KBzsuRAle4EhV3lte2k1f450wOg29O9jrBYqmYG+VjzEt9cDlL1VOIlOk4ZKrPMP0YGsS0su5PupbrVaqKqtcoL7fR76uNHDkScN2fUSaTYTQGvpt0pUUuh67joHZzWP4MHFlDy4sP0FD2LKekBMCb5eXc/kxvNPHy9/9Yez7aolHKrdUBSjLsRdArGLgSKnfi9cB8c1y0bvVIet6Q6PG6+daAvfmr7dbykiRYN4N+R2eADP5SpTIkbwT5RNHcZg1iUvw1y8vfiqo2hEyqhMlkcvsjhKuU3NDPHMivVpeInCOsUI/nJvk+wJvl5Rzzyrxa7FK4wBwHsXThNvhgeZ2+XGitCCAILK7cem8xL1cpDe7wmOelK4Tvh5gtfuBjwz38t/W75GO2smxFpoHV8iqdeJlMEvm68kmVEK3PQoX6N8LwtWjrpFBDls9XqukMUPzmNlUCXC8R8mTe21pe3mYbT18u5Na319Jhyhp/fxOBD7gSqkDEvCy4zbDPPQef3w17f0SSqxirH840w2OkNKplPcQyIw0lbuPFfF2pmmcU6AzWtJCguY1FRUVkZGRw3333ATBu3Di02pKebgqFgilTphARUYo2SQIzsXW50n85f84eSC/FZmaoPoE1KpcF3sB1KWhPT8gIlQKlwuI2ep5t3HrcvIrCsXy0IDC4trw8p0r4E58sCdjbvM/Z7ebmr/mZEBnPqe4fsXipnmoRSprWjrEeZhtYrx6lolqEkrxiA2evFNEswfcKKVDiMqoUMqfJqLLi89W+/PJLPvroI+vruXPnsmnTJnbs2MGOHTv4+uuvmT9/fkAHVxXRRETznH4Us/QPmjdsnQ+LHoZi50XvrmYbz+bYW162FTDVSjkKuflPbjB5try8fZFsycorLlMeUFXE1ccbUMvLGrC/tmHPD2aLKz/TnCw9/DdWXTVXhGhWJ4bqNik5jhaSxfoqzYxjSbzLdXWUsuCzeH3zzTc89dRTdtsWLVrE2rVrWbt2Le+88w5Lly4N6OCqIubZQBlzjP2YFj0WlJFwNAM+6QaX7BMerTEvD26j2qbsboRKgfKamOkMrqtKWGanHL9IJpPEGyv30vfDjVzILbYTxbRpGdzy1lqn+NiEFXu45/0NbktVVyZW7DzLlmOXfD7eZZ5XAK1caw04yQhrp8P3Q8FQjKFpNxi2hrPyROb+dgSAx25qRGxkiWBFq+1zDa3pEqXI9SqPrkEWfBavI0eO0KZNSa/biIgI5PKS09PS0ti3b19gR1cFsRWbfTW6wtDVUK2eubDhx3fAsZIcO1eWl6MFpFbaipfNbKO1kqo9Fs1yjMn8uOMsX2w6wY5TOTy/eIfLmuZnHG7u/2w+yb7zufyyN9Pp2MrEoQt5PL94JwP8yHkr95iXWkEEWl4tfMcamF9ouJc+l0ZDRCyT/28vRXojaY3j6du+vl2V0xiHwHpS/DXLqxQzjrnlVEUV/BCvnJwcuxhXdnY2ycnJ1tcmk8luv6/MmzeP5ORkIiIi6NSpE3/++afbY/fu3csDDzxAcrI532X27Nl+v1+oo1TIrdZRXJQa6qWYS+vU7wDFOfB1P/jrU6Ak5mW7PMjRbbQVL42yxPJy5zZavlSOzSBsrYodp3Ks17HFnVfgbx31cMP2S/3Waue2gK5wlX/lbbbRNj656chFt8cp5TKiirNYqp5MF8NGkKtYljSWNw2PsiezgLUHsvhl7wWUchlTet+ATCZDqZBbBcZRaJJquCkH7QPllSYBfohXgwYN2LNnj9v9//zzDw0aNPDrzZcsWUJ6ejoTJ05k+/bttGvXjh49epCVleXy+MLCQpo0acKMGTNITPSc7xLOWAKb1qVB1RLNxQ3bPAQmA/w3HX56iSil+Wa3WF4Go4nzOcUurwUOlpebhdmWL5DjF8n2pdZgcml5uROvSq5ddhbT/HVHnRq3usKVTnmzvGw/30c+2er2uA7KY7T+b2/ayo+TQywMWsmGmJIqxxNX7gVg6C2NaZ5YEoC3xL2cxCu+LDGv8ikBDX6I1z333MOECRMoLi522ldUVMSkSZO4917/Oua8++67DB8+nCFDhtCqVSsWLFhAVFSU287bHTt25J133mHAgAFoNBq/3iucsHTNtluUrYo0l5e+Y7z59Z8LuX/3c8SSb40pXcjTOn0B3FpebprOGt3FvBwUyKXl5WbmsrKLl+NSH52L+mqOuE5SdT7PWy8CR+6Tb+ZL+STURVkcNDVgROQ70Ohmu96Jpy4XkhgbwfN3NrM7t0frRBrUiOSG+vZNdSy5XqVxGy1Lg6qVg9vo8xVfffVVli5dSvPmzRk1ahTXX389AAcPHmTu3LkYDAZeffVVn99Yp9Oxbds2xo0bZ90ml8vp1q0bmzdv9uNX8IxWq7VzZy3t2kIZS9zLqV+jTAa3jTFn5C97inqXt/Cj+iQfF7wJdOCsC7PePmAvtxYp1LtpOmsJGjt+kRy/bAq579PelVy7nILvBm+VIHFXVaL0cTAZJv6tXMbzymUAXE26gwcOP0J8jDl/y1KZ18KE+1sR7SAoE+5vxfj7Wjo90CwB+9xiA1eL9HYzk94ICbcxISGBTZs20bJlS8aOHUvfvn3p27cv48aNo1WrVvzxxx8kJCT4/MYXL17EaDQ6nZOQkGBXcqesTJ8+nerVq1t/kpKSAnbt8sKy/tDt0qCW98PQXyiKrEtT+XlePz8ajq61BustNxvYu42uLS/7S5fF8nJHZa+w6yjs7lY4eDrH3TZfyuREoGWu6gOrcC1R9eF4t0/IJ8r6dzt5qcRquu362tztZpmRq3SGaI2SmtHme9HflJiQcBsBGjduzOrVq8nOzmbLli1s2bKF7OxsVq9eTZMm3ruIBINx48Zx9epV68/p06eDPSSvOMW8XFG3LX91/4FtpmbESPnw9QPE7/0SgLYNShoj2H4fIlRyFApLhr3lS2F/s1q+LM4xL0fLy/eYVyBKvYQyjp+NL52vS2N5ufp4E7nEd+pJ3Kv4E52k4CX9U3wR86S5kxVml/1qoZ6r17qv160ewZTerf3OuSptukRIWF62xMfHk5aWRlpaGvHx8aV641q1aqFQKLhw4YLd9gsXLgQ0GK/RaKzNccOlSe4DNzagTf3qdGzs+bNVxCbwiO41flV1BcnI7UffZoryM1rWLuluXGCzpMPW8rJkzjtqkOVBb/ulkSTJKalSpXAV83LN2GW7K3X1CUex0pfSbXSV5+VJCNvJjrBCM5428hNckqrxiO41vjPeToRKjvyaOJkkiZOXzQ02alfTsHncnTSqGe32mu5oEF+6NY6W+mKBrqIKQVzbqFar6dChAxkZGdZtJpOJjIwMOnfuHKxhhQRPd2nK/42+xWtsIUKlQIuaycrnoNsbmJDxuPJX+h/6N9UxL+ItsJn5ss/zcr0w2+I22lpLJsk5riP3w/ICWPJX6Fu8pcVRiHxZUuVK31xbXiUH2u6+X76JJeopJMhyOGBKorduCn9LLQCIsCloKUklLmOj+JKHmr+UtjROyLiNgSY9PZ2PP/6YL7/8kv379/PMM89QUFDAkCFDAHjiiSfsAvo6nY6dO3eyc+dOdDodZ8+eZefOnRw5ciRYv0JQsSapGkxwywu8rhlHgaShzsUtLFePp6nsLIXakhww2wx7dyVxLE9/2ye+0SQ5zXq5jnlVzfI6jkLkS5zKleXlKlZm+3cwGE1m0/i3acxRzyVCpudXY3se1E3kjFTHepzG0fK6ZLa8GtYsg3iVsjROebqNgb+iH/Tv35/s7GwmTJhAZmYmKSkprF692hrEP3XqlF0W/7lz52jfvr319cyZM5k5cyZdunRh3bp1FT38oGO7PMhkkvguvw3bTZNYVWsOjfPO8qN6IqP0z7GetoClqoSltZo3y8v2iS+5CNg7P/c8WV6+tOAKVxyt0mXbz9K2QZzHc1ylQOS5KD9kFy/UF5o7Vu83d+9aYLiPtw0DMDnYIBFKBZY/j8nO8vLfXbTQoJTrG/NCIVWivBg1ahSjRo1yuc9RkJKTkyv9zJU/2C4PysrTojdKHJE3guFr+WvmfXSUH+Jz1VtMMTzOF8YeaFQKLJkT7jPsTXb7wSxejp6Qq4B9VcVRiL7YdIIXul/v0e13tbYxt8hZvCwPmUQu8fDuSVBwEOQqxmiH8r2xi8tr28a8JEni5DVrqVFZLC+bLHtJkjwG/CVJ4ts/T6M3mqwJu+XhNgZdvASlxyJeBpPEsYvmGFdi9QiUsQk8JZvIa8aPeFCxnjdU/+F62Rki5O9ZLS/LE93JbTTB+OV7+GbrKes2k+Sc7uBytjFwv1pY4SpWVaw3uuynacHVDKxtIqntcSmyIyxUv0udghyIqgn9v+H7+ZfdXlujVFj/FhJw6prlVRa3sf418SrUGblcoKNmjOsk8XytgTFLd7HaYT1ryMw2CkKDmAil1XLad86cfGuZ0v7u2dvZ3n4a5zu9hkmS8YjyN27ZPJwYo0NpHQfFMZhMfLXlpN02kyQ5fdl2ns5xGo/np7HztsqSQuHKBdx+8go3Tc9wcbQZV5+Hq6q1kQd/ZIl6CnVkOWRFNjF3nmrkeUIrQiW3/i2KdEYyc80JqmUJ2GuUChJizYJ12k3Q/vjFAvrO2+gkXCDES+CAQi4j/loi664zZlGqH2e+Qa+rE8Ob/doS0/UFntS/SL4UQXz2n/TfOYjrZGes15A7CI6rWLPJJHEw03uLM38sr5m/HCRl0v+sweRwI/NqMWN/+If953NduoCL/jzl4qwSXAl3bpGN5WUywW9TafDbaDQyPWuMN/LJ9R9BjUZexxahUlhTYCzrXmM0SuKjy9ZXNcnDMqG1B7LoNfcPDmflU6eaxi7XUCaDaLUQL4EDlhvynzM5gH12PZgD67+ZbqSfbhLFMQ2oXnyWZeqJ3C7fCTgLjquZMpMEOS7iMa5YsfOsT7XW5649Qp7WwHtrDvl03VDjuW93sPiv09z9/gaXQuQtCdSV4BXpjeYYl67AHJhf/w4ACwz387Q+nSKZb5aTRil3eig1qhlV5mKASfHO6RKSJDH3t8MM/fIv8ooNdGhUg1Wjb6Fby5KVMzFqpcvUmrIixCvMsYiXZUbJUbwssalDUhKHe60kq8aNxMqK+FT1DsMUPyFzmAV0NYVvkiSfJkp+2nOe5xfv5M5Zvvf1DNcq0/szS9bI7jnrXOXW21fV3YLrooun4LOe5hlFhZpj/3qHGYaBmJB7XOdoW0BQo1I4TcSUJVhvIcmhh2O+1sAzX29n5v8OIUnw2E0N+Xb4TdSJjaBZnZKy0uXhMoIQr7CnlkPgtL6T5VVyFyur1eafO/7DYsPtKGQS41VfMyBzJipK3BVXGd0mk+S1eSnAql3n/Rx9+Nb6stWGn/c4x3i8GTmucrpSZEeI/rI7ZP4DUbVg0P+R2aSfdb+n0tyRNm6Z2W20H0DDMqRJWLCtLmEb31Ir5Lz1QBum9mljrWJiW+vecQF4oBDiFeY4xjEscQkLcrnMGv+IUCmoWT2GsYbhTNE/hlGS0fnqf/la/SY1MFsSrtwZX+Pq+877X7EjXFNfyuoGPb94p93rXvKNLFFPQVGYBXVaw/DfoOFN9kmqniwvm44/5oC9/f5AWF4NriWqbjh80RrfSojVsPjpm+jfsaHb9yuv9nlCvMIcW/GSy8ypEo7ERamRy8zF5syWmoxPjfcwTP8SRfJoOskPsEI9nmayMy7jN+VpHYXrjKM36fJV2mSYeFG5lA/U89DI9OQ26g7DfrEG5m0/H0+fVZSN5aVROlteZZlptGD7YMwrNpDaqAb/N/oWbmxYw+lYlU0ppgu5/ldY9gWR5xXm1IwpEa/E2Ai7m8bCgsc6kFOoIz5aTYSqZP86UwofNP6QAYdfopE8i2XqiRw/HQfE2J1fngITptrlNfjty68VSTHvqebTU/EXYA7M33jbe+hOadl7Lounbmtit8jb0yJt25iXbZKqhbLkeFmoa/NgrB8XyaLhN9kVu6xohOUV5tSMLol5Najh+gZNaxzPXa3NlTqi1EprN2WA7IjG9NFNZoupJdVkRbRZ/zTDFauw/foFwvBy5x6Gq9vozbLy9mvV5RLfqyfRU/EXWknJi7oRzDAMpNAg8dinW5n+8wHWHsyye3B4WjMZZRNXilDaB+xVChl1q0e6OMs/lAo5nRrHE6NR8ungVK/C1a99fQDuaFHH43GlHk+5XFVQYdi6jY7BenfUjFFTeK0ukwy4QiyP68YxSfk5jyjX8ppqEc1kZ3ndMBQdKpdxsEARTm5jkc5oXU9alqyDFNkRPlbPorbsKhelWJ7SpbNdut76HhZOXiq0y2T39FnZzzbax7ySakQFbDnXV8M6UaQzeuzkbmFq3xu4qWlNu7SJQCIsrzDH1m10TJNwe46NtWa5yfUoedXwJHvbvopRkvGw8ne+Vr9JTa6Wa8wrXLRr/aFsWk5YzZyMw4B3t9Gxi5MFS2C+tuwq+00N6a2dYhUusO8EZTRJdjOMngL2kXZuo33MKxAuowW1Uu6TcIHZyn84NanMybHuEOIV5tjeGL6Kl216hf3aRhlHmjzGEP3L5EpRpMkPskIzHv05912j/OVsThFjvttlfR0uqRKv/rgbgFnXkmq92TFHsuwTdWWYGKNcYg3MrzF24AHdG5yltt1xhTbFIyXJvjaYZ8vL3m20Fa9ABOtDESFeYU6NKLXVerIsDfJGLRtrzdGA0Bsl1pva0Vc3ieOmBBrILtJgWW+6ybeVaZx7zubyx+GLjPxmO99vK1meFC7iZZtU+tuBC365jVEUM1/1PqOUKwCYb7ifp/QvUIjzzLCt5TXtp/3MX1fSJd1ThdYojYPbaLOvYSkqp4YDQrzCHIVcRuOa0SjlMpolxHg/AQfLy+FL+Pa1pqlHpfr00U1ho7E1MbJiFqre5WnF/1HaXkCr92by2Kdb2eWwoHvjkUthEbS3NXqGfvG3z9P/9bhoF5hP143gLcNAJDdfvUKHShTHL5as/fSYKqFyb3klB9BtDCWEeFUCvhyaxg/P3ExCrPOT3BW2cTJHBygrr+RLeZUYPk2eydeGO5HLJMapvmWWagFqApt06CpDPdQozaTFjbJDrNCMp5X8JNlSLAN1r7PMdJvHczwldPqVpGrzzQ5EgmooIsSrEpAUH0W7pDifj7edwbKdhGoYH0UNh2Bsv47JvG4Yxnj9YAySnAcUG/hWPZVaOK/ns2XoF3/5PJ6M/a47pIcKkiSRnedfomVv+R98q55mDcz3cQjMu+OqhwXwHmNeGvsk1Qilgii1ghiN0m0KTbgjUiWqIO5iXvHRampX07Dt5BXrNsvM5FfGuzgu1WWe6n06yA+zXDOe4boX2S+5LtHy2wHfBcmXbjsVSaHOwPu/HubuNnUxGE08uMD3JsjmwPxSnlWaSzX/z9iBf+ufdRnfcoWraqoWPCWpRjmkSqiVchY/dRMKuYwIlcLteeGMEK8qiLvZRoVcZi04Z8HWxfzD1IY+uil8oppJU/l5vle/wQv6kfzP1LFM4/GlYUVFMvvXwyxcf4yP1h/z67woinlP9SE9FH8D8KGhF+8YHnYb33KFJ8vL0+dkqaork5X0/fRWRz/cEW5jFaSmTXqFbSxHIZNRp5q9haCUy4izcSWPS3Xpq5vEBuMNRMu0LFS/x0jFCkobyAfQGUJLvPaXYoG5JTDfQ/E3WknJC7pneNswwC/hAoeChA5YYl6uyulYst01SnmZ63aFC0K8qiA1okrE64pNM1iz5WUvXjKZzCnJMJcYButf4QvDXQC8rFrCe6oP0VC6xrK6ECvq5a+YmgPzr9sF5n803Vqq987XuhcvS8zLVTkdy5pWjbJyuoiuEOJVBbEt53LJRryUCme3UYa9pWbBiII3DIN5XT8EgySnr2Iji9VTqU2O3+MxhFDM65Xv/2HrcffNLRzpI/+Db9VTqS3LZZ+pkc+BeXe4y8yHkpiXqxihZfmP7cL7yk7V+U0FLrmUr7XGSLpcX5tEB8tL7sLysuVrY3ce148jR4qmvfwIKzSv01p2wq8xhFLAfsnfvnX2lmHiZeViZqs/RCMz8IsxlQd1E50y5gOJxfJy1ZE79lprMVururIjAvZVnCuFetaOuZ0/j1/mvrZ1OXHJvrmCTAbx0a7bXFnYbGpNH91kPr0WyP9OPYkX9M/wiynNpzGEmtvojSiKma2ax10K86qDeYZezPQzMF8aDCaJIp2Rhz9ynv1sWbcak3u3pnW92HIdQyghLK8qzuUCHfXiIunTvj5KhZyGLtbBuXIbHTkh1aWvbjLrjW2Ikmn5SD2bUYof8SWQv+t0jss68BWNq47VjtQnmx/Ub3CXYhtaScW/dSN5pxSB+dJgMJn4bONxp3WTYI5NPtE5mQ6N4st9HKGCEK8qiiXXyzH7Wq2U8/PzJcFms+XlmyuSSzRD9C/zuaEHAGNU3/G+ap5PgfyHFmzmSJb39mrlyS1vrfW4/0bZIZZrxtNSfupaYP41lptuqaDRgdHougVdr3b1KmwMoYQQryrK4qc606tdPT55ItVpn62lJZPJHJYTecaIgkmGQYzTD0MvKeit2MQS9WTqcMXjeUV6I93eXc+KnWd9/yVKwZebTvDjjjMu93nKseor32AXmO+tnVqmwLwjbz/Q1usxWoOJP45cBLCuhEiKj+T9ASkBG0c4IWJeVZTr6sTwwcD2LvdpbDKyjUbJrv6Xr3xrvJPjUl3mq2aTIj/GCs14huvS2SM18Xje84t30q5BHMm1Al8J4fTlQiau3AtA3/YN7Pa5m+UzB+aX8Izy/wD4xZjKC/qRPmfM+0q/G+vz8g//eDxGZzRxuUBHXJSKP1/rhtEkoZTLqkxelyPC8hI4YTvdrjUYS11MboupFb11Uzhsqk9d2WW+U0/mHvkWr+c9MH9Tqd7PG7aWVb7WwMYjF60zeAtsSs9YiKKYj1TvWYVrrqE3I/T/DrhwgbnEsq90b5mASiEnQqXw67zKRtX9zQVuUdt8IRRy/9xGR05JCfTTTWKtsR2RMh0fqj/gecUPeArk2+aelRdPfLqVRz/ZyoLfzaIlOYzHMTD/vG4kMw39KyQw7wqlXEanxvEM+VcyL97VPChjCDWEeAmckMlkvHZPS56+rQlNase4zB26sWEc1/tYPyyPKIbpX+Jjwz0AvKD6gbmqOUTgvlLD01/9zYCFm912li4r20/lANgVRrRgH5ivzgDd66yogMD8zIfaud03re8NLHm6MxPvb+2yvV1VRMS8BC4ZfltJbMqxS8yBKT2JUCm49e3ffL6eCTnTDI9xWKrPVOVn3KfYQkPZBYbrXuQCztP7v+y9AMCR7Hyut+m+HGiOXyxg28nLbD+ZA0A/+Xqmqz5BIzOw19SI4boXOUetcnnvhFgNF3K1NK1tju892KEBCjm8sMRcJvv+dvXYd+4qpy8X0SKx6uRv+YoQL4FfTOrV2lpipUjnf2b8UmNXTpgSWaB+j7by46zUvM5w3Yv8IzV1ebynMjCB4oH5m5FjYqxyCSOuxbdWGzvygv4ZinyMb8VGKBl3T0te/XG317Zn61/qSv0akeiNJn7ec55bm5Vk5d/fth6bj16iY3I8D6UmIUkSWoOp0pa1KQvCbRT4xPsDUujbvj4D0pKs24r1Rg9nuOdPqSW9dFM5aGpAgiyHperJ3C93HaQPVGs0k0li3znX1SKiKeIj1XtW4Zpj6MMz+ud9Fi4wxwYHpjVkzQuuK6W2rBvLfW3r8t2IzjSsGWWts9W3fQO7EkVKhZy3H2zHQ6nmz1kmq7z1uMqKsLwEPtE7pT69U+rbbSsqpXgBnJHq8IDuDd5XzeNOxQ7mqOdyneEssw0P2AXFA1Xr64Y3fnGqDw/mwPwn6pm0lJ9GK6l4Sf8UK03/8vv62muVKK6rU+LiRqsVjL2nJWnJ8TRPLD/Xt6oiLC9BqfFmFblaamRLPlEM17/IR4Z7AXhe+SPzVB8QSbH1mEB0F7qYr3UpXB1k5tZuLeWnyZLi6K8bXyrhAtdWaIHOyOM3NRLCVU4I8RKUmtn9Uzzu96U8iwk50w2PMkb/NDpJwT2KP/lOPZlELgGwcP0xvtx0wq6TtD9cLtCROvVXp+0PyNezSD2NWrJc9poa0Vs7hZ3SdaV6D7DvLnTLdeYAf7eW5dPmXmBGJoVD36kAkpubS/Xq1bl69SqxsWIGp6zkFev58/hlhn35t9O+6f3aMG7Zbp+v1UF2kI/U71FLlkuWFMdTunQ7QUnvfj0Ppyb5lSrwf7vOMfrbHdbX8mulbEYoVwHws7Ej6X4E5t3Ro3UCHz1uXmqVU6hj+Y6z9EqpX27doiszvn5HhXgJAsLaA1kMcegYtGr0LdSLi6R6pIqj2fnc9d56r9dpIMvmY5XnGNS7D7ej340N3FzBnk5v/mrtsRhNEbNV8+iu2A7AB4Y+vGd4MCCJp7sm3kX1SJX3AwVe8fU7KtxGQUDo2qIOTWrbr0fUKOXER6tRyGVcn1CN/ZN7er3OGak2D+reYI2xAxqZng/U8xijXIKMksB9+tJdPrUiW/rXaatwNZCZM+a7K7ajlVQ8p3uWd/2swbVvcg/6ta/vcp8QropHiJcgYPzv37eR8WIX62vH5NZItYI9k3p4vU4BkTylf4EPDb0AGKVcwXzV+0TZBPJtO0m7YvupK9aFzqmyAyxXj6dFGQPzUWol7/ZPoXY1/xeqCwKPEC9BwFAq5NSrHml97aq8c4xGyc1Na3q9loSctw0DeEH3DFpJSU/FX3yvnkQ9zCVhHvtkK54iHv0+NOeNPaj43RqY32NKppcfgflZbpbrrBp9C+8+3I6MF7tQPVLF6DtKH+gXlB4hXoKAEqlWcH1CDPWqR9Aw3nVZm6+HdeLNvm18ut6PplsZqHudbCmWVvKTrNC8zo2yQ+iMJuatPeL2PHPG/CJmqj5CLTPykzGNh3QTyMS7cALERal4oENJXK2aTUfqhNgI+t3YgKa1Y9gxvrtYKB0khHgJAs7Pz9/G2pdud3IbLcjlMh5KbUD3Vgl225vVcb3Qe7t0PX20U9hnakRtWS7fqqfSV76Bmf87RO95GzmWXVIWWZIksi9dZKFqlnVG8X1DX57VP+d1RvG6OjF8P6IzPVonsGq0eSH29H5tUMplfPjYjW5/F0FwCAnxmjdvHsnJyURERNCpUyf+/PNPj8d/9913tGjRgoiICNq0acNPP/1UQSMV+IJCLvPaP1ClkPPxE6lsH98dgKdua8IrPVu4Pf4stXlQN5FfjKloZAbeU8/nFeW3/HP6Mk99ZW6Ecf5qEbe++iWX3r+dbood1wLzo3jP8JDbwPwN9WO5vXltnr+zGStH/YvU5Hg+ejyVBjXMCbYD0xpyYEpPu/WHgtAg6MuDlixZQnp6OgsWLKBTp07Mnj2bHj16cPDgQerUcU7y27RpEwMHDmT69Oncd999LFq0iD59+rB9+3ZuuOGGIPwGgrIQH63mxAxzhr238jeFRDBC/29elL5jlHIFzyj/j+tk50jPHokkSYye8SEr1O9RU5ZHlhTHcF06u9zEt568pTH/3X2er4Z2ooaXXKyqXPAvlAl6nlenTp3o2LEjc+fOBcBkMpGUlMTo0aMZO3as0/H9+/enoKCAVatWWbfddNNNpKSksGDBAq/vJ/K8Qp/LBToenL+JYx5mFHvL/+Bt1cdoZHr2m5L4wXgbLysXo5YZ2W1KZrjuRaf4Vqu6sbzzUFta1Y1FJpMhSVKVLaEcyvj6HQ2q5aXT6di2bRvjxo2zbpPL5XTr1o3Nm5170wFs3ryZ9PR0u209evRg+fLlLo/XarVotSU5Qbm5risLCEKH+Gg1v4253fpakiRMEsz830HmXyvXvMJ0Cyd1iSxUv0tL+Wlel38DwH+NaYzRj7DGtx7t1JCxd7cgSq20dpW2IIQrvAmqeF28eBGj0UhCgn3gNiEhgQMHDrg8JzMz0+XxmZmZLo+fPn06kyZNCsyABUFBJpOhkMErPVvwSs8WHM3O5/1fD7NyF/TWTmGhehZt5Cd439CX2YYHqBahZssrd4jE0UpO0GNe5c24cePsLLXc3FySkpI8nCEIdZrWNnc+snQ/yi18mMPnTvP8dc14PshjE1QcQRWvWrVqoVAouHDhgt32CxcukJiY6PKcxMREv47XaDRoNCIjujITGxVB7HXNgj0MQQUT1GkUtVpNhw4dyMjIsG4zmUxkZGTQuXNnl+d07tzZ7niANWvWuD1eIBBUToLuNqanpzNo0CBSU1NJS0tj9uzZFBQUMGTIEACeeOIJ6tevz/Tp0wF4/vnn6dKlC7NmzeLee+9l8eLF/P333yxcuDCYv4ZAIKhggi5e/fv3Jzs7mwkTJpCZmUlKSgqrV6+2BuVPnTqFXF5iIN58880sWrSI119/nVdffZVmzZqxfPlykeMlEFQxgp7nVdGIPC+BILQR9bwEAkGlRoiXQCAIS4R4CQSCsCToAfuKxhLiE8uEBILQxPLd9BaOr3LilZeXByCy7AWCECcvL4/q1au73V/lZhtNJhPnzp2jWrVqYb8w17LU6fTp01V65lR8DmYqy+cgSRJ5eXnUq1fPLk3KkSpnecnlcho08K1tVrgQGxsb1jdroBCfg5nK8Dl4srgsiIC9QCAIS4R4CQSCsESIVxij0WiYOHFila+aIT4HM1Xtc6hyAXuBQFA5EJaXQCAIS4R4CQSCsESIl0AgCEuEeAkEgrBEiFeYcfnyZR599FFiY2OJi4tj2LBh5Ofnezxn4cKF3H777cTGmvsV5uTkVMxgA4joqm7Gn89h7969PPDAAyQnJyOTyZg9e3bFDbQCEOIVZjz66KPs3buXNWvWsGrVKtavX89TTz3l8ZzCwkJ69uzJq6++WkGjDCyWruoTJ05k+/bttGvXjh49epCVleXyeEtX9WHDhrFjxw769OlDnz592LNnTwWPPLD4+zkUFhbSpEkTZsyY4bZBTVgjCcKGffv2SYD0119/Wbf9/PPPkkwmk86ePev1/LVr10qAdOXKlXIcZeBJS0uTnn32Wetro9Eo1atXT5o+fbrL4x9++GHp3nvvtdvWqVMn6emnny7XcZY3/n4OtjRq1Eh67733ynF0FY+wvMKIzZs3ExcXR2pqqnVbt27dkMvlbN26NYgjKz8sXdW7detm3eZLV3Xb48HcVd3d8eFAaT6Hyo4QrzAiMzOTOnXq2G1TKpXEx8e77Rge7njqqu7ud/a3q3o4UJrPobIjxCsEGDt2LDKZzOPPgQMHgj1MgSCkqHIlcUKRF198kcGDB3s8pkmTJiQmJjoFZw0GA5cvX66cAVkqpqt6OFCaz6GyIyyvEKB27dq0aNHC449araZz587k5OSwbds267m//fYbJpOJTp06BfE3KD9EV3UzpfkcKj3BnjEQ+EfPnj2l9u3bS1u3bpX++OMPqVmzZtLAgQOt+8+cOSM1b95c2rp1q3Xb+fPnpR07dkgff/yxBEjr16+XduzYIV26dCkYv4LfLF68WNJoNNIXX3wh7du3T3rqqaekuLg4KTMzU5IkSXr88celsWPHWo/fuHGjpFQqpZkzZ0r79++XJk6cKKlUKmn37t3B+hUCgr+fg1arlXbs2CHt2LFDqlu3rjRmzBhpx44d0uHDh4P1KwQUIV5hxqVLl6SBAwdKMTExUmxsrDRkyBApLy/Puv/48eMSIK1du9a6beLEiRLg9PP5559X/C9QSubMmSM1bNhQUqvVUlpamrRlyxbrvi5dukiDBg2yO37p0qXS9ddfL6nVaql169bSf//73woecfngz+dguRccf7p06VLxAy8HREkcgUAQloiYl0AgCEuEeAkEgrBEiJdAIAhLhHgJBIKwRIiXQCAIS4R4CQSCsESIl0AgCEuEeAkEgrBEiJcgJBg8eLDLaho9e/YM9tAEIYqoKiEIGXr27Mnnn39ut81d92e9Xo9KpbLbptPpUKvVfr9vac8TBBdheQlCBo1GQ2Jiot1PjRo1AJDJZMyfP59evXoRHR3NtGnTeOONN0hJSeGTTz6hcePGREREAHDq1Cl69+5NTEwMsbGxPPzww3alZNydJwgvhHgJwoY33niDvn37snv3boYOHQrAkSNH+OGHH1i2bBk7d+7EZDLRu3dvLl++zO+//86aNWs4duwY/fv3t7uW43mC8EO4jYKQYdWqVcTExNhte/XVV61djx555BGGDBlit1+n0/Gf//yH2rVrA+a6Xbt37+b48eMkJSUB8J///IfWrVvz119/0bFjR5fnCcIPIV6CkKFr167Mnz/fblt8fLz137aNRyw0atTIToD2799PUlKSVbgAWrVqRVxcHPv377eKl+N5gvBDiJcgZIiOjua6667zuN+Xbb6+lyC8ETEvQaWiZcuWnD59mtOnT1u37du3j5ycHFq1ahXEkQkCjbC8BCGDVqt1auOlVCqpVauWz9fo1q0bbdq04dFHH2X27NkYDAZGjhxJly5dXLqdgvBFWF6CkGH16tXUrVvX7ueWW27x6xoymYwVK1ZQo0YNbrvtNrp160aTJk1YsmRJOY1aECxEGWiBQBCWCMtLIBCEJUK8BAJBWCLESyAQhCVCvAQCQVgixEsgEIQlQrwEAkFYIsRLIBCEJUK8BAJBWCLESyAQhCVCvAQCQVgixEsgEIQlQrwEAkFY8v/y7yJC59kfzAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.018825, + "end_time": "2024-03-24T18:03:03.919922", + "exception": false, + "start_time": "2024-03-24T18:03:03.901097", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4452.398135, + "end_time": "2024-03-24T18:03:06.660126", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/lct_gan/42/mlu-eval.ipynb", + "output_path": "eval/contraceptive/lct_gan/42/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/contraceptive/lct_gan/42", + "path_prefix": "../../../../", + "random_seed": 42, + "single_model": "lct_gan" + }, + "start_time": "2024-03-24T16:48:54.261991", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/lct_gan/model.pt b/contraceptive/lct_gan/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..6aa42be17b6c36654fef44b8523331b3eab1eac2 --- /dev/null +++ b/contraceptive/lct_gan/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6785c703d4ee8c5add91cabf03b6ea2bd3d4c747dee8755337a5808a74b4c23c +size 30465390 diff --git a/contraceptive/lct_gan/params.json b/contraceptive/lct_gan/params.json new file mode 100644 index 0000000000000000000000000000000000000000..18bb60501cb6fc3c87f401c6ae0b9c8b7c8eeaec --- /dev/null +++ b/contraceptive/lct_gan/params.json @@ -0,0 +1 @@ +{"gradient_penalty_mode": "ALL", "loss_balancer_beta": 0.7999999999999999, "loss_balancer_r": 0.95, "tf_pma_low_exp_2": 3, "grad_loss_fn": "mae", "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.7, "bias_weight_decay": 0.05, "head_activation": "relu6", "tf_activation": "tanh", "dataset_size_exp_2": 11, "batch_size_exp_2": 2, "epochs": 100, "lr_mul": 0.07, "n_warmup_steps": 200, "Optim": "amsgradw", "fixed_role_model": "lct_gan", "mse_mag_target": 0.5, "g_loss_mul": 0.1, "d_model_exp_2": 8, "attn_activation": "leakyhardtanh", "tf_d_inner_exp_2": 9, "tf_n_layers_enc": 3, "tf_n_head_exp_2": 6, "tf_activation_final": "leakyhardtanh", "tf_num_inds_exp_2": 7, "ada_d_hid_exp_2": 9, "ada_n_layers": 9, "ada_activation": "softsign", "ada_activation_final": "leakyhardsigmoid", "head_d_hid_exp_2": 9, "head_n_layers": 9, "head_n_head_exp_2": 6, "head_activation_final": "leakyhardsigmoid", "mse_mag": true, "mse_mag_multiply": true, "single_model": true, "models": ["lct_gan"], "max_seconds": 3600} \ No newline at end of file diff --git a/contraceptive/realtabformer/eval.csv b/contraceptive/realtabformer/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..80f6623be8b934a68c22bc6f616bf1853c4f5f57 --- /dev/null +++ b/contraceptive/realtabformer/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +realtabformer,0.012081380597755924,0.01889093526957178,0.0019067052836180664,1.5411152839660645,0.22686973214149475,6.461758613586426,0.38246825337409973,3.961230959248496e-06,3.9902100563049316,0.0328068844974041,0.09182539582252502,0.04366584122180939,0.09260228276252747,0.0068534864112734795,5.531325340270996 diff --git a/contraceptive/realtabformer/history.csv b/contraceptive/realtabformer/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..8efa04887f3c91907a73897fbff0c3aa15d6b33e --- /dev/null +++ b/contraceptive/realtabformer/history.csv @@ -0,0 +1,11 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.011365748913376592,0.9511739558440473,0.0006854779220273954,1.2036508678562112,0.0,0.0,0.0,0.0,0.011510453373646467,900,225,244.78535199165344,1.087934897740682,0.2719837244351705,0.10629495567538672,0.004436042588980248,1.158144313332901,5.6492201652799975e-05,0.0,0.0,0.0,0.0,0.0,0.004436042588980248,450,113,100.92714786529541,0.8931606005778355,0.22428255081176757,0.06720254934532979 +1,0.004198840709804143,0.8141998785290303,3.387297497191669e-05,0.2937400783267286,0.0,0.0,0.0,0.0,0.004241808241511333,900,225,244.76988172531128,1.0878661410013835,0.27196653525034586,0.0953530696950232,0.00333135591690532,2.725961913635608,1.3745906455597812e-05,0.0,0.0,0.0,0.0,0.0,0.00333135591690532,450,113,99.83114504814148,0.883461460603022,0.22184698899586996,0.04183473708664388 +2,0.002762561462489733,0.5484214044140991,8.805304913856541e-06,0.2618754346999857,0.0,0.0,0.0,0.0,0.0027956583087668857,900,225,242.95946621894836,1.0798198498619926,0.26995496246549816,0.10219329664483666,0.0027281873274801506,2.8626933845729767,6.899153320190609e-06,0.0,0.0,0.0,0.0,0.0,0.0027281873274801506,450,113,100.66479754447937,0.8908389163228263,0.22369955009884304,0.04004584675329218 +3,0.002585010585737311,0.5059408610661492,1.07776368556805e-05,0.2515256485177411,0.0,0.0,0.0,0.0,0.0026162271208078083,900,225,244.41886401176453,1.086306062274509,0.2715765155686273,0.1017372147883806,0.0025077687115925883,2.664003259325225,7.708884488491681e-06,0.0,0.0,0.0,0.0,0.0,0.0025077687115925883,450,113,100.92277240753174,0.8931218797126702,0.22427282757229275,0.04433754623756307 +4,0.0022580953761239977,0.45952041590186543,8.792309877011795e-06,0.2588796340343025,0.0,0.0,0.0,0.0,0.0022885854777915586,900,225,244.1408052444458,1.0850702455308703,0.27126756138271757,0.10189315725092052,0.0023599671401704353,3.2095572816661777,7.228522299237366e-06,0.0,0.0,0.0,0.0,0.0,0.0023599671401704353,450,113,101.14786124229431,0.8951138163034895,0.22477302498287624,0.04659274682213049 +5,0.0021870362648365293,0.25351457023726504,7.473293455636948e-06,0.2523958419428931,0.0,0.0,0.0,0.0,0.002216868234035145,900,225,244.6163468360901,1.087183763715956,0.271795940928989,0.10260770147045453,0.002132210925208508,2.034111335401322,4.895658492241791e-06,0.0,0.0,0.0,0.0,0.0,0.002132210925208508,450,113,100.4099771976471,0.8885838690057265,0.22313328266143798,0.05016624194001202 +6,0.002052127151619061,0.18765673593578922,6.5528008938834434e-06,0.277902038457493,0.0,0.0,0.0,0.0,0.002083149238023907,900,225,243.84846901893616,1.083770973417494,0.2709427433543735,0.10590767532587052,0.0024004279846849387,2.7415380834772916,5.895928737795737e-06,0.0,0.0,0.0,0.0,0.0,0.0024004279846849387,450,113,100.27250218391418,0.8873672759638424,0.2228277826309204,0.04519922711136463 +7,0.0018999467526898822,0.20382699480085795,5.589238350358375e-06,0.25682228529618845,0.0,0.0,0.0,0.0,0.001928712559650497,900,225,245.35998964309692,1.0904888428582085,0.2726222107145521,0.10311477472798691,0.001989704634373387,2.2423227408280177,4.893121499534396e-06,0.0,0.0,0.0,0.0,0.0,0.001989704634373387,450,113,100.96781826019287,0.893520515576928,0.22437292946709528,0.05078682927332357 +8,0.0017601978679401024,0.4031062119189409,4.838700413782498e-06,0.26518729818363984,0.0,0.0,0.0,0.0,0.0017887732903990481,900,225,245.00445175170898,1.08890867445204,0.27222716861301,0.10410975835182601,0.002103634206901511,1.6868754704122124,6.431579865517962e-06,0.0,0.0,0.0,0.0,0.0,0.002103634206901511,450,113,100.77440333366394,0.8918088790589729,0.2239431185192532,0.056379431404329966 +9,0.0016397621689894651,0.11264491108698306,6.2875129506191465e-06,0.2709155303819312,0.0,0.0,0.0,0.0,0.001668296587401225,900,225,244.95206952095032,1.088675864537557,0.27216896613438923,0.10980390641838313,0.001868966455739509,1.625626234592526,4.7393164126034105e-06,0.0,0.0,0.0,0.0,0.0,0.001868966455739509,450,113,101.65219712257385,0.8995769656864943,0.22589377138349745,0.05243304344017751 diff --git a/contraceptive/realtabformer/mlu-eval-load.ipynb b/contraceptive/realtabformer/mlu-eval-load.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..7430e1d507f7661c2048ca62bbb51fad0d027716 --- /dev/null +++ b/contraceptive/realtabformer/mlu-eval-load.ipynb @@ -0,0 +1,1892 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:40.831378Z", + "iopub.status.busy": "2024-03-26T07:26:40.830526Z", + "iopub.status.idle": "2024-03-26T07:26:40.863942Z", + "shell.execute_reply": "2024-03-26T07:26:40.863194Z" + }, + "papermill": { + "duration": 0.047495, + "end_time": "2024-03-26T07:26:40.865965", + "exception": false, + "start_time": "2024-03-26T07:26:40.818470", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:40.889233Z", + "iopub.status.busy": "2024-03-26T07:26:40.888808Z", + "iopub.status.idle": "2024-03-26T07:26:40.895597Z", + "shell.execute_reply": "2024-03-26T07:26:40.894790Z" + }, + "papermill": { + "duration": 0.020549, + "end_time": "2024-03-26T07:26:40.897522", + "exception": false, + "start_time": "2024-03-26T07:26:40.876973", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:40.919116Z", + "iopub.status.busy": "2024-03-26T07:26:40.918330Z", + "iopub.status.idle": "2024-03-26T07:26:40.922735Z", + "shell.execute_reply": "2024-03-26T07:26:40.921901Z" + }, + "papermill": { + "duration": 0.017084, + "end_time": "2024-03-26T07:26:40.924716", + "exception": false, + "start_time": "2024-03-26T07:26:40.907632", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:40.946265Z", + "iopub.status.busy": "2024-03-26T07:26:40.945723Z", + "iopub.status.idle": "2024-03-26T07:26:40.949723Z", + "shell.execute_reply": "2024-03-26T07:26:40.948751Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.01676, + "end_time": "2024-03-26T07:26:40.951524", + "exception": false, + "start_time": "2024-03-26T07:26:40.934764", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:40.973484Z", + "iopub.status.busy": "2024-03-26T07:26:40.973168Z", + "iopub.status.idle": "2024-03-26T07:26:40.979014Z", + "shell.execute_reply": "2024-03-26T07:26:40.978206Z" + }, + "papermill": { + "duration": 0.019579, + "end_time": "2024-03-26T07:26:40.980840", + "exception": false, + "start_time": "2024-03-26T07:26:40.961261", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "18969463", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:41.003839Z", + "iopub.status.busy": "2024-03-26T07:26:41.003169Z", + "iopub.status.idle": "2024-03-26T07:26:41.009013Z", + "shell.execute_reply": "2024-03-26T07:26:41.008148Z" + }, + "papermill": { + "duration": 0.019386, + "end_time": "2024-03-26T07:26:41.010884", + "exception": false, + "start_time": "2024-03-26T07:26:40.991498", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"realtabformer\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 3\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/realtabformer/3\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.009782, + "end_time": "2024-03-26T07:26:41.030508", + "exception": false, + "start_time": "2024-03-26T07:26:41.020726", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:41.052720Z", + "iopub.status.busy": "2024-03-26T07:26:41.052387Z", + "iopub.status.idle": "2024-03-26T07:26:41.062837Z", + "shell.execute_reply": "2024-03-26T07:26:41.061827Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.024195, + "end_time": "2024-03-26T07:26:41.065000", + "exception": false, + "start_time": "2024-03-26T07:26:41.040805", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/realtabformer/3\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:41.087450Z", + "iopub.status.busy": "2024-03-26T07:26:41.087128Z", + "iopub.status.idle": "2024-03-26T07:26:43.063141Z", + "shell.execute_reply": "2024-03-26T07:26:43.062126Z" + }, + "papermill": { + "duration": 1.989828, + "end_time": "2024-03-26T07:26:43.065599", + "exception": false, + "start_time": "2024-03-26T07:26:41.075771", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:43.090641Z", + "iopub.status.busy": "2024-03-26T07:26:43.090146Z", + "iopub.status.idle": "2024-03-26T07:26:43.104202Z", + "shell.execute_reply": "2024-03-26T07:26:43.103139Z" + }, + "papermill": { + "duration": 0.028664, + "end_time": "2024-03-26T07:26:43.106468", + "exception": false, + "start_time": "2024-03-26T07:26:43.077804", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:43.128949Z", + "iopub.status.busy": "2024-03-26T07:26:43.128689Z", + "iopub.status.idle": "2024-03-26T07:26:43.135518Z", + "shell.execute_reply": "2024-03-26T07:26:43.134835Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.020033, + "end_time": "2024-03-26T07:26:43.137478", + "exception": false, + "start_time": "2024-03-26T07:26:43.117445", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:43.159190Z", + "iopub.status.busy": "2024-03-26T07:26:43.158944Z", + "iopub.status.idle": "2024-03-26T07:26:43.255223Z", + "shell.execute_reply": "2024-03-26T07:26:43.254262Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.109664, + "end_time": "2024-03-26T07:26:43.257415", + "exception": false, + "start_time": "2024-03-26T07:26:43.147751", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:43.280813Z", + "iopub.status.busy": "2024-03-26T07:26:43.280521Z", + "iopub.status.idle": "2024-03-26T07:26:47.886826Z", + "shell.execute_reply": "2024-03-26T07:26:47.885973Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.62073, + "end_time": "2024-03-26T07:26:47.889253", + "exception": false, + "start_time": "2024-03-26T07:26:43.268523", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-26 07:26:45.486000: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-26 07:26:45.486056: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-26 07:26:45.487635: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:47.913147Z", + "iopub.status.busy": "2024-03-26T07:26:47.912080Z", + "iopub.status.idle": "2024-03-26T07:26:47.918464Z", + "shell.execute_reply": "2024-03-26T07:26:47.917570Z" + }, + "papermill": { + "duration": 0.020614, + "end_time": "2024-03-26T07:26:47.920637", + "exception": false, + "start_time": "2024-03-26T07:26:47.900023", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:47.944393Z", + "iopub.status.busy": "2024-03-26T07:26:47.944073Z", + "iopub.status.idle": "2024-03-26T07:26:56.168151Z", + "shell.execute_reply": "2024-03-26T07:26:56.166875Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.239005, + "end_time": "2024-03-26T07:26:56.170980", + "exception": false, + "start_time": "2024-03-26T07:26:47.931975", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-26T07:26:56.197862Z", + "iopub.status.busy": "2024-03-26T07:26:56.196876Z", + "iopub.status.idle": "2024-03-26T07:26:56.205023Z", + "shell.execute_reply": "2024-03-26T07:26:56.204004Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.023939, + "end_time": "2024-03-26T07:26:56.207023", + "exception": false, + "start_time": "2024-03-26T07:26:56.183084", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 46,\n", + " 'realtabformer': (24, 72, Embedding(72, 672), True),\n", + " 'lct_gan': 40,\n", + " 'tab_ddpm_concat': 10}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:56.230833Z", + "iopub.status.busy": "2024-03-26T07:26:56.230245Z", + "iopub.status.idle": "2024-03-26T07:26:56.303119Z", + "shell.execute_reply": "2024-03-26T07:26:56.302032Z" + }, + "papermill": { + "duration": 0.087354, + "end_time": "2024-03-26T07:26:56.305549", + "exception": false, + "start_time": "2024-03-26T07:26:56.218195", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/realtabformer/all inf False\n", + "../../../../ml-utility-loss/aug_test/contraceptive 0\n", + "Caching in ../../../../contraceptive/_cache_bs_test/realtabformer/all inf False\n", + "../../../../ml-utility-loss/bs_test/contraceptive 0\n", + "Caching in ../../../../contraceptive/_cache_synth_test/realtabformer/all inf False\n", + "../../../../ml-utility-loss/synthetics/contraceptive 600\n", + "600\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:56.331135Z", + "iopub.status.busy": "2024-03-26T07:26:56.330816Z", + "iopub.status.idle": "2024-03-26T07:26:56.654533Z", + "shell.execute_reply": "2024-03-26T07:26:56.653664Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.338682, + "end_time": "2024-03-26T07:26:56.656561", + "exception": false, + "start_time": "2024-03-26T07:26:56.317879", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'loss_balancer_beta': 0.7999999999999999,\n", + " 'loss_balancer_r': 0.96,\n", + " 'tf_pma_low': 16,\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7999999999999999,\n", + " 'bias_weight_decay': 0.05,\n", + " 'head_activation': torch.nn.modules.activation.ReLU6,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.09,\n", + " 'n_warmup_steps': 80,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'fixed_role_model': 'realtabformer',\n", + " 'g_loss_mul': 0.1,\n", + " 'd_model': 256,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 512,\n", + " 'ada_n_layers': 8,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 512,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'single_model': True,\n", + " 'models': ['realtabformer'],\n", + " 'max_seconds': 3600,\n", + " 'Body': 'twin_encoder',\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 16,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 1.0, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "# params = {\n", + "# **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "# }\n", + "params = getattr(PARAMS, dataset_name).BEST_DICT[gp][gp_multiply][single_model]\n", + "if isinstance(params, (list, tuple)):\n", + " params = params[param_index]\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-26T07:26:56.681474Z", + "iopub.status.busy": "2024-03-26T07:26:56.680881Z", + "iopub.status.idle": "2024-03-26T07:26:57.096125Z", + "shell.execute_reply": "2024-03-26T07:26:57.095205Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.429989, + "end_time": "2024-03-26T07:26:57.098235", + "exception": false, + "start_time": "2024-03-26T07:26:56.668246", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n", + "[*] Embedding True True\n", + "['realtabformer'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:57.123623Z", + "iopub.status.busy": "2024-03-26T07:26:57.122990Z", + "iopub.status.idle": "2024-03-26T07:26:57.127382Z", + "shell.execute_reply": "2024-03-26T07:26:57.126473Z" + }, + "papermill": { + "duration": 0.019295, + "end_time": "2024-03-26T07:26:57.129303", + "exception": false, + "start_time": "2024-03-26T07:26:57.110008", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:57.153047Z", + "iopub.status.busy": "2024-03-26T07:26:57.152297Z", + "iopub.status.idle": "2024-03-26T07:26:57.159482Z", + "shell.execute_reply": "2024-03-26T07:26:57.158638Z" + }, + "papermill": { + "duration": 0.021132, + "end_time": "2024-03-26T07:26:57.161396", + "exception": false, + "start_time": "2024-03-26T07:26:57.140264", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "8630553" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:57.185355Z", + "iopub.status.busy": "2024-03-26T07:26:57.185067Z", + "iopub.status.idle": "2024-03-26T07:26:57.267717Z", + "shell.execute_reply": "2024-03-26T07:26:57.266767Z" + }, + "papermill": { + "duration": 0.097317, + "end_time": "2024-03-26T07:26:57.269982", + "exception": false, + "start_time": "2024-03-26T07:26:57.172665", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 16128] --\n", + "├─Adapter: 1-1 [2, 1179, 16128] --\n", + "│ └─Embedding: 2-1 [2, 1179, 24, 672] (48,384)\n", + "│ └─TensorInductionPoint: 2-2 [24, 1] 24\n", + "│ └─Sequential: 2-3 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 512] 344,576\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 256] 131,328\n", + "│ │ │ └─LeakyHardsigmoid: 4-16 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 16128] (recursive)\n", + "│ └─Embedding: 2-4 [2, 294, 24, 672] (recursive)\n", + "│ └─TensorInductionPoint: 2-5 [24, 1] (recursive)\n", + "│ └─Sequential: 2-6 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-9 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-18 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-10 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-32 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-7 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-6 [2, 16, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 16] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-12 [2, 1179, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-18 [2, 16, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 16] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-24 [2, 1179, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-30 [2, 16, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 16] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-36 [2, 1179, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-42 [2, 16, 256] --\n", + "│ └─Encoder: 2-8 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-8 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-21 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-22 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-48 [2, 16, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-23 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 16] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-54 [2, 294, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-25 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-60 [2, 16, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 16] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-66 [2, 294, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-31 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-72 [2, 16, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 16] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-78 [2, 294, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-14 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-40 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-84 [2, 16, 256] --\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-9 [2, 1] --\n", + "│ │ └─FeedForward: 3-19 [2, 512] --\n", + "│ │ │ └─Linear: 4-39 [2, 512] 2,097,664\n", + "│ │ │ └─ReLU6: 4-40 [2, 512] --\n", + "│ │ └─FeedForward: 3-20 [2, 512] --\n", + "│ │ │ └─Linear: 4-41 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-42 [2, 512] --\n", + "│ │ └─FeedForward: 3-21 [2, 512] --\n", + "│ │ │ └─Linear: 4-43 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-44 [2, 512] --\n", + "│ │ └─FeedForward: 3-22 [2, 512] --\n", + "│ │ │ └─Linear: 4-45 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-46 [2, 512] --\n", + "│ │ └─FeedForward: 3-23 [2, 512] --\n", + "│ │ │ └─Linear: 4-47 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-48 [2, 512] --\n", + "│ │ └─FeedForward: 3-24 [2, 512] --\n", + "│ │ │ └─Linear: 4-49 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-50 [2, 512] --\n", + "│ │ └─FeedForward: 3-25 [2, 512] --\n", + "│ │ │ └─Linear: 4-51 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-52 [2, 512] --\n", + "│ │ └─FeedForward: 3-26 [2, 512] --\n", + "│ │ │ └─Linear: 4-53 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-54 [2, 512] --\n", + "│ │ └─FeedForward: 3-27 [2, 1] --\n", + "│ │ │ └─Linear: 4-55 [2, 1] 513\n", + "│ │ │ └─LeakyHardsigmoid: 4-56 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 8,678,937\n", + "Trainable params: 8,630,553\n", + "Non-trainable params: 48,384\n", + "Total mult-adds (M): 26.78\n", + "========================================================================================================================\n", + "Input size (MB): 0.28\n", + "Forward/backward pass size (MB): 611.80\n", + "Params size (MB): 34.72\n", + "Estimated Total Size (MB): 646.80\n", + "========================================================================================================================" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = test_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "357e54f8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:57.298334Z", + "iopub.status.busy": "2024-03-26T07:26:57.297625Z", + "iopub.status.idle": "2024-03-26T07:26:57.351148Z", + "shell.execute_reply": "2024-03-26T07:26:57.350264Z" + }, + "papermill": { + "duration": 0.070214, + "end_time": "2024-03-26T07:26:57.353219", + "exception": false, + "start_time": "2024-03-26T07:26:57.283005", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torch\n", + "model.load_state_dict(torch.load(\"model.pt\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:26:57.378317Z", + "iopub.status.busy": "2024-03-26T07:26:57.378000Z", + "iopub.status.idle": "2024-03-26T07:28:55.431010Z", + "shell.execute_reply": "2024-03-26T07:28:55.430034Z" + }, + "papermill": { + "duration": 118.068266, + "end_time": "2024-03-26T07:28:55.433593", + "exception": false, + "start_time": "2024-03-26T07:26:57.365327", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:28:55.460773Z", + "iopub.status.busy": "2024-03-26T07:28:55.460451Z", + "iopub.status.idle": "2024-03-26T07:28:55.480918Z", + "shell.execute_reply": "2024-03-26T07:28:55.480077Z" + }, + "papermill": { + "duration": 0.036167, + "end_time": "2024-03-26T07:28:55.482901", + "exception": false, + "start_time": "2024-03-26T07:28:55.446734", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
realtabformer0.0120810.0188910.0019071.5411150.226876.4617590.3824680.0000043.990210.0328070.0918250.0436660.0926020.0068535.531325
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "realtabformer 0.012081 0.018891 0.001907 1.541115 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss pred_duration \\\n", + "realtabformer 0.22687 6.461759 0.382468 0.000004 3.99021 \n", + "\n", + " pred_mae pred_mape pred_rmse pred_std std_loss \\\n", + "realtabformer 0.032807 0.091825 0.043666 0.092602 0.006853 \n", + "\n", + " total_duration \n", + "realtabformer 5.531325 " + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:28:55.508032Z", + "iopub.status.busy": "2024-03-26T07:28:55.507773Z", + "iopub.status.idle": "2024-03-26T07:28:55.882843Z", + "shell.execute_reply": "2024-03-26T07:28:55.881912Z" + }, + "papermill": { + "duration": 0.39035, + "end_time": "2024-03-26T07:28:55.885037", + "exception": false, + "start_time": "2024-03-26T07:28:55.494687", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:28:55.912203Z", + "iopub.status.busy": "2024-03-26T07:28:55.911912Z", + "iopub.status.idle": "2024-03-26T07:31:07.173898Z", + "shell.execute_reply": "2024-03-26T07:31:07.173011Z" + }, + "papermill": { + "duration": 131.278434, + "end_time": "2024-03-26T07:31:07.176509", + "exception": false, + "start_time": "2024-03-26T07:28:55.898075", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/realtabformer/all inf False\n", + "Caching in ../../../../contraceptive/_cache_bs_test/realtabformer/all inf False\n", + "Caching in ../../../../contraceptive/_cache_synth_test/realtabformer/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:31:07.205481Z", + "iopub.status.busy": "2024-03-26T07:31:07.205109Z", + "iopub.status.idle": "2024-03-26T07:31:07.227326Z", + "shell.execute_reply": "2024-03-26T07:31:07.226670Z" + }, + "papermill": { + "duration": 0.039158, + "end_time": "2024-03-26T07:31:07.229440", + "exception": false, + "start_time": "2024-03-26T07:31:07.190282", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:31:07.255797Z", + "iopub.status.busy": "2024-03-26T07:31:07.255492Z", + "iopub.status.idle": "2024-03-26T07:31:07.261027Z", + "shell.execute_reply": "2024-03-26T07:31:07.260145Z" + }, + "papermill": { + "duration": 0.021277, + "end_time": "2024-03-26T07:31:07.263161", + "exception": false, + "start_time": "2024-03-26T07:31:07.241884", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'realtabformer': 0.40327791005373004}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:31:07.289048Z", + "iopub.status.busy": "2024-03-26T07:31:07.288773Z", + "iopub.status.idle": "2024-03-26T07:31:07.722179Z", + "shell.execute_reply": "2024-03-26T07:31:07.721284Z" + }, + "papermill": { + "duration": 0.449164, + "end_time": "2024-03-26T07:31:07.724405", + "exception": false, + "start_time": "2024-03-26T07:31:07.275241", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABEQ0lEQVR4nO3deXhU5dk/8O+ZfTKZJfsC2QgQEGRfRHahCqKCVhS1JREBa6GWUt6f0ustSK0GrSLUtlRbTaAutPqitqUoboBVNtkRDGsWQjayzWQms57n98fJDAzZJpOZOZnJ/bmuuZI588w598wk9zznOc/CMcYYCCFERBKxAyCEEEpEhBDRUSIihIiOEhEhRHSUiAghoqNERAgRHSUiQojoKBERQkRHiYgQIjpKRKRdzzzzDDiOC+g+CwsLwXEcvv32W5/K/+53v0O/fv0glUoxYsSIgMZCeg5KRKRLnn/+eXz44YchOdauXbvw//7f/8PEiRNRUFCA559/PiTHJaEnEzsAEl6ef/553H///Zg3b17Qj/XFF19AIpHgjTfegEKhCPrxiHioRhQhzGaz2CEEXHV1NdRqdcCSEGMMzc3NAdlXd1gsFrFD6HEoEYUhd9vN6dOn8fDDDyMmJgaTJk0CALz11lsYPXo01Go1YmNjsWDBApSVlXk9/6uvvsL8+fORnp4OpVKJtLQ0/OIXv+j0n5TjOJjNZmzZsgUcx4HjOOTl5QEASkpK8NOf/hQ5OTlQq9WIi4vD/PnzUVxc3Oa+LBYLHn/8ccTFxUGn02HhwoWor6/3OlZBQQHMZrPnWIWFhQAAp9OJZ599FtnZ2VAqlcjMzMSvfvUr2Gw2r2NkZmbirrvuwieffIIxY8ZArVbjtddew+7du8FxHP7xj39g3bp16NOnD7RaLe6//340NjbCZrNhxYoVSExMRHR0NB599NFW+/b1vZ42bRqGDh2Kw4cPY8qUKYiKisKvfvWrDt/n3ohOzcLY/PnzMWDAADz//PNgjOG5557Dr3/9azzwwANYvHgxampq8Oqrr2LKlCk4evQoDAYDAOC9996DxWLBE088gbi4OBw8eBCvvvoqLl++jPfee6/d4/3tb3/D4sWLMW7cOCxduhQAkJ2dDQA4dOgQvvnmGyxYsAB9+/ZFcXExNm/ejGnTpuH06dOIiory2tfy5cthMBjwzDPPoKioCJs3b0ZJSYknSfztb3/D66+/joMHD+Kvf/0rAODWW28FACxevBhbtmzB/fffj1/+8pc4cOAA8vPzcebMGXzwwQdexykqKsJDDz2Exx9/HEuWLEFOTo7nsfz8fKjVajz99NM4f/48Xn31VcjlckgkEtTX1+OZZ57B/v37UVhYiKysLKxZs8bzXF/fawCora3F7NmzsWDBAvzoRz9CUlJSFz/pXoCRsLN27VoGgD300EOebcXFxUwqlbLnnnvOq+zJkyeZTCbz2m6xWFrtMz8/n3Ecx0pKSlod53oajYbl5ua2en5b+9y3bx8DwLZu3erZVlBQwACw0aNHM7vd7tn+4osvMgDso48+8mzLzc1lGo3Ga5/Hjh1jANjixYu9tq9atYoBYF988YVnW0ZGBgPAPv74Y6+yX375JQPAhg4d6hXDQw89xDiOY7Nnz/YqP2HCBJaRkeG535X3eurUqQwA+/Of/9zq/SHX0KlZGPvJT37i+X379u3geR4PPPAArl696rklJydjwIAB+PLLLz1l1Wq153ez2YyrV6/i1ltvBWMMR48e9SuW6/fpcDhQW1uL/v37w2Aw4MiRI63KL126FHK53HP/iSeegEwmw3/+858Oj+N+fOXKlV7bf/nLXwIAduzY4bU9KysLd9xxR5v7WrhwoVcM48ePB2MMixYt8io3fvx4lJWVwel0Aujaew0ASqUSjz76aIevq7ejU7MwlpWV5fn93LlzYIxhwIABbZa9/h+utLQUa9aswT//+U+vdhkAaGxs9CuW5uZm5Ofno6CgAOXl5WDXTfzZ1j5vjDM6OhopKSnttim5lZSUQCKRoH///l7bk5OTYTAYUFJS4rX9+vfoRunp6V739Xo9ACAtLa3Vdp7n0djYiLi4uC691wDQp08fuurXCUpEYez6WgjP8+A4Djt37oRUKm1VNjo6GgDgcrnwgx/8AHV1dXjqqacwaNAgaDQalJeXIy8vDzzP+xXLz372MxQUFGDFihWYMGEC9Ho9OI7DggUL/N5nR3ztaHn9e3Sjtt6njra7k6uv77UvMRABJaIIkZ2dDcYYsrKyMHDgwHbLnTx5EmfPnsWWLVuwcOFCz/ZPP/3Up+O0lwDef/995Obm4uWXX/Zss1qtaGhoaLP8uXPnMH36dM/9pqYmVFRU4M477+zw+BkZGeB5HufOncPgwYM926uqqtDQ0ICMjAyfXkd3+PpeE99RG1GEuO+++yCVSrFu3Tqv0yJA+Cavra0FcO3b/voyjDFs2rTJp+NoNJo2k4tUKm113FdffRUul6vN/bz++utwOBye+5s3b4bT6cTs2bM7PL47UW3cuNFr+4YNGwAAc+bM6ewldJuv7zXxHdWIIkR2djZ++9vfYvXq1SguLsa8efOg1Wpx6dIlfPDBB1i6dClWrVqFQYMGITs7G6tWrUJ5eTl0Oh3+7//+r1VbUXtGjx6Nzz77DBs2bEBqaiqysrIwfvx43HXXXfjb3/4GvV6Pm266Cfv27cNnn32GuLi4Nvdjt9sxY8YMPPDAAygqKsKf/vQnTJo0Cffcc0+Hxx8+fDhyc3Px+uuvo6GhAVOnTsXBgwexZcsWzJs3z6uWFSy+vtfEd5SIIsjTTz+NgQMH4pVXXsG6desACA2vt99+u+cfXC6X41//+heefPJJ5OfnQ6VS4d5778Xy5csxfPjwTo+xYcMGLF26FP/7v/+L5uZm5ObmYvz48di0aROkUinefvttWK1WTJw4EZ999lm7V6z+8Ic/4O2338aaNWvgcDjw0EMP4fe//71PbT9//etf0a9fPxQWFuKDDz5AcnIyVq9ejbVr13bh3eoeX95r4juO3Vi3JISQEKM2IkKI6CgREUJER4mIECI6SkSEENFRIiKEiI4SESFEdGHdj4jneVy5cgVarTbgk7wTQrqPMQaTyYTU1FRIJO3Xe8I6EV25cqXVSGlCSM9TVlaGvn37tvt4WCcirVYLQHiROp1O5GgIITcyGo1IS0vz/K+2J6wTkft0TKfTUSIipAfrrOmEGqsJIaKjREQIER0lIkKI6MK6jcgXjDE4nc52J+gi7ZNKpZDJZNQ1ggRdRCciu92OiooKWlmzG6KiopCSkkKTv5OgithExPM8Ll26BKlUitTUVCgUCvpm7wLGGOx2O2pqanDp0iUMGDCgww5phHRHxCYiu90OnueRlpbWapVR4hu1Wg25XI6SkhLY7XaoVCqxQwqYc1UmHCtrQHZiNEalx4gdTq8XsYnIjb7FuycS3z/GGE5XGHG5vhmX65uREK1EWix9WYkp8v7KCOkEx3G4Z3gqEnVKAMDRsgZxAyKUiEjvxHEcfnBTEgCgtNYMhyvwi0AS31Ei6uUyMzNbrREWyexOHi5eWC8iIVqJaKUMDhdDldEqcmS9GyUi0qscK2vAn/dcwIGLteA4Dkl6oQGeEpG4KBFFALvdLnYIYeNyvQV2Jw+lXFjxNlnnTkQ2McPq9XplIrI7+XZvzhvaCjoqe2O7Qltl/DFt2jQsX74cy5cvh16vR3x8PH796197ljfOzMzEs88+i4ULF0Kn02Hp0qUAgP/+97+YPHky1Go10tLS8OSTT8JsNnv2W11djbvvvhtqtRpZWVl4++23/YovXDHGPAknpaUmNCAxGveMSMWUgQlihtbriX75vry8HE899RR27twJi8WC/v37o6CgAGPGjAnaMf/45fl2H8uK12DeyD6e+6/vvQCHq+01KPvGqDF/zLWJ2d78+hKa7d5DSX7xg4F+xbhlyxY89thjOHjwIL799lssXboU6enpWLJkCQDgpZdewpo1azyrm164cAGzZs3Cb3/7W7z55puoqanxJLOCggIAQF5eHq5cuYIvv/wScrkcTz75JKqrq/2KLxwZrU5YHS5IJRziNEJP8RiNAjEa6jUuNlETUX19PSZOnIjp06dj586dSEhIwLlz5xATQx3M0tLS8Morr4DjOOTk5ODkyZN45ZVXPInotttuwy9/+UtP+cWLF+ORRx7BihUrAAADBgzA73//e0ydOhWbN29GaWkpdu7ciYMHD2Ls2LEAgDfeeAODBw8O+WsTS41JaAeKi1ZAJu2VJwM9lqiJ6IUXXkBaWprnGxsAsrKygn7cZdP7t/uY5IZRIEunZLdb9sYRI4smBi72W265xWtIyoQJE/Dyyy97Bu/eWGM8fvw4Tpw44XW6xRjzDHU5e/YsZDIZRo8e7Xl80KBBMBgMAYu5p3OfliVqvXuIF181o9JoRb8ETavHSGiImoj++c9/4o477sD8+fOxZ88e9OnTBz/96U893/o3stlssNmuNSoajUa/jquQ+f5tGKyy3aXRaLzuNzU14fHHH8eTTz7Zqmx6ejrOnj0bqtB6rKtN7kSk9Np+6kojzlU1QSGTUCISiaj104sXL2Lz5s0YMGAAPvnkEzzxxBN48sknsWXLljbL5+fnQ6/Xe26RPHH+gQMHvO7v378fAwYMgFQqbbP8qFGjcPr0afTv37/VTaFQYNCgQXA6nTh8+LDnOUVFRWhoaAjmy+hR+saokZ0YjSSdd7LRqeQAAJPVKUZYBCInIp7nMWrUKDz//PMYOXIkli5diiVLluDPf/5zm+VXr16NxsZGz62srCzEEYdOaWkpVq5ciaKiIrz77rt49dVX8fOf/7zd8k899RS++eYbLF++HMeOHcO5c+fw0UcfYfny5QCAnJwczJo1C48//jgOHDiAw4cPY/HixVCr1aF6SaIbnRGLe4anIlnvnYiiVcKJgcnqECMsApETUUpKCm666SavbYMHD0ZpaWmb5ZVKpWei/EifMH/hwoVobm7GuHHjsGzZMvz85z/3XKZvy7Bhw7Bnzx6cPXsWkydPxsiRI7FmzRqkpqZ6yhQUFCA1NRVTp07Ffffdh6VLlyIxMTEUL6dH07UkoiaqEYlG1DaiiRMnoqioyGvb2bNnkZGRIVJEPYdcLsfGjRuxefPmVo8VFxe3+ZyxY8di165d7e4zOTkZ//73v722/fjHP+5WnOHC6nDBxTNEKaSt5qXS0qmZ6EStEf3iF7/A/v378fzzz+P8+fN455138Prrr2PZsmVihkUi0JkKI17fexE7T1W2ekzbUiMy252ecWgktERNRGPHjsUHH3yAd999F0OHDsWzzz6LjRs34pFHHhEzLBKBGpqF9h930rmeWi6FTMKBMTo9E4voPavvuusu3HXXXWKH0aPs3r1b7BAiToNFGI9nULfuRc1xHOaN7IMohdTTcE1Ci9510is0WIQakSFK3ubjNEOjuKifO4l4Lp7B2CyccrWXiIi4qEZEIp6x2QGeMcilHKKVbf/Jlzc043KdBQlaJfolRIc4QkI1IhLx6lvah/RR7S8pVVZnwTcXanGhxtzm4yS4qEZEIp5WJceojBhEKdoeHgPA85jFTlfNxECJiES8BK0SU7UdT3wWpRD+FSx2WppcDHRqRgiurxFRIhIDJSIS8aqMVjTbXZ6pdtuicdeIbM4Oy5HgoEREIpqLZ3j3YCn+vOdCh7UddUuNyMkz2Pyca5z4r3clIsYApz30ty58w27duhVxcXFeE8ABwLx583rNANVAamx2gDFh0rqOGqsVMolnYrsb5x0nwde7GqtdDuCrl0N/3Mm/BGS+TdA+f/58PPnkk/jnP/+J+fPnAxBW39ixY0eHI+tJ2zyX7tXydi/du903qg/kUkmb49FIcPWuGlEYUKvVePjhh73m8X7rrbeQnp6OadOmiRdYmHIP7YiJ6vyLIEWvRny0kibWF0HvSv1SuVA7EeO4XbBkyRKMHTsW5eXl6NOnDwoLC5GXl9fpNzppzTPYlYZ29Gi9KxFxnM+nSGIaOXIkhg8fjq1bt+L222/Hd999hx07dogdVljqbLDr9UprLbjS2Iw+BjUNgg2x3pWIwsjixYuxceNGlJeXY+bMmRG9UEAw1XtqRJ1/AV242oRjpQ0YlxVLiSjE6GS4h3r44Ydx+fJl/OUvf8GiRYvEDidsjc2MxYh0A2J9SEQqmXBVja6ahR4loh5Kr9fjhz/8IaKjozFv3jyxwwlbw9MMmJ6T6Okn1BF3GauTElGoUSLqwcrLy/HII49AqVR2Xpj4zlQFNFW32qySUz8isVAbUQ9UX1+P3bt3Y/fu3fjTn/4kdjhhq7bJBruLR0yUAiq5VOhYWrQTqDguFOgzChh4h6e8Wu6uEVHP6lCjRNQDjRw5EvX19XjhhReQk5Mjdjhh62hpA06WN2J8Vixu7R8PVBy7loQAoPwIYMgAEgcBgJCsAFipRhRylIh6oPbWLSNd4165wxClAFxOoPi/wgPZtwFOK1DyDVD8FZCQA3DctUTkEAbIUr+t0KFERCKWV2fG2nOArQlQRgN9RgO8E7h8CDBfBRpKgZgMRCtluH90X58atklgRXxjNU3p0D3h+v45XLxn5daYKAVQ9Z3wQPIwQCoD5CogQTglw9WzAACphENabBTio5VUGwoxURPRM888A47jvG6DBg0KyL7lcqEnrcViCcj+eiv3++d+P8OFu0e1Ui6BirMDdReFB5KGXCuU0NL+VlPUpRkSSOCJfmo2ZMgQfPbZZ577MllgQpJKpTAYDKiuFi7TRkVF0bdcFzDGYLFYUF1dDYPBAKk0vE5XGpuF07KYKAW4hlKAdwFRcYAm/lqhmCyhdmQzAeYaIDoRRZUm1FvsGJikRaym5w8HihSiJyKZTIbk5GSfytpsNq95eoxGY4fl3ft1JyPSdQaDwefPpyfxjDFTy4H6M8LGmEzvQlIZoOsL1BcDDWVAdCKOX25AeX0zYjUKSkQhJHoiOnfuHFJTU6FSqTBhwgTk5+cjPT29zbL5+flYt26dz/vmOA4pKSlITEyEw+EIVMi9hlwuD7uakFtGnAYSCSckoguXhI2xWa0LGtKFRNRYCvQdDWXL5Gh26ksUUqImovHjx6OwsBA5OTmoqKjAunXrMHnyZJw6dQparbZV+dWrV2PlypWe+0aj0afBoFKpNGz/oYh/ErRKJGiVgLURaK4HOAmgb+NvxdCyraEMYAxK2bVL+CR0RE1Es2fP9vw+bNgwjB8/HhkZGfjHP/6Bxx57rFV5pVJJwx1I1zSWCz+jE4UrZTfSpghJym4GbCYoW4Z50LzVodWjLt8bDAYMHDgQ58+fFzsUEsYcLh7fVxpRZbSCGa8IG3WpbReWygFNnPB7U5Xn1MxGA19DqkcloqamJly4cAEpKSlih0LCWL3Fjp0nK7H9SDk4U4Wwsb1EBADRLY3xpkrPqZnNQTWiUBI1Ea1atQp79uxBcXExvvnmG9x7772QSqV46KGHxAyLhDn3FbO4KAnQVCls1HaQiLQticirRkSJKJREbSO6fPkyHnroIdTW1iIhIQGTJk3C/v37kZDQ8fLAhHSk3iz0IUqQNgljzGRKICq2/SdEJwk/TZXIHKjB/DF9Ea0U/YJyryLqu71t2zYxD08iVH1LjSiB1QkbtCnCfOXtcScimwnRnB3RMTRNbKj1qDYiQgLBPdg1hq8XNmiTOn6CTAGodMLvltogRkbaQ4mIRBx3jUjrahA2aBI7f1KUMPTDYarBsbIGHLxUF6ToSFsoEZGI0mx3CZ0RGYPG0VK7ifYhEbVcwmfmq/jy+2p8ff4qnC5qsA4VapEjEUUm5TB3RCospjpIS+xCZ8WouM6f2FJGZq0DkAkAsLt4WvU1RCgRkYgil0rQLyEa4Fou20fFAhIfhve0nJpJmmuhkElgd/KwOXj4sAoRCQBK9yQymWuEn76clgHXak1WI9QSoVc19SUKHUpEJKKcqzLhbJUJ1oaWHtUaH/ukKaKEGwAdE6aXoWEeoUOJiESU/ZfqsONEBZrq3InIxxoR4KkVaXl3IqIaUahQIiIRgzGGBrMdYDw0rpZJ8zQ+NFS7qWOEp/AmADQVSChRYzWJGCabE06eIYo3QyVjgEQOKPW+70BlAAAMjeWRmdVXmHSfhAQlIhIxGswtQztkZmF+crUBkHSh0t9SIzLADAMN8wgpOjUjEaO+ZWhHvKRJ2NDRQNe2qA3CT2tDwGIivqFERCKGOxHFcGZhQ0sNx2ct5ZubGnG8uBrnq02BDI90gBIRiRielTvQkkDUXawRydWAXAWT1YH9313AyfLGAEdI2kNtRCRiTB2YgKF99IgvsgBOdP3UDABUBsgkRiidJlrJI4QoEZGIEaNRIEYlAVwtbURdrREBgDoGUkkZVLZG6kcUQnRqRiKLtUFYPlqmABSarj9fbYBMwkHlNNG81SFENSISEaqNVpTUWdDXWY4UQKgN+bPEuDoGUgkHldMIO00DEjJUIyIRobTOgv+eu4rissvCBn/ahwBAZYBUwnnaiFw8C1yQpF2UiEhEcM/KaODcV8y6eOneTaWDVMJB4WoCGKMG6xChUzMSEdx9iPTMz0v3bkodJBIJbkqKQubwWChk9F0dCpSISERwT5gfzZoADv6fmkmkgFKLGMYQo7IDEj/amUiX9Zh0v379enAchxUrVogdCgkzNqcLZpsLUt6OKGYRNvpbIwIAZcuKHlbq0BgqPSIRHTp0CK+99hqGDRsmdigkDLl7VMdwTZBJJMIEZ3KV/ztU6dHQbMe5snLUtSzWSIJL9ETU1NSERx55BH/5y18QE+NnAyPp1dztQwmyANSGAEClR0WjFd9dKENFY3M3oyO+ED0RLVu2DHPmzMHMmTM7LWuz2WA0Gr1uhGQnRONHt2RgnHsdRX/bh9xUOshaLuFT7+rQEH3J6SNHjuDQoUM+lc/Pz8e6deuCHBUJN3KpBAlaJcB1Y2jH9VR6zyV8unwfGqLViMrKyvDzn/8cb7/9NlQq387nV69ejcbGRs+trKwsyFGSsNLcsjqrv32I3DydGptgo+liQ0K0GtHhw4dRXV2NUaNGeba5XC7s3bsXf/jDH2Cz2SCVeq9HpVQqoVQqQx0q6cEYY9hdVAOdSorhTbXCH3R3T82UOsgkEkiZFVarORBhkk6IlohmzJiBkydPem179NFHMWjQIDz11FOtkhAhbbHYXThW1gA5b8VIuU0YX9bdGpFUBk6pAWABo0v4IeFXIrp48SL69evXrQNrtVoMHTrUa5tGo0FcXFyr7YS0xzM9rNQMCccBKh0glXd7v5zKAKAGrJkSUSj41UbUv39/TJ8+HW+99RasVmugYyLEZ+4+RAnSADVUtzDExiEnWYsxyaJfWO4V/HqXjxw5gmHDhmHlypVITk7G448/joMHD3Y7mN27d2Pjxo3d3g/pPdw1ojj3PNXdbR9qodbGIjZKgVipLSD7Ix3zKxGNGDECmzZtwpUrV/Dmm2+ioqICkyZNwtChQ7FhwwbU1NQEOk5C2uQeda/3d57q9qha1kOjFT1Colv1TplMhvvuuw/vvfceXnjhBZw/fx6rVq1CWloaFi5ciIqKikDFSUib3INdde5R9wGqETnlWtQ02VBWWRWQ/ZGOdSsRffvtt/jpT3+KlJQUbNiwAatWrcKFCxfw6aef4sqVK5g7d26g4iSkFcYYGi0OgDHPMtHdvmLWwqXQ4nx1E4ovV8BJMzUGnV9XzTZs2ICCggIUFRXhzjvvxNatW3HnnXdC0rKqZlZWFgoLC5GZmRnIWAnxwnEclk7tB2N9HZQnXQAnvXZK1U0KjQEcABlvhc1mhSyKVn4NJr8S0ebNm7Fo0SLk5eUhJSWlzTKJiYl44403uhUcIZ1RyqRIkJlb+g8ZhPmEAoCTq4UJ+J122M2N0FAiCiq/EtGnn36K9PR0Tw3IjTGGsrIypKenQ6FQIDc3NyBBEtIhi3toR4Aaqlu4lHpInDVwWhoAtP2FSwLDrzai7OxsXL16tdX2uro6ZGVldTsoQnxxrKwBX3xfhZqalosiUYGdRoZXaAGgJRGRYPKrRsRY2ysbNDU1+TyAlZDuunS1CcVXLRigaPlSDHCNyD1To9NCvauDrUuJaOXKlQCERsI1a9Yg6rrzZpfLhQMHDmDEiBEBDZCQ9tSbhT5E0XzLvFQBunTvoRISEU/DPIKuS4no6NGjAIQa0cmTJ6FQKDyPKRQKDB8+HKtWrQpshIS0weniYbQ6AMZDwzcJjQwBrhENzOgLuVULtYJ6VwdblxLRl19+CUAYJb9p0ybodLqgBEVIZxqbHWAM0MIMuYQJA12V2oAeIyk+AShTADxNBRJsfrURFRQUBDoOQrrEPbQjSWYGh5apP/xZYrojnmEeRoDnAQkNgA0WnxPRfffdh8LCQuh0Otx3330dlt2+fXu3AyOkI+6hHfGSJoABiIoL/DFcCjjMDigkgN7e5GkzIoHncyLS6/XgWr5x9PrA9F4lxF8mmxMAYOBaElGgr5gBKKu3obaWR7LCBr3NSIkoiHxORNefjtGpGRHb9JxETOgXB8mJg4ARgb9iBkApl8AmjYaLtwqnZ/T9GzR+nfQ2NzfDYrF47peUlGDjxo3YtWtXwAIjpDMquRQKW71wJwg1IoVUArssGi6eATZauiqY/EpEc+fOxdatWwEADQ0NGDduHF5++WXMnTsXmzdvDmiAhLTLaQdsLTMzBrFG5OSZUCMiQeP3DI2TJ08GALz//vtITk5GSUkJtm7dit///vcBDZCQG9WYbNh+5DIOnTkvbFBEAXJ1wI+jlElhk2mFGhFNoh9Ufl2+t1gs0GqFPhu7du3CfffdB4lEgltuuQUlJSUBDZCQG9WabSiptUAjrQA4BOW0DAAUMncbEQOzNiLAnQPIdfyePP/DDz9EWVkZPvnkE9x+++0AgOrqaurkSILOPbQjThLYeapvpJQJbUQMgIuGeQSVX4lozZo1WLVqFTIzMzF+/HhMmDABgFA7GjlyZEADJORG7j5EBvc81UHoQwQAMgmHKTdnIydJC4nLBjhpqEew+HVqdv/992PSpEmoqKjA8OHDPdtnzJiBe++9N2DBEdIWd69qzzzVQTo14zgON6XFAyU6wNFyCT86ISjH6u387rOenJyMkSNHek2ONm7cOAwaNMjnfWzevBnDhg2DTqeDTqfDhAkTsHPnTn9DIr0AY0xYQogxRLtaTpeCdGrm0TIdCF3CDx6/akRmsxnr16/H559/jurqavC89+TiFy9e9Gk/ffv2xfr16zFgwAAwxrBlyxbMnTsXR48exZAhQ/wJjUQ4i90Fu5OHgrdAJXMBnARQGYJ2vPKGZsCqQJzTBRUtLRQ0fiWixYsXY8+ePfjxj3+MlJQUz9CPrrr77ru97j/33HPYvHkz9u/fT4mItMlsd0KtkCLW7l5i2gBI/foz9sm+C7WQVjgxSe2EivoSBY1fn+DOnTuxY8cOTJw4MWCBuFwuvPfeezCbzZ7G7xvZbDbYbNcaDI1G+sPobRK1KvxkajYcpfXABQT9tEwpk8Aspd7VweZXG1FMTAxiYwPzB3Dy5ElER0dDqVTiJz/5CT744APcdNNNbZbNz8+HXq/33NLS0gISAwk/cmvLhPma+KAex30J30W9q4PKr0T07LPPYs2aNV7jzfyVk5ODY8eO4cCBA3jiiSeQm5uL06dPt1l29erVaGxs9NzKysq6fXwSpswty5pHBTcRuTs1OnmeakRB5Nep2csvv4wLFy4gKSkJmZmZkMvlXo8fOXLE530pFAr0798fADB69GgcOnQImzZtwmuvvdaqrFKphFKp9CdkEiE+OlYOAPhBQxWiAEAT3MvpSpnU07saNhNNkBYkfiWiefPmBTiMa3ie92oHIsSN5xlKay2AwwKJxALIpUHrzOimlEvgkEbBySQAa2knUhuCeszeyK9EtHbt2oAcfPXq1Zg9ezbS09NhMpnwzjvvYPfu3fjkk08Csn8SWUw2J5w8g8HVAKVCIiQEmaLT53WHQioBOA7NEg0ASkTB4vd1z4aGBrz//vu4cOEC/ud//gexsbE4cuQIkpKS0KdPH5/2UV1djYULF6KiogJ6vR7Dhg3DJ598gh/84Af+hkUimHtoR6LEKHQZCXL7EAD0jVHjjiHJSL6UDNgrqME6SPxKRCdOnMDMmTOh1+tRXFyMJUuWIDY2Ftu3b0dpaalnrqLOvPHGG/4cnvRS7qEdcZKWoR1BvmIGAIYoBQxRCqAxHqisoAbrIPGr1W3lypXIy8vDuXPnvFZ2vfPOO7F3796ABUfI9epbakSxrGVoRwgSkYd7vmqqEQWFXzWiQ4cOtXlVq0+fPqisrOx2UIS0paFljJmOdyei4A9Adbp4lNZZIGmSIROgGlGQ+FUjUiqVbfZqPnv2LBISaHQyCQ6pRAINZ0UUZxfWMAvyFTMAcLgYPjp2BV9csoIxmqkxWPxKRPfccw9+85vfwOEQztk5jkNpaSmeeuop/PCHPwxogIS43TM8FUtGRkGrkglJSCrv/EndpJAJ/yI2mXvu6kbhMj4JKL8S0csvv4ympiYkJCSgubkZU6dORf/+/aHVavHcc88FOkZCPLimKmFl1+ikkBxPKuEgl3Kwuzs1uhyA0xqSY/cmfrUR6fV6fPrpp/j6669x/PhxNDU1YdSoUZg5c2ag4yPEW1OV8FObHLJDKmVSNLkYnBI1ALvQYB2Eyfp7sy4nIp7nUVhYiO3bt6O4uBgcxyErKwvJyclgjPk9JQghHTlwsRbfV5ows7EYfVQIWY0IaDk9swF2eTTgqhMarLWhO35v0KVTM8YY7rnnHixevBjl5eW4+eabMWTIEJSUlCAvL4+miSVBc7XJjkajEZytpbE4hIlI2dJOZJdGCxvoEn7AdalGVFhYiL179+Lzzz/H9OnTvR774osvMG/ePGzduhULFy4MaJCE1Jlt0DhqoZZLhSEWclWnzwkUpfxagzXsAGx05SzQulQjevfdd/GrX/2qVRICgNtuuw1PP/003n777YAFRwggDHattzigsV+FWiENaW0IAEakxeD2IUmIjWvpmkKX8AOuS4noxIkTmDVrVruPz549G8ePH+92UIRcr6HZARfPEOOsEU6TtCkhPX5WvAZDUvXQ6lv6LdGpWcB1KRHV1dUhKan9b6OkpCTU19d3OyhCrldntgGMIYG/Kly61/s2qDrgaDWPoOlSG5HL5YJM1v5TpFIpnE5nt4Mi5Hq1TXYoXU3QSWwApwx5jchodeCqyYYoTo5kALCbAd4FSKQhjSOSdSkRMcaQl5fX7iyJNKEZCQaVXIoMeR00Sqlw2TwEPaqvd7HGjC+/r8aARA3uksgA3tkyL1FMSOOIZF1KRLm5uZ2WoStmJNCGpxkwvJkB5WpA1zfkx3dfvrc5mTAK31IntBNRIgqYLiWigoKCYMVBSMcaWxZKEKF9yDPezMkL7USWOrpyFmA0Czjp0RwuHi6rCWiqFjYY0kMeg6dDo9MFqPTCRmqwDihKRKRHO33FiG27vsKlq2ahfUihCXkMSpnQKG1z8jRBWpBQIiI9Wo3JBq3lMqQSDojJFCWGVqdmANWIAowSEenRqo1W6K2XhStmMVmixOA+NXPxDE65VthINaKA8nsVD0KCzeni0VxXDoXLgih1PKAXZ4lxpUyCqTkJQkJStfSTs7VMkEazTQQEJSLSY1WbbDCYL0Au5aBKygGk4vy5chyHUektl+pdzms/HRZR2qwikainZvn5+Rg7diy0Wi0SExMxb948FBUViRkS6UGu1FsQa7kErUoOLnGQ2OEIpDJASdOBBJqoiWjPnj1YtmwZ9u/fj08//RQOhwO33347zGazmGGRHqK24hJUTiOio1RAbLaosVSbrLhY04Qmm5MarINA1FOzjz/+2Ot+YWEhEhMTcfjwYUyZMkWkqEhPMYi/AKtGAU3fm4O+tHRndhfVoLy+GXOGpWCgSgcYr1CNKIB6VBtRY6PQWzU2NrbNx202m9d4traWNCIRwm5GhqsUSNIC2ePEjua6To3XX8Kn3tWB0mMu3/M8jxUrVmDixIkYOnRom2Xy8/Oh1+s9t7Q0ca6ikBAo3S8MLtUmA7pUsaO5brzZdb2rqUYUMD0mES1btgynTp3Ctm3b2i2zevVqNDY2em5lZWUhjJCEjLURtWf3w+p0AVlTesQlck/vasd1NSIabxYwPeLUbPny5fj3v/+NvXv3om/f9kdXK5XKdqcgIRGCMdhO78C5inoYlcm4JSoNOrFjwnW9q138dTUiSkSBImqNiDGG5cuX44MPPsAXX3yBrCxxes6SHqR0H+pKv4eLk8KUPhM6tbiN1G6eUzMHL0zeDwCOZsBBiy0Ggqg1omXLluGdd97BRx99BK1Wi8rKSgDCAo5qNS1g16swBpQdBLu4GzUmG0oMt2BwZobYUXlcG/jqAmRKoSOj3Qw01wPy0M4YGYlErRFt3rwZjY2NmDZtGlJSUjy3v//972KGRULN5QTOfgJc+AKNzQ6cUw1FnWEoBiX3hJMyQYpBhak5CRje1yBscE+K1kxztAeCqDUixpiYhyc9QXM98N2HgKkSPIB93EiU6nMwso9eWDqoh4iPViI++rr2SXUM0HgZsDaIFlMk6RGN1aSXaiwHTv5DaGeRq3E4aiK+dwoJaFxW233Jegx3OxHViAKCEhERR+Nl4Pg2wOUAdCnAkHsR3yQDV1eOmYMTEaXoWX+aLp6hymiF1eFCv4RoOjULsB7Tj4j0Is0NcJ54D2VXG3HWHgcMfxhQ6ZEVr8HD49PRP1ErdoStOHkefz9Uho+OXYHDxVMiCrCe9bVDIh/Pw3jkfZy/WIF6LgZFuknIgBTu1pdEbejWtO8KhVQCqYSDi2dodrggdyciWxPgtIs+Fi7cUY2IhFTd2a9RdLYIFl6KirQ7MWNoGuSSnv9nyHEc1HKh8dxqdwFyNSBvSZrUYN1tPf8vgEQMa1M9Lh7cCSfPYEmbhvmThyInWQuJRPwhHL5QtVzFa3a4hA2e07MGcQKKIJSISMic+vo/cDjscGlTMWXKDE8nwXDhrhG1TkTUTtRdlIhISFRXlMFx+Rg4ADkT50LVw66K+cKTiOwtiUhlEH5SIuq28PtrIGEpoe4wJIka1EdlIaGPuLMt+kutEL63qUYUeJSISPA1N4CrKRJ6J4+5Q+xo/NY/QQtDlAKp+pZxkFEtnS4tteIFFSEoEZGg4nkGVnoAUsYLCyRqk8UOyW/pcVFIj4u6tiEqTvhpMwFOmzAYlviF2ohIUJ29XI0jB/ag2mQF0seLHU5gydXXlhOy1IkbS5ijRESChucZik98BafDDrMiQbSVWgPF7uRxpaEZZXWWaxvdtSI6PesWSkQkaM5W1EFTcwwyCYc+w6b2iClfu6PBYsffD5Xh41OV1zZ6EtFVcYKKEJSISFAwxnDh5D7IeCsSExKhSB4idkjddn2HRs8UNlQjCghKRCQozlcZoak6DJmEQ/LQqUAYDOPojLsfkYtnsLt4YaPGnYiojag7wv+vg/Q4jDF8f/IQlE4TEuIMUPYdIXZIASGXSiCXCqeXnk6N7hpRcz3Au0SKLPxRIiIBV9nYjKiKQ5BKOKQMnhxRI9Pd8ySZ3YlIqQOkMiEJ0Zgzv1EiIgGX4qrALYlOZCbqocwYI3Y4AaVRCqdnFptT2MBx1E4UAJSISOCV7odGKUPiwFsARVTn5cOIRnlDjQigK2cBQD2ryTVOG1B5CrCbAG0qENe/S43MjDE0VZ2HtqEUkEiBtLFBDFYcg1N06GNQo4/huuWuNIkATgNN1aLFFe4oERGBsQI4+Z6wVpebNgm4ad61MVWdOF9lQuln2zFAbUH6sCnXVkSNINkJ0a03RicKP801oQ0mgoh6arZ3717cfffdSE1NBcdx+PDDD8UMp/dqbgBObBOSkDoGSBkuzD5oqgKOvuXTN72LZzh9bB+ibDWAVA5k3Br8uHsKdyKy1AqLAZAuEzURmc1mDB8+HH/84x/FDKN3Ywwo2iks6aNNBsY8Cgy6Exi7WPgHs5uBY+8ATR1/258qqURMxX8hl3JIvnn6tTFYEcY9zKO09rphHopoYdwZY4CZ2on8IWoimj17Nn7729/i3nvvFTOMXq22+DiqS79HnY3BmXP3tRHkSi0w4mEhOTmagePvtPtPZrY6UHHoQ8j5ZiSn9IUiK3JrQ55hHt9VXNvIcdednlE7kT/C6qqZzWaD0Wj0uhH/OFw8dp4ox/Gv/o0LNWZ83pSFwmNGFF+9ro1IrgaGL2ipGVmEmlEbPYhP7vsYetMFaJRypI7/odBQHaHcV80sdhd4/rqVit2JiBqs/RJWiSg/Px96vd5zS0tLEzuksHW2yoSai8cR5WiARqOBMW44TFYnPjhajsMl1804KFcDwx8CohOE07SjfwNqioTTEKcd5d/+G+ziHnAA+oy9CxJDX9FeUyio5VJwnPDyPTM1Ai1XzkCJyE9hddVs9erVWLlypee+0WikZOSnIal6xOvLoFLooB88HYPTB2Hv2RqcuNyI/RdrMShZ6/n2hyJKSEbHtwn/aKe2C6dwvAtKoxlSCQfVgOmIGxi5p2RuEgmHKIUUZpsLZpvz2nt0/akZY2E/00CohVUiUiqVUCppFryAMFUiCXWARgWkjoRcKsGMwUnQqeVINaiv/YO5KTTAqFyg5L9A+WGhzxGA+MQUKEdMRlSfwb3mn0+jlMFsc8FkcyLRvTEqHuAkQqO/zRiRXReCKawSEem+01eMSItVQ1t+RNiQkAMor/WNGZvp3WeotskGvVoOmVQijKnqNw3NqRPQVF+FBL2wBry2lyQgN51KjmqjDSar89pGqQzQxAs1RmMFJaIuEjURNTU14fz58577ly5dwrFjxxAbG4v09HQRI4tMTTYnPjtTBTlvxSLpSSg5AKmj2i1fbbLi/cOXEa2UYVxWLGI1ClQ12nDgUi2cPMOCsdEw9LIkBAA6tRwAYGy+oc+Qrk9LIioHEgeJEFn4EjURffvtt5g+fbrnvrv9Jzc3F4WFhSJFFbmOlzXAxTPcxIqh5HihAVrffuNys90FKcehtsmOnScrvR4zRMnhuv6qUS+SnaCBTiVDil7t/YAuFbhyFDBVtP1E0i5RE9G0adOuzXRHgsrp4nHiciPAGIZLLgobU0d12K6TEafBwgmZOFpWjws1ZphtTuhUcuQkR2NYXwPk0rC66BowfWOi0DemjcG8uj7CT1OFMC1IBHdjCDRqI+olLtSYYXW4kIoqxEvMwlWvpKGdPk+tkOLW7Hjcmh0fgijDXFSs8L46bcK4szBeOinUeudXWi90uqIRADBCegkcxwlJKIImLAslxhgqGptxtsoEp3vKWECoXepShd+N5eIEF6YoEfUCJqsDJbUWyJ1mpPMt/yCpI8UNKsxtP1KOHScqvK+cAdcSUSMloq6gRNQLVJtskHIcbpIUQy3jhAbq6MTOn0jaxHEctCqhVcNoveHKmb6lg21DqdCxkfiEElEvkJ0QjSWTMnCLslTYQLWhbtO3XMJvsNyYiPoKjdQ2kzChPvEJJaJeQlVfhChmEXpIJ1Afl+4yRAnta/UWu/cDUvm107OGkhBHFb4oEUU4s80pnCKU7hc29B0r9AIm3RLbXiICAEOG8LOeEpGvKBFFMIeLR+E3xfjXF7thN1YLl5b7tN+TmvguRiOcmtWZ25iRMcadiIoBnm/9OGmFElEEu1hjhsPhQHzVN8LCgH1GX5v4jHRLrEaoEZmsDjhcNyQbXR/hfXY0A6YrIkQXfqiOHsFOVzQi2XQKfZVWcIoEIP0WsUOKGGq5FNNyEmCIUqBV33SJFIjLBqpOA1fPdjiMhgioRhShmmxOVFVeQVrjt4jXKoGsKVQbCiCO4zAyPQZZ8RphZoIbxQ0Qfl493/ox0gologhVdPkqBtR8Dr2Sgzqpv7AyBwmd2H7C/ESWWppQ3weUiCIQczlgObYdUY5axBkMwKA5vWbSslCyOV04V2XC8bKG1g/KVUIyAoDKkyGNKxxRIoo0ViPqvymErOEiOKkMMbc8LKzIQQKuyerEv09U4KtzNd4T6bsl3yz8rDpFV886QYkokjSUAYcLEeOowcA+8dCNfQiqOJpgLlhiohRQyCRwuBjq2upPFNdfqBnZmoD6S6EPMIxQIooEjAGXDwvL/djN4LSJiJuyFDcNoXahYJJIOCTpVACAy/XNrQtIZdemWrn8bQgjCz+UiMKdywkU/Qc4twtgPPiEwcDIhT6vV0+6JyNOmCCtpNbcdoG+Y4T2ubqLwhLepE2UiMKZtVFYZ6ziBMBxsKRPxZt1N+NAqantNgsScOmxQiK6XN/c9tS56phrY/tKvg5hZOGFElG4qrsEfFsAmCqFRRCHPYh99myYbC4U15rpIlmIJGqVUCuksDt5lLd1egYAGbcKtaKaIhp/1g5KROGGMaBkH3Di78IQAm0SMDoP1bIUnCwXZmG8NTtemIWRBB3HcRiQKCzHVN7QTiKKTrw29cq5XcLpNPFCQzzCidMGfP9voOascD9lGDDgdjghxa7jZWAMyEnWIi22jYndSdCMyYjFqPQYxGg6mHo3czJQfUbo3Hj+MyBnVugCDAOUiMJFfQnw/Q6hXUgiBQb8AEgZAQZg95lq1JhsUCukmDIwQexIex19lLzzQoooYPDdwMn3hCWH1AYa+3edHnFq9sc//hGZmZlQqVQYP348Dh48KHZIPYfdDBR9DBx/V0hCagMw4hGhqs9x2HexFifLG8FxwB1DkhF941LRJKSqTVbUm9voUwQIA2Gzpgq/X/hSuFFHRwA9IBH9/e9/x8qVK7F27VocOXIEw4cPxx133IHq6mqxQxNXcz1w4QvgwJ+Fb1DGhPFiYxYB+j6eYvHRwkDW6TmJyIrXiBUtAfB9pRHvHijDR8fKW89l7ZYxAcicJPxeuh84UgjUXuj181tzTOQVDsePH4+xY8fiD3/4AwCA53mkpaXhZz/7GZ5++ukOn2s0GqHX69HY2AidTheKcAOP5wFns1DzMdcATVXCFbGma4mYaZNhz5wGkyoVlY1W8IxhWF+D5/FqoxWJLR3riHhMVgf+fqgMJqsTSrkEI9IM6J8QjRiNovVilNVnhP5fzpbak9og9MTW9RH6gKn0gEwV9mMEff0fFbUeb7fbcfjwYaxevdqzTSKRYObMmdi3b1+r8jabDTabzXPfaDT6dqC6S8CFz9v+1vHaxjre3sa2MxVGmG2Oa890l2GATMphZJrBU/b7ChOM1uuq7YyHlHcAjEHCASPTYzwPnatuQhkSURU9BLV8OhxHHQCES78KmQQ5yVooZcJKopSEegatSo75Y9Lwn5MVqGy04sDFOhy4WAcAiItWYOGETE/Z/1QZ0IjZSDQdQaypCBK+AThXDACQSTgMSdULSUgqR1GNDSabC4wTkhmDBIyTYGCSFjG+tE8Fy7AHAVVgKgCiJqKrV6/C5XIhKSnJa3tSUhK+//77VuXz8/Oxbt26rh/IaQOaavwNs+NdWxrhsLVzOVbCAfZrfyi83QyXzbvK7nI/JlUJE2hpEgB9HxRpVThX35LUWpoRFDIJErRKpOrVvXbd+Z5Or5bjwTFpOFfdhDMVRpQ3NMPu5OFweX9ejc0OVFplqFSPg0Q5EgZrGfTWK9DYryKabxIKMQY47XA2G+G4cf00AExjBZiIi2SywLVvhVXL5urVq7Fy5UrPfaPRiLS0tM6faEgDhi8QfvdUda+r8npVfzvZfsO2pIE2xLp4r+3uPjxSCQfo1J6yyWYb4nkGBk4oznFC9VumBiQSQHutZjNZ78AtLh4cAJlEgiiltNeuNR9uJBIOOcla5CRrwRiDzcnD6nB5lbltUCLszuv/kbM8v3EcAJ1c+AJ12pDUaILB4Wr5x2dCgmIMao0MkEtD8prapAhcm6SoiSg+Ph5SqRRVVd5jcKqqqpCc3HrdcKVSCaXSj1kGFRogNqvzcn5I7ELNNDba97LudbNIeOM4Diq5FKobEkaSL6fTUjmgjEa8Ji5I0fUcon7FKhQKjB49Gp9//rlnG8/z+PzzzzFhwgQRIyOEhJLop2YrV65Ebm4uxowZg3HjxmHjxo0wm8149NFHxQ6NEBIioieiBx98EDU1NVizZg0qKysxYsQIfPzxx60asAkhkUv0fkTdERH9iAiJYL7+j9JlGEKI6CgREUJER4mIECI60Ruru8PdvOXzUA9CSEi5/zc7a4oO60RkMpkAwLfe1YQQ0ZhMJuj1+nYfD+urZjzP48qVK9BqtT1yalT3EJSysrKIu6pHry08hfq1McZgMpmQmpoKiaT9lqCwrhFJJBL07dtX7DA6pdPpIu4P2o1eW3gK5WvrqCbkRo3VhBDRUSIihIiOElEQKZVKrF271r8ZA3o4em3hqae+trBurCaERAaqERFCREeJiBAiOkpEhBDRUSIihIiOElEXdXVV2vfeew+DBg2CSqXCzTffjP/85z9ejzPGsGbNGqSkpECtVmPmzJk4d+5cMF9Cu7ry2v7yl79g8uTJiImJQUxMDGbOnNmqfF5eHjiO87rNmiXOmu9deW2FhYWt4lapvOeYDtfPbdq0aa1eG8dxmDNnjqeMKJ8bIz7btm0bUygU7M0332TfffcdW7JkCTMYDKyqqqrN8l9//TWTSqXsxRdfZKdPn2b/+7//y+RyOTt58qSnzPr165ler2cffvghO378OLvnnntYVlYWa25uDtXLYox1/bU9/PDD7I9//CM7evQoO3PmDMvLy2N6vZ5dvnzZUyY3N5fNmjWLVVRUeG51dXWhekkeXX1tBQUFTKfTecVdWVnpVSZcP7fa2lqv13Xq1CkmlUpZQUGBp4wYnxsloi4YN24cW7Zsmee+y+ViqampLD8/v83yDzzwAJszZ47XtvHjx7PHH3+cMcYYz/MsOTmZ/e53v/M83tDQwJRKJXv33XeD8Ara19XXdiOn08m0Wi3bsmWLZ1tubi6bO3duoEPtsq6+toKCAqbX69vdXyR9bq+88grTarWsqanJs02Mz41OzXzkXpV25syZnm0drUoLAPv27fMqDwB33HGHp/ylS5dQWVnpVUav12P8+PHt7jMY/HltN7JYLHA4HIiNjfXavnv3biQmJiInJwdPPPEEamtrAxp7Z/x9bU1NTcjIyEBaWhrmzp2L7777zvNYJH1ub7zxBhYsWACNxnuNslB/bpSIfNTRqrSVlZVtPqeysrLD8u6fXdlnMPjz2m701FNPITU11eufYtasWdi6dSs+//xzvPDCC9izZw9mz54Nl8vVwZ4Cy5/XlpOTgzfffBMfffQR3nrrLfA8j1tvvRWXL18GEDmf28GDB3Hq1CksXrzYa7sYn1tYj74nPcP69euxbds27N6926tRd8GCBZ7fb775ZgwbNgzZ2dnYvXs3ZsyYIUaoPpkwYYLXunq33norBg8ejNdeew3PPvusiJEF1htvvIGbb74Z48aN89ouxudGNSIfdXVVWgBITk7usLz7Z1f2GQz+vDa3l156CevXr8euXbswbNiwDsv269cP8fHxOH/+fLdj9lV3XpubXC7HyJEjPXFHwudmNpuxbds2PPbYY50eJxSfGyUiH/mzKu2ECRO8ygPAp59+6imflZWF5ORkrzJGoxEHDhwI6Uq3/q64++KLL+LZZ5/Fxx9/jDFjxnR6nMuXL6O2thYpKSkBidsXgVhN2OVy4eTJk564w/1zA4RuJTabDT/60Y86PU5IPreQNo2HuW3btjGlUskKCwvZ6dOn2dKlS5nBYPBc2v3xj3/Mnn76aU/5r7/+mslkMvbSSy+xM2fOsLVr17Z5+d5gMLCPPvqInThxgs2dO1e0y8BdeW3r169nCoWCvf/++16XeU0mE2OMMZPJxFatWsX27dvHLl26xD777DM2atQoNmDAAGa1Wnv0a1u3bh375JNP2IULF9jhw4fZggULmEqlYt99953X6w/Hz81t0qRJ7MEHH2y1XazPjRJRF7366qssPT2dKRQKNm7cOLZ//37PY1OnTmW5uble5f/xj3+wgQMHMoVCwYYMGcJ27Njh9TjP8+zXv/41S0pKYkqlks2YMYMVFRWF4qW00pXXlpGRwQC0uq1du5YxxpjFYmG33347S0hIYHK5nGVkZLAlS5a06o8TKl15bStWrPCUTUpKYnfeeSc7cuSI1/7C9XNjjLHvv/+eAWC7du1qtS+xPjeaBoQQIjpqIyKEiI4SESFEdJSICCGio0RECBEdJSJCiOgoERFCREeJiBAiOkpEhBDRUSIiYaWwsBAGg8Fz/5lnnsGIESM89/Py8jBv3ryQx0W6hxIRaZN73uKf/OQnrR5btmwZOI5DXl6eV/lAJ4DMzExs3LjRa9uDDz6Is2fPtvucTZs2obCw0HN/2rRpWLFiRUDjIoFHiYi0Ky0tDdu2bUNzc7Nnm9VqxTvvvIP09HRRYlKr1UhMTGz3cb1e71VjIuGBEhFp16hRo5CWlobt27d7tm3fvh3p6ekYOXJkt/bdVk1l3rx5nlrWtGnTUFJSgl/84heelSSA1qdmN7q+ZpaXl4c9e/Zg06ZNnn1cunQJ/fv3x0svveT1vGPHjoHjuJDOlUSuoUREOrRo0SIUFBR47r/55pt49NFHg37c7du3o2/fvvjNb36DiooKVFRUdHkfmzZtwoQJE7BkyRLPPtLT01u9JgAoKCjAlClT0L9//0C9BNIFlIhIh370ox/hv//9L0pKSlBSUoKvv/7ap8m0uis2NhZSqRRarRbJycl+zXyo1+uhUCgQFRXl2YdUKkVeXh6Kioo86385HA688847WLRoUaBfBvERzVlNOpSQkIA5c+agsLAQjDHMmTMH8fHxYofVLampqZgzZw7efPNNjBs3Dv/6179gs9kwf/58sUPrtahGRDq1aNEiFBYWYsuWLQGrNUgkEtw4FZbD4QjIvn2xePFiT0N8QUEBHnzwQURFRYXs+MQbJSLSqVmzZsFut8PhcOCOO+4IyD4TEhK82n1cLhdOnTrlVUahUHR7CZv29nHnnXdCo9Fg8+bN+Pjjj+m0TGR0akY6JZVKcebMGc/v7WlsbMSxY8e8tsXFxSEtLa1V2dtuuw0rV67Ejh07kJ2djQ0bNqChocGrTGZmJvbu3YsFCxZAqVT6dUqYmZmJAwcOoLi4GNHR0YiNjYVEIvG0Fa1evRoDBgwI6aT3pDWqERGf6HQ66HS6Dsvs3r0bI0eO9LqtW7euzbKLFi1Cbm4uFi5ciKlTp6Jfv36YPn26V5nf/OY3KC4uRnZ2NhISEvyKe9WqVZBKpbjpppuQkJCA0tJSz2OPPfYY7HZ7SK4Cko7RnNWk1/rqq68wY8YMlJWVtVotlYQWJSLS69hsNtTU1CA3NxfJycl4++23xQ6p16NTM9LrvPvuu8jIyEBDQwNefPFFscMhoBoRIaQHoBoRIUR0lIgIIaKjREQIER0lIkKI6CgREUJER4mIECI6SkSEENFRIiKEiO7/AzqYjt7au0iOAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:31:07.754067Z", + "iopub.status.busy": "2024-03-26T07:31:07.753691Z", + "iopub.status.idle": "2024-03-26T07:31:08.135937Z", + "shell.execute_reply": "2024-03-26T07:31:08.135080Z" + }, + "papermill": { + "duration": 0.399431, + "end_time": "2024-03-26T07:31:08.138041", + "exception": false, + "start_time": "2024-03-26T07:31:07.738610", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABBKElEQVR4nO3deXxTZb4/8M/JnrRZuqQbXdl3KCIIooAyboioVwe3EQR1HGHmIj9G4d4ZGJcZ0FEH5upllBGqd0RURJ1R2UTBAQGRHYECha50o1vSNPt5fn+cJG3oQpsmOU36fb9eebU5OTn5nqb55nme8ywcY4yBEEJEJBE7AEIIoURECBEdJSJCiOgoERFCREeJiBAiOkpEhBDRUSIihIiOEhEhRHQysQPoDp7ncenSJWi1WnAcJ3Y4hJArMMZgNpuRlpYGiaT9ck9EJ6JLly4hIyND7DAIIVdRUlKC9PT0dh+P6ESk1WoBCCep0+lEjoYQciWTyYSMjAzfZ7U9EZ2IvNUxnU5HiYiQHuxqTSfUWE0IER0lIkKI6CgREUJER4mIECI6SkSEENFRIiKEiC6iL98TEqhzlWYcLalHv6RYjMmMEzucXo8SEel1GGM4VW5CaZ0VpXVWGGOVyIjXiB1Wr0ZVM9LrcByHu0alIUmnBAAcKakXNyBCiYj0ThzH4WdDkwEAxTUWON28yBH1bpSISK/icPFw88IKWsZYJWKVMjjdDJUmm8iR9W6UiEivcrSkHn/bXYADF2rAcRyS9SoAoEQkMkpEpFcprWuCw8VDKZcCAFJ03kRkFzOsXo+umpFegzHmSzipnpLQgKRYJMQqkOxJSEQcopeIysrK8MgjjyAhIQFqtRojRozAjz/+KHZYJAqZbC7YnG5IJRwSYhQAgLgYBfoZYxGrpO9kMYn616+rq8P111+PqVOnYsuWLTAajTh37hzi4qiDGQm+arPQDpQQq4BMKvp3MGlB1ET08ssvIyMjA+vXr/dty8nJETEiEs281bIkrX81rPCyBRUmG/oaY1o9RsJD1K+Ff/7znxg7dizuv/9+JCUlITc3F2vXrm13f7vdDpPJ5HcjpLMuN3oTkdJv+8lLDdhXUIPSOqsYYRGInIguXLiANWvWYMCAAdi2bRt+9atf4Te/+Q3efffdNvdfsWIF9Hq970YT55OuSI9To19SbKuGaZ1KDgAw21xihEUAcIwxJtaLKxQKjB07Ft9//71v229+8xscPHgQ+/bta7W/3W6H3d58mdU7MXdDQwPNWU0Cdri4DrvzqzEgORZ3jkwTO5yoYjKZoNfrr/oZFbVElJqaiqFDh/ptGzJkCIqLi9vcX6lU+ibKpwnzSbDoVEJTaSOViEQjaiK6/vrrkZ+f77ft7NmzyMrKEikiEq1sTjcsdhfaqgBoqWomOlET0TPPPIP9+/fjT3/6E86fP48NGzbg7bffxvz588UMi0Sh0+UmvP3dBWw5WdHqMa2nRGRxuHzj0Eh4iZqIrr32Wnz66af44IMPMHz4cLz44otYtWoVHn74YTHDIlGo3uoE0Jx0WlLLpZBJODBG1TOxiN6d9M4778Sdd94pdhgkytU3OQAABrWi1WMcx+Hu3D7QKKSIbSNRkdCjvzrpFeqbhBKRQSNv83GaoVFc1M+dRD03z2CyClWu9hIREReViEjUM1md4BmDXMq1O7i1rN6K0tomGLVK9DXGhjlCQiUiEvXqPO1Deo0CHMe1uU9JbRO+L6hBQbUlnKERDyoRkainVckxJisOGoW03X28jzU56KqZGCgRkahn1CoxWWvscB+NQvgoNDnc4QiJXIGqZoSgZYmIEpEYKBGRqFdpssHqcLc5vMMrxlsiamcYCAktSkQkqrl5hg9+KMbfdhd0WNpRe0pELp7B7qI1zsKNEhGJag1WJxgDFDJJh43VCpkECpnwcbBS9SzsqLGaRDXfpXu1vN1L9173jukDuVTS5ng0Elr0FydRzTu0I07TeozZlVL16lCHQ9pBVTMS1XyDXWloR49GJSIS1a422LWl4pomXGqwoo9BTYNgw4xKRCSq1flKRFevmhVcbsS+ghoU1zaFOixyBSoRkah2bXY8apsciO9EIlLJhKtqdNUs/CgRkag2KsPQ6X29fYlsLkpE4UZVM9L7mCuBxqpWm1Vy6kckFioRkahV02iHw80jTqOASi4FGAPytwDlx4Qd+owBBt7q218t95aIqGd1uFGJiEStI8X12PhDCQ4X1Qkbyo82JyEAKDsMVJ3x3VV5ExGViMKOEhGJWt6VOwwaBeB2AYV7hAf63QRkTRR+L/y3UFJCi0Tk7HiALAk+qpqRqOXXmbHmHGBvBJSxQJ9rAN4FlB4ELJeB+mIgLguxShnuuybd12hNwkfUEtEf/vAHcBzndxs8eLCYIZEo4XTzvpVb4zQKoPIn4YGUkYBUBshVgNHzv3b5LABAKuGQEa9BYqzyquPSSHCJXiIaNmwYvv76a999mUz0kEgU8PaoVsolUHEOoPaC8EDysOadjIOAihNAdT7QfxpAyUc0on/qZTIZUlJSxA6DRJkGq1Ati9MowNUXA7wb0CQAMYnNO8XlCKUjuxmwVAOxScivMKOuyYGByVrEx1y9EyQJDtEbq8+dO4e0tDT07dsXDz/8MIqLi9vd1263w2Qy+d0IaYtvjJlaDtQVChvjsv13ksoAXbrnCSUAgGOl9dhXUIPLjfbwBEoAiJyIxo8fj7y8PGzduhVr1qzBxYsXccMNN8BsNre5/4oVK6DX6323jIyMMEdMIkVWQgwmDzJiUIoWqL0obIzPab2jIVP42SB8ASo9k6M5qC9RWIlaNbv99tt9v48cORLjx49HVlYWPvroI8ybN6/V/kuXLsWiRYt8900mEyUj0iajVgmjVgnYGgBrHcBJAH0b/ysGz7b6EoAxKGXNl/BJ+IjeRtSSwWDAwIEDcf78+TYfVyqVUCqVYY6KRLSGMuFnbJJwpexK2lQhSTksgN0MpWeYB81bHV6itxG11NjYiIKCAqSmpoodColgTjePMxUmVJpsYKZLwkZdWts7S+VATILwe2Olr2pmp4GvYSVqIlq8eDF2796NwsJCfP/997jnnnsglUrx4IMPihkWiXB1TQ5sOVGBzYfLwJnLhY3tJSIAiPVctTVX+KpmdieViMJJ1KpZaWkpHnzwQdTU1MBoNGLSpEnYv38/jMaOV+UkpCPeK2YJGgnQWCFs1HaQiLQpQn+ixkoo40cCoKpZuImaiDZu3Cjmy5MoVWcR+hAZpY3CGDOZEtDEt/+E2GThp7kC2QNjcP/YdMQqe1TzadSjvzaJOnWeEpGR1QobtKkd95r2JiK7GbGcA7FxNF91uPWoxmpCgsE72DWO90z/oU3u+AkyBaDSCb831YQwMtIeSkQk6nhLRFp3vbAhJunqT9IIQz+c5mocLanHDxdrQxQdaQslIhJVrA630BmRMcQ4PaWb2E4kIs8lfGa5jG/PVGHv+ctwuanBOlyojYhEFZmUw8zRaWgy10Ja5BA6K2oSrv5Ezz4yWy2AbACAw81DJqXv6nCgRESiilwqQV9jLMB5Lttr4gFJJyY681TNJNYaKGQSOFw87E4enViFiAQBpXsSnSzVws/OVMuA5lKTzQS1ROhVTX2JwocSEYkq5yrNOFtphq3e06M6ppOdYxUa4QZAx4TpZWiYR/hQIiJRZf/FWnx5vByNtd5E1MkSEeArFWl5byKiElG4UCIiUYMxhnqLA2A8YtyeSfNiOtFQ7aWOE57CC/Nh0VQg4UON1SRqmO0uuHgGDW+BSsYAiRxQ6jt/AJUBADA8nkd2Trow6T4JC0pEJGrUWzxDO2QWYRUOtQGQdKHQ7ykRGWCBgYZ5hBVVzUjUqPMM7UiUNAobOhro2ha1Qfhpqw9aTKRzKBGRqOFNRHGcRdjgKeF0mmd/a2MDjhVW4XxV23Onk+CjRESihm/lDngSiLqLJSK5GpCrYLY5sf+nApwoawhyhKQ91EZEosbkgUYM76NHYn4T4ELXq2YAoDJAJjFB6TLTSh5hRImIRI24GAXiVBLA7Wkj6mqJCADUcZBKSqCyN1A/ojCiqhmJLrZ6gDFhjiFFTNefrzZAJuGgcplp3uowohIRiQpVJhuKapuQ7ipDKiCUhgJZy14dB6mEg8plgoOmAQkbKhGRqFBc24Q95y6jsKRU2BBI+xAAqAyQSjhfG5GbZ8ELkrSLEhGJCt5ZGQ2c94pZFy/de6l0kEo4KNyNAGPUYB0mVDUjUcHbh0jPArx076XUQSKRYGiyBtmj4qGQ0Xd1OPSYv/LKlSvBcRwWLlwodigkAnknzI9lAfaq9pJIAaUWcRoF+qgckEoCaGciXdYjEtHBgwfx1ltvYeTIkWKHQiKQ3eWGxe6GlHdAw5qEjYGWiABA6VnRw0YdGsNF9ETU2NiIhx9+GGvXrkVcXID1etKreXtUx3GNkEkkwgRnclXgB1TpUW914FxJGWo9izWS0BI9Ec2fPx/Tp0/HtGnTxA6FRChv+5BRFoTSEACo9ChvsOGnghKUN1i7GR3pDNGXnD58+DAOHjzYqf3tdjvsdrvvvslkClVoJIL0M8bikeuyIC0uBaoQePuQl0oHmecSPvWuDg/RSkQlJSX4z//8T7z//vtQqTpXjF6xYgX0er3vlpGREeIoSSSQSyUwapWI57oxtKMlld53CZ8u34eHaIno0KFDqKqqwpgxYyCTySCTybB792789a9/hUwmg9vdeprOpUuXoqGhwXcrKSkRIXLSY1k9q7MG2ofIy9epsRF2mi42LESrmt188804ceKE37bHHnsMgwcPxnPPPQeptPVaVEqlEkqlMlwhkgjAGMOu/GroVFKMaqwR/qG7WzVT6iCTSCBlNthslmCESa5CtESk1WoxfPhwv20xMTFISEhotb273G43nE5nUI9JeoYmhwv5ZTWQMQeGyGRwSWQApwZstm4d160xAo01YDRbY1gElIguXLiAvn37BjuWoGOMoaKiAvX19WKHQkLExfPIjXNDyjEUYbywxHRxabeP604aD3WcE1LODYfDAYWCJtIPpYASUf/+/TF58mTMmzcP9913X6cbm69m165dQTmOlzcJJSUlQaPRCBOqk6hic7jQ6HBDBSdi0QRI5d1vIwLAW+vhdthRXmdBeXk5MjMz6f8nhAJKRIcPH8b69euxaNEiLFiwALNmzcK8efMwbty4YMcXMLfb7UtCCQldWNuKRBQnnJDDDQ3HQ8XkQmfGoHwxagApAy9V4FJ1PVwuF+RyeRCOS9oS0FWz0aNHY/Xq1bh06RLWrVuH8vJyTJo0CcOHD8frr7+O6urqYMfZZd42IY2GloWJZt5pOqTwXGaXBKnZUyJcLFFIhVJQW1dxSfB06/K9TCbDvffei48//hgvv/wyzp8/j8WLFyMjIwOPPvooysvLgxVnwKg4Hd1c3kTEPIlC0vpqayAYJ4WTZ3BSAgqLbiWiH3/8EU8//TRSU1Px+uuvY/HixSgoKMCOHTtw6dIlzJw5M1hxEtIKY0woETFAAk/C4IJTImKcBHanG06nC4zR5GihFlAiev311zFixAhMnDgRly5dwnvvvYeioiK89NJLyMnJwQ033IC8vDwcPnw42PGSABUWFoLjOBw9ejQsx9m7dy9GjBgBuVyOu+++u1uv2R6O44Qe1RopOADgELQSEeet4jGeElEYBPT1sWbNGsydOxdz5sxBampqm/skJSXhnXfe6VZwJHR27dqFqVOnoq6uDgaDIejHX7RoEUaPHo0tW7YgNjY26Mf3knAcJJynfYiTBTZPdRs4iaT5WJSIQi6gRLRjxw5kZmZCcsW64owxlJSUIDMzEwqFArNnzw5KkL1dJPZjKSgowFNPPYX09PSAj9Hp8+Zdws9ulIYYY3C73ZDJWnwkJFIATjBG481CLaCqWb9+/XD58uVW22tra5GTk9PtoHq7KVOmYMGCBVi4cCESExNx66234uTJk7j99tsRGxuL5ORk/OIXv/B7D7Zu3YpJkybBYDAgISEBd955JwoKCto8fmFhIaZOnQoAiIuLA8dxmDNnTpeOc+bMGUycOBEqlQrDhw/H7t27fcfmOA41NTWYO3cuOI5DXl4eAGD37t0YN24clEolUlNTsWTJErhcrg7Pe9euXeA4Dtu2bUNubi7UajVuuukmVFVV4dN/foFBgwdDl5iKh+Y9jSZb89xBPM9jxYoVyMnJgVqtxqhRo7Bp0ybf497jbtmyBddccw2USiX27Nnjd46M8yQ2SkShxwLAcRyrrKxstb2wsJBpNJpADhmQhoYGBoA1NDS0esxqtbJTp04xq9Xa6jG7093uzelyd3pfRyf2DcTkyZNZbGws++1vf8vOnDnD9u/fz4xGI1u6dCk7ffo0O3z4MPvZz37Gpk6d6nvOpk2b2CeffMLOnTvHjhw5wmbMmMFGjBjB3G4hhosXLzIA7MiRI8zlcrFPPvmEAWD5+fmsvLyc1dfXd+k46enpbNOmTezUqVPs8ccfZ1qtll2+fJm5XC5WXl7OdDodW7VqFSsvL2dNTU2stLSUaTQa9vTTT7PTp0+zTz/9lCUmJrLly5e3e95nzpxh3377LQPArrvuOrZnzx52+PBh1r9/fzZ58mQ29eZpbMd3+9jOrf9kCfFxbOUfX/Ad66WXXmKDBw9mW7duZQUFBWz9+vVMqVSyXbt2McaY77gjR45k27dvZ+fPn2c1NTV+74Op7jK7XHaBHT96qM3/I3J1HX1GW+pS1WzRokUAhEbCZcuW+fXRcbvdOHDgAEaPHh2cDBlCb357vt3HchJjcHduH9/9t78rgNPddhtBepwa949tnopk3d6LsDr8L/c+87OBAcU4YMAAvPLKKwCAl156Cbm5ufjTn/7U/Frr1iEjIwNnz57FwIED8R//8R9+z1+3bh2MRiNOnTrVauyeVCpFfLwwMDQpKcmvjaizx1mwYIFv3zVr1mDr1q1455138OyzzyIlJQUcx0Gv1yMlJQUA8L//+7/IyMjAG2+8AY7jMHjwYFy6dAnPPfccli1b5qvmtzxvAL4uIC+99BKuv/56AMC8efOwdOlSHDx2ChnZOUhAA+6beSe+3f1vPPdfwrxVf/rTn/D1119jwoQJAIC+fftiz549eOuttzB58mTf8V944QX87Gc/a/tN8Fb1eCoRhVqXEtGRI0cACPXpEydO+NXfFQoFRo0ahcWLFwc3wl7qmmuu8f1+7NgxfPvtt202+hYUFGDgwIE4d+4cli1bhgMHDuDy5cvgPR+e4uLiLg0i7uxxvB9wQOhPNnbsWJw+fbrd454+fRoTJkzw69d1/fXXo7GxEaWlpcjMzGx13i21nM88OTkZGo0GGdk5AGOQcDySkxLxw9GTAIDz58+jqampVYJxOBzIzc312zZ27Nh2Y1Yp5OBkUkg5aqwOtS4lom+//RaAMF3H6tWrodPpQhJUqM2f2r/dx65ctOHJG/u1u++VF2jmXh+89rGYmOblkhsbGzFjxgy8/PLLrfbzXrWcMWMGsrKysHbtWqSlpYHneQwfPhwOR9fmXA7WcQLV8rxbajm8guM4330JeHAAOE7iS5qNjcIEaV9++SX69Onjd5wrp5Fp7/UAQC6Twy0BOFAiCrWArpqtX78+2HGEVVfWqgrVvl0xZswYfPLJJ8jOzva/quNRU1OD/Px8rF27FjfccAMAtGp4bRWrpzTbcuhCV46zf/9+3HjjjQAAl8uFQ4cOYcGCBe2+3pAhQ/DJJ5+AMeYrFe3duxdarbZbV9bkHO/pQ9T8rTB06FAolUoUFxf7VcO6TNKisZqqZyHV6UR07733Ii8vDzqdDvfee2+H+27evLnbgZFm8+fPx9q1a/Hggw/i2WefRXx8PM6fP4+NGzfi73//O+Li4pCQkIC3334bqampKC4uxpIlSzo8ZlZWFjiOwxdffIE77rgDarW6S8d58803MWDAAAwZMgR/+ctfUFdXh7lz57b7ek8//TRWrVqFX//611iwYAHy8/OxfPlyLFq0qFU3kK6QcTzAIEz/4aHVarF48WI888wz4HkekyZNQkNDA/bu3QudTtfpbiUuxsHJAzxjgLMJAI1bDJVO/wfo9XrfN1nLeaPbupHgSktLw969e+F2u3HLLbdgxIgRWLhwIQwGAyQSCSQSCTZu3IhDhw5h+PDheOaZZ/DnP/+5w2P26dMHzz//PJYsWYLk5GQsWLCgS8dZuXIlVq5ciVGjRmHPnj345z//icTExA5f76uvvsIPP/yAUaNG4amnnsK8efPwu9/9rlt/G6lvaIf/v/KLL76I3//+91ixYgWGDBmC2267DV9++WWXupc43Dwcbs/AWkdjt+IkHeMYi9xuoyaTCXq9Hg0NDa3aq2w2Gy5evIicnJygzZdEehaeZ4C1FhK3A1DrAXlwSyw2pxvmy5dQXlaCgSk6qNKDO3Nob9DRZ7SlgMrEVqsVTU1NvvtFRUVYtWoVtm/fHsjhCAmIRMJBwjwdIoM02LUlDsLgV8ZAJaIQCygRzZw5E++99x4AoL6+HuPGjcNrr72GmTNnYs2aNUENkJB2tWxEDtY8RC1wHAfe+xGxUyIKpYAS0eHDh31XVTZt2oSUlBQUFRXhvffew1//+tegBkjIlZxuHnUWByxWz2KbEolwCzIJJwzzYACViEIsoK+RpqYmaLVaAMD27dtx7733QiKR4LrrrkNRUVFQAyTkSi6eweHmIYFnZZYQVMsAz6R6nkZwZjeH5DWIIKCvkf79++Ozzz5DSUkJtm3bhltuuQUAUFVVFbGdHEnkcHuG3Mi8038EaQ6iK3EcwHuuFLttVCIKpYAS0bJly7B48WJkZ2dj/Pjxvu7+27dvb9WFnpBgc3vahXyX7kPQPgQIjdUxSgXkUg4S3gG47CF5HRJg1ey+++7DpEmTUF5ejlGjRvm233zzzbjnnnuCFhwhbWk9T3XoqmYqhRwSTiIM/bGZgFhjSF6rtwu4hS8lJQW5ubl+vWLHjRuHwYMHd/oYa9aswciRI6HT6aDT6TBhwgRs2bIl0JBIL8Damqc6RFUzH29nSbsptK/TiwX0VWKxWLBy5Urs3LkTVVVVvsGGXhcuXOjUcdLT07Fy5UoMGDAAjDG8++67mDlzJo4cOYJhw4YFEhqJcp4cBN+FdQ4hKxEBgMPFw8UAu4uHipafDpmA3sHHH38cu3fvxi9+8QukpqYGvGTPjBkz/O7/8Y9/xJo1a7B//35KRKRNPGPgOEAGt2ewqzRo81S3pcnhgsMNWBxu6G1UIgqVgBLRli1b8OWXX/omqgoGt9uNjz/+GBaLxW+um5bsdjvs9uYGQ5MpOv8xpkyZgtGjR2PVqlVihwKgZ8Ujl0qQpFWBt7sBO0JaGgK8fYk4oTpIVbOQCaiNKC4uzjfDX3edOHECsbGxUCqVeOqpp/Dpp59i6NChbe67YsUKv8G1GRkZbe5HELb5g8TiG9oR4kTEcRwYJMK4NioRhU4g89D+3//9H7vvvvuYxWIJ5Ol+7HY7O3fuHPvxxx/ZkiVLWGJiIvvpp5/a3Ndms7GGhgbfraSkJKA5q3uy2bNnMwjNIL7b+fPn2dy5c1l2djZTqVRs4MCBbNWqVa2eN3PmTPbSSy+x1NRUlp2dzRhjbO/evWzUqFFMqVSya665hn366ae+uau9Tpw4wW677TYWExPDkpKS2COPPMKqq6vbjefixYvh+nO0z3KZsYZLjNm7/z/Ykcv1Jnbg0GF2/vMVjH3/RkhfKxqFZM5qr9deew0FBQVITk5Gdna23+x5ALq0sKJCoUD//sKMiddccw0OHjyI1atX46233mq1r1KpbDXDXpcwBridgT8/UFJ5p9sxVq9ejbNnz2L48OF44YUXAAgl0PT0dHz88cdISEjA999/jyeffBKpqan4+c9/7nvuzp07odPpsGPHDgBC1XXGjBm44447sGHDBhQVFWHhwoV+r1dfX4+bbroJjz/+OP7yl7/AarXiueeew89//nN88803bcZjNIp3Cbu+SSjpaV1OSAFAGtoSkac1XCgR2c3C2LYQDCfp7QJ6F0O1cicgLAPTsh0oqNxO4N+vhebYHbnh/wGyzq1LptfroVAooNFofBPPA8Dzzz/v+z0nJwf79u3DRx995JeIYmJi8Pe//903++Lf/vY3cByHtWvXQqVSYejQoSgrK8MTTzzhe84bb7xx1Yn524pHDIwx2F08wHjowAvJPeRVM4CBg4tJhEG2dhOgNoT0NXujgN7F5cuXB+XFly5dittvvx2ZmZkwm83YsGEDdu3ahW3btgXl+NHkzTffxLp161BcXAyr1QqHw9FqxZQRI0b4LWiQn5+PkSNH+s3HNG7cOL/ndGZi/p7CzZo7MnISTug/xIW2dOI9ul2iBmChRBQiAX+d1NfXY9OmTSgoKMBvf/tbxMfH4/Dhw0hOTm41YXl7qqqq8Oijj6K8vBx6vR4jR47Etm3b2l/epbukcqF0Em5S+dX36cDGjRuxePFivPbaa5gwYQK0Wi3+/Oc/48CBA377dTQRfHs6MzF/T+HmvWPMPJfuQ1waAgCZVAK1QookoxGwW6jBOkQCeiePHz+OadOmQa/Xo7CwEE888QTi4+OxefNmFBcX++Yqupp33nknkJcPHMd1uookJoVC4Tep/d69ezFx4kQ8/fTTvm3treLa0qBBg/CPf/wDdrvd17Z28OBBv32uNjF/W/GIxZeIQjzGrCWZVAK5VAKD3gBUgS7hh0hA5dpFixZhzpw5OHfunF+x/4477sB3330XtOB6q+zsbBw4cACFhYW4fPkyBgwYgB9//BHbtm3D2bNn8fvf/75VQmnLQw89BJ7n8eSTT+L06dPYtm0bXn31VQDwdUKdP38+amtr8eCDD+LgwYMoKCjAtm3b8Nhjj/mSz5XxXNmTPlxcVyaikDdUt6D0VF2pRBQSASWigwcP4pe//GWr7X369EFFRUW3g+rtFi9eDKlUiqFDh8JoNOLWW2/Fvffei1mzZmH8+PGoqanxKx21R6fT4V//+heOHj2K0aNH47//+7+xbNkyAPB9gVxtYv624ikuLg7dyXfAO8asedR996q8ncEYg8vNo6zJk/SoRBQSAU2en5SUhG3btiE3NxdarRbHjh1D3759sWPHDsydOxclJSWhiLUVmjy/695//3089thjaGhogFqtFjucLqlvcsDpciGe1UMq4YDY5JA3Vjc1WXHizDmcr6jFQ6rvwcUagXFPXP2JBECIJ8+/66678MILL8DpFPrkcByH4uJiPPfcc63WTifieu+997Bnzx5cvHgRn332ma+PUKQlIQAwaBRIVEsg8V62D3ESApq7fzmkGqFqaGsQ+qORoAronXzttdfQ2NgIo9EIq9WKyZMno3///tBqtfjjH/8Y7BhJN1RUVOCRRx7BkCFD8Mwzz+D+++/H22+/LXZYAeN4p+eKWeirZYDwJctxgFOqEaqGbifgsoXltXuTgFr79Ho9duzYgb179+LYsWNobGzEmDFjMG3atGDHR7rp2WefxbPPPit2GMHDe8aYdbNLRNdw4DkpXFI1AIfQYC2PvBJlT9blRMTzPPLy8rB582YUFhaC4zjk5OQgJSXFb11zQoKp0e6CzemGzm2HIsRzEF3JVz2TxwLuWqHBWpscttfvDbpUNWOM4a677sLjjz+OsrIyjBgxAsOGDUNRURHmzJnTI6eJDaAtnvRALjcvdCfgw3fFDBD+f4Q8xMEhpUv4odKlr5W8vDx899132LlzJ6ZOner32DfffIO7774b7733Hh599NGgBhkI70DcpqamiGyYJf7cPIOEuTwN1dKwDTx1OBxggDBLoywWcACwN4TltXuTLiWiDz74AP/1X//VKgkBwE033YQlS5bg/fff7xGJSCqVwmAwoKqqCgCg0Wio2hihGGOw2hyQua1wSF1wQQrYQt9gzPM8qquroVZrMGVIAuKttUAThCtnJKi6lIiOHz+OV155pd3Hb7/99h610qt3tLg3GZHIxPMMjXYX5LwNdVIekKkAWW1YXlsikWBgTo4wmLg6ASgBVc1CoEuJqLa2FsnJ7TfSJScno66urttBBQvHcUhNTUVSUpKvzxOJPEU1Fhw5XYVr63cgxygDhswAdGlheW2FQtG8Uo3S0yGPelcHXZcSkdvtbndgJCBUh1wuV7eDCjapVAqpNMRLzpCQqbdbwDuboHfXQsV0QEJGWC/fm2xOXDbboeHkSAEAh0VoNA/1Mka9SJcSEWMMc+bMaXeWxJBNaEZ6NZVciix5LWKUUuGyeVj7EAEXqi349kwVBiTF4E6JTOjLZDcB6riwxhHNupSIZs+efdV9ekJDNYkuozIMGGVlQJka0KWH/fWVMqFqZncxQKUDmmqFdiJKREHTpUS0fv36UMVBSMcaPAOp9Z2bdC+YFL5ExAvtRE21dOUsyGgWcNKjOd083DYz0Oi58mnIDHsM3hKRw+UGVHphIzVYBxUlItKjnbpkwsbt/8bFyxahfUjR9elwu0spExql7S5eqJoBdAk/yCgRkR6t2myHtqlUmH8oLluUGFpVzQAqEQUZJSLSo1WZbNDbSoUrZnE5osTgrZq5eQaXXCtspBJRUIVx0l9Cusbl5mGtLYPC3QSNOhHQi7PEuFImweRBRiEhqTz95OyeCdJo2FBQUCIiPVaV2Q6DpQByKQdV8qDwTpbfAsdxGJPpuVTvdjX/dDaJ0mYVjUStmq1YsQLXXnsttFotkpKScPfddyM/P1/MkEgPcqmuCfFNF6FVycElDRY7HIFURit6hICoiWj37t2YP38+9u/fjx07dsDpdOKWW26BxWIRMyzSQ9SUX4TKZUKsRgXE9xM1liqzDReqG9Fod1GDdQiIWjXbunWr3/28vDwkJSXh0KFDuPHGG0WKivQUg/kC2GIUiEkfIfrCmLvyq1FWZ8X0kakYqNIBpktUIgqiHtVG1NAg9FaNj49v83G73e43ns1kon+EqOWwIMtdDCRrgX7jxI6mRafGlpfwqXd1sPSYy/c8z2PhwoW4/vrrMXz48Db3WbFiBfR6ve+WkSHOVRQSBsX7hcGl2pSwTfnRkebxZi16V1OJKGh6TCKaP38+Tp48iY0bN7a7z9KlS9HQ0OC7hWshRxJmtgbUnN0Pm8sN5NzYIy6R+3pXO1uUiGi8WdD0iKrZggUL8MUXX+C7775Denr7o6uVSmW7U5CQKMEY7Ke+xLnyOpiUKbhOk4H21wcNH1/vajffokREiShYRC0RMcawYMECfPrpp/jmm2+QkyNOz1nSgxTvQ23xGbg5KcyZ06BTi9tI7eWrmjl5QG0QNjqtgJMWWwwGUUtE8+fPx4YNG/D5559Dq9WioqICgLCAI6280cswBpT8AHZhF6rNdhQZrsOQ7Cyxo/JpHvjqBmRKoSOjwwJY6wB5qsjRRT5RS0Rr1qxBQ0MDpkyZgtTUVN/tww8/FDMsEm5uF3B2G1DwDRqsTpxTDUetYTgGp/SESpkg1aDC5EFGjEo3CBu8k6JZe84c7ZFM1BIRLX5IYK0DfvoMMFeAB7CPy0WxfhBy++ihVvScOaETY5VIjG3RPqmOAxpKAVu9aDFFkx7RWE16qYYy4MRHQjuLXI1DmutxxiUkoHE5bfcl6zG87URUIgoKSkREHA2lwLGNgNsJ6FKBYfcgsVEGrrYM04YkQaPoWf+abp6h0mSDzelGX2MsVc2CrMf0IyK9iLUeruMfo+RyA846EoBRDwEqPXISY/DQ+Ez0T9KKHWErLp7HhwdL8PnRS3C6eUpEQdazvnZI9ON5mA5vwvkL5ajj4pCvm4QsSOFtfUnSqkQNrz0KqQRSCQc3z2B1uiH3JiJ7I+ByiD4WLtJRiYiEVe3Zvcg/m48mXoryjDtw8/AMyCU9/9+Q4zio5ULjuc3hBuRqQO5JmtRg3W09/z+ARA1bYx0u/LAFLp6hKWMK7r9hOAalaCGRiD+EozNUnqt4Vqdb2OCrntWLE1AUoUREwubk3q/gdDrg1qbhxhtv9nUSjBTeElHrRETtRN1FiYiERVV5CZylR8EBGHT9TKh62FWxzvAlIocnEakMwk9KRN0Wef8NJCIZaw9BkhSDOk0OjH3EnW0xUGqF8L1NJaLgo0REQs9aD646X+idPPZWsaMJWH+jFgaNAml6zzhIjafTZVONeEFFCUpEJKR4noEVH4CU8cICidoUsUMKWGaCBpkJmuYNmgThp90MuOzCYFgSEGojIiF1trQKhw/sRpXZBmSOFzuc4JKrm5cTaqoVN5YIR4mIhAzPMxQe/zdcTgcsCqNoK7UGi8PF41K9FSW1Tc0bvaUiqp51CyUiEjJny2sRU30UMgmHPiMn94gpX7ujvsmBDw+WYOvJiuaNvkR0WZygogQlIhISjDEUnNgHGW9DkjEJipRhYofUbS07NPqmsKESUVBQIiIhcb7ShJjKQ5BJOKQMnwxEwDCOq/H2I3LzDA43L2yM8SYiaiPqjsj/7yA9DmMMZ04chNJlhjHBAGX6aLFDCgq5VAK5VKhe+jo1ektE1jqAd4sUWeSjRESCrqLBCk35QUglHFKH3BBVI9O98yRZvIlIqQOkMiEJ0ZizgFEiIkGX6i7HdUkuZCfpocwaK3Y4QRWjFKpnTXaXsIHjqJ0oCCgRkeAr3o8YpQxJA68DFJqr7x9BYpRXlIgAunIWBNSzmjRz2YGKk4DDDGjTgIT+XWpkZoyhsfI8tPXFgEQKZFwbwmDFMSRVhz4GNfoYWix3FZME4BTQWCVaXJGOEhERmMqBEx8La3V5aZOBoXc3j6m6ivOVZhR/vRkD1E3IHHlj84qoUaSfMbb1xtgk4aelOrzBRBFRq2bfffcdZsyYgbS0NHAch88++0zMcHovaz1wfKOQhNRxQOooYfZBcyVw5B+d+qZ38wynju6Dxl4NSOVA1sTQx91TeBNRU42wGADpMlETkcViwahRo/Dmm2+KGUbvxhiQv0VY0kebAox9DBh8B3Dt48IHzGEBjm4AGjv+tj9ZVIG48j2QSzmkjJjaPAYryniHeRTXtBjmoYgVxp0xBlionSgQoiai22+/HS+99BLuueceMcPo1WoKj6Gq+Axq7QyuQTOaR5ArtcDoh4Tk5LQCxza0+yGz2JwoP/gZ5LwVKanpUOREb2nIN8zjp/LmjRzXonpG7USBiKirZna7HSaTye9GAuN089hyvAzH/v0FCqot2NmYg7yjJhRebtFGJFcDox7wlIyahJJRGz2IT+zbCr25ADFKOdLG/4fQUB2lvFfNmhxu8HyLlYq9iYgarAMSUYloxYoV0Ov1vltGRobYIUWss5VmVF84Bo2zHjExMTAljILZ5sKnR8pwqKjFjINyNTDqQSDWKFTTjvwfUJ0vVENcDpT9+AXYhd3gAPS59k5IDOminVM4qOVScJxw+r6ZGgHPlTNQIgpQRF01W7p0KRYtWuS7bzKZKBkFaFiaHon6EqgUOuiHTMWQzMH47mw1jpc2YP+FGgxO0fq+/aHQCMno2Ebhg3Zys1CF491QmiyQSjioBkxFwsDorZJ5SSQcNAopLHY3LHZX89+oZdWMsYifaSDcIioRKZVKKJU0C15QmCuQjFogRgWk5UIuleDmIcnQqeVIM6ibP2BeihhgzGygaA9QdkjocwQgMSkVytE3QNNnSK/58MUoZbDY3TDbXUjybtQkApxEaPS3m6Ky60IoRVQiIt136pIJGfFqaMsOCxuMgwBlc9+Ya7P9+wzVNNqhV8shk0qEMVV9p8CaNgGNdZUw6oU14LW9JAF56VRyVJnsMNtczRulMiAmUSgxmsopEXWRqImosbER58+f992/ePEijh49ivj4eGRmZooYWXRqtLvw9elKyHkb5kpPQMkBSBvT7v5VZhs2HSpFrFKGcTnxiI9RoLLBjgMXa+DiGR64NhaGXpaEAECnlgMATNYr+gzp+ngSURmQNFiEyCKXqInoxx9/xNSpU333ve0/s2fPRl5enkhRRa9jJfVw8wxDWSGUHC80QOvbb1y2OtyQchxqGh3YcqLC7zGDRg53y6tGvUg/Ywx0KhlS9Wr/B3RpwKUjgLm87SeSdomaiKZMmdI80x0JKZebx/HSBoAxjJJcEDamjemwXScrIQaPTsjGkZI6FFRbYLG7oFPJMSglFiPTDZBLI+qia9Ckx2mQHtfGYF5dH+GnuVyYFiSKuzEEG7UR9RIF1RbYnG6koRKJEotw1St5+FWfp1ZIMbFfIib2SwxDlBFOEy/8XV12YdxZBC+dFG698yutFzpV3gAAGC29CI7jhCQURROWhRNjDOUNVpytNMPlnTIWEEqXujThd1OZOMFFKEpEvYDZ5kRRTRPkLgsyec8HJC1X3KAi3ObDZfjyeLn/lTOgORE1UCLqCkpEvUCV2Q4px2GopBBqGSc0UMcmXf2JpE0cx0GrElo1TLYrrpzpPR1s64uFjo2kUygR9QL9jLF4YlIWrlMWCxuoNNRtes8l/PqmKxNRutBIbTcLE+qTTqFE1Euo6vKhYU1CD2kj9XHpLoNGaF+ra3L4PyCVN1fP6ovCHFXkokQU5Sx2l1BFKN4vbEi/VugFTLolvr1EBACGLOFnHSWizqJEFMWcbh553xfiX9/sgsNUJVxa7tN+T2rSeXExQtWs1tLGjIxx3kRUCPB868dJK5SIotiFagucTicSK78XFgbsc03zxGekW+JjhBKR2eaE031FstH1Ef7OTitgviRCdJGHyuhR7FR5A1LMJ5GutIFTGIHM68QOKWqo5VJMGWSEQaNAq77pEimQ0A+oPAVcPtvhMBoioBJRlGq0u1BZcQkZDT8iUasEcm6k0lAQcRyH3Mw45CTGCDMTXClhgPDz8vnWj5FWKBFFqfzSyxhQvRN6JQd1cn9hZQ4SPvF9hfmJmmpoQv1OoEQUhZjbiaajm6Fx1iDBYAAGT+81k5aFk93lxrlKM46V1Ld+UK4SkhEAVJwIa1yRiBJRtLGZUPd9HmT1F8BJZYi77iFhRQ4SdI02F744Xo5/n6v2n0jfK2WE8LPyJF09uwpKRNGkvgQ4lIc4ZzUG9kmE7toHoUqgCeZCJU6jgEImgdPNUNtWf6KE/kLJyN4I1F0Mf4ARhBJRNGAMKD0kLPfjsIDTJiHhxicxdBi1C4WSRMIhWacCAJTWWVvvIJU1T7VS+mMYI4s8lIgindsF5H8FnNsOMB68cQiQ+2in16sn3ZOVIEyQVlRjaXuH9LFC+1ztBWEJb9ImSkSRzNYgrDNWfhzgODRlTsa62hE4UGxuu82CBF1mvJCISuusbU+dq45rHttXtDeMkUUWSkSRqvYi8ON6wFwhLII4chb2OfrBbHejsMZCF8nCJEmrhFohhcPFo6yt6hkAZE0USkXV+TT+rB2UiCINY0DRPuD4h8IQAm0ycM0cVMlScaJMmIVxYr9EYRZGEnIcx2FAkrAcU1l9O4koNql56pVz24XqNPFDQzwiicsOnPkCqD4r3E8dCQy4BS5Isf1YCRgDBqVokRHfxsTuJGTGZsVjTGYc4mI6mHo3+wag6rTQufH818Cg28IXYASgRBQp6oqAM18K7UISKTDgZ0DqaDAAu05Xodpsh1ohxY0DjWJH2uvoNfKr76TQAENmACc+FpYcUhto7F8LPaJq9uabbyI7OxsqlQrjx4/HDz/8IHZIPYfDAuRvBY59ICQhtQEY/bBQ1Oc47LtQgxNlDeA44NZhKYi9cqloElZVZhvqLG30KQKEgbA5k4XfC74VbtTREUAPSEQffvghFi1ahOXLl+Pw4cMYNWoUbr31VlRVVYkdmrisdUDBN8CBvwnfoIwJ48XGzgX0fXy7JcYKA1mnDkpCTmKMWNESAGcqTPjgQAk+P1rWei5rr6wJQPYk4ffi/cDhPKCmoNfPb80xkVc4HD9+PK699lq88cYbAACe55GRkYFf//rXWLJkSYfPNZlM0Ov1aGhogE6nC0e4wcfzgMsqlHws1UBjpXBFrLE5ETNtChzZU2BWpaGiwQaeMYxMN/gerzLZkOTpWEfEY7Y58eHBEphtLijlEozOMKC/MRZxMYrWi1FWnRb6f7k8pSe1QeiJresj9AFT6QGZKuLHCHb2MypqOd7hcODQoUNYunSpb5tEIsG0adOwb9++Vvvb7XbY7XbffZPJ1LkXqr0IFOxs+1vHbxvreHsb206Xm2CxO5uf6d2HATIph9wMg2/fM+VmmGwtiu2Mh5R3AoxBwgG5mXG+h85VNaIESaiMHYYaPhPOI04AwqVfhUyCQSlaKGXCSqKUhHoGrUqO+8dm4KsT5ahosOHAhVocuFALAEiIVeDRCdm+fb+qNKABtyPJfBjx5nxI+HrgXCEAQCbhMCxNLyQhqRz51XaY7W4wTkhmDBIwToKByVrEdaZ9KlRGzgJUwSkAiJqILl++DLfbjeTkZL/tycnJOHPmTKv9V6xYgeeff77rL+SyA43VgYbZ8aGbGuC0t3M5VsIBjuZ/FN5hgdvuX2R3ex+TqoQJtGKMgL4P8rUqnKvzJDVPM4JCJoFRq0SaXt1r153v6fRqOWaNzcC5qkacLjehrN4Kh4uH0+3/fjVYnaiwyVChHgeJMhcGWwn0tkuIcVxGLN8o7MQY4HLAZTXBeeX6aQBYjA1gIi6SyYLXvhVRLZtLly7FokWLfPdNJhMyMjKu/kRDBjDqAeF3X1G3RZHXr/h7le1XbEseaEe8m/fb7u3DI5VwgE7t2zfFYkciz8DACbtznFD8lqkBiQTQNpdsbtA7cZ2bBwdAJpFAo5T22rXmI41EwmFQihaDUrRgjMHu4mFzuv32uWlwEhyulh/kHN9vHAdAJxe+QF12JDeYYXC6PR98JiQoxqCOkQFyaVjOqU2K4LVJipqIEhMTIZVKUVnpPwansrISKSmt1w1XKpVQKgOYZVARA8TnXH2/ACR1oWQaH9v5fb3rZpHIxnEcVHIpVFckjOTOVKelckAZi8SYhBBF13OI+hWrUChwzTXXYOfOnb5tPM9j586dmDBhgoiREULCSfSq2aJFizB79myMHTsW48aNw6pVq2CxWPDYY4+JHRohJExET0SzZs1CdXU1li1bhoqKCowePRpbt25t1YBNCIleovcj6o6o6EdESBTr7GeULsMQQkRHiYgQIjpKRIQQ0YneWN0d3uatTg/1IISElfezebWm6IhORGazGQA617uaECIas9kMvV7f7uMRfdWM53lcunQJWq22R06N6h2CUlJSEnVX9ejcIlO4z40xBrPZjLS0NEgk7bcERXSJSCKRID09Xewwrkqn00XdP7QXnVtkCue5dVQS8qLGakKI6CgREUJER4kohJRKJZYvXx7YjAE9HJ1bZOqp5xbRjdWEkOhAJSJCiOgoERFCREeJiBAiOkpEhBDRUSLqoq6uSvvxxx9j8ODBUKlUGDFiBL766iu/xxljWLZsGVJTU6FWqzFt2jScO3culKfQrq6c29q1a3HDDTcgLi4OcXFxmDZtWqv958yZA47j/G633SbOmu9dObe8vLxWcatU/nNMR+r7NmXKlFbnxnEcpk+f7ttHlPeNkU7buHEjUygUbN26deynn35iTzzxBDMYDKyysrLN/ffu3cukUil75ZVX2KlTp9jvfvc7JpfL2YkTJ3z7rFy5kun1evbZZ5+xY8eOsbvuuovl5OQwq9UartNijHX93B566CH25ptvsiNHjrDTp0+zOXPmML1ez0pLS337zJ49m912222svLzcd6utrQ3XKfl09dzWr1/PdDqdX9wVFRV++0Tq+1ZTU+N3XidPnmRSqZStX7/et48Y7xsloi4YN24cmz9/vu++2+1maWlpbMWKFW3u//Of/5xNnz7db9v48ePZL3/5S8YYYzzPs5SUFPbnP//Z93h9fT1TKpXsgw8+CMEZtK+r53Yll8vFtFote/fdd33bZs+ezWbOnBnsULusq+e2fv16ptfr2z1eNL1vf/nLX5hWq2WNjY2+bWK8b1Q16yTvqrTTpk3zbetoVVoA2Ldvn9/+AHDrrbf69r948SIqKir89tHr9Rg/fny7xwyFQM7tSk1NTXA6nYiPj/fbvmvXLiQlJWHQoEH41a9+hZqamqDGfjWBnltjYyOysrKQkZGBmTNn4qeffvI9Fk3v2zvvvIMHHngAMTH+a5SF+32jRNRJHa1KW1FR0eZzKioqOtzf+7MrxwyFQM7tSs899xzS0tL8PhS33XYb3nvvPezcuRMvv/wydu/ejdtvvx1ut7uDIwVXIOc2aNAgrFu3Dp9//jn+8Y9/gOd5TJw4EaWlpQCi53374YcfcPLkSTz++ON+28V43yJ69D3pGVauXImNGzdi165dfo26DzzwgO/3ESNGYOTIkejXrx927dqFm2++WYxQO2XChAl+6+pNnDgRQ4YMwVtvvYUXX3xRxMiC65133sGIESMwbtw4v+1ivG9UIuqkrq5KCwApKSkd7u/92ZVjhkIg5+b16quvYuXKldi+fTtGjhzZ4b59+/ZFYmIizp8/3+2YO6s75+Yll8uRm5vrizsa3jeLxYKNGzdi3rx5V32dcLxvlIg6KZBVaSdMmOC3PwDs2LHDt39OTg5SUlL89jGZTDhw4EBYV7oNdMXdV155BS+++CK2bt2KsWPHXvV1SktLUVNTg9TU1KDE3RnBWE3Y7XbjxIkTvrgj/X0DhG4ldrsdjzzyyFVfJyzvW1ibxiPcxo0bmVKpZHl5eezUqVPsySefZAaDwXdp9xe/+AVbsmSJb/+9e/cymUzGXn31VXb69Gm2fPnyNi/fGwwG9vnnn7Pjx4+zmTNninYZuCvntnLlSqZQKNimTZv8LvOazWbGGGNms5ktXryY7du3j128eJF9/fXXbMyYMWzAgAHMZrP16HN7/vnn2bZt21hBQQE7dOgQe+CBB5hKpWI//fST3/lH4vvmNWnSJDZr1qxW28V63ygRddH//M//sMzMTKZQKNi4cePY/v37fY9NnjyZzZ4922//jz76iA0cOJApFAo2bNgw9uWXX/o9zvM8+/3vf8+Sk5OZUqlkN998M8vPzw/HqbTSlXPLyspiAFrdli9fzhhjrKmpid1yyy3MaDQyuVzOsrKy2BNPPNGqP064dOXcFi5c6Ns3OTmZ3XHHHezw4cN+x4vU940xxs6cOcMAsO3bt7c6lljvG00DQggRHbUREUJER4mIECI6SkSEENFRIiKEiI4SESFEdJSICCGio0RECBEdJSISUfLy8mAwGHz3//CHP2D06NG++3PmzMHdd98d9rhI91AiIm3yThf61FNPtXps/vz54DgOc+bM8ds/2AkgOzsbq1at8ts2a9YsnD17tt3nrF69Gnl5eb77U6ZMwcKFC4MaFwk+SkSkXRkZGdi4cSOsVqtvm81mw4YNG5CZmSlKTGq1GklJSe0+rtfr/UpMJDJQIiLtGjNmDDIyMrB582bfts2bNyMzMxO5ubndOnZbJZW7777bV8qaMmUKioqK8Mwzz/gmcAdaV82u1LJkNmfOHOzevRurV6/2HePixYvo378/Xn31Vb/nHT16FBzHhXWKEtKMEhHp0Ny5c7F+/Xrf/XXr1uGxxx4L+etu3rwZ6enpeOGFF1BeXo7y8vIuH2P16tWYMGECnnjiCd8xMjMzW50TAKxfvx433ngj+vfvH6xTIF1AiYh06JFHHsGePXtQVFSEoqIi7N27t1Nz2HRXfHw8pFIptFotUlJSAppwTK/XQ6FQQKPR+I4hlUoxZ84c5Ofn+5bdcTqd2LBhA+bOnRvs0yCdRFPFkg4ZjUZMnz4deXl5YIxh+vTpSExMFDusbklLS8P06dOxbt06jBs3Dv/6179gt9tx//33ix1ar0UlInJVc+fORV5eHt59992glRokEgmunIHG6XQG5did8fjjj/sa4tevX49Zs2ZBo9GE7fWJP0pE5Kpuu+02OBwOOJ1O3HrrrUE5ptFo9Gv3cbvdOHnypN8+CoWi2ytHtHeMO+64AzExMVizZg22bt1K1TKRUdWMXJVUKsXp06d9v7enoaEBR48e9duWkJCAjIyMVvvedNNNWLRoEb788kv069cPr7/+Ourr6/32yc7OxnfffYcHHngASqUyoCphdnY2Dhw4gMLCQsTGxiI+Ph4SicTXVrR06VIMGDAgrHNNk9aoREQ6RafTQafTdbjPrl27kJub63d7/vnn29x37ty5mD17Nh599FFMnjwZffv2xdSpU/32eeGFF1BYWIh+/frBaDQGFPfixYshlUoxdOhQGI1GFBcX+x6bN28eHA5HWK4Cko7RVLGk1/r3v/+Nm2++GSUlJa0WKSThRYmI9Dp2ux3V1dWYPXs2UlJS8P7774sdUq9HVTPS63zwwQfIyspCfX09XnnlFbHDIaASESGkB6ASESFEdJSICCGio0RECBEdJSJCiOgoERFCREeJiBAiOkpEhBDRUSIihIiOEhEhRHT/H6IfBNcAQ9zPAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:31:08.167352Z", + "iopub.status.busy": "2024-03-26T07:31:08.167032Z", + "iopub.status.idle": "2024-03-26T07:31:08.397180Z", + "shell.execute_reply": "2024-03-26T07:31:08.396233Z" + }, + "papermill": { + "duration": 0.247481, + "end_time": "2024-03-26T07:31:08.399501", + "exception": false, + "start_time": "2024-03-26T07:31:08.152020", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEnCAYAAAD4jcW1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAtkUlEQVR4nO3de1wTZ74/8A8BEu6IAgERibcKXgCFRbHtUbsCXaurx26X2guI1m5VWmsO1bI9hWP1FK2K1NbK0VO8tLvV06rVVYtSlG5VvIG4aAELoqByVQEhEELy/P7wx9RIgKCBySTf9+vFC2bmmcl3mOHDzGTyjAVjjIEQQgRGxHcBhBDyOCi8CCGCROFFCBEkCi9CiCBReBFCBInCixAiSBRehBBBovAihAgShRchRJCs+C6gr2k0Gty+fRuOjo6wsLDguxxCyCMYY7h//z4GDhwIkaiL4yvGs88//5z5+PgwiUTCQkJC2NmzZ7tsf+/ePbZ48WLm4eHBxGIxGzFiBDt8+LDer1deXs4A0Bd90ZeRf5WXl3f5t8zrkdeePXsgl8uRmpqKCRMmICUlBRERESgqKoK7u3uH9q2trQgLC4O7uzu+++47eHl54caNG+jXr5/er+no6AgAKC8vh5OTk6FWxeioVCocO3YM4eHhsLa25rscYgDmsk0bGhrg7e3N/a12htfwSk5OxsKFCxETEwMASE1NxeHDh5GWlob333+/Q/u0tDTcvXsXp0+f5jaeTCbr0Wu2nyo6OTmZfHjZ2dnBycnJpHd0c2Ju27S7yzq8hVdraytycnIQHx/PjROJRJg2bRqys7N1znPw4EGEhoZiyZIlOHDgANzc3PDKK69gxYoVsLS01DmPUqmEUqnkhhsaGgA82BFUKpUB18i4tK+bKa+juTGXbarv+vEWXrW1tVCr1ZBKpVrjpVIpCgsLdc5z7do1HD9+HK+++iqOHDmC4uJiLF68GCqVComJiTrnSUpKwsqVKzuMP3bsGOzs7J58RYxcRkYG3yUQAzP1bapQKPRqJ6h3GzUaDdzd3bF161ZYWloiKCgIt27dwrp16zoNr/j4eMjlcm64/Xw6PDzc5E8bMzIyEBYWZhanGObAXLZp+9lRd3gLL1dXV1haWqKqqkprfFVVFTw8PHTO4+npCWtra61TRD8/P1RWVqK1tRVisbjDPBKJBBKJpMN4a2trk94B2pnLepoTU9+m+q4bbzepisViBAUFITMzkxun0WiQmZmJ0NBQnfM8/fTTKC4uhkaj4cZdvXoVnp6eOoOLEGK6eL3DXi6XY9u2bdi5cycKCgqwaNEiNDU1ce8+RkVFaV3QX7RoEe7evYulS5fi6tWrOHz4MD7++GMsWbKEr1UghPCE12tekZGRqKmpQUJCAiorKxEYGIj09HTuIn5ZWZnWHbbe3t44evQoli1bBn9/f3h5eWHp0qVYsWIFX6tASK9QKBQd3rhqbFbidH4JXFwvwMH2t0shvr6+ZvHm06MsGDOvB3A0NDTA2dkZ9fX1Jn/B/siRI5g+fbpJXx8xVbm5uQgKCtKrbU5ODsaPH9/LFfUdff9GBfVuIyHmwtfXFzk5OVrjiirqIP82H8kvjcVIz35abc0RhRchRsjOzq7D0ZToxh1Ifm6G35gABPoM4Kky40Fd4hBCBInCixAiSBRehBBBovAihAgShRchRJAovAghgkThRQgRJAovQoggUXgRQgSJwosQIkgUXoQQQaLwIoQIEoUXIUSQKLwIIYJE4UUIESQKL0KIIFF4EUIEicKLECJIFF6EEEGi8CKECBKFFyFEkCi8CCGCROFFCBEkCi9CiCBReBFCBInCixAiSBRehBBBovAihAgShRchRJAovAghgkThRQgRJCu+CwCAzZs3Y926daisrERAQAA+++wzhISE6Gy7Y8cOxMTEaI2TSCRoaWnpi1KNlkKhQGFhITfc2KzE6fwSuLhegIOtRKutr68v7Ozs+rpEQgyK9/Das2cP5HI5UlNTMWHCBKSkpCAiIgJFRUVwd3fXOY+TkxOKioq4YQsLi74q12gVFhYiKCiow/hPdLTNycnB+PHje78oQnoR7+GVnJyMhQsXckdTqampOHz4MNLS0vD+++/rnMfCwgIeHh59WabR8/X1RU5ODjdcVFEH+bf5SH5pLEZ69uvQlhCh4zW8WltbkZOTg/j4eG6cSCTCtGnTkJ2d3el8jY2N8PHxgUajwfjx4/Hxxx9j9OjROtsqlUoolUpuuKGhAQCgUqmgUqkMtCb8s7a2xtixY7lhjfNdSDyaMcJ3NMYO7t+hvSmtu7loa2vjvpvy9tN33XgNr9raWqjVakilUq3xUqlU6/rNw0aOHIm0tDT4+/ujvr4e69evx6RJk3DlyhUMGjSoQ/ukpCSsXLmyw/hjx46Z9HWf8kYAsMKZM2dw6zLf1RBDMJdtqlAo9GrH+2ljT4WGhiI0NJQbnjRpEvz8/PA///M/WLVqVYf28fHxkMvl3HBDQwO8vb0RHh4OJyenPqmZD5fK7gL5FzBx4kQE6DjyIsJjLtu0/eyoO7yGl6urKywtLVFVVaU1vqqqSu9rWtbW1hg3bhyKi4t1TpdIJJBIJB3GW1tbw9rauudFC4SVlRX33ZTX05yYyzbVd914vc9LLBYjKCgImZmZ3DiNRoPMzEyto6uuqNVq5Ofnw9PTs7fKJIQYId5PG+VyOaKjoxEcHIyQkBCkpKSgqamJe/cxKioKXl5eSEpKAgB89NFHmDhxIoYPH466ujqsW7cON27cwBtvvMHnahBC+hjv4RUZGYmamhokJCSgsrISgYGBSE9P5y7il5WVQST67QDx3r17WLhwISorK+Hi4oKgoCCcPn0ao0aN4msVCCE84D28ACA2NhaxsbE6p2VlZWkNb9y4ERs3buyDqgghxow+20gIESSjOPIihACltU1oUrZ1Or2kpon73v7O46PsJVYY4mrfK/UZGwovQoxAaW0Tpq7P0qvtf3yX3+X0E3FTzCLAKLwErKv/1Pr8lwbM6z+1MWvfjimRgRju7qC7TbMSh7KyMWNKKOxtO967WFzdiHf35HV59GZKKLwESt//1N39lwbM5z+1EAx3d8AYL2ed01QqFSrdgPE+LiZ9k6q+KLwEqrv/1N39lwbM7z81MS0UXgLX2X9q+i9NTB3dKkEIESQKL0KIIFF4EUIEicKLECJIFF6EEEGi8CKECBKFFyFEkCi8CCGCROFFCBEkCi9CiCBReBFCBInCixAiSBRehBBBol4lCDECSnULRDa3UNpQBJGN7s4I29racLvtNgruFujsYLK0oREim1tQqlsA6O4TzJRQeAlUdzt7dzs6YH47uzG73XQD9kM+w1/Pdd/2i/QvOp1mPwS43RSIIEgNWJ1xovASKH139q52dMC8dnZjNtDeB02lb+PTyEAM66Qb6La2Npw6eQpPP/O0zn9IJdWNWLonDwOn+vR2uUaBwkugutvZu9vRAfPb2Y2ZxNIGmhYvDHEaiVEDOu8GutSqFH79/XR2MKlpqYempQYSS5veLtcoUHgJVHc7e3c7OmB+OzsxLfRuIyFEkCi8CCGCROFFCBEkCi9CiCBReBFCBKnH4XXt2rXeqIMQQnqkx+E1fPhwTJ06FV9//TVaWlp6oyZCCOlWj8MrNzcX/v7+kMvl8PDwwF/+8hecO6fHZxq6sHnzZshkMtjY2GDChAl6L2/37t2wsLDA7Nmzn+j1CSHC0+PwCgwMxKefforbt28jLS0NFRUVeOaZZzBmzBgkJyejpqamR8vbs2cP5HI5EhMTkZubi4CAAERERKC6urrL+a5fv464uDg8++yzPV0FQogJeOwL9lZWVpgzZw6+/fZbrF27FsXFxYiLi4O3tzeioqJQUVGh13KSk5OxcOFCxMTEYNSoUUhNTYWdnR3S0tI6nUetVuPVV1/FypUrMXTo0MddBUKIgD32x4MuXLiAtLQ07N69G/b29oiLi8OCBQtw8+ZNrFy5ErNmzer29K+1tRU5OTmIj4/nxolEIkybNg3Z2dmdzvfRRx/B3d0dCxYswM8//9zlayiVSiiVSm64oaEBwIOPz6hUKn1W1Si1tbVx33WtR/u4rtaxu2WQvqPPtuhum5rK9tS39h6HV3JyMrZv346ioiJMnz4du3btwvTp0yESPTiIGzJkCHbs2AGZTNbtsmpra6FWqyGVavdoIJVKUVhYqHOekydP4ssvv0ReXp5e9SYlJWHlypUdxh87dgx2dnZ6LcMYlTcCgBVOnjyJG7o7IQAAZGRkPPEySO/rybbobJuayvZUKBR6tetxeG3ZsgXz58/HvHnz4OnpqbONu7s7vvzyy54uulv379/H66+/jm3btsHV1VWveeLj4yGXy7nhhoYGeHt7Izw8HE5OTgavsa9cud2A9fln8Mwzz2D0wI7roVKpkJGRgbCwsE4/mN3dMkjf0WdbdLdNTWV7tp8ddafH4ZWRkYHBgwdzR1rtGGMoLy/H4MGDIRaLER0d3e2yXF1dYWlpiaqqKq3xVVVV8PDw6NC+pKQE169fx8yZM7lxGo3mwYpYWaGoqAjDhg3TmkcikUAikXRYlrW1dad/1ELQ3s2NlZVVl+vR1XrquwzS+3qyLTrbpqayPfWtvccX7IcNG4ba2toO4+/evYshQ4b0aFlisRhBQUHIzMzkxmk0GmRmZiI0NLRDe19fX+Tn5yMvL4/7+uMf/4ipU6ciLy8P3t7ePV0dQohA9fjIizGmc3xjYyNsbHreL5RcLkd0dDSCg4MREhKClJQUNDU1ISYmBgAQFRUFLy8vJCUlwcbGBmPGjNGav1+/fgDQYTwhxLTpHV7t140sLCyQkJCgdbFbrVbj7NmzCAwM7HEBkZGRqKmpQUJCAiorKxEYGIj09HTuIn5ZWVmHU1RCCNE7vC5evAjgwZFXfn4+xGIxN00sFiMgIABxcXGPVURsbCxiY2N1TsvKyupy3h07djzWaxJChE3v8Dpx4gQAICYmBp9++qmg36kjhAhfj695bd++vTfqIISQHtErvObMmYMdO3bAyckJc+bM6bLtvn37DFIYIYR0Ra/wcnZ2hoWFBfczIYTwTa/wevhUkU4bCSHGgO5BIIQIkl5HXuPGjeNOG7uTm5v7RAURQog+9Aov6qmUEGJs9AqvxMTE3q6D9FCzSg0AuHyrXuf0pmYlLtQAHjfuwd624wfTAaC4urHX6iOktz12Z4SEXyX/P3je35ffRSsrfFV8vttl2UtoNyDCo9de279/f1y9ehWurq5wcXHp8vrX3bt3DVYc6Vz46AddBg1zd4CttWWH6UUV9fiP7/Kx4U9jMdKz89tb7CVWGOJq32t1EtJb9AqvjRs3wtHRkftZ34v3pPf0txfj5ZDBnU5v7xJ4mJs9xnjRvXnE9OgVXg93LDhv3rzeqoUQQvTW4/u8LC0tdT6W7M6dO7C07Hj6QgghvaHH4dVZZ4RKpVKrmxxCCOlNer/NtGnTJgAPOiP83//9Xzg4/PZ4ErVajX/+85/w9fU1fIWEEKKD3uG1ceNGAA+OvFJTU7VOEcViMWQyGVJTUw1fISFmoLv79oDu790zt/v29A6v0tJSAMDUqVOxb98+uLi49FpRhJgb/e7bA/S5d89c7tvr8Vq296hKCDGc7u7bA/S7d8+c7tvrcXjNnz+/y+lpaWmPXQwh5qq7+/YAunfvUT0Or3v37mkNq1QqXL58GXV1dXjuuecMVhghhHSlx+G1f//+DuM0Gg0WLVrU4WnVhBDSWwzSGaFIJIJcLufekSSEkN5msJ5US0pKuHNyQgjpbT0+bWx/cnY7xhgqKipw+PBhrc9AEkJIb+pxeLU/ObudSCSCm5sbNmzY0O07kYQQYih0nxchRJDo6UGEEEGi8CKECBKFFyFEkCi8CCGCZLDwunnzJt58801DLY4QQrpksPC6c+cOvvzyS0MtjhBCumQUp42bN2+GTCaDjY0NJkyYgHPnznXadt++fQgODka/fv1gb2+PwMBAfPXVV31YLSHEGPAeXnv27IFcLkdiYiJyc3MREBCAiIgInQ/5AB48Q/KDDz5AdnY2/vWvfyEmJgYxMTE4evRoH1dOCOET7+GVnJyMhQsXIiYmBqNGjUJqairs7Ow67RdsypQp+Pd//3f4+flh2LBhWLp0Kfz9/XHy5Mk+rpwQwie977CfM2dOl9Pr6up6/OKtra3IyclBfHw8N04kEmHatGnIzs7udn7GGI4fP46ioiKsXbtWZxulUgmlUskNNzQ0AHjQD5lKpepxzULR/iH5trY2k15Pc2Iu21TfddM7vJydu+650dnZGVFRUfouDgBQW1sLtVoNqVSqNV4qlaKwsLDT+err6+Hl5QWlUglLS0t88cUXCAsL09k2KSkJK1eu7DD+2LFjsLOz61G9QlLeCABWOHPmDG5d5rsaYgjmsk0VCoVe7fQOr+3btz92MYbm6OiIvLw8NDY2IjMzE3K5HEOHDsWUKVM6tI2Pj9fqCaOhoQHe3t4IDw+Hk5NTH1bdty6V3QXyL2DixIkIGNyf73KIAZjLNm0/O+oOr48ZcXV1haWlJaqqqrTGV1VVwcPDo9P5RCIRhg8fDgAIDAxEQUEBkpKSdIaXRCKBRNLxMVHW1tawtrZ+shUwYlZWVtx3U15Pc2Iu21TfddM7vPTt7qYnD+AQi8UICgpCZmYmZs+eDeBBl9KZmZmIjY3VezkajUbruhYhxPTpHV47duyAj48Pxo0bB8aYwQqQy+WIjo5GcHAwQkJCkJKSgqamJsTExAAAoqKi4OXlhaSkJAAPrmEFBwdj2LBhUCqVOHLkCL766its2bLFYDURQoyf3uG1aNEifPPNNygtLUVMTAxee+019O//5OfdkZGRqKmpQUJCAiorKxEYGIj09HTuIn5ZWRlEot/u6GhqasLixYtx8+ZN2NrawtfXF19//TUiIyOfuBZCiHBYsB4cRimVSuzbtw9paWk4ffo0XnjhBSxYsADh4eGwsLDozToNpqGhAc7OzqivrzfpC/Z5N+5g9pYz+H7RRAT6DOC7HGIA5rJN9f0b7dFNqhKJBHPnzkVGRgZ++eUXjB49GosXL4ZMJkNjY+MTF00IIfp67DvsRSIRLCwswBiDWq02ZE2EENKtHoWXUqnEN998g7CwMDz11FPIz8/H559/jrKyMjg4OPRWjYQQ0oHeF+wXL16M3bt3w9vbG/Pnz8c333wDV1fX3qyNEEI6pXd4paamYvDgwRg6dCh++ukn/PTTTzrb7du3z2DFEUJIZ/QOr6ioKMG8o0gIMX09ukmVEEKMBe/9eRFCyOOg8CKECBKFFyFEkCi8CCGCROFFCBEkCi9CiCBReBFCBInCixAiSBRehBBBovAihAgShRchRJAovAghgkThRQgRJAovQoggUXgRQgSJwosQIkgUXoQQQaLwIoQIEoUXIUSQKLwIIYJE4UUIESQKL0KIIFF4EUIEicKLECJIFF6EEEGi8CKECJJRhNfmzZshk8lgY2ODCRMm4Ny5c5223bZtG5599lm4uLjAxcUF06ZN67I9IcQ08R5ee/bsgVwuR2JiInJzcxEQEICIiAhUV1frbJ+VlYW5c+fixIkTyM7Ohre3N8LDw3Hr1q0+rtx4qdVqXMg+iaZffsKF7JNQq9V8l0SI4TGehYSEsCVLlnDDarWaDRw4kCUlJek1f1tbG3N0dGQ7d+7Uq319fT0DwOrr6x+rXmO3d+9e5uPjwwBwXz4+Pmzv3r18l0ae0MXrtcxnxSF28Xot36X0Kn3/Rq34DM7W1lbk5OQgPj6eGycSiTBt2jRkZ2frtQyFQgGVSoX+/fvrnK5UKqFUKrnhhoYGAIBKpYJKpXqC6o2LQqFAamoq3n///Q7Tbty4gRdffBFr1qzB1KlTMXLkSNjZ2fFQJXkSbW1t3HdT2ncfpe+68RpetbW1UKvVkEqlWuOlUikKCwv1WsaKFSswcOBATJs2Tef0pKQkrFy5ssP4Y8eOmdQf8K+//qozuB7WPn3Dhg0YNmxYX5RFDKi8EQCscObMGdy6zHc1vUehUOjVjtfwelJr1qzB7t27kZWVBRsbG51t4uPjIZfLueGGhgbuOpmTk1NfldrrDh06pFe7TZs2ISoqyqSC21xcKrsL5F/AxIkTETBY95mGKWg/O+oOr+Hl6uoKS0tLVFVVaY2vqqqCh4dHl/OuX78ea9aswY8//gh/f/9O20kkEkgkkg7jra2tYW1t/XiFG6G9e/fq1e7s2bN4++23e7ka0husrKy476a07z5K33Xj9d1GsViMoKAgZGZmcuM0Gg0yMzMRGhra6XyffPIJVq1ahfT0dAQHB/dFqUbv2rVrBm1HiLHj/bRRLpcjOjoawcHBCAkJQUpKCpqamhATEwMAiIqKgpeXF5KSkgAAa9euRUJCAv7+979DJpOhsrISAODg4AAHBwfe1oNvzc3NBm1HiLHjPbwiIyNRU1ODhIQEVFZWIjAwEOnp6dxF/LKyMohEvx0gbtmyBa2trfjTn/6ktZzExET813/9V1+WblQovIi54T28ACA2NhaxsbE6p2VlZWkNX79+vfcLEqBHrxs+aTtCjB3vd9gTw2hpaTFoO0KMHYWXiXj41NoQ7QgxdrQnmwgXFxeDtiPE2FF4mQhXV1eDtiPE2FF4mQhbW1uDtiPE2FF4mQgKL2JuKLxMhEajMWg7QoydUdznRZ5caWmpQdsRfikUig49qxRV1EFZWYyCy7bQ3OnHjff19TXLD9pTeJkIfbsR0bcd4VdhYSGCgoJ0Tntlp/ZwTk4Oxo8f3wdVGRcKLxNhb2+P2tpaAA+uaz38MaCHh+3t7Xmpj/SMr68vcnJytMY1Nitx+EQ2XpgaCgdbiVZbc0ThZSL69++PGzduAOj4+cWHhzvrcZYYFzs7uw5HUyqVCvdqqxEaEmzSXeLoiy7Ymwhvb2+DtiPE2FF4mYjJkycbtB0xLq2trdi0aRO2bt2KTZs2obW1le+SeGfBGGN8F9GXGhoa4OzsjPr6epPqBrq5uVmvd5wUCgXd6yUwy5cvx8aNG7kHcAAPelNdtmwZPvnkEx4r6x36/o3SkZeJOHv2rEHbEeOwfPlyrFu3DgMGDEBqaiq2b9+O1NRUDBgwAOvWrcPy5cv5LpE3FF4mory8HMCDC70WFhZa00QiEXdU1t6OGL/W1lZs3LgRUqkUN2/exPz58+Hi4oL58+fj5s2bkEql2Lhxo9meQlJ4mYj2I6p33nkHLS0tWL9+PaZPn47169ejubmZ6+yRjryE44svvkBbWxtWr17NPXyjnZWVFT766CO0tbXhiy++4KlCftGtEiai/dJlTk4OrKys8M4772D48OGYPn06LC0tcfHiRa12xPiVlJQAAGbMmKFzevv49nbmho68TMSIESMAABkZGZg9ezbOnDmD5uZmnDlzBrNnz0ZGRoZWO2L82h8MfOjQIdTU1GDEiBF4+eWXMWLECNTU1HDP6jTXBwjTu40morW1Ffb29rC3t0e/fv24G1YBQCaT4d69e2hqakJTUxPEYjGPlRJ9tW/Th99lfJSVlZXJbVN6t9HMiMViLFu2DPX19WhpacG7776LN998E++++y6am5tRX1+PZcuWmdROburEYrFWt90eHh54++23tR7ILBKJzHab0pGXiTG3e4JMWU1NDdzd3bttV11dDTc3tz6oqG/QkZeZ+uSTT9DU1KT1bmNTUxMFlwCFhIQAAEaPHg2lUqm1TZVKJfz8/LTamRs68jJRKpUKR44cwfTp0+lDvALl4OCApqYmHDp0CC+88EKHbbp//37MmTMH9vb2aGxs5Ltcg6EjL0IErv1UcMWKFTqnf/DBB1rtzA2FFyFG6ty5cwCAK1euoKGhQWtaQ0MDCgoKtNqZGwovQoyUm5sbnJ2dAQDOzs4ICAhAdnY2AgICtMab65EX3WFPiBGrq6tDv379UF9fj4KCAu5oC3gQXHV1dfwVxzM68iLEyNXV1aG6uho+Pj6wsbGBj48PqqurzTq4ADryIkQQHBwc8Ic//AFnzpzBxIkT4eDgwHdJvKMjL0KM3OzZs2FnZ4fU1FTk5eUhNTUVdnZ2mD17Nt+l8YrCixAjNnv2bBw4cABisRjLly/Hli1bsHz5cojFYhw4cMCsA4z38Nq8eTNkMhlsbGwwYcKELt/2vXLlCl588UXIZDJYWFggJSWl7wolpI81NzdzwXX//n2sXr0anp6eWL16Ne7fv88F2KNPizIXvIbXnj17IJfLkZiYiNzcXAQEBCAiIgLV1dU62ysUCgwdOhRr1qzR+nAqIabovffeAwDI5fIOH74Wi8V49913tdqZG17DKzk5GQsXLkRMTAxGjRrFncunpaXpbP+73/0O69atw8svvwyJRKKzDSGm4tdffwUAvPHGGzqnL1iwQKudueHt3cbW1lbk5OQgPj6eGycSiTBt2jRkZ2cb7HWUSiWUSiU33H6nskqlgkqlMtjrGJv2dTPldTR1Q4cOBQBs3boVq1ev7rBNt23bxrUzpe2s77rwFl61tbVQq9WQSqVa46VSKQoLCw32OklJSVi5cmWH8ceOHdPrUWFC196DKhGeKVOmIDU1FcnJyQgODuZOHTMyMtDa2spd850yZQqOHDnCY6WGpVAo9Gpn8vd5xcfHQy6Xc8MNDQ3w9vZGeHi4yfcqkZGRgbCwMOpVQsBmzpyJf/zjH3jttdcQGxuLYcOGoaSkBJ9//jna2towc+ZMzJkzh+8yDerRz3F2hrfwcnV1haWlJaqqqrTGV1VVGfRivEQi0Xl9zNra2iz+qM1lPU3VwYMHudslkpOTtabNmjUL33//PT+F9SJ991feLtiLxWIEBQUhMzOTG6fRaJCZmYnQ0FC+yiLE6Hz//fdQKBR46623EBgYiLfeegsKhcIkg6sneD1tlMvliI6ORnBwMEJCQpCSkoKmpibExMQAAKKiouDl5YWkpCQADy7y//LLL9zPt27dQl5eHhwcHDB8+HDe1oOQ3mZra4tNmzZRB5MP4TW8IiMjUVNTg4SEBFRWViIwMBDp6encRfyysjKtBxDcvn0b48aN44bXr1+P9evXY/LkycjKyurr8gkhPOL9gn1sbCz3NOdHPRpIMpmMHppKCAFgBB8PIoSQx8H7kVdfaz9y0/ftWKFSqVRQKBRoaGig6yMmwly2afvfZndnWWYXXvfv3wcAeHt781wJIaQr9+/f57q71sXsHn2m0Whw+/ZtODo6wsLCgu9yek37zbjl5eUmfTOuOTGXbcoYw/379zFw4ECtN+weZXZHXiKRCIMGDeK7jD7j5ORk0ju6OTKHbdrVEVc7umBPCBEkCi9CiCBReJkoiUSCxMRE6vfMhNA21WZ2F+wJIaaBjrwIIYJE4UUIESQKL0KIIFF48eT69euwsLBAXl5enyzn1KlTGDt2LKytrc36WX/EdFB4GYmsrCxYWFigrq6uV5Yvl8sRGBiI0tJS7Nixo1deg3Q0ZcoU7hFlxsDY6nkSFF6PobW1le8SeqykpATPPfccBg0ahH79+j3WMvpyvRljaGtr67PXM2ZC3N/6BCPdmjx5MluyZAlbunQpGzBgAJsyZQrLz89nzz//PLO3t2fu7u7stddeYzU1Ndw8P/zwA3v66aeZs7Mz69+/P3vhhRdYcXExN720tJQBYBcvXuR+fvgrOjq6R8v55ptvWGhoKJNIJGz06NEsKytLa/rDX9u3b2eMMZaVlcV+97vfMbFYzDw8PNiKFSuYSqXqcr1PnDjBALD09HQWGBjIbGxs2NSpU1lVVRU7cuQI8/X1ZY6Ojmzu3LmsqamJW5ZarWYff/wxk8lkzMbGhvn7+7Nvv/2Wm96+3CNHjrDx48cza2trduLECUNuxj4XHR3d4XdfXFzM5s+fz/0ennrqKZaSktJhvlmzZrHVq1czT09PJpPJGGOMnTp1igUEBDCJRMKCgoLY/v37uX2oXVf7pa56SktL++rXYXAUXnqYPHkyc3BwYO+99x4rLCxkZ86cYW5ubiw+Pp4VFBSw3NxcFhYWxqZOncrN891337G9e/eyX3/9lV28eJHNnDmTjR07lqnVasaYdni1tbWxvXv3MgCsqKiIVVRUsLq6uh4tZ9CgQey7775jv/zyC3vjjTeYo6Mjq62tZW1tbayiooI5OTmxlJQUVlFRwRQKBbt58yazs7NjixcvZgUFBWz//v3M1dWVJSYmdrrehYWFXMhMnDiRnTx5kuXm5rLhw4ezyZMns/DwcJabm8v++c9/sgEDBrA1a9Zwy1q9ejXz9fVl6enprKSkhG3fvp1JJBIuZNuX6+/vz44dO8aKi4vZnTt3envT9qq6ujoWGhrKFi5cyCoqKlhFRQVraWlhCQkJ7Pz58+zatWvs66+/ZnZ2dmzPnj3cfNHR0czBwYG9/vrr7PLly+zy5cusvr6e9e/fn7322mvsypUr7MiRI+ypp57SCq979+51uV/qqqetrY2PX41BUHjpYfLkyWzcuHHc8KpVq1h4eLhWm/Lyci58dKmpqWEAWH5+PmNMO7wY++2P9969e13W0tlyHg4KlUrFBg0axNauXcuNc3Z25o64GGPsr3/9Kxs5ciTTaDTcuM2bNzMHBwcuGB9d74fr/PHHH7lxSUlJDAArKSnhxv3lL39hERERjDHGWlpamJ2dHTt9+rTWshYsWMDmzp2rtdzvv/++y/UXmsmTJ7OlS5d22WbJkiXsxRdf5Iajo6OZVCplSqWSG7dlyxY2YMAA1tzczI3btm2b1j6kz36pTz1CQde89BQUFMT9fOnSJZw4cQIODg7cl6+vL4AH15aAB49gnzt3LoYOHQonJyfIZDIAD/rl7wl9l/PwE5esrKwQHByMgoKCTpdbUFCA0NBQrW6Bnn76aTQ2NuLmzZs61/th/v7+3M9SqRR2dnbcE57bx1VXVwMAiouLoVAoEBYWpvU727VrF/f7ahccHNxpzaZi8+bNCAoKgpubGxwcHLB169YO23Ps2LHcQ2YBoKioCP7+/rCxseHGhYSEaM2jz35pSsyuS5zHZW9vz/3c2NiImTNnYu3atR3aeXp6AnjwsFAfHx9s27YNAwcOhEajwZgxY3p88dVQy3lcD6/3wx7uydPCwqJDz54WFhbQaDQAHvy+AODw4cPw8vLSavfo5/Q6ez1TsXv3bsTFxWHDhg0IDQ2Fo6Mj1q1bh7Nnz2q1e5zfgz77pSmh8HoM48ePx969eyGTyWBl1fFXeOfOHRQVFWHbtm149tlnAQAnT57scpnt/2XVavVjLefMmTP4t3/7NwBAW1sbcnJyOn2wCQD4+flh7969YIxxR1+nTp2Co6Ojwfs7GzVqFCQSCcrKyjB58mSDLtvYicVirW166tQpTJo0CYsXL+bG6XNUNHLkSHz99ddQKpVc4J8/f16rTXf7pa56hIxOGx/DkiVLcPfuXcydOxfnz59HSUkJjh49ipiYGKjVari4uGDAgAHYunUriouLcfz4ccjl8i6X6ePjAwsLCxw6dAg1NTVobGzs0XI2b96M/fv3o7CwEEuWLMG9e/cwf/78Tl9v8eLFKC8vx9tvv43CwkIcOHAAiYmJkMvlXfZe+TgcHR0RFxeHZcuWYefOnSgpKUFubi4+++wz7Ny506CvZWxkMhnOnj2L69evo7a2FiNGjMCFCxdw9OhRXL16FR9++GGHENLllVdegUajwZtvvomCggIcPXoU69evBwDun093+6WuetqPjoWIwusxDBw4EKdOnYJarUZ4eDjGjh2Ld999F/369YNIJIJIJMLu3buRk5ODMWPGYNmyZVi3bl2Xy/Ty8sLKlSvx/vvvQyqVIjY2tkfLWbNmDdasWYOAgACcPHkSBw8ehKura5evd+TIEZw7dw4BAQF46623sGDBAvznf/7nE/1uOrNq1Sp8+OGHSEpKgp+fH55//nkcPnwYQ4YM6ZXXMxZxcXGwtLTEqFGj4ObmhoiICMyZMweRkZGYMGEC7ty5o3UU1hknJyf84x//QF5eHgIDA/HBBx8gISEBALjrYN3tl7rq6ek1WGNCXeIQIlB/+9vfEBMTg/r6etja2vJdTp+ja16ECMSuXbswdOhQeHl54dKlS1ixYgX+/Oc/m2VwARRehAhGZWUlEhISUFlZCU9PT7z00kv47//+b77L4g2dNhJCBIku2BNCBInCixAiSBRehBBBovAihAgShRchRJAovEivmjdvHiwsLLgPb0ulUoSFhSEtLa1HH03ZsWPHY/cA+yTmzZtHff4bKQov0uuef/55VFRU4Pr16/jhhx8wdepULF26FDNmzKCunsnj47U3MWLy2rs0flRmZiYDwLZt28YYY2zDhg1szJgxzM7Ojg0aNIgtWrSI3b9/nzH2W0eFD3+19/i6a9cuFhQUxBwcHJhUKmVz585lVVVV3OvcvXuXvfLKK8zV1ZXZ2Niw4cOHs7S0NG56WVkZe+mll5izszNzcXFhf/zjH7mukRMTEzu8rtC7pjYldORFePHcc88hICAA+/btAwCIRCJs2rQJV65cwc6dO3H8+HEsX74cADBp0iSkpKTAyckJFRUVqKioQFxcHABApVJh1apVuHTpEr7//ntcv34d8+bN417nww8/xC+//IIffvgBBQUF2LJlC/eBdZVKhYiICDg6OuLnn3/GqVOn4ODggOeffx6tra2Ii4vDn//8Z+7IsaKiApMmTerbXxTpHN/pSUxbZ0dejDEWGRnJ/Pz8dE779ttv2YABA7jh7du3M2dn525f7/z58wwAd9Q2c+ZMFhMTo7PtV1991aErbKVSyWxtbdnRo0e7rZ/wi468CG/YQx0h/vjjj/j9738PLy8vODo64vXXX8edO3egUCi6XEZOTg5mzpyJwYMHw9HRkevssL2rl0WLFmH37t0IDAzE8uXLcfr0aW7eS5cuobi4GI6Ojly3yf3790dLS4tJdptsaii8CG8KCgowZMgQXL9+HTNmzIC/vz/27t2LnJwcbN68GUDXzyxsampCREQEnJyc8Le//Q3nz5/H/v37teb7wx/+gBs3bmDZsmW4ffs2fv/733OnnI2NjQgKCkJeXp7W19WrV/HKK6/08tqTJ0W9ShBeHD9+HPn5+Vi2bBlycnKg0WiwYcMGrtO8//u//9Nqr6v74sLCQty5cwdr1qyBt7c3AODChQsdXsvNzQ3R0dGIjo7Gs88+i/feew/r16/H+PHjsWfPHri7u8PJyUlnnabUbbKpoSMv0uuUSiUqKytx69Yt5Obm4uOPP8asWbMwY8YMREVFYfjw4VCpVPjss89w7do1fPXVV0hNTdVahkwmQ2NjIzIzM1FbWwuFQoHBgwdDLBZz8x08eBCrVq3Smi8hIQEHDhxAcXExrly5gkOHDsHPzw8A8Oqrr8LV1RWzZs3Czz//jNLSUmRlZeGdd97hnqAkk8nwr3/9C0VFRaitrYVKpeqbXxrpHt8X3Yhpe/gpzVZWVszNzY1NmzaNpaWlcc+HZIyx5ORk5unpyWxtbVlERATbtWtXh+dYvvXWW2zAgAFat0r8/e9/ZzKZjEkkEhYaGsoOHjzY4VmGfn5+zNbWlvXv35/NmjWLXbt2jVtmRUUFi4qKYq6urkwikbChQ4eyhQsXsvr6esYYY9XV1SwsLIw5ODjQrRJGhvrzIoQIEp02EkIEicKLECJIFF6EEEGi8CKECBKFFyFEkCi8CCGCROFFCBEkCi9CiCBReBFCBInCixAiSBRehBBBovAihAjS/wN3n2z0frF/2AAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:31:08.432786Z", + "iopub.status.busy": "2024-03-26T07:31:08.431820Z", + "iopub.status.idle": "2024-03-26T07:31:08.729544Z", + "shell.execute_reply": "2024-03-26T07:31:08.728578Z" + }, + "papermill": { + "duration": 0.316237, + "end_time": "2024-03-26T07:31:08.731572", + "exception": false, + "start_time": "2024-03-26T07:31:08.415335", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABlvklEQVR4nO2dd3hU1daH3zMtvVCTAKFJ700QUAEFQbFgRVQUBCuogIigfiCK4lUUFAVERFRE9F4VFBEFBJEiSgnSgtJrIJRUkkw73x+TM3NmMj0zKbDf55knkzOn7Gm/WWvttdeSZFmWEQgEgkqGprwHIBAIBMEgxEsgEFRKhHgJBIJKiRAvgUBQKRHiJRAIKiVCvAQCQaVEiJdAIKiUCPESCASVEl15D6CssVqtnDx5kri4OCRJKu/hCAQCF2RZJjc3l1q1aqHReLavLjvxOnnyJKmpqeU9DIFA4INjx45Rp04dj49fduIVFxcH2F6Y+Pj4ch6NQCBwJScnh9TUVPt31ROXnXgprmJ8fLwQL4GgAuMrrCMC9gKBoFIixEsgEFRKhHgJBIJKyWUX86osyLKM2WzGYrGU91AEgpCi1WrR6XSlTlUS4lUBMRqNnDp1iosXL5b3UASCsBAdHU1KSgoGgyHocwjxqmBYrVYOHTqEVqulVq1aGAwGkUwruGSQZRmj0UhmZiaHDh2icePGXhNRvSHEK8TIssxLS3ZRKzGKEb0aBXy80WjEarWSmppKdHR0GEYoEJQvUVFR6PV6jhw5gtFoJDIyMqjzCPEKMf+czuOLzUcBGNC+NrUTo4I6T7C/RgJBZSAUn2/xDQkxJ7MK7Pe/TztZjiMRCC5thHiFmJPZDvFamnaiHEciEFzaCPEKMaeyCu330zNySc/IKcfRVH4OHz6MJEmkpaWVyXk2bNhA69at0ev1DBgwoFTXvFxZu3YtkiSRlZUV1usI8QoxassLYGk5uY5FJgsHM/PIKzKXy/XDRbi/GGPGjKFdu3YcOnSIBQsWhOUagtAgxCvEKJbXDS2SAFvcy2r1r6+vv/v5w5HzF8krMnMwMy9k5wwUo9FYbtcOlgMHDnDddddRp04dEhMTgzpHWT5vJZk51FSG906IV4g5VWx53X9VPWIjdJzIKmDLkQs+jzuXV0SXqauZvvKfEo/JssxFozmgW26BiUKThUKTJeBj1bdAGqr37NmTkSNHMmrUKKpXr07fvn3ZtWsXN954I7GxsSQlJTF48GDOnj1rP2bFihVcffXVJCYmUq1aNW6++WYOHDjg9vyHDx+mV69eAFSpUgVJkhgyZEhA50lPT6dbt25ERkbSqlUrfvvtN/u5JUni3LlzPPzww0iSZLe8fvvtNzp37kxERAQpKSmMHz/eSTDcPW/FQvz5559p3749UVFRXHfddZw5c4affvqJ5s2bEx8fz3333eeUjGy1Wpk6dSoNGjQgKiqKtm3b8r///c/+uHLen376iY4dOxIREcH69et9vjdTpkyhZs2axMXFMXz4cMaPH0+7du3sjw8ZMoQBAwbw2muvUatWLZo2bQrA559/TqdOnYiLiyM5OZn77ruPM2fOOJ17+fLlNGnShKioKHr16sXhw4d9jicUiFSJECLLMqeybZZXg2ox9GuVzP+2HmdJ2gk6N6jq9djdJ3PIzC1i/f6zDGiU4vRYgclCi4k/h23c3tjzSl+iDf5/TD799FOeeOIJNmzYQFZWFtdddx3Dhw9n+vTpFBQU8Pzzz3PPPffw66+/ApCfn8+YMWNo06YNeXl5TJw4kdtvv520tLQS0+mpqal888033Hnnnezbt4/4+HiioqICOs9zzz3HjBkzaNGiBe+88w633HILhw4dIjU1lVOnTtG0aVNeeeUVBg4cSEJCAidOnOCmm25iyJAhfPbZZ6Snp/PII48QGRnJyy+/7PZ5A5w6dQqAl19+mffff5/o6Gjuuece7rnnHiIiIli0aBF5eXncfvvtzJw5k+effx6AqVOnsnDhQubMmUPjxo1Zt24dDzzwADVq1KBHjx72640fP55p06bRsGFDqlSp4vU9+eKLL3jttdeYNWsW3bt3Z/Hixbz99ts0aNDAab/Vq1cTHx/PypUr7dtMJhOvvvoqTZs25cyZM4wZM4YhQ4awfPlywFYX74477mDEiBE8+uijbNmyhWeffdb3ByUECPEKIefzjRSZrQAkJUQwoF1t/rf1OMt3nuLlW1pi0Hk2dJXYVNZFY0DWTkWjcePGvPnmm4Dt1759+/a8/vrr9sfnz59Pamoq//zzD02aNOHOO+90On7+/PnUqFGDPXv20KpVK6fHtFotVavafgRq1qzp5Nb5e56RI0fa9509ezYrVqzg448/Zty4cSQnJyNJEgkJCSQnJwMwa9YsUlNTef/995EkiWbNmnHy5Emef/55Jk6caBdG9fMGh3hNmTKF7t27AzBs2DAmTJjAgQMHaNiwIQB33XUXa9as4fnnn6eoqIjXX3+dVatW0bVrVwAaNmzI+vXr+fDDD53E65VXXqFPnz6+3xBg5syZDBs2jKFDhwIwceJEfvnlF/LynEMKMTExzJs3z2nJzsMPP2y/37BhQ9577z2uvPJK8vLyiI2NZfbs2VxxxRW8/fbbADRt2pSdO3fyn//8x6+xlQYhXiFEsbqqx0YQodPS9Ypq1IyL4ExuEb/9k0mf4jiYO/IKHW6IxSX2FaXXsueVvgGNZe/JHCzFItiqdkJAx7peOxA6duxov79jxw7WrFlDbGxsif0OHDhAkyZN+Pfff5k4cSKbN2/m7NmzWK028T969GgJ8fKGv+dRRAFAp9PRqVMn9u7d6/G8e/fupWvXrk5LtLp3705eXh7Hjx+nbt26JZ63mjZt2tjvJyUlER0dbRcuZduff/4JwP79+7l48WIJUTIajbRv395pW6dOnTyO2ZV9+/bx5JNPOm3r3Lmz3fpVaN26dYm1hlu3buXll19mx44dXLhwwel1bdGiBXv37qVLly5Ox6hf43AixCuEKAmqtRJtyx20Golb2tbi4/WHWJJ2wqt45RSa7PctLpaXJEkBuW4AEXot1uLzBHpsaYiJibHfz8vL45ZbbnH7K5ySYnONb7nlFurVq8dHH31ErVq1sFqttGrVKuCAcajOEyzq561Gr9fb70uS5PS/sk0RBMUS+vHHH6ldu7bTfhEREX5drzS4njM/P5++ffvSt29fvvjiC2rUqMHRo0fp27dvhQjoi4B9CFEsr5QEx1qtAe1sH8JVe06TqxIoV9QpDa6WV2WlQ4cO7N69m/r169OoUSOnW0xMDOfOnWPfvn289NJLXH/99TRv3pwLF7xPbiiWgbpUUCDn+eOPP+z3zWYzW7dupXnz5h6v17x5czZt2uTkym/YsIG4uDivzSGCoUWLFkRERHD06NESr1dpmsY0bdqUv/76y2mb6//uSE9P59y5c7zxxhtcc801NGvWrESwvnnz5nbLUUH9GocTIV4hxCFejvWMrWrH07BGDEVmKz/vPu3xWG9uY2VlxIgRnD9/nkGDBvHXX39x4MABfv75Z4YOHYrFYqFKlSpUq1aNuXPnsn//fn799VfGjBnj9Zz16tVDkiSWLVtGZmYmeXl5AZ3ngw8+4LvvviM9PZ0RI0Zw4cIFp7iOK08++STHjh3jqaeeIj09naVLlzJp0iTGjBkT8vWncXFxjB07ltGjR/Ppp59y4MABtm3bxsyZM/n000+DPu9TTz3Fxx9/zKeffsq///7LlClT+Pvvv31WK6lbty4Gg4GZM2dy8OBBvv/+e1599VWnfR5//HH+/fdfnnvuOfbt28eiRYvKLD9OiFcIUdIkFLcRbG7BbW1t1pe35UK5l6B41apViw0bNmCxWLjhhhto3bo1o0aNIjExEY1Gg0ajYfHixWzdupVWrVoxevRo3nrrLa/nrF27NpMnT2b8+PEkJSUxcuTIgM7zxhtv8MYbb9C2bVvWr1/P999/T/Xq1b1eb/ny5fz555+0bduWxx9/nGHDhvHSSy+V6rXxxKuvvsr//d//MXXqVJo3b06/fv348ccfS8wMBsL999/PhAkTGDt2LB06dODQoUMMGTLEZzWHGjVqsGDBAv773//SokUL3njjDaZNm+a0T926dfnmm29YsmQJbdu2Zc6cOU4TNGFFLkdef/11uVOnTnJsbKxco0YN+bbbbpPT09N9Hvf111/LTZs2lSMiIuRWrVrJP/74o9/XzM7OlgE5Ozu7NEN3y92zN8r1nl8mf592wmn7ocw8ud7zy+QG45fJp3MK3B775MKtcr3nl8ndpvwk//7ndrmgwP1+/rLzeJa849gFecexC6U6j+DSpHfv3vIDDzxQbtcvKCiQ9+zZ4/Zz7u93tFwtr99++40RI0bwxx9/sHLlSkwmEzfccAP5+fkej9m4cSODBg1i2LBhbN++nQEDBjBgwAB27dpVhiN3z0k3lhdA/eoxtEtNxCrDsh2n3B6bewnGvAQVg4sXL/LOO++we/du0tPTmTRpEqtWreKhhx4q76GVinIVrxUrVjBkyBBatmxJ27ZtWbBgAUePHmXr1q0ej3n33Xfp168fzz33HM2bN+fVV1+lQ4cOvP/++2U48pJYrTKnc0rGvBQGtKsFeHYd89SzjUK8BAHQsmVLYmNj3d6++OILJEli+fLlXHvttXTs2JEffviBb775ht69e5f30EtFhUqVyM7OBrAnIrpj06ZNJYKxffv2ZcmSJW73LyoqoqioyP5/Tk54qjyczSvCZJHRSFAzLqLE4ze3rcWrP+5lx/FsDp3Np0F152lpp5iXXPp1jkL+Lh+WL1+OyeR+JjspKYmoqChWrVpVxqMKPxVGvKxWK6NGjaJ79+5ekxMzMjJISnLOl0pKSiIjI8Pt/lOnTmXy5MkhHas7ThbPNCbFR6LTljRoq8dGcHWj6vz2TyZLtp9gdJ8mTo+7Vn8wW6zhG6zgkqJevXrlPYRyocLMNo4YMYJdu3axePHikJ53woQJZGdn22/Hjh0L6fkVThUnqKpzvFwZ0N7hOsouiahKqoS+WPhMViFeAoE3KoTlNXLkSJYtW8a6det8Jv4lJydz+rRzvtTp06fta9FciYiIKJGdHA5OusnxcuWGFslE6Xdx+NxFdhzPpl1qImBzEfOMNvFKrWprumGyCMdPIPBGuVpesiwzcuRIvvvuO3799Ve/clm6du3K6tWrnbatXLmyzNZTecIfyysmQmdfIrRkuyNwn280oxhi9avZxMssgvYCgVfKVbxGjBjBwoULWbRoEXFxcWRkZJCRkUFBgaMa6YMPPsiECRPs/z/zzDOsWLGCt99+m/T0dF5++WW2bNnCyJEjy+Mp2LFn1/voFnRb8azjjzsdKRNKvEuvlezdhkTMSyDwTrmK1+zZs8nOzqZnz56kpKTYb1999ZV9n6NHj9rLiwB069aNRYsWMXfuXHuhtiVLlgRUgSAc2HO8vFheAFc1rAZAZm4R+cWipcw0xkboqBlvc3FLbXkJw01wiVOuMS/XoLU71q5dW2Lb3Xffzd133x2GEQVPhp+WV0yEjii9lgKThbN5RcRE6OziFRepp0ZsJIUXLs+YV/369Rk1ahSjRo0CbEurvvvuO9EIQ+CWChGwr+yYLVZ7gqovywugepyBY+cLOJtXRL1qMXa3UbG8jl4QbiPYCvr5qhIquHypMKkSlZkzuUVYZVvMqnqs75lNZZ/MXFtNJCVNIjZSR43iBFerLGMpVbpE5bfckpOTy2Sm2BtymBpcCEqPEK8QoFSTSIqPRKPxXmYEHOJ1Ns+W+a/U+YqL0BFt0KGcwu46yjIY8wO7mS4iFd8CPlZ9C7AkdW5uLvfffz8xMTGkpKQwffp0evbsaXcFA0GSJPvKCaVBxrfffkuvXr2Ijo6mbdu2bNq0yemY9evXc8011xAVFUVqaipPP/2001pZXw0lgm1wISh7hNsYAk5mKS6j93iXgqt4KW5jXKTt7dAWq5fJYiVSrwXTRXi9VkBjah3Q3l544SQY/K/aOWbMGDZs2MD3339PUlISEydOZNu2bU6dakrDiy++yLRp02jcuDEvvvgigwYNYv/+/eh0Og4cOEC/fv2YMmUK8+fPJzMzk5EjRzJy5Eg++eQTwHdDCYVAGlwIygchXiFAsbxSEn3HuwBqxNqqgTosL4fbCM7iVZnIzc3l008/ZdGiRVx//fUAfPLJJ9SqFZjwemPs2LH0798fgMmTJ9OyZUv2799Ps2bNmDp1Kvfff7/dymvcuDHvvfcePXr0YPbs2URGRvpsKKEQSIMLQfkgxCsEKJaXt+x6NdWL41pni2NejlQJW31zRbyMituoj7ZZQAGw80S2/X7rUjTgQB/t964HDx7EZDLRuXNn+7aEhAR7D8BQoG5oodTBP3PmDM2aNWPHjh38/ffffPHFF/Z9ZFnGarVy6NAhmjdv7rOhhEIgDS4E5YMQrxDgroKqN0q6jcUxr2LLS6dYXsVt1JCkgFw3AFmvqjIQ4LEVGdeGFoBTA4vHHnuMp59+usRxdevWDaihRDgaXAhCixCvEOCudr03Aol5VSYaNmyIXq/nr7/+srcEy87O5p9//uHaa68N+/U7dOjAnj17aNSokdvHd+7caW8ooTS02LJlS9jHJQgPYrYxBDjcRv8sr2r2mJer21gsXpIiXpUr3SEuLo6HHnqI5557jjVr1rB7926GDRuGRqPx2ewhFDz//PNs3LiRkSNHkpaWxr///svSpUvtS8f8aSghqDwI8SolRWaL3YLyV7wUyyuvyEyhyVJSvFSWlz+rECoS77zzDl27duXmm2+md+/edO/enebNm/ts9hAK2rRpw2+//cY///zDNddcQ/v27Zk4caJ9wsCfhhKCyoNwG0vJ6WybcEXoNFSNMfjY20Z8pA6DVoPRYiUzt0jlNjoC9laURFUZnTb8VkuoiIuLcwqY5+fnM3nyZB599FGfxx4+fNjpf7Vw169fv4SQJyYmlth25ZVX8ssvv3i8xqBBgxg0aJDH6/Ts2bPS/WBcrgjxKiXKguyUhEi/XSNJkqgea+BkdiFn84rsGfZKzEuSJLQaDVZsrqNOG5ahh4Xt27eTnp5O586dyc7O5pVXXgHgtttuK+eRCS41hHiVkowAg/UK1eMiOJldyLk8oz3DXnEbwTbjaMTmOkZRidQLmDZtGvv27cNgMNCxY0d+//139u7dy4033ujxGKXVvUDgL0K8SsnJABNUFZS415ncIvKNttb1NsvL5rLotRJGS+WbcWzfvr3b7k8FBQWkpaWV/YAElyxCvErJqQCXBilUL55xPHLOse4uNlKHbLZZYTqtBiqheHkiKirKYwqDQBAMYraxlAS6NEhBsbwOnbWJl0GrIUIV3FIaEFW2dAmBwB9CMSkixMsDtmUlss8GsIEuylZwFS8lWK9kkMtG2yym8RKxvAQCNRcvXgScV0wEinAbPbDrRA63vL+elIRINk243u0+ZouVYxdsb0LAllfx+sYj523H2xdla7UkJiZy5txZ5Ih4iuRICgsDf5tks2O5S2FhYcDHCwThQJZlLl68yJkzZ0hMTESrDX4ySoiXB5SsB2/W7fr9Z8ktNFM1xsAVNWI97+gGJeZlLF6/qJ5pTE5OJrvAxMETp9FrNUh5gVl1AGcuOJqYGAoCP14gCCeJiYke2xX6ixAvH8heKpIuTbNVeri5TYq9Way/1HCpuKoWL0mSuKJeHW7/ZBeJERq+faIbVfxMgFUY/u1a+/3Vz/YM6FiBIJzo9fpSWVwKQrw8oCk2vTxZXvlFZlbsygBgQPvaAZ/ftVy0kl2voNdqiIs0cCq3iLMFMinVAnNLT+Ra7PfLYmmOQFDWiIC9B+xuo4fHV+45TYHJQr1q0bQv7nwdCAlRenvpG3AE7NUonYiUXDKBQOBAiJcHHDEv9/K1JM3W8fq2drWDqpig0UhOayHVbqOC0olI6cYtEAgcCPHygIRntzEzt4jf/z0LwIB2wZc4VruObi2v4vQLpV5YqLhUEl8FlzdCvDyg8eI2Lvv7JBarTNvURBoGOMuoRkmXAEeqhBqlMuvJEIrXqj2naTnpZ5YWW44CQWVFiJcHvLmNS4pnGUtjdYEjXQJsbc9csVteIXQbtx29gNFsZcvhCyE7p0BQHgjx8khxfXQX7TqYmceOY1loNRI3tymdeNVwchtLZhonKzGvEFpelmIxNpeqoa1AUP4I8fKAJ8tLsbquaVzd3t06WNQxL7cB+2K3MSOn0OcyJTVHz130+JileK2kWayZFFRyhHh5wJ7npdomy7I9VjSgXeC5Xa5Uj1PNNrqJedWMi0SrkbBYZTJzi/w+77VvrfH4mNkqO/31hizLHD13UVQWFVRIhHh5wJ78oPrebj+WxZFzF4k2aLmhZVKpr+HL8tJqJJKKrbtTIcr1shYLkT8zjm+sSOfat9Ywa+2BkFxbIAglQrw84C5Jdcl2m9XVt2Uy0YbSL05Qi1e8m5gXOBJVQxX3sltefriNH/52EIC3ft4HwKYD59h/RlQ8FQTOZ5sO88Ga/V5DGoFSqm9gXl6eveGnQnx8fKkGVFGQ7AF7h6Wy7O9TANxWyllGBSfLy43bCI6ORCdDNONoj3kFGLDffyaPQR/9AcDhN/qHZCyCy4fPNx3h3zN5tK+bSN1q/ndh90bA4nXo0CFGjhzJ2rVrnUqtyLKMJElYLBYvR1ceXKtK/P5vJufzjVSPNXB1o+ohuUa1GANt6iRglWUSo9xbXrXCZHkFWuRw/5nckFxfcHmSU9ynwZOHEQwBi9cDDzyALMvMnz+fpKSkMmkmWh443Ebbl3zJdqWCRC1bieYQoNFILHmyu/2+O1Ls6RKhjXkFanlpLtH3WVA25BTYOmQlePiRDoaAxWvHjh1s3bqVpk2bhmwQFRFJVVUir8jML3tsFSRuD6KChDc8iZaCkqiqVGz1hdlHIN5TzKvAaCG3yETNOPcVKIR4CYLFaLZSYLJ5ZKG0vAI2Ia688kqOHTsWsgFUVJSvqizDz7syKDRZaVg9hjZ1Esp0HEqul7+WV6HZu3hZii0u11SJwR9v5ur/rOFCvtHdYWjE1I4gSJTWfuA5thsMAZ9p3rx5PP7445w4cYJWrVqVqEHdpk2bkA2uPHHkecmlriBRGhTL60xuESaL1WfRw0KT95ijxW55OYvcwbP5GM1WMnIK3RY+vFTDA4Lwk6M0VY7QofXhaQRCwL+nmZmZHDhwgKFDh3LllVfSrl072rdvb/8bCOvWreOWW26hVq1aSJLEkiVLvO6/du1aJEkqccvIyAj0afhE+a6aLDIb9hdXkGgfmlnGQFBiBLIMF42+J0OKfFpe7gP2vvK+hNsoCJacguJgfQjjXRCE5fXwww/Tvn17vvzyy1IH7PPz82nbti0PP/wwd9xxh9/H7du3zyklo2bNmkGPwRPqZ2WVoUPdROpViwn5dXyh1zpGYvQhTODb8nJk2DufS4mBqZPpJcnxfwh/MAWXGcpMo7uyT6Uh4LMdOXKE77//PiQNRG+88UavLeA9UbNmTRITE0t9fa+4fFmDKfUckmFIEgadBqPZ6lcbNP/dRmfLy93so16jsV9TKywvQZDkFruNoQzWQxBu43XXXceOHTtCOohAadeuHSkpKfTp04cNGzZ43beoqIicnBynmz+o3SSdRqJ/65RSjbk0RBTHufyxvPx2G1ViJcuy27wvdXxCbWGLtY6CQHC4jeVsed1yyy2MHj2anTt30rp16xIB+1tvvTVkg3MlJSWFOXPm0KlTJ4qKipg3bx49e/Zk8+bNdOjQwe0xU6dOZfLkyQFfS21n9GhSg2qxpasgURoi9Bpyi0LsNqrESj3xqO6WpNNKUDxRpHYbrTJohSEm8JNwJKhCEOL1+OOPA/DKK6+UeCzcGfZNmzZ1yi/r1q0bBw4cYPr06Xz++eduj5kwYQJjxoyx/5+Tk0NqaqrPa6ktjdvKyWVUMBRbXkVmPwL2Ju8CZ3VTVcJTsF7dIESdj2aVZbSufrVA4AElQbXcA/auaxnLm86dO7N+/XqPj0dERBAREbjVFBuho3qsAa1Gok/z0leQKA0GXSBuo7+Wl7XENnAO2KtXEqgtL4tVRl/6tnuCywSH5VWObqPJZCIqKoq0tDRatWoV0oEES1paGikpoY9HGXQalj99DUgQZSjfb2og4lXow/JyF7D3lJWv8xjz8jkMgcBOhUiV0Ov11K1bN2SuYV5eHvv377f/f+jQIdLS0qhatSp169ZlwoQJnDhxgs8++wyAGTNm0KBBA1q2bElhYSHz5s3j119/5ZdffgnJeFypGV8xmrUq4lUUwtlGdcDe0yJtnSqwpZ5ttAr1EgRATkWZbXzxxRd54YUXOH/+fKkvvmXLFtq3b29Pbh0zZgzt27dn4sSJAJw6dYqjR4/a9zcajTz77LO0bt2aHj16sGPHDlatWsX1119f6rFUZAxhmG1UW16eSkzrVGuC1JkSFiFeggCoMLON77//Pvv376dWrVrUq1ePmBjnxM1t27b5fa6ePXt6nXZfsGCB0//jxo1j3LhxAY33UiAwt9FXzMuxtlEpY+RXwF7tNlassKegguNIUi3ngP2AAQNCOgCBbyJ0tphbKGNeYBMwvVbyWM9e60G8hNsoCAT7bGN5i9ekSZNCOgCBb+wxrxDMNqpdPmXWUB2wV+uSehG4cBsFwaJUlSh3t1Fh69at7N27F4CWLVsGvCi7UnBwLcSlQI3yrV3mcBt9T5T4tLwszvldkXqtx4C9c4a9Y7uwvAT+YrZYyTeGvpYXBCFeZ86c4d5772Xt2rX29YVZWVn06tWLxYsXU6NGjZAOsNzY+wN8/RAk1oVHfoXoquU2FPvyIH9mG/3M8wJH0F69rlGdYa9eFK7WK6FdAn9R1jVC6BdmBzzb+NRTT5Gbm8vu3bs5f/4858+fZ9euXeTk5PD000+HdHDlSt2ukFAbLhyCrx8Es/sifWVBQEmqvjLsVcpj8lCYUMFT7aVAGuAKLm+UYH2MQRuy8ukKAZ9txYoVzJo1i+bNm9u3tWjRgg8++ICffvoppIMrV2Kqw6CvwBALh3+H5WPLzeQIaLYxGMvLg9voqfChcBsF/hKupUEQhHhZrdYSi7HBlsBa0ZYOlZqkFnDXfECCbZ/C5jnlMoyIAJJUi3wlqVrciZf7gL06VUK4jZUbWZZ54budfLTuYJleN1yLsiHIkjjPPPMMJ0+etG87ceIEo0ePvjSTRZv0hRum2O7//AL8u7LMh2CfbfThEoIfSapu3EaTpyRVleWldhWF21j5+PdMHos2H2XaL/vKtKRRuBJUIQjxev/998nJyaF+/fpcccUVXHHFFTRo0ICcnBxmzpwZ8gFWCLqOgPaDbdmZ/x0KZ/aW6eUN2uI8rxAsD3LvNro/rzpgr15OJNzGyofStLjIbPWrnHioCKflFbAcpqamsm3bNlatWkV6ejoAzZs3p3fv3iEfXIVBkqD/O3D+IBzZAIsGwiNrIKZamVw+HAuzwTnbXkEtS1rV8iCTWS1ePochqGBkqJoWn883EhMRekvIHUrMK9QzjRBknpckSfTp04c+ffqEejwVF50B7vkc5l0HFw7D14Nh8BLb9jATqpI4siw7i5ePgL065qUWOFfLKz0jh0c+28KYPk24vX0dn2MUlD0nVeJ1Lt9IatXoMrmuI0G1AlheAKtXr2b16tWcOXOmRJB+/vz5IRlYhSSmmm0G8uM+Ngvsx9Fw6/vOGZxhIFSWl6vF5LC8nEtCK6iflnr9o6t4PbFwG8fOFzD6qx1CvCooGaq+n556c4aDcFWUgCBiXpMnT+aGG25g9erVnD17lgsXLjjdLnlqNoO7PgFJA9sXwqYPwn7JgJJUvcS8XJtsKJn1njLs1T6kU9lol2EcPX/R57gE5cspF8urrAhnwD7gM86ZM4cFCxYwePDgkA+m0tC4N/R9HVaMh19egmqNoGm/sF0uQh+akjiuouMrYK/Gm+UlZh8rPuqYV9laXhUoVcJoNNKtW7eQD6TS0eVx6DgEkOGbYXB6T9guFUgN+4AsLw8B+72ncjBZrE7Be5OXmJeg4pNRbpZXBUpSHT58OIsWLQr5QCodkgQ3TYP614AxD74cCHmZYbmUvzEvWZa9Wl6uFpI7y+vzTUe48d3fefKLbS77+p5trBFXfh2WBJ7JLTSRW+RYY3g+vwhZlslTbQsXFSpVorCwkLlz57Jq1SratGlTItv+nXfeCdngKjxaPdzzGcy73pZG8dUD8ND3oAvtl9jfkji+Hnddw2hxY3l9t/0EACv3nGZAu1qOYz1UXlUH+GsK8aqQqK0ugPP5Jt76eR9z1x3k68e70qFulbBdW4l5pWRtgbVp0OP5kE1wBSxef//9N+3atQNg165dTo9Jl2NX5eiqthnIeb3h2B/wwzMwYHZIZyANfgbs/W17puAzYK/e18OMpPrXW1heFZNTJcSriOU7czFbZXafyA6veBWaSZVOU3/1K1B4AaKqQJfHQnLugMVrzZo1IbnwJUWNJnDPAlh4F+z4Emo0g6tHhez0/rqNBQFk19v+L7a8PIiiU8zLQ5Lqmdwi+/3ocu6yJHCPYnnFRejILTJz8Gw+WRdtFlF+GLPtzRYrUlEOHxumoS28ALU6QIcHQ3b+0NaouJy54jq48T+2+6tehvTlITu1v+LlK4bhGvOyW15+zBaaPaxtPJNT5G53QQXiZHGOV/Na8QB24QLCulQor6CQmfqZNNGcQI6rBfcuAn1UyM4vxCuUdH4EOg3DNgM5HDJ2+TzEH+w17H24jYGKlxLHsnioBqKeVFS7lmq38Uyus0siqHgolleLlPgSj10MY9Beu2oiPbU7KJANSIMWQXxo+6sK8Qo1N/4HGvQAUz58eS/knSn1KSP8rCqRV+j9g+jZbfQj5uVhtjEzV1heFR0l5tU8JQ7X+pIXfYQagmbLJ8SlfQTAK/qnoVboy8QL8Qo1Wj3c86ktcTX7GCy+H0yls07sbqNPy8vk9XGPbqMH8VJvVcfFLE6WlxCvio5iedVKjKJKtPNa3LBYXofW2Yp3AtNMd7Ml+trQX4MgxGvdunWYzSWfsNlsZt26dSEZVKUnqoptBjIyAY7/CT88XaoKfspso8Uqe81mz/VgeSlunuuxFjdrGz1hVC8PUotXjnAbKzpKzCslIZIqMc7iFfKA/bkD8NVgsJo5mXoz71sGhCVBFYIQr169erntlp2dnU2vXr1CMqhLguqNbDlgkhb+/grWB5//plhe4D1o7ynmpWhNwJaXSqScq626t7xE4n3FI6/IbP9RS06IomqxeCnuY0Eoxasgy1YuqjALandiU6uXAYn4MJTDgSDES+my7Mq5c+dKdM++7GnYE256y3Z/9Su2jkRB4K945XsSr+K/rhaWr4C9075Os42O7cJtrNjY0yQidcRG6EiKjwSgXWoiAPnGELmNFjP8byic+xfi68C9i7hgtE00hcvy8lsS77jjDsCWiDpkyBAiIhwJiRaLhb///luseXTHlcMgcx/8+SF8+yg8vAJS2gZ0Cp1GQiPZAuVFFgvg/sOQ69HykgGpxJpEXwF7pzwvDwuzswu8x9kE7rFYZdbuO0Pb1ESqx4YvuVcRr5QEm2iN7NWIWgmRtE1N5MkvtoXO8vr5BTjwK+ijYdCXEJdETuE/QHiWBkEA4pWQkADYvghxcXFERTnyNQwGA1dddRWPPPJI6Ed4KdD3ddsv0oFf4ctBtiqscUl+Hy5JEgadhkKT1euMo6fZRrvlZfHgNvqT5+UhVaIs66FfSnyx+QgTl+6mRlwEf70YvirESrwrOcH2fW2aHMeEm5qz7aitfFVILK+/5tl+nAHumAspbYDwlsOBAMTrk08+AaB+/fqMHTtWuIiBoNXZaoDN620TscX3wZBlASXsGbQ28fI24+gp5mX1ELBX4lgeS+I45XmpZhsvsSZR5cHKPaeB8Kea2C2vYndRQVkNcbGolJbXgTWwfJzt/vUTofkt9oeURdlxYbK8Ao55TZo0SQhXMEQlwn1fQWQinNgCS0cGFOE2KImq3gL2HmcbbX8tJdzGQNY2up9tFIZXxUbJ8UpJdBavGIPNbilVhv3Z/fDfh0C2QJuBcPUYp4ft5XAqinidPn2awYMHU6tWLXQ6HVqt1ukm8EK1K2Dg56DRwa7/wbppfh8a4ccSIU8xLwXXJFXFmvKYYa8yvcweYl5Cu4JDU0ZFDDJUaRJqFMurwGQpsWDfLwouwKJ7oDAb6nSGW94rUYzAUb++nN1GhSFDhnD06FH+7//+j5SUlMuzkkRpaHAt9H/bVn1izRSo3hhaDvB5mGui6t5TORjNVtoWzxqBw/KKMWid8nfslpeLhaW4ka6i5g6Tmzwv12C9sML8p6y+NorlpcS8FKINjq9+gckSWDchiwm+fgjOH4CEVLj3C9BHltgtnPXrIQjxWr9+Pb///ru9LI4gCDoOsc1A/jELvnscqtTzuXzCXhbHbMVqlRn00R8UmiykTbyBSL3tV1SJecVE6JzFq9g+cnUbHXlevtc2qtMsrFZYvfc0wz7d4seTFbijrCyvUy6zjQqReg2SZHuP843mwMTrp+fh0G+gj4FBiyG2ptvdHAH7CuI2pqamihmmUNDnVWjUB8wFthnInFNed1fXsS80W8i6aKLQZHXK7VLEK9blg+gpSTX4tY0yry8v28a7lxplIV0XjWa7dewqXpIkEV38oxdQusSfH8GWjwEJ7pwHya087uqoolpBklRnzJjB+PHjOXz4cBiGcxmh1cFdH9tqf+WegsWDwOi5C4+6jr36w6bWI7t4uXxYHEmq7qtKeEqV8FRVwuohUVngP2Xx+ikzjbEROrczftHFP3L5/s447l9ts7oAer8MzW7yuKvV6igzXWEsr4EDB7J27VquuOIK4uLiqFq1qtNNEACRCTazO6oqnNwOS5/0GDhSl4IuNJdcqlNkttiD+SUtL/eZ9CZfqRIq/KlhL/CfstB+R7yrZDwKVOkS/uR6Zf4D/x1qm1lsex90f8br7rlFZvtHORzdsiGImNeMGTPCMIzLmKoNYOBC+Ow22P2dzRLrOb7EbuqChOoOQYqQqH891cFYcFherhqlWGKeFns7zTa6pEoIu6t0lMXr5ynepRDtb7rExfO2mcWibEi9Cm6Z4VN9lXhXpF5jr0cXagIWr4ceeihkF1+3bh1vvfUWW7du5dSpU3z33XcMGDDA6zFr165lzJgx7N69m9TUVF566SWGDBkSsjGVC/W7w83vwPdPwdqpthnIVnc67aKuY692G5UgvDLTGG3QonWxpx0xL/eWl6eA/c+7T6v2VYmXVS6z2bJLlbJ4/ZQ0ieR49+IV44/lZTbC1w/ChUOQWNc2s+hHg5lwdg1SCKqe14EDB3jppZcYNGgQZ87Yiu399NNP7N69O6Dz5Ofn07ZtWz74wL+u04cOHaJ///706tWLtLQ0Ro0axfDhw/n5558Dfg4Vjg4PQteRtvtLnoQTW50eVlte6v6NSo5ObnEtL1eXEbCbXiW7BwWSKuHsNkpubAdZZH35TVnMNjoSVN2v5Iiyi5cHy0uWbXW5Dv8OhlhbmaeY6n5dW0lQDZfLCEGI12+//Ubr1q3ZvHkz3377LXl5eQDs2LGDSZMmBXSuG2+8kSlTpnD77bf7tf+cOXNo0KABb7/9Ns2bN2fkyJHcddddTJ8+PdCnUTHp8wo07gvmQvjyPsg+YX8oQpVhX2BUx7xsfxXLyzVYDw5RcU1GdPRt9Gdto3p5kH8ilZlbxKo9p4NLgrzEKcuYlye3Ucmy91jTa/Mc2PYpIMFd8yGphd/XdiSoViDLa/z48UyZMoWVK1diMDgKm1133XX88ccfIR2cK5s2baJ3b+dFrH379mXTpk0ejykqKiInJ8fpVmHRaG3TzzVbQF5G8QxkPuA55mV3G4tnduIidCVi/lYXyyuyOO1CaWfmyW1UU3K20ffTuf7ttQz/bAtfbznme+fLjLKYbfQ7YO9uZca/q2yVIgBumAJN+gZ07XAnqEIQ4rVz5063llLNmjU5e/ZsSAbliYyMDJKSnKsxJCUlkZOTQ0FBgdtjpk6dSkJCgv2Wmpoa1jGWmsh42wxkdHU4tQOWPAFWq6OOfXGel4LVRbzcJRu6VlJVrDhHPS8/LC9rSWvPF8oH+Nf00tfxv9Qoi5Chp6VBCtERHtzGM+m22lyyFdo/AF1HBHztcCeoQhDilZiYyKlTJRMqt2/fTu3atUMyqFAyYcIEsrOz7bdjxyqBFVClnm0GUqOHPUth7VSn5UEFTkt/imNehe4TVEE926iIV7Hl5SNgr0Zteblm6gsCJ9wxr0KThQvFLc5S4t3HvByzjSrLK/8cfDkQinKgXnfoPz0oHzfcCaoQhHjde++9PP/882RkZCBJElarlQ0bNjB27FgefDB0DSXdkZyczOnTp522nT59mvj4eKf6YmoiIiKIj493ulUK6nWFW9613V/3Jm3O/wIoGfYlc648JaiCw1JyuI1ap//9CtibnTPsA3F7hNSVJNxeo+IyRhu0HhdGR7sG7M1G+HowXDgMifXgns9BZ3B7rC/sFSUqkuX1+uuv06xZM1JTU8nLy6NFixZce+21dOvWjZdeeikcY7TTtWtXVq9e7bRt5cqVdO3aNazXLTfa329PBux3YArtpP02t1GdKlEsPPnqmJfLaexrG11iXo56Xv6UxPHtNgqDzH/CbXmdshchjPT4Q+NUFkeW4cfRcGQDRMTDfV9DTLWgr18hUyUMBgMfffQRBw4cYNmyZSxcuJD09HQ+//zzgEvi5OXlkZaWRlpaGmBLhUhLS+Po0aOAzeVTW3OPP/44Bw8eZNy4caSnpzNr1iy+/vprRo8eHejTqDxcPwma3oRONvKR4W2iC065JKm6uI3uzHSXtY1KzMtRSdWfDHuV22gVSaqlJdyvn2v5Z3dEqfO8Nn0A2xeCpLHNLNZsVqrrh7uKKgSRpKpQt25d6tatW6qLb9myxanj0JgxtmJmDz30EAsWLODUqVN2IQNo0KABP/74I6NHj+bdd9+lTp06zJs3j759A5sJqVRotHDHXM7PvI4aef8w9Oh4/ldlnv1h2dVtjCj5S+ca81IsL4tVxmqV/bKYXDPsBaUj3LON9plGD/EugJjigH3j7I3wy0Tbxr6vQ+M+pb5+WVhefonXmDFjePXVV4mJibELjCfeecf/Fl89e/b0WqFiwYIFbo/Zvn2739e4JIiIY12n9+i+5h7qFB2g3z+TeJdHkdE4ZhtVlpfrS+o55mX1y+pyxSqXXT2qS5Xwx7xsbmOtRC+Wl15HE+kYT557HZBtpZq6PB6S65dFkqpfZ96+fTsmk8l+3xOi0kD4MMXW4THjGL6KeI1mWesYq0vkLfO9dmtKneflij1JVXadbZT9ine5IpYHlZ6ychs95XgBJMpZfKyfRrRcAPWvgZumhUxVc8ogSdUv8VqzZo3b+4Kyw6DTsE1uwtwqoxlx4U1G6L7nX2sdrLKt3Vyuh1peoLK8LK55XtbgxEuW3S4P+mXPaTd7i0C+O8pqttFjzMtcRMv1I4jTZHJcSqbOPZ+BNnRCk1sRk1QF5YNiLa0x9GJFlfsA+I9+LtGnbWsg84p/6dwvD7KhLMyOsGfYy0G7jZ44lyea0PqDO/EPJRneYl6yDMtGE3d6CzlyNM9oJkB06MpZWa1y2OvXg5+Wl9Jw1h++/fbboAcj8Iw6SfW/8UOQzv5DX+0WGv36KDRZ61JF1VldlLhiiZhXaSwvD989f3LGBKAJo9lQZLZwLt8IeLC8Nr4HaV8gS1pGGJ9mr5wc0uvnG832H7hyt7zUy2vi4+NZvXo1W7Y46pdv3bqV1atX2xvTCkKPQetYmH3RJDPa9CS7rfXQF56DRfdiLcwFvLuNrjEvq+xfdr0rVi+pEsJF9JfwWV7qXowlYk7py2GlrYBCXq8p/G5tw0VjkB2EPKAsCzPoNPYfynDgl+WlNJwFeP7557nnnnuYM2eOPa/LYrHw5JNPVp7s9UqIaw37i0Qy3DiW3xInYzizm9flmTzGaO8Z9hZnywtwyhnzF6uMmG4sJZowvnzKD5JGAq36Qhm74JvhgAydhqHt8ggst63cKDRbShSxDBZ7jlcYrS4IIuY1f/58xo4d65SQqtVqGTNmDPPnzw/p4AQOHDXsHWsbT1GNPT0+RNZG0Ee7lXG6r4h1U1XCNcNesbwACk3BxLyEeVVawqn9RcVLuQyq95m8M/DlvWDKhwY94Mb/EKl3iJXfdez9oCwSVCEI8TKbzaSnp5fYnp6ejjWI4K/AP9Q17ItU6wyzq7Ul64YZADyu+4GIXV+WONa1Y7a6LG9BUJaXZ/HyVZCw0GRh2d8nyb5o8rrfpU44lwcplpdeKalrKoSvHoDsY1D1CrjnU9Dq0WgkR/PZ0nTOdqEsyuFAEBn2Q4cOZdiwYRw4cIDOnTsDsHnzZt544w2GDh0a8gEKbDjqeVkoNDk++FZZ5mz9W/jM/AvP6L5D+mEUjWq+yWocpYMUOXGt5wXBi1ewX73Xl+/ls01HaFsngaUjrw7yLJWfcDrdyrIvg1Zj++X64Rk4ttnW8OW+ryCqin3faIOWi0YL+f404fCTsiiHA0GI17Rp00hOTubtt9+2l8ZJSUnhueee49lnnw35AAU21DXsNSbnkji5RWZmmO+kTUQGvSybGHnmZX6UXuG4XMO+Dzg6Zus0ElqNhMUqB/WL6y3G794oc2xcst1WHXbH8eyAr3spEc6EbqPabVw/Hf5eDJIW7v7U1h9BhS3OZfTdhCMAlATVcGbXQxBuo0ajYdy4cZw4cYKsrCyysrI4ceIE48aNC3hhtsB/Ijx1D7LalgbJaHg37llIaUucNZt5+mnEYusDac/zKlYWrUaDrjiQq66H7y+yl1QJEQ3zj3DGvIzFvy7XyX/C6sm2jTe9CVf0KrFvQO3P/KQsElShlEmqlao+ViVHiVNZZecgu0V2NPc0RMbCvV+Spa1GM80x3tV/gAZriY7ZOo1kj4cEY3kt/usY249mBfU8xBIyG+GOebWUDvOSsbi3Q+dH4crhbvdVxKsyBuyDOvv//vc/vv76a44ePYrRaHR6bNu2bSEZmMAZp5kjFbIs2xdlx0RoIaE279WczLiTo7leu53x8pdAT8AR89JqJHRa25cnmJhXaRDaZSOcL4Oce5qPDNOIogiuuA76TvW4r1I2vMAUwphXGVSUgCAsr/fee4+hQ4eSlJTE9u3b6dy5M9WqVePgwYPceOON4RijAM/iZZVV6xqLPyyHDE151vQEAI/qfmTxh6/Zy99AsXgVp3iHWry8VQkROAibiJsKafHb49SSznNcWwfu+gS0nm2UKH04LK/wV1GFIMRr1qxZzJ07l5kzZ2IwGBg3bhwrV67k6aefJjv78g7ChhNtcZDdFass26uoqrPrf7RexXSTrXHtOPNcjmz9xd5EwyZetnMFk+dVGoThZSNY93nrkQuMWrydMzmFJR+UZfh+JAnnd5Alx/BG4mSISvR6PrvlFYaAfTjr10MQ4nX06FG6dbNVMoiKiiI317YsZfDgwXz5ZckcI0HoMLi2wsYWx7KXwyn+sHRpaCvf+67lDn6wXIVBslB31WNUM54EbDEvxW0sCrnlFdLTXbIEa3kt/OMIS9JO8v2OkyUf/H0a7PwvVknHE6ZRnI+s4/N8SjXVkKZKlEE5HAhCvJKTkzl//jxgq6aq9Go8dOiQcBnCjDvXUZZLdg4adnUDpt7RGpAYa3qcNGtDdEUXeP7Cy8RxEY06YF/mMS9he0HwVSWUmcQLF51jzexZCr9OAWB7qxfZZG3pSFL1QoyvrtlBYHcbK1rM67rrruP7778HbAmro0ePpk+fPgwcONDvzteC4HAnXlbVbKMiXnqthkGdbSW6izDwqPFZiqKSqWs5ykz9TPRY7G5jKN0FfxDSZUMdAQjoR7941yz1CoWTafDtY7b7XZ7gn9S7APwSL7ftz0qJYnklVLTZxrlz59qXAY0YMYJq1aqxceNGbr31Vh577LGQD1DgIMKNeFmsstdaXgBnqMK27h/QftV99NTu4Njfb6LT2j7gZWF5CYO8JGoD1CqD1k9VV5Zf2cUr5xR8OQjMBdCoN9wwBePm44D7z4srjq7ZofkcyLJsT5WIq0iWl9lsZsqUKWRkZNi33Xvvvbz33ns89dRTGAzB9XgT+Icnt9FbCWiFjJhmTIu2dVlK3beAm00rgOCqSnjDl1AJr9GGOs8rEMtL2TWrwAjGi7B4EOSehOpNbV1/tDrV2kbfL3Z0hKr9WQjIN1rKpJYXBCheOp2ON998E7M5dCamwH/cBeytsuy97Vkx+UUWftN1Y5rpbgAey5tNV83uMp9tFNhQy0ogpbTs4pVvhKUj4OR2iKoK9y22rV3EERfzy23UhzZgr1RQ1WslpzW04SDgs19//fX89ttv4RiLwAfu3ACrTImYl0L12Aj7/YtGMxarzPuWAZxrcCs6LMzWzyCx4CihxFdVCRH1sqGeuAikxJDy+t6e+wXs/hY0ehi4EKo2tO+jrG3U++E2Ku3PQhX7VAfrwz05E3DM68Ybb2T8+PHs3LmTjh07EhMT4/T4rbfeGrLBCZxx5zZa3ATsFb58pAvDPt3C0fMXyS+yFGfYSxy5+j+cP/4PjU3pPH/hZdYxiRxiSpw7GAKJb32+6TCDu9YPyXUrG5JTwN7/42QZ+mv+YLh5sW3Dze9A/e5O+yhuoztL3RUlYJ8fKvEqozQJCEK8nnzyScB9f0ZJkrBYynb26nLC7WyjVZWk6uI2Nk6K48ZWyXy47iAFJot9baPGEM17NScz/viTpFqP877+PYaaxmEh/Avr1V/a/1u6+7IVL02Qlle9wnSe1c8GwNJlBNoOD5bYx14SJ5CAfYjcRkcV1fDONEIQbqPVavV4E8IVXtRFBBUKTRb7h9Vd/Xr7L2uR2Wlh9kV9NR4xPksBEVyr3clLuoVhHLkDV0di3P922DvdXE44x7wc4rXrRDY3vfs7a/adKXlQzklGnJ5IpGTiV0s7znV70e257W6jPwF7Q2gD9mVpeYnWZ5UId26A8mEBiHFTg1yJaVw0WkoszN4j1+dF6SkAhup+5gHtylKPMdCsiK+3HOepLy/9xfzfbT9OvxnrOHIuHyiZKqGwau9p9pzK4Zutx51PYLwIX95LouUc+6x1eNo0kuxC95MtRrvb6NuSdqRKhMryKpsEVQjAbSwoKGD16tXcfPPNAEyYMIGiIkePPq1Wy6uvvkpkpOcOvYLS4c4NyC4202MjdGjcrH1Uln9k5hbZf+Ft4mU710/mK0m2DmSc/ite1n3KITmZDdbWIR23WtDcxXCV8jp7TubwwZr9jLmhCVfUiA3pGMqb0V/tAOClJbv4fFgXp2B21kUjLy3ZxR0datstoNPqtYtWKyx5HE7tIFeTwLCiseQRzQUPpbRN9oC9P6kSxeJlshTXaStdkL2syuFAAJbXp59+yocffmj///3332fjxo1s376d7du3s3DhQmbPnh2WQQpsuBMv5ZfOncsIDmts/f6znC/u5afVSOiVDHuThVmWW/nWcjU6ycos/bs0kE4FPcZglogpFuGAWRv4cecphnzyZ9DXr+go4qTWiOkr/+GHHScZ+slf9smXU2pXeu1U2/IfjZ73a77McbkmYBM9dxiDCNjLsvdF+kfPXfSr70BZlcOBAMTriy++4NFHH3XatmjRItasWcOaNWt46623+Prrr0M+QIEDt+LlI7tecQvU2MpAq88lMcE0nK3WxiRIF/lY/xbx5AU1Rk/StTTtBEvTTnhd06fEao6dLwjq2pUBRdzVr0OGyspS3LfTOYW2EkY7/wfr3rQ9eMu7/BPRyr5vVoEHy8uiKgPtgyhVGzxPuV5px7K49q01PPq5rVfrmZxCj6kVyo9puEtAQwDitX//flq3drgTkZGRaFRfgM6dO7Nnz57Qjk7ghNuYl8ptdEeMm+0aSSoRzC3CwGPGMRyXq9NQk8Es/bvoCE0cJK/IzDOL03hmcVpI19BdKuhU36O84mU6JotM9v5NsMQ2u0+3p6H9/U4/Dh4tL7NtL3+SVLUayS5gngTp1WW27/XmQ+c5kJlH9//86jFOmVtUAQP2WVlZTjGuzMxM6tevb//farU6PS4IPe6SVLMLvDc7cGt5aR0lcdScJYHhxrHkyxFcrd3NJN1npRyxDXXZHXXbtssZdTKv+r1QxD2Fc8R+9yBYiqDJjdD7ZdtxKvXK8hTzCiDDHlSloN38sJzPN7L1yAX7/2v3ZWKyyKxOP2MPQ6gpy4C93+JVp04ddu3a5fHxv//+mzp1fNcPEgSPWryUD5zSI8/dTCO4t7zUlVRdSZfr8oxpJFZZYrBuFYO1vwQ0RnchL3UQ+HIRL4tVdrtu1J1brX4v8ovMRFHIPMM09AWZULMl3PkRaLQljvfkNhrdNZ31QnSE57I4P7jUDduw/6xtHDKs+yezxP6OVIkK5DbedNNNTJw4kcLCkjk5BQUFTJ48mf79+4d0cAJnDG7EKzeImJfWjduoZpW1I/8x3wvAJN1nXKP5O+gxw+W5GHvABxto/8pKewKxK2qRV78X+YVGputn01JzhAJD8ZrFiDjVcY4DPbmNjgx7/1746OLO2e4qS3y7zTllY/PBc/b7v6aXzEVzJKlWIMvrhRde4Pz58zRt2pS33nqLpUuXsnTpUt58802aNm3KhQsXeOGFF8I51sseZ/GyfeCUGSJfs41qdBqNPVXCEx9abuZ/lmvRSVY+0L/HFdIJP0dZ0rYIZ6ecisrOE9kUmCxsUblcgNuuSyeyHBMUDxQspJ/2L4pkHd80/g8k1vV4jZC5jRHus+wPZOax43g2Wo1EUrxtnax6GdFv/2Ridmniae+WXZGWByUlJbFx40aeeOIJxo8f75g1kST69OnDrFmzSEpK8nEWQWlQB+xdLSqPMa8IN5aX1lHD3jMSL5iGUU/K4ErNP3ysn8YA4ytkEef1KHduo89LUXEbd+w/k8up7EKuaVwjqOONfrjJfxc34B2gWc8QyzcAjDc9gmRtwgMu+/oT8yoK1G30UE31u222H6weTWpgslg5nWOLaRu0GqIMWrILTKQdy6JT/arFY5MrpuUF0KBBA1asWEFmZiZ//PEHf/zxB5mZmaxYsYKGDRv6PoGgVBhUy4NcxcuT5eVuhlIreY55qTGiL56BrEF9zWlm699F72MGUgZ2Hs92+hX3p+SxssSpotH7nXUM/vhP9pzMCep4k4f24q5i3UH6h//oPwLgA/OtfGe9xjnXSzkO/91G/wP2yuJsx3tmtcp8V9zd/I4OtZ0qlDROiqVHE5uYq13HApNjFUeFinmpqVq1Kp07d6Zz585UrVo11GMSeED9S+oaiPcU83KXMa3u2+iL88QzWjuBPDmSrto9TNZ9grdFQD/+fYpb3l/PHbM2qgbh+zpKZ6OKyp5T/ouXVbXexx/LqzaZfGh4hwjJxM+WTkwz3wM4538pOFleHvO8/E+VAEcde3WqxJ+Hz3Miq4C4CB29mydRLcZRaLRZcjy9mtnEa80+R9BemWnUqdIvwkmFWNv4wQcfUL9+fSIjI+nSpQt//uk5w3rBggVIkuR0u1yWJLkL2Ct4srzcYeuY7X8c6ri+Pk+ZnsIqS9ynW8NQ7QqP+3673RbgTc/ItW/T+hHzqqiWl4IlAHE1+yFeighFU8g8w9vUkHLYY63HaNOTyMVfy1PZBSUsNPW/F40Wiswlg+zKNf0pAw0QZV+87ziX4jLe1DqFSL2WqrEO8WqeEkePJjWRJNh7KodT2baYnTLTGBepK5NGK+UuXl999RVjxoxh0qRJbNu2jbZt29K3b1/OnHGzqr6Y+Ph4Tp06Zb8dOXKkDEdcfkS4Cdgr+JvRLEmg0UhuK1R4u+4aa3teM98HwEu6hfTUpLk/vxszKzHad/zDk3tVUTAHUO7Uotq3yMvzkrDyrv4DmmuOkiknMNz4LBeJtP+wFJqs9jw+T7hbshOo22jvIFTcNbvQZGH5TtsSsTs61AageozDbWyeEk/VGAPtUhMBWJNus74c6xrDH++CCiBe77zzDo888ghDhw6lRYsWzJkzh+joaObPn+/xGEmSSE5Ott8ul4kC75aXfx8YxQry91fZtq/tWh9bbmKxuSdaSeY9/UwaScdL7OsuOO+PeJkruOVlDUC81C6wyY3l9fmmw+zNyGGc7iv6aLdSJOt51DiGk1QHICFKb3fTXONerpVq3bmOxgBq2EPJJhwr95wmt8hM7cQoriwOxleLVbuNtkmb65ra1lgq5XuUcuRlEayHchYvo9HI1q1b6d27t32bRqOhd+/ebNq0yeNxeXl51KtXj9TUVG677TZ2797tcd+ioiJycnKcbpWVCC+zjf66jUrXbX9nogAi7LXIJf7P/DCbrc2Ilwr4WD+NKji/nu6SUP35Ja7olpc7t9Y1tcBqlTl0Nt9JiI1untf/Ld1N1O6veUL3AwDPmR5lu9zY/nhMhI6keFsoxLXWmeuk7AU3We6BJ6k6B+yV3K7b29e2VypJSYgq/htJteLgfa9mNvHasP8sRWZLmSaoQjmL19mzZ7FYLCUsp6SkJKcORWqaNm3K/PnzWbp0KQsXLsRqtdKtWzeOHy9pBQBMnTqVhIQE+y01NTXkz6OsUD6MkgSR+uDES0mRCNRtVDCh43HjKI5aa1BPc4Y5hhlOM5DuZsj8qW4QiFtWHlhcxjdx6S5aTPyZd1b+w6y1+5FlmQnf7qTXtLXM33DIvp+7mFdHaR+v6+cB8J55AN9bncs4Rxt0pCTYxKuk5eWMO8srkDLQtus5AvaZuUWs+9eWRX97scsItjjX5FtbMmNgO/u2lrXiqRkXwUWjhc0Hz5dpmgRUALcxULp27cqDDz5Iu3bt6NGjB99++y01atRwKtejZsKECWRnZ9tvx44dK+MRhw5FvKL02hKJn946B6nRBGN5uQjdBeJ52PQcOXIUXTTpTNHNx9sMpD8pXIfP5vs9nvLAVVw/22SLs763+l/eXLGP3/7J5Kstts/WzF/32/dz7YtZR8rkQ8N0IiQzyy2dmW6+q8S1Ygxaku3iVcDWIxcc6whdXkvXmJfFKtuLGwaeKmHhhx0nsVhl2qYmOtVUkySJh7rVp0vDak7beqlcx5zLyW2sXr06Wq2W06dPO20/ffo0ycnJfp1Dr9fTvn179u/f7/bxiIgI4uPjnW6VFeVDkRClLyFeMW6SURWGXd3Aft9hefn/1rsTuv1yHZ42PYVFlhioW8sw7XKPx/tTo/2VZRW7Iomv2UZ3Fic4px/EUMA8/TSqSznsstbnWdPj9plFNTEROupViwZgwYbD3Dl7I73fsXXsUmJeSirCBZdcL7Wl5+8PlCNVwmyfLb6jfW1vh9ixp0yknynTQoRQzuJlMBjo2LEjq1evtm+zWq2sXr2arl27+nUOi8XCzp07SUlJCdcwKwz1q8fwfze34PU7WjsFxg06jVc3UJ1gqNTxCixg737ftdZ2TDHbcsBf1C3iOo37Mimz1h7weY2KXirHl1u78cA5t9sV8dJg5V39+zTTHOOMnMhw47MU4D7FJyZCS6tath6MucVrIxXLS/kdqFoc0Hd1G9UxNn8tL6Xa7r6MXHadyEGnkbilbS2/ju3eqDp6rcThcxdJO5YFXCaWF8CYMWP46KOP+PTTT9m7dy9PPPEE+fn5DB06FIAHH3yQCRMm2Pd/5ZVX+OWXXzh48CDbtm3jgQce4MiRIwwfPry8nkKZMuzqBvRqWtOp5LO3TtkA6s+wcj8wt9Hzvp9Y+rHIfB0aSeY9/fs0kYJzy/1J5ixPLFYZWZaZunwv/3OtLw9k5rq3vJT6Vs/rvqS3djuFsp5HjGPIoJrb/cG2HrVlsXi5okioMoPrukTI5CRe/s02KgnPitvXs2lNuzj6Ii5Sb5+R3HzoPFB2qRJlY995YeDAgWRmZjJx4kQyMjJo164dK1assAfxjx496lT08MKFCzzyyCNkZGRQpUoVOnbsyMaNG2nRokV5PYVyQe02+op3qffV2S2vQAL23vaVmGgeQgMpg67aPXysn8Ztxlc5T2DuuetsXpHZEtAYw43ZKrPpwDk+XHfQ7eOeFp9fyDfBts95TPcjAGNNj7NDbuT1WjEROhKi9dSpEsXxC85VZZWk1SrRxZaXB7fRoNX4nSjqmg1/Rwf/XEaF65rVdLI8Lwu3UWHkyJEcOXKEoqIiNm/eTJcuXeyPrV27lgULFtj/nz59un3fjIwMfvzxR9q3b18Ooy5f1G6jr5lG9RerdKkS7jGj4wnTMxy2JpGqyWSOYToGfNc7V+OaUnDrzA0BHR8O1NntFqvscTkOOF5XV2rnbINlowGYYb6DZVbf4RAlftnKjfWljKiK4jZ6sLwCWUGhXmoWH6njuuIUCH/p2dR5/zg/cw5LS4UQL0HgOFlePt1GteUVeMDen0XcWcQxzDSWHDmazpp9vKb7mEAaobm6jftO53rYs+xQh7mMZqtbd9EbqdJpXsp7Hawmllmu4l3zHX4dp8z+tapd0npV9LSK4jYWeBCvAN5fdc5g/za1SqTh+OKKGjGkVo2y/3/ZZNgLgsMp5uXLbVTtq9z3ZU2p8XeZ2gG5NiNMT2OWNdytW8ej2mV+X6Miop4lXfjHEbfF9xRKpK5wkY/100gkF7lWe8aaHnM7s+gOZfavZW3PlleiB7exyBxYjhc4W16BuoxgS5lIiVeL12XkNgoCJxC3USu5sbz8aEgaDL9b2/CKeTAA43WL6a3ZGpbrlAVq8fI126jWLg1WZupn0kRzggy5CoV3fk4hEZ4PdkERk5a13MQNi8dU1WPAPrCKEmD7/DzcvQH3dalLp3pV/D5Ojdo4v2xmGwXBof6ld1enXo3zbGMQlldgQ+Mzyw18bu6NRpJt6QHS0QDPUDEIpD6i+v14QfcFvbQ7KJANDDc+y3lt9YCuq7yfNeNKplK4xrwKTBanWvmBtD1TM/GWFrx+e+ugq0GoVyAIt1HgFSfLK4DZRnvA3scv82M9SlNcUmKy+UHWW1oSIxUxzzCN6mSX4nzlgz/JtQpKs9h7tb8yXPcTAM+aHmeX3NDt+kNvePsxUoYUG6Gzv5fqyhPGINzGUKAupxPjpm9COBDiVUkJJM/LnXhpvNRm3jelHxNubF6q8ZnR8aTpGQ5YU6gjnbUV2yOwL3F5E8hyyz8PnecqzR5e1X0CwNumu1huvQrwXK7ZE96+/EqGvUaSSIgq6TraK0royrZvgLoKa1nU8gIhXpWWQGYb1fGIqtG+kw9d86uqxfofr1GTQyzDTWPJlqPpqPm3eDFyxV6ArSYQy6uelMFs/Qz0koXvLV2Zabnd/pjrEh5fqC2vd+9tBzh+oOxDktSJqo7zKyV4Aol5hYJ8N52Hwo0Qr0qKs9voPcaQGOUQrGdvaOp1X3UKxQf3deD29rUZ2r1+UGMEOCSn8KTpGcyyhju163lC+0PQ5yprZD+T/uPJt5UHkvJIs17Bc6bHUEcKAxYvVaHJtnUSAYeQKuIlAYnFltcFleWlBOzL2m0sj+VdQrwqKVIAltfVjavz6oBWrBx9LS1UM1juPEd1jk//NilMH9gu4LwfVzZYW/Oy+SEAntcvpq/mL7+OW5p2oly7CvljeWmx8L7+PRppTnJSrsojxjEU4WzdXsgP0G1ULbJX3HyLIl7F2yVJsqdLZBc4xNFosVlAgQbsS4u7hrXhRohXJUWd/uArz0uv1TD4qno0TnJuW+Yu+TTSwyykpwxyf1lo6cMC8w0ATNfPoqV02OcxzyxOY/Vez7lV4cYf8XpJt5BrtTu5KEfwiHEsmZRMNUg7dsHNkZ5Ru43K664UtbC3HMThNjpZXubAUyUqK5f+M7xEUetOIM03PJ1DwZOVFUhGvideNQ9mnaU10VIRHxmmUYMsn8f8fdz3PuHCV8D+fu0qhup+BmC06Ql2y/Xd7qfusOMLjeT8WrtaXgqS5AgHuA3YB7A8qLIixKuSEsjCbH/OoRDpYTF0KMTLgpaRpqc5YE2hlnSeuX7MQJZneN+by9pVs5vJugUAvGm6h5+tnUt1rWsa23LBYiKcO+8o75FS1cIR85LslpeT22gvAV1xFrWHCyFelZRAZhs94a4lmSe3MVQxlBxieNj0HFlyDO01+3lTPxdvEvXtthMlWsqXFa7WjkID6RSz9TPQSVa+s3RnluW2oK+RkhDJwmFdeL5fM6BksxK1u26VHakSkuRY3/jln8foNW0t17z5KzNW/QOUveWlZOb3b1N2dfWEeFVSQiFe7nK9PLuNofslPyIn84RpFCZZy23ajYzQLvW474msAu6ftzlk1w4Ed25jPHnM008jUcpnm7UR402P4GkNwrS72/q8xqjejbm6cXVa1opnVO/GTLq5pdPj6h8Ym/Vluy8BCaq0l0Nn8zl2vsBek6t5ctlWDP5wcEdev701b9zRusyuWe71vATBoeiOJJXsJBToOcA2tW60WOnRtIbbfUM9e7XJ2pKJ5iFM1X/Mc/qvOSDXYoUH10spclcWyLJsd9tc253pMDNL/y5XaE5xQq7Go8ZnS8wsqkmtEuXxMQWlLpckSYzq3aTE4+q4pFVWNT6THKkSAONvbEbnBlXRazRER2hpWD3G57VDSbXYCO7rUrdMrynEq5KifMFiI4LvTqx2SVY/24ONB85yR4c6bvd1jXm1rp3AzhOlW/LzpeV6GksneFi3gun6WRwz1mC33MD3gWFi+9EL3D5rI8/2aUJ0hI43ftrr9PhE3edcrd1NvhzBcONYzuK+2qlCzXjfndzV/RDdoX6PLFbZvnZRQrK7jQBdGlSlfd3gFlVXVoTbWElRPtS+lgZ5Q+16plaNZuCVdT1OsbtaXiN6XRH0ddW8Zr6ftZa2RElG5hnepiaBpRWEktFfpQHw9sp/eHXZHqfqroO1v/CgbiVWWeIZ00j2yvV8ni8lIdJnOaHkBO/Wmfo9+mLzEY6cu4hWI1G3WrR9eRA49ym4XBDiVUlRfpCDnWmEwHK3XC0vdSz77o51GH9js6DGYEHLU6an+NdamxTpPB8Z3iaSoqDOFS6u1uxkku4zAN40D2SVtaNfx0XqtcQaPL8/CVF6aid6Fy/1e/T68nQAxvVtSu3EKBJjhHgJKiENa8Si10q0rp0Y9Dk81V13h2vAXh0Nal+3Clc3Cqzsi5pcohlmGst5OZa2moNM03+I6wzk+G/+5vM/jgR9DX9wJ+ZXSCeYpX8XnWTlG8s1zLHcEtA5TV5apt3pwUV3GpPLe9SraQ0eucZW8SM+Us/HD3Xik6FX2jsAXU6ImFclpUH1GLa82Mdndr03/KjubMdbwL593cQSzVUD5aicxOPG0Sw0vM7N2j/411qbdy132h9f/Ncx+OsYg6/y7a4Fy4FM58a3CcUzi/HSRf6yNmGCaTiBVjcrNLkXr7apiYzq09jn8a4zwi/c1Nxp2/XNk1wPuWwQllclJiFa77W0jS/c5Xl5wlW89FoND1xVl0Gd69IsOY7WtRMCbtzgyp9yc14yPwzAaP039Nf8UarzBYJrQqoOM7P1M2igOc1xuTqPG0djJHRF9paO6B5wxdE6VaJKLPG6nBHidRkTiPC5xrysssyUAa2Zeoet+qZeq2H+kCtLPaavLb2Ya+4PwNv62bSRvDes/ft4FoM/3szuk6Wb+SxyagAi84puAd20e8iTIxlmHMs5HzOL/tAkKbZUxw/qXLapCBUdIV6XMYFYXiViXh6yz5c/fU2pxgTwhnkQqy3tiZRMfGR4myQ853kN+GADv/97lvs+Cj6R9VR2AW0m/2L/f4j2Z+7T/YpVlnjaNJJ9culFo3ZiFLV8BOc98WDXelzTuDrDri6/NJKKiBCvy5jSzDZqPQTMWrhrGhEgVjQ8YxrBPmsdkqQs5hmm2Wcgf00/7bxvsYZme+mp6IsFGw/b1wT20Ozg/3SfA/C6+T5+tXYI+rxqqscagq708Mptrfh8WJdSlya61BDidRkTiCXQU5V5371RNXp5yMQPFXnFM5Dn5Dhaaw7ztn42ElYeXrCl1C6iK0qN+SukE8zUv4dWkvna3IN5lptCdo12qYmXRaWHskSI12XMa7e3okeTGiwY6jtW1bNpTRYN78KfL1zPF8OvQhegFRHMF/e4XJPHjKMxylr6a/9klO4bAFbtOROyIoVZF418veU4ieQyX/8W8VIBm63NiicOSi82XwzvwsBOqYzr1+yyqLFVlohX8zImJSGKTx/uXKJduye6Naru15KXULJFbsYL5uEAPKP7jls1G5i+6h+ufWtNCQvshe92Bnz+dq+sRI+ZOYYZ1NOc4ai1Bk8YR4VsZrF7o+r85642xETohHiFGJHnJQg5/Voms2J3Bnd0qI3JIlO/WjRzfvM+a+iN/1l60Eg6yeO6H3hLP5ejxiTSzjei/3vrnfZbtPkoY/o08Zlt/vu/mXz117Hi0sUyr+rmc5VmL7lyFMNMz3Ge0sXt6lSJYmj3ksH1si7NfKkjxEsQct4Z2Ja7D9She6Pq9iDz7LXBixfYluVcIZ2kj3Yrcw3vcFvRq5yiWon9/PEmB3/8p/3+MO1P3Ktbi0WWbMuUZN9Z7954+vrGjOlTsjoEgL6UpbQFzoifAkHIiTbouL55ktfZsYXDugR0TisaRpmeZK81lZrFM5BRFJbYrzCATP+emu28oPsCgNfMD7DW2i6gMSlcUcNRfqZNbc/5YMJtDC3i1RSUCa45Slc3DnwtZD5RDDeOJVOOp6XmCNOLZyDVXPPmGq/nWL3XlmrRWDrOTP37aCWZReZezLf0C3g89nM+25M1Y3vywX0duL655/ihXriNIUW8moIy4bm+TUkOQbD/BDV4zDiGIllHP+1fPKv7r9/HGs1Whn26hark8LH+LeKkAv6wNmeSeSj+zizOGNjO7fYG1WPo3ybFa221/q1tJZLrVYv2e8wCzwjxEpQJOq2G70Z0o1lyHK/d3qpU59omNykuvwwjdUsZoFnv4wgb47/5GwMm5himU1eTyWFrEo8bR2HyM/Q7+Kp6NE0Ofm1hq9oJ/D6uFz+PujbocwgcCPESlBkpCVGsGHUt93ep57I9cIvsO+s1fGC+FYD/6OfSQfrH5zHfbj/OFN18Omv2kSNHMcw0liz8F6PODarSLDmOId3qBzxehdSq0SJTPkQI8RKUOze1TuH29rUBeOCquqRN7ENvL7EjhWnme/jZ0okIycyHhneoja0/omvtebCtxXxE+yP36H7DIku2Fmxybb/GN6BdLd6+uy03F7uFL9/aklpBCK4gtAjxEpQbDYtn6W5uk8LUO1qzaHgXJt7cksRoA/MeupLDb/T3eryMhtGmJ9ltrUcNKYd5hreJppA3f95Hkdkx65hfZGb4i68yQfclAK+YH2Sd1X1nnzvaOwta1RgD0+5uy50d6zjFszo3qAoE3/xEUHokOVTrLCoJOTk5JCQkkJ2dTXx82baHEjhTaLJwOqeQetU8d7rJKTRx30d/oJUkdhx3v6YxhXN8H/ESNaRsVlo68qhpNC/2b8nw4oqja9etpdPqgcRKhSw0X+916c+KUdfQb8bv9v89CWh2gYlPNx7m1ra1qF/GnXoudfz9jlYIy+uDDz6gfv36REZG0qVLF/7880+v+//3v/+lWbNmREZG0rp1a5YvX15GIxWEkki91qtwga3U8bKnrmHpyKs97nOKajxifJYiWU8f7VbG6b6yC13GyaM0Wj2MWKmQDZaWvGx+CE/Cteypq2mWHE+z4qC8t+KKCVF6nr6+sRCucqTcxeurr75izJgxTJo0iW3bttG2bVv69u3LmTNn3O6/ceNGBg0axLBhw9i+fTsDBgxgwIAB7Nq1q4xHLihrNo6/jldua+k2HpYmN+I506MAPKH7AcPOxaQfz+T4nDupI53loDWZJ03PYFbNLN7cJoU5D3RgzdieHH6jP62KE0y/fbIbs+/vwHuD2pfNExMERbm7jV26dOHKK6/k/fffB8BqtZKamspTTz3F+PHjS+w/cOBA8vPzWbZsmX3bVVddRbt27ZgzZ47P6wm38dIgp9BEgdGCVZZ54dudrNlnC9aP0X3N07olGGUtp+Rq1NOcIVuO5nbjKxyUawEwrl9TYiN0DL6qXtA9LwXho1K4jUajka1bt9K7d2/7No1GQ+/evdm0aZPbYzZt2uS0P0Dfvn097l9UVEROTo7TTVD5iY/UkxQfSUpCFJ8M7czHD3UCYLr5LpZbOmOQLNTTnMEiS4wwPWMXriUjuvNkz0Y82LW+EK5KTrkuzD579iwWi4WkJOcOKElJSaSnp7s9JiMjw+3+GRkZbvefOnUqkydPDs2ABRWW65sncfiN/uQWmnhuUQK1Dz9DW81BJpmHcLLaVeweeTUxpWjQK6h4XPLv5oQJExgzZoz9/5ycHFJTU8txRIJwEhepZ87D15J3cSOHTxxgSuPSZfMLKi7lKl7Vq1dHq9Vy+rRzXfLTp0+TnJzs9pjk5OSA9o+IiCAi4vLrJny5ExsdRawQrkuaco15GQwGOnbsyOrVq+3brFYrq1evpmvXrm6P6dq1q9P+ACtXrvS4v0AguDQpd7dxzJgxPPTQQ3Tq1InOnTszY8YM8vPzGTp0KAAPPvggtWvXZurUqQA888wz9OjRg7fffpv+/fuzePFitmzZwty5c8vzaQgEgjKm3MVr4MCBZGZmMnHiRDIyMmjXrh0rVqywB+WPHj2KRtVmq1u3bixatIiXXnqJF154gcaNG7NkyRJatRIugkBwOVHueV5ljcjzEggqNpUiz0sgEAiCRYiXQCColAjxEggElZJyD9iXNUqITywTEggqJsp301c4/rITr9zcXACRZS8QVHByc3NJSPDcSu6ym220Wq2cPHmSuLi4S3JhrrL86dixY5f1bKp4HRxUttdClmVyc3OpVauWU5qUK5ed5aXRaKhTp3RdkSsD8fHxleKDGm7E6+CgMr0W3iwuBRGwFwgElRIhXgKBoFIixOsSIyIigkmTJl32lTTE6+DgUn0tLruAvUAguDQQlpdAIKiUCPESCASVEiFeAoGgUiLESyAQVEqEeFVyzp8/z/333098fDyJiYkMGzaMvLw8r8fMnTuXnj17Eh8fjyRJZGVllc1gQ4zotO4gkNdi9+7d3HnnndSvb2v/NmPGjLIbaAgR4lXJuf/++9m9ezcrV65k2bJlrFu3jkcffdTrMRcvXqRfv3688MILZTTK0CM6rTsI9LW4ePEiDRs25I033vDYuKZSIAsqLXv27JEB+a+//rJv++mnn2RJkuQTJ074PH7NmjUyIF+4cCGMowwPnTt3lkeMGGH/32KxyLVq1ZKnTp3qdv977rlH7t+/v9O2Ll26yI899lhYx1kWBPpaqKlXr548ffr0MI4ufAjLqxKzadMmEhMT6dSpk31b79690Wg0bN68uRxHFl7KotN6ZSGY1+JSQYhXJSYjI4OaNWs6bdPpdFStWtVjB/FLAW+d1j0970A7rVcWgnktLhWEeFVAxo8fjyRJXm/p6enlPUyBoFy57EriVAaeffZZhgwZ4nWfhg0bkpycXCIoazabOX/+fOUOxPqgLDqtVxaCeS0uFYTlVQGpUaMGzZo183ozGAx07dqVrKwstm7daj/2119/xWq10qVLl3J8BuFFdFp3EMxrcclQ3jMGgtLRr18/uX379vLmzZvl9evXy40bN5YHDRpkf/z48eNy06ZN5c2bN9u3nTp1St6+fbv80UcfyYC8bt06efv27fK5c+fK4ykExeLFi+WIiAh5wYIF8p49e+RHH31UTkxMlDMyMmRZluXBgwfL48ePt++/YcMGWafTydOmTZP37t0rT5o0Sdbr9fLOnTvL6ymEjEBfi6KiInn79u3y9u3b5ZSUFHns2LHy9u3b5X///be8nkJQCPGq5Jw7d04eNGiQHBsbK8fHx8tDhw6Vc3Nz7Y8fOnRIBuQ1a9bYt02aNEkGStw++eSTsn8CpWDmzJly3bp1ZYPBIHfu3Fn+448/7I/16NFDfuihh5z2//rrr+UmTZrIBoNBbtmypfzjjz+W8YjDRyCvhfKZcL316NGj7AdeCkRJHIFAUCkRMS+BQFApEeIlEAgqJUK8BAJBpUSIl0AgqJQI8RIIBJUSIV4CgaBSIsRLIBBUSoR4CQSCSokQL0GFYMiQIW6rZ/Tr16+8hyaooIiqEoIKQ79+/fjkk0+ctnnq8mwymdDr9U7bjEYjBoMh4OsGe5ygfBGWl6DCEBERQXJystOtSpUqAEiSxOzZs7n11luJiYnhtdde4+WXX6Zdu3bMmzePBg0aEBkZCcDRo0e57bbbiI2NJT4+nnvuucepZIyn4wSVCyFegkrDyy+/zO23387OnTt5+OGHAdi/fz/ffPMN3377LWlpaVitVm677TbOnz/Pb7/9xsqVKzl48CADBw50OpfrcYLKh3AbBRWGZcuWERsb67TthRdesHc5uu+++xg6dKjT40ajkc8++4waNWoAthpdO3fu5NChQ6SmpgLw2Wef0bJlS/766y+uvPJKt8cJKh9CvAQVhl69ejF79mynbVWrVrXfVzcaUahXr56TAO3du5fU1FS7cAG0aNGCxMRE9u7daxcv1+MElQ8hXoIKQ0xMDI0aNfL6uD/b/L2WoHIjYl6CS4rmzZtz7Ngxjh07Zt+2Z88esrKyaNGiRTmOTBBqhOUlqDAUFRWVaNel0+moXr263+fo3bs3rVu35v7772fGjBmYzWaefPJJevTo4dbtFFRehOUlqDCsWLGClJQUp9vVV18d0DkkSWLp0qVUqVKFa6+9lt69e9OwYUO++uqrMI1aUF6IMtACgaBSIiwvgUBQKRHiJRAIKiVCvAQCQaVEiJdAIKiUCPESCASVEiFeAoGgUiLESyAQVEqEeAkEgkqJEC+BQFApEeIlEAgqJUK8BAJBpUSIl0AgqJT8P+lR5mH4mr+pAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.014658, + "end_time": "2024-03-26T07:31:08.761379", + "exception": false, + "start_time": "2024-03-26T07:31:08.746721", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 272.095697, + "end_time": "2024-03-26T07:31:11.497251", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/realtabformer/3/mlu-eval-load.ipynb", + "output_path": "eval/contraceptive/realtabformer/3/mlu-eval-load.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/contraceptive/realtabformer/3", + "path_prefix": "../../../../", + "random_seed": 3, + "single_model": "realtabformer" + }, + "start_time": "2024-03-26T07:26:39.401554", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/realtabformer/mlu-eval.ipynb b/contraceptive/realtabformer/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..f9958b2cff574cb9ed6e5c10e337dfb45d883ed3 --- /dev/null +++ b/contraceptive/realtabformer/mlu-eval.ipynb @@ -0,0 +1,2278 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:38:42.697393Z", + "iopub.status.busy": "2024-03-24T02:38:42.697005Z", + "iopub.status.idle": "2024-03-24T02:38:42.739151Z", + "shell.execute_reply": "2024-03-24T02:38:42.738116Z" + }, + "papermill": { + "duration": 0.059702, + "end_time": "2024-03-24T02:38:42.741766", + "exception": false, + "start_time": "2024-03-24T02:38:42.682064", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:38:42.772261Z", + "iopub.status.busy": "2024-03-24T02:38:42.771077Z", + "iopub.status.idle": "2024-03-24T02:38:42.780958Z", + "shell.execute_reply": "2024-03-24T02:38:42.779962Z" + }, + "papermill": { + "duration": 0.027878, + "end_time": "2024-03-24T02:38:42.783212", + "exception": false, + "start_time": "2024-03-24T02:38:42.755334", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:38:42.810448Z", + "iopub.status.busy": "2024-03-24T02:38:42.809731Z", + "iopub.status.idle": "2024-03-24T02:38:42.814145Z", + "shell.execute_reply": "2024-03-24T02:38:42.813233Z" + }, + "papermill": { + "duration": 0.020359, + "end_time": "2024-03-24T02:38:42.816339", + "exception": false, + "start_time": "2024-03-24T02:38:42.795980", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:38:42.842505Z", + "iopub.status.busy": "2024-03-24T02:38:42.841716Z", + "iopub.status.idle": "2024-03-24T02:38:42.846312Z", + "shell.execute_reply": "2024-03-24T02:38:42.845228Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.019981, + "end_time": "2024-03-24T02:38:42.848520", + "exception": false, + "start_time": "2024-03-24T02:38:42.828539", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:38:42.875273Z", + "iopub.status.busy": "2024-03-24T02:38:42.874920Z", + "iopub.status.idle": "2024-03-24T02:38:42.881580Z", + "shell.execute_reply": "2024-03-24T02:38:42.880631Z" + }, + "papermill": { + "duration": 0.022356, + "end_time": "2024-03-24T02:38:42.883691", + "exception": false, + "start_time": "2024-03-24T02:38:42.861335", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "52b90da1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:38:42.910031Z", + "iopub.status.busy": "2024-03-24T02:38:42.909716Z", + "iopub.status.idle": "2024-03-24T02:38:42.915242Z", + "shell.execute_reply": "2024-03-24T02:38:42.914247Z" + }, + "papermill": { + "duration": 0.02196, + "end_time": "2024-03-24T02:38:42.917465", + "exception": false, + "start_time": "2024-03-24T02:38:42.895505", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"realtabformer\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 3\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/realtabformer/3\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.012314, + "end_time": "2024-03-24T02:38:42.942034", + "exception": false, + "start_time": "2024-03-24T02:38:42.929720", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:38:42.969716Z", + "iopub.status.busy": "2024-03-24T02:38:42.968811Z", + "iopub.status.idle": "2024-03-24T02:38:42.979545Z", + "shell.execute_reply": "2024-03-24T02:38:42.978554Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.027554, + "end_time": "2024-03-24T02:38:42.981814", + "exception": false, + "start_time": "2024-03-24T02:38:42.954260", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/realtabformer/3\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:38:43.010808Z", + "iopub.status.busy": "2024-03-24T02:38:43.010428Z", + "iopub.status.idle": "2024-03-24T02:38:45.304295Z", + "shell.execute_reply": "2024-03-24T02:38:45.303185Z" + }, + "papermill": { + "duration": 2.311808, + "end_time": "2024-03-24T02:38:45.306872", + "exception": false, + "start_time": "2024-03-24T02:38:42.995064", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:38:45.336937Z", + "iopub.status.busy": "2024-03-24T02:38:45.335841Z", + "iopub.status.idle": "2024-03-24T02:38:45.351618Z", + "shell.execute_reply": "2024-03-24T02:38:45.350784Z" + }, + "papermill": { + "duration": 0.032502, + "end_time": "2024-03-24T02:38:45.354031", + "exception": false, + "start_time": "2024-03-24T02:38:45.321529", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:38:45.382841Z", + "iopub.status.busy": "2024-03-24T02:38:45.382459Z", + "iopub.status.idle": "2024-03-24T02:38:45.391119Z", + "shell.execute_reply": "2024-03-24T02:38:45.390230Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.025034, + "end_time": "2024-03-24T02:38:45.393217", + "exception": false, + "start_time": "2024-03-24T02:38:45.368183", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:38:45.424353Z", + "iopub.status.busy": "2024-03-24T02:38:45.423724Z", + "iopub.status.idle": "2024-03-24T02:38:45.530029Z", + "shell.execute_reply": "2024-03-24T02:38:45.529081Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.125359, + "end_time": "2024-03-24T02:38:45.532690", + "exception": false, + "start_time": "2024-03-24T02:38:45.407331", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:38:45.561868Z", + "iopub.status.busy": "2024-03-24T02:38:45.561057Z", + "iopub.status.idle": "2024-03-24T02:38:50.856524Z", + "shell.execute_reply": "2024-03-24T02:38:50.855232Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 5.313135, + "end_time": "2024-03-24T02:38:50.859430", + "exception": false, + "start_time": "2024-03-24T02:38:45.546295", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-24 02:38:48.116544: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-24 02:38:48.116610: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-24 02:38:48.118474: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:38:50.889598Z", + "iopub.status.busy": "2024-03-24T02:38:50.888895Z", + "iopub.status.idle": "2024-03-24T02:38:50.895658Z", + "shell.execute_reply": "2024-03-24T02:38:50.894813Z" + }, + "papermill": { + "duration": 0.024441, + "end_time": "2024-03-24T02:38:50.897998", + "exception": false, + "start_time": "2024-03-24T02:38:50.873557", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:38:50.925180Z", + "iopub.status.busy": "2024-03-24T02:38:50.924526Z", + "iopub.status.idle": "2024-03-24T02:39:00.369372Z", + "shell.execute_reply": "2024-03-24T02:39:00.368208Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 9.461598, + "end_time": "2024-03-24T02:39:00.372274", + "exception": false, + "start_time": "2024-03-24T02:38:50.910676", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-24T02:39:00.403750Z", + "iopub.status.busy": "2024-03-24T02:39:00.402960Z", + "iopub.status.idle": "2024-03-24T02:39:00.410518Z", + "shell.execute_reply": "2024-03-24T02:39:00.409593Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.025599, + "end_time": "2024-03-24T02:39:00.412753", + "exception": false, + "start_time": "2024-03-24T02:39:00.387154", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 46,\n", + " 'realtabformer': (24, 72, Embedding(72, 672), True),\n", + " 'lct_gan': 40,\n", + " 'tab_ddpm_concat': 10}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:39:00.440362Z", + "iopub.status.busy": "2024-03-24T02:39:00.440053Z", + "iopub.status.idle": "2024-03-24T02:39:00.445820Z", + "shell.execute_reply": "2024-03-24T02:39:00.444729Z" + }, + "papermill": { + "duration": 0.022368, + "end_time": "2024-03-24T02:39:00.448186", + "exception": false, + "start_time": "2024-03-24T02:39:00.425818", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:39:00.478355Z", + "iopub.status.busy": "2024-03-24T02:39:00.477968Z", + "iopub.status.idle": "2024-03-24T02:39:01.025228Z", + "shell.execute_reply": "2024-03-24T02:39:01.024152Z" + }, + "papermill": { + "duration": 0.565158, + "end_time": "2024-03-24T02:39:01.027760", + "exception": false, + "start_time": "2024-03-24T02:39:00.462602", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/realtabformer/all inf False\n", + "../../../../ml-utility-loss/aug_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_bs_test/realtabformer/all inf False\n", + "../../../../ml-utility-loss/bs_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_synth_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:39:01.061792Z", + "iopub.status.busy": "2024-03-24T02:39:01.060781Z", + "iopub.status.idle": "2024-03-24T02:39:01.430674Z", + "shell.execute_reply": "2024-03-24T02:39:01.429588Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.390462, + "end_time": "2024-03-24T02:39:01.433161", + "exception": false, + "start_time": "2024-03-24T02:39:01.042699", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'loss_balancer_beta': 0.7999999999999999,\n", + " 'loss_balancer_r': 0.96,\n", + " 'tf_pma_low': 16,\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7999999999999999,\n", + " 'bias_weight_decay': 0.05,\n", + " 'head_activation': torch.nn.modules.activation.ReLU6,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.09,\n", + " 'n_warmup_steps': 80,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'fixed_role_model': 'realtabformer',\n", + " 'g_loss_mul': 0.1,\n", + " 'd_model': 256,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 512,\n", + " 'ada_n_layers': 8,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 512,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'single_model': True,\n", + " 'models': ['realtabformer'],\n", + " 'max_seconds': 3600,\n", + " 'Body': 'twin_encoder',\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 16,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 1.0, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "# params = {\n", + "# **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "# }\n", + "params = getattr(PARAMS, dataset_name).BEST_DICT[gp][gp_multiply][single_model]\n", + "if isinstance(params, (list, tuple)):\n", + " params = params[param_index]\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:39:01.465746Z", + "iopub.status.busy": "2024-03-24T02:39:01.465371Z", + "iopub.status.idle": "2024-03-24T02:39:01.592669Z", + "shell.execute_reply": "2024-03-24T02:39:01.591421Z" + }, + "papermill": { + "duration": 0.147122, + "end_time": "2024-03-24T02:39:01.595269", + "exception": false, + "start_time": "2024-03-24T02:39:01.448147", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_train/realtabformer/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/contraceptive [400, 0]\n", + "Caching in ../../../../contraceptive/_cache_aug_val/realtabformer/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/contraceptive [0, 200]\n", + "Caching in ../../../../contraceptive/_cache_bs_train/realtabformer/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/contraceptive [100, 0]\n", + "Caching in ../../../../contraceptive/_cache_bs_val/realtabformer/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/contraceptive [0, 50]\n", + "Caching in ../../../../contraceptive/_cache_synth/realtabformer/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/contraceptive [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-24T02:39:01.629134Z", + "iopub.status.busy": "2024-03-24T02:39:01.628700Z", + "iopub.status.idle": "2024-03-24T02:39:02.145260Z", + "shell.execute_reply": "2024-03-24T02:39:02.144040Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.537093, + "end_time": "2024-03-24T02:39:02.148039", + "exception": false, + "start_time": "2024-03-24T02:39:01.610946", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding True True\n", + "['realtabformer'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:39:02.182993Z", + "iopub.status.busy": "2024-03-24T02:39:02.182428Z", + "iopub.status.idle": "2024-03-24T02:39:02.188560Z", + "shell.execute_reply": "2024-03-24T02:39:02.187248Z" + }, + "papermill": { + "duration": 0.027321, + "end_time": "2024-03-24T02:39:02.191569", + "exception": false, + "start_time": "2024-03-24T02:39:02.164248", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:39:02.224551Z", + "iopub.status.busy": "2024-03-24T02:39:02.223879Z", + "iopub.status.idle": "2024-03-24T02:39:02.232450Z", + "shell.execute_reply": "2024-03-24T02:39:02.231344Z" + }, + "papermill": { + "duration": 0.027014, + "end_time": "2024-03-24T02:39:02.234838", + "exception": false, + "start_time": "2024-03-24T02:39:02.207824", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "8630553" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:39:02.267778Z", + "iopub.status.busy": "2024-03-24T02:39:02.267362Z", + "iopub.status.idle": "2024-03-24T02:39:02.366344Z", + "shell.execute_reply": "2024-03-24T02:39:02.365227Z" + }, + "papermill": { + "duration": 0.118353, + "end_time": "2024-03-24T02:39:02.368824", + "exception": false, + "start_time": "2024-03-24T02:39:02.250471", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 16128] --\n", + "├─Adapter: 1-1 [2, 1179, 16128] --\n", + "│ └─Embedding: 2-1 [2, 1179, 24, 672] (48,384)\n", + "│ └─TensorInductionPoint: 2-2 [24, 1] 24\n", + "│ └─Sequential: 2-3 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 512] 344,576\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 256] 131,328\n", + "│ │ │ └─LeakyHardsigmoid: 4-16 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 16128] (recursive)\n", + "│ └─Embedding: 2-4 [2, 294, 24, 672] (recursive)\n", + "│ └─TensorInductionPoint: 2-5 [24, 1] (recursive)\n", + "│ └─Sequential: 2-6 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-9 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-18 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-10 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-32 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-7 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-6 [2, 16, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 16] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-12 [2, 1179, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-18 [2, 16, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 16] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-24 [2, 1179, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-30 [2, 16, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 16] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-36 [2, 1179, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-42 [2, 16, 256] --\n", + "│ └─Encoder: 2-8 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-8 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-21 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-22 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-48 [2, 16, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-23 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 16] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-54 [2, 294, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-25 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-60 [2, 16, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 16] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-66 [2, 294, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-31 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-72 [2, 16, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 16] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-78 [2, 294, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-14 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-40 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 16, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-84 [2, 16, 256] --\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-9 [2, 1] --\n", + "│ │ └─FeedForward: 3-19 [2, 512] --\n", + "│ │ │ └─Linear: 4-39 [2, 512] 2,097,664\n", + "│ │ │ └─ReLU6: 4-40 [2, 512] --\n", + "│ │ └─FeedForward: 3-20 [2, 512] --\n", + "│ │ │ └─Linear: 4-41 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-42 [2, 512] --\n", + "│ │ └─FeedForward: 3-21 [2, 512] --\n", + "│ │ │ └─Linear: 4-43 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-44 [2, 512] --\n", + "│ │ └─FeedForward: 3-22 [2, 512] --\n", + "│ │ │ └─Linear: 4-45 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-46 [2, 512] --\n", + "│ │ └─FeedForward: 3-23 [2, 512] --\n", + "│ │ │ └─Linear: 4-47 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-48 [2, 512] --\n", + "│ │ └─FeedForward: 3-24 [2, 512] --\n", + "│ │ │ └─Linear: 4-49 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-50 [2, 512] --\n", + "│ │ └─FeedForward: 3-25 [2, 512] --\n", + "│ │ │ └─Linear: 4-51 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-52 [2, 512] --\n", + "│ │ └─FeedForward: 3-26 [2, 512] --\n", + "│ │ │ └─Linear: 4-53 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-54 [2, 512] --\n", + "│ │ └─FeedForward: 3-27 [2, 1] --\n", + "│ │ │ └─Linear: 4-55 [2, 1] 513\n", + "│ │ │ └─LeakyHardsigmoid: 4-56 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 8,678,937\n", + "Trainable params: 8,630,553\n", + "Non-trainable params: 48,384\n", + "Total mult-adds (M): 26.78\n", + "========================================================================================================================\n", + "Input size (MB): 0.28\n", + "Forward/backward pass size (MB): 611.80\n", + "Params size (MB): 34.72\n", + "Estimated Total Size (MB): 646.80\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:39:02.403230Z", + "iopub.status.busy": "2024-03-24T02:39:02.402822Z", + "iopub.status.idle": "2024-03-24T03:45:01.465570Z", + "shell.execute_reply": "2024-03-24T03:45:01.464454Z" + }, + "papermill": { + "duration": 3959.099226, + "end_time": "2024-03-24T03:45:01.484387", + "exception": false, + "start_time": "2024-03-24T02:39:02.385161", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding True True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.011365748913376592, 'avg_role_model_std_loss': 0.9511739558440473, 'avg_role_model_mean_pred_loss': 0.0006854779220273954, 'avg_role_model_g_mag_loss': 1.2036508678562112, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011510453373646467, 'n_size': 900, 'n_batch': 225, 'duration': 244.78535199165344, 'duration_batch': 1.087934897740682, 'duration_size': 0.2719837244351705, 'avg_pred_std': 0.10629495567538672}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004436042588980248, 'avg_role_model_std_loss': 1.158144313332901, 'avg_role_model_mean_pred_loss': 5.6492201652799975e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004436042588980248, 'n_size': 450, 'n_batch': 113, 'duration': 100.92714786529541, 'duration_batch': 0.8931606005778355, 'duration_size': 0.22428255081176757, 'avg_pred_std': 0.06720254934532979}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004198840709804143, 'avg_role_model_std_loss': 0.8141998785290303, 'avg_role_model_mean_pred_loss': 3.387297497191669e-05, 'avg_role_model_g_mag_loss': 0.2937400783267286, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004241808241511333, 'n_size': 900, 'n_batch': 225, 'duration': 244.76988172531128, 'duration_batch': 1.0878661410013835, 'duration_size': 0.27196653525034586, 'avg_pred_std': 0.0953530696950232}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00333135591690532, 'avg_role_model_std_loss': 2.725961913635608, 'avg_role_model_mean_pred_loss': 1.3745906455597812e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00333135591690532, 'n_size': 450, 'n_batch': 113, 'duration': 99.83114504814148, 'duration_batch': 0.883461460603022, 'duration_size': 0.22184698899586996, 'avg_pred_std': 0.04183473708664388}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002762561462489733, 'avg_role_model_std_loss': 0.5484214044140991, 'avg_role_model_mean_pred_loss': 8.805304913856541e-06, 'avg_role_model_g_mag_loss': 0.2618754346999857, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0027956583087668857, 'n_size': 900, 'n_batch': 225, 'duration': 242.95946621894836, 'duration_batch': 1.0798198498619926, 'duration_size': 0.26995496246549816, 'avg_pred_std': 0.10219329664483666}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0027281873274801506, 'avg_role_model_std_loss': 2.8626933845729767, 'avg_role_model_mean_pred_loss': 6.899153320190609e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0027281873274801506, 'n_size': 450, 'n_batch': 113, 'duration': 100.66479754447937, 'duration_batch': 0.8908389163228263, 'duration_size': 0.22369955009884304, 'avg_pred_std': 0.04004584675329218}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002585010585737311, 'avg_role_model_std_loss': 0.5059408610661492, 'avg_role_model_mean_pred_loss': 1.07776368556805e-05, 'avg_role_model_g_mag_loss': 0.2515256485177411, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0026162271208078083, 'n_size': 900, 'n_batch': 225, 'duration': 244.41886401176453, 'duration_batch': 1.086306062274509, 'duration_size': 0.2715765155686273, 'avg_pred_std': 0.1017372147883806}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0025077687115925883, 'avg_role_model_std_loss': 2.664003259325225, 'avg_role_model_mean_pred_loss': 7.708884488491681e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0025077687115925883, 'n_size': 450, 'n_batch': 113, 'duration': 100.92277240753174, 'duration_batch': 0.8931218797126702, 'duration_size': 0.22427282757229275, 'avg_pred_std': 0.04433754623756307}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0022580953761239977, 'avg_role_model_std_loss': 0.45952041590186543, 'avg_role_model_mean_pred_loss': 8.792309877011795e-06, 'avg_role_model_g_mag_loss': 0.2588796340343025, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022885854777915586, 'n_size': 900, 'n_batch': 225, 'duration': 244.1408052444458, 'duration_batch': 1.0850702455308703, 'duration_size': 0.27126756138271757, 'avg_pred_std': 0.10189315725092052}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023599671401704353, 'avg_role_model_std_loss': 3.2095572816661777, 'avg_role_model_mean_pred_loss': 7.228522299237366e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023599671401704353, 'n_size': 450, 'n_batch': 113, 'duration': 101.14786124229431, 'duration_batch': 0.8951138163034895, 'duration_size': 0.22477302498287624, 'avg_pred_std': 0.04659274682213049}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0021870362648365293, 'avg_role_model_std_loss': 0.25351457023726504, 'avg_role_model_mean_pred_loss': 7.473293455636948e-06, 'avg_role_model_g_mag_loss': 0.2523958419428931, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002216868234035145, 'n_size': 900, 'n_batch': 225, 'duration': 244.6163468360901, 'duration_batch': 1.087183763715956, 'duration_size': 0.271795940928989, 'avg_pred_std': 0.10260770147045453}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002132210925208508, 'avg_role_model_std_loss': 2.034111335401322, 'avg_role_model_mean_pred_loss': 4.895658492241791e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002132210925208508, 'n_size': 450, 'n_batch': 113, 'duration': 100.4099771976471, 'duration_batch': 0.8885838690057265, 'duration_size': 0.22313328266143798, 'avg_pred_std': 0.05016624194001202}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002052127151619061, 'avg_role_model_std_loss': 0.18765673593578922, 'avg_role_model_mean_pred_loss': 6.5528008938834434e-06, 'avg_role_model_g_mag_loss': 0.277902038457493, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002083149238023907, 'n_size': 900, 'n_batch': 225, 'duration': 243.84846901893616, 'duration_batch': 1.083770973417494, 'duration_size': 0.2709427433543735, 'avg_pred_std': 0.10590767532587052}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0024004279846849387, 'avg_role_model_std_loss': 2.7415380834772916, 'avg_role_model_mean_pred_loss': 5.895928737795737e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0024004279846849387, 'n_size': 450, 'n_batch': 113, 'duration': 100.27250218391418, 'duration_batch': 0.8873672759638424, 'duration_size': 0.2228277826309204, 'avg_pred_std': 0.04519922711136463}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0018999467526898822, 'avg_role_model_std_loss': 0.20382699480085795, 'avg_role_model_mean_pred_loss': 5.589238350358375e-06, 'avg_role_model_g_mag_loss': 0.25682228529618845, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001928712559650497, 'n_size': 900, 'n_batch': 225, 'duration': 245.35998964309692, 'duration_batch': 1.0904888428582085, 'duration_size': 0.2726222107145521, 'avg_pred_std': 0.10311477472798691}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001989704634373387, 'avg_role_model_std_loss': 2.2423227408280177, 'avg_role_model_mean_pred_loss': 4.893121499534396e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001989704634373387, 'n_size': 450, 'n_batch': 113, 'duration': 100.96781826019287, 'duration_batch': 0.893520515576928, 'duration_size': 0.22437292946709528, 'avg_pred_std': 0.05078682927332357}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0017601978679401024, 'avg_role_model_std_loss': 0.4031062119189409, 'avg_role_model_mean_pred_loss': 4.838700413782498e-06, 'avg_role_model_g_mag_loss': 0.26518729818363984, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0017887732903990481, 'n_size': 900, 'n_batch': 225, 'duration': 245.00445175170898, 'duration_batch': 1.08890867445204, 'duration_size': 0.27222716861301, 'avg_pred_std': 0.10410975835182601}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002103634206901511, 'avg_role_model_std_loss': 1.6868754704122124, 'avg_role_model_mean_pred_loss': 6.431579865517962e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002103634206901511, 'n_size': 450, 'n_batch': 113, 'duration': 100.77440333366394, 'duration_batch': 0.8918088790589729, 'duration_size': 0.2239431185192532, 'avg_pred_std': 0.056379431404329966}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0016397621689894651, 'avg_role_model_std_loss': 0.11264491108698306, 'avg_role_model_mean_pred_loss': 6.2875129506191465e-06, 'avg_role_model_g_mag_loss': 0.2709155303819312, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001668296587401225, 'n_size': 900, 'n_batch': 225, 'duration': 244.95206952095032, 'duration_batch': 1.088675864537557, 'duration_size': 0.27216896613438923, 'avg_pred_std': 0.10980390641838313}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001868966455739509, 'avg_role_model_std_loss': 1.625626234592526, 'avg_role_model_mean_pred_loss': 4.7393164126034105e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001868966455739509, 'n_size': 450, 'n_batch': 113, 'duration': 101.65219712257385, 'duration_batch': 0.8995769656864943, 'duration_size': 0.22589377138349745, 'avg_pred_std': 0.05243304344017751}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0014940200044495416, 'avg_role_model_std_loss': 0.2755374942968874, 'avg_role_model_mean_pred_loss': 3.1630159733497656e-06, 'avg_role_model_g_mag_loss': 0.27379964623186326, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0015222167474591211, 'n_size': 900, 'n_batch': 225, 'duration': 246.83059191703796, 'duration_batch': 1.0970248529646132, 'duration_size': 0.2742562132411533, 'avg_pred_std': 0.10100599300530222}\n", + "Time out: 3711.764585494995/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'realtabformer', 'n_size': 1050, 'n_batch': 263, 'role_model_metrics': {'avg_loss': 0.0016764628703683792, 'avg_g_mag_loss': 0.026020999058271363, 'avg_g_cos_loss': 0.010928842550446613, 'pred_duration': 7.159314393997192, 'grad_duration': 2.6302547454833984, 'total_duration': 9.78956913948059, 'pred_std': 0.10647688060998917, 'std_loss': 0.00215106806717813, 'mean_pred_loss': 3.3525986964377807e-06, 'pred_rmse': 0.04094462841749191, 'pred_mae': 0.03046504035592079, 'pred_mape': 0.09343661367893219, 'grad_rmse': 0.43987467885017395, 'grad_mae': 0.24295493960380554, 'grad_mape': 7.0997748374938965}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0016764628703683792, 'avg_g_mag_loss': 0.026020999058271363, 'avg_g_cos_loss': 0.010928842550446613, 'avg_pred_duration': 7.159314393997192, 'avg_grad_duration': 2.6302547454833984, 'avg_total_duration': 9.78956913948059, 'avg_pred_std': 0.10647688060998917, 'avg_std_loss': 0.00215106806717813, 'avg_mean_pred_loss': 3.3525986964377807e-06}, 'min_metrics': {'avg_loss': 0.0016764628703683792, 'avg_g_mag_loss': 0.026020999058271363, 'avg_g_cos_loss': 0.010928842550446613, 'pred_duration': 7.159314393997192, 'grad_duration': 2.6302547454833984, 'total_duration': 9.78956913948059, 'pred_std': 0.10647688060998917, 'std_loss': 0.00215106806717813, 'mean_pred_loss': 3.3525986964377807e-06, 'pred_rmse': 0.04094462841749191, 'pred_mae': 0.03046504035592079, 'pred_mape': 0.09343661367893219, 'grad_rmse': 0.43987467885017395, 'grad_mae': 0.24295493960380554, 'grad_mape': 7.0997748374938965}, 'model_metrics': {'realtabformer': {'avg_loss': 0.0016764628703683792, 'avg_g_mag_loss': 0.026020999058271363, 'avg_g_cos_loss': 0.010928842550446613, 'pred_duration': 7.159314393997192, 'grad_duration': 2.6302547454833984, 'total_duration': 9.78956913948059, 'pred_std': 0.10647688060998917, 'std_loss': 0.00215106806717813, 'mean_pred_loss': 3.3525986964377807e-06, 'pred_rmse': 0.04094462841749191, 'pred_mae': 0.03046504035592079, 'pred_mape': 0.09343661367893219, 'grad_rmse': 0.43987467885017395, 'grad_mae': 0.24295493960380554, 'grad_mape': 7.0997748374938965}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:45:01.520372Z", + "iopub.status.busy": "2024-03-24T03:45:01.519964Z", + "iopub.status.idle": "2024-03-24T03:45:01.524884Z", + "shell.execute_reply": "2024-03-24T03:45:01.523879Z" + }, + "papermill": { + "duration": 0.02466, + "end_time": "2024-03-24T03:45:01.526806", + "exception": false, + "start_time": "2024-03-24T03:45:01.502146", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:45:01.562409Z", + "iopub.status.busy": "2024-03-24T03:45:01.562060Z", + "iopub.status.idle": "2024-03-24T03:45:01.640792Z", + "shell.execute_reply": "2024-03-24T03:45:01.639690Z" + }, + "papermill": { + "duration": 0.099037, + "end_time": "2024-03-24T03:45:01.643399", + "exception": false, + "start_time": "2024-03-24T03:45:01.544362", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:45:01.682076Z", + "iopub.status.busy": "2024-03-24T03:45:01.681124Z", + "iopub.status.idle": "2024-03-24T03:45:01.998932Z", + "shell.execute_reply": "2024-03-24T03:45:01.997894Z" + }, + "papermill": { + "duration": 0.340062, + "end_time": "2024-03-24T03:45:02.001370", + "exception": false, + "start_time": "2024-03-24T03:45:01.661308", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAEYCAYAAAAJVKDwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8/klEQVR4nO3de1xUdf4/8NfMwAz34T7DIFcFMUUwCELdrJWVWqwody1z1fy6ai2WRW1Gqdh2wVtt6yXd6vfV+m4EumW1ahah1iZ446IQQl5QUBwuIjPIZYCZz++PgaMjA8zAMMPI+/l4nMfEOZ9z5j3EvPyc2+fwGGMMhBBiJfiWLoAQQoxBoUUIsSoUWoQQq0KhRQixKhRahBCrQqFFCLEqFFqEEKtCoUUIsSoUWoQQq0KhRQixKgMKra1btyIwMBB2dnaIjY3F8ePH+2y/e/duhIWFwc7ODuHh4di/f7/O8i+//BIzZsyAh4cHeDweioqKdJY3NDTgueeew9ixY2Fvbw9/f388//zzUCgUAymfEGLFjA6trKwspKSkIC0tDQUFBYiIiEBCQgJqa2v1ts/NzcWcOXOwaNEiFBYWIikpCUlJSSgpKeHaNDc3Y+rUqVi3bp3ebVRXV6O6uhobN25ESUkJdu7ciQMHDmDRokXGlk8IsXbMSDExMSw5OZn7Wa1WM5lMxtLT0/W2nz17NktMTNSZFxsby5YuXdqjbUVFBQPACgsL+61j165dTCgUso6ODuM+ACHEqtkYE3Dt7e3Iz89HamoqN4/P5yM+Ph55eXl618nLy0NKSorOvISEBHz11VdGB+ytFAoFXFxcYGOj/yOoVCqoVCruZ41Gg4aGBm4XlBAyvDDG0NTUBJlMBj6/951Ao0Krvr4earUaEolEZ75EIkFZWZnedeRyud72crncmLfuUcebb76JJUuW9NomPT0db7zxxoDfgxBiGVVVVRg1alSvy40KreFAqVQiMTERd911F9asWdNru9TUVJ0enkKhgL+/P6qqquDi4mKGSgkhxlAqlfDz84Ozs3Of7YwKLU9PTwgEAtTU1OjMr6mpgVQq1buOVCo1qn1fmpqa8OCDD8LZ2Rl79uyBra1tr21FIhFEIlGP+S4uLhRahAxj/R2+MersoVAoRFRUFHJycrh5Go0GOTk5iIuL07tOXFycTnsAyM7O7rV9b5RKJWbMmAGhUIhvvvkGdnZ2Rq1PCLkzGL17mJKSggULFiA6OhoxMTF4//330dzcjIULFwIA5s+fD19fX6SnpwMAli9fjmnTpuHdd99FYmIiMjMzcfLkSXz44YfcNhsaGlBZWYnq6moAQHl5OQBtL00qlXKB1dLSgn/9619QKpVQKpUAAC8vLwgEgsH9Fggh1mMgpxw3b97M/P39mVAoZDExMezo0aPcsmnTprEFCxbotN+1axcLDQ1lQqGQjR8/nu3bt09n+Y4dOxiAHlNaWhpjjLFDhw7pXQ6AVVRUGFSzQqFgAJhCoRjIRyaEDDFDv6M8xkbGgy2USiXEYjF3qQQxjlqtRkdHh6XLIFbM1ta2z70iQ7+jVnf2kJgXYwxyuRyNjY2WLoXcAVxdXSGVSgd1rSSFFulTd2B5e3vDwcGBLswlA8IYQ0tLC3e7n4+Pz4C3RaF1mxuqTpy5qkSnmiFutIely7EotVrNBZaHx8j+XZDBs7e3BwDU1tbC29t7wCfQaGia2+Seq8cft+fh7f2lli7F4rqPYTk4OFi4EnKn6P5bGszxUQqt2wR7OQEAKuqaMULOUfSLdgmJqZjib4lC6zb+7g4Q8HloblejRqnqfwVCiFlRaN1GaMOHv7u2C3uh/oaFqyHWisfjDXokk+FmzZo1iIyMtHQZFFr6BHk6AgAu1DVbuBJCBm7nzp1wdXU12fZefvnlHrfkWQKFlh7BFFpkBGlvbzeonZOT07A4i0yhpUf3wXjaPbReBw4cwNSpU+Hq6goPDw/MnDkT58+fBwBMnjwZK1as0GlfV1cHW1tb/PTTTwCAq1evIjExEfb29ggKCkJGRgYCAwPx/vvvD6ie4uJi/Pa3v4W9vT08PDywZMkS3Lhx8+/r8OHDiImJgaOjI1xdXTFlyhRcunQJAHDq1Ck88MADcHZ2houLC6KionDy5Mk+3+/w4cNYuHAhFAoFeDweeDweN5RTYGAg3nzzTcyfPx8uLi7cuHQrVqxAaGgoHBwcEBwcjFWrVumc5bt99/Dpp59GUlISNm7cCB8fH3h4eCA5OXnI75yg67T0CPbS9rQq6qmndSvGGFo71BZ5b3tbgVFnnpqbm5GSkoKJEyfixo0bWL16NR577DEUFRVh7ty5WL9+PdauXcttMysrCzKZDL/5zW8AaG/8r6+vx+HDh2Fra4uUlJRen4NgSC0JCQmIi4vDiRMnUFtbiz//+c9YtmwZdu7cic7OTiQlJWHx4sX4/PPP0d7ejuPHj3O1zZ07F5MmTcK2bdsgEAhQVFTU57BMgDaY33//faxevZobgMDJyYlbvnHjRqxevRppaWncPGdnZ+zcuRMymQzFxcVYvHgxnJ2d8corr/T6PocOHYKPjw8OHTqEc+fO4YknnkBkZCQWL148oN+VISi09OjePaxqaIGqUw2RDY0iAQCtHWrctfo7i7x36d8S4CA0/M911qxZOj//7//+L7y8vFBaWorZs2fjhRdewM8//8yFVEZGBubMmQMej4eysjL88MMPOHHiBKKjowEAH3/8MUJCQgZUe0ZGBtra2vDpp5/C0VH7t7VlyxY8/PDDWLduHWxtbaFQKDBz5kyMHj0aADBu3Dhu/crKSvz1r39FWFgYABhUh1AohFgsBo/H0zt23W9/+1u89NJLOvNWrlzJ/XdgYCBefvllZGZm9hlabm5u2LJlCwQCAcLCwpCYmIicnJwhDS3aPdTDy1kEJ5ENNAyovNZi6XLIAJw9exZz5sxBcHAwXFxcEBgYCEAbAF5eXpgxYwY+++wzAEBFRQXy8vIwd+5cANqhkWxsbHD33Xdz2xszZgzc3NwGVMuZM2cQERHBBRYATJkyBRqNBuXl5XB3d8fTTz+NhIQEPPzww/jHP/6Bq1evcm1TUlLw5z//GfHx8Vi7di23mzsY3WF8q6ysLEyZMgVSqRROTk5YuXIlKisr+9zO+PHjda5s9/HxGXCP1FDU09KDx+Mh2MsRpy8rcL6uGSGSvod/HSnsbQUo/VuCxd7bGA8//DACAgLw0UcfQSaTQaPRYMKECdxB57lz5+L555/H5s2bkZGRgfDwcISHhw9F6QbZsWMHnn/+eRw4cABZWVlYuXIlsrOzce+992LNmjV46qmnsG/fPnz77bdIS0tDZmYmHnvssQG/360BCoAL7TfeeAMJCQkQi8XIzMzEu+++2+d2bt9N5fF40Gg0A67LENTT6kX3LiId17qJx+PBQWhjkcmY41nXrl1DeXk5Vq5cienTp2PcuHG4fv26TptHH30UbW1tOHDgADIyMrheFgCMHTsWnZ2dKCws5OadO3euxzYMNW7cOJw6dQrNzTf/lo4cOQI+n4+xY8dy8yZNmoTU1FTk5uZiwoQJyMjI4JaFhobixRdfxPfff4/HH38cO3bs6Pd9hUIh1GrDjkHm5uYiICAAr7/+OqKjoxESEsKdCBhuKLR6EeTZdQaxjs4gWhs3Nzd4eHjgww8/xLlz53Dw4MEej7FzdHREUlISVq1ahTNnzmDOnDncsrCwMMTHx2PJkiU4fvw4CgsLsWTJEtjb2w/oNpS5c+fCzs4OCxYsQElJCQ4dOoTnnnsO8+bNg0QiQUVFBVJTU5GXl4dLly7h+++/x9mzZzFu3Di0trZi2bJlOHz4MC5duoQjR47gxIkTOse8ehMYGIgbN24gJycH9fX1aGnp/VBHSEgIKisrkZmZifPnz2PTpk3Ys2eP0Z/VHCi0etF9BvEC9bSsDp/PR2ZmJvLz8zFhwgS8+OKL2LBhQ492c+fOxalTp/Cb3/wG/v7+Oss+/fRTSCQS3HfffXjssce4M2kDeTaBg4MDvvvuOzQ0NOCee+7BH/7wB0yfPh1btmzhlpeVlWHWrFkIDQ3FkiVLkJycjKVLl0IgEODatWuYP38+QkNDMXv2bDz00EMGPR5v8uTJeOaZZ/DEE0/Ay8sL69ev77XtI488ghdffBHLli1DZGQkcnNzsWrVKqM/qznQyKW9+KVagcRNP8PNwRaFq2eYocLhp62tDRUVFQgKChrxDxK5fPky/Pz88MMPP2D69OmWLsdq9fU3RSOXDlL3rTzXWzpwvbkdbo5CC1dEzOngwYO4ceMGwsPDcfXqVbzyyisIDAzEfffdZ+nSRjzaPeyFg9AGMrH2XwLaRRx5Ojo68Nprr2H8+PF47LHH4OXlxV1o+tlnn8HJyUnvNH78eLPV+NBDD/VaxzvvvGO2OsyNelp9CPJyRLWiDRfqbiAqYGDX6BDrlJCQgIQE/Zd3PPLII4iNjdW7rL8r1U3p448/Rmtrq95l7u7uZqvD3Ci0+hDs6YQj565RT4vocHZ27vfR7ebg6+tr6RIsgnYP+8CdQaTLHggZNii0+sANvUw9LUKGDQqtPnRfFX/xWgvUmhFxZQghwx6FVh9krvYQ2vDR3qnBlev6D3gSQsyLQqsPAj4PQR7a3tZ5GhCQkGGBQqsf3ICANPQyMcKd+GCL4YJCqx/cQy6op0WsjKkfbAFoh3Hm8XhobGw06XaNQaHVD268eOppETIsUGj14+a1WhRa1oQebNH7gy1UKhVefvll+Pr6wtHREbGxsTh8+DC37qVLl/Dwww/Dzc0Njo6OGD9+PPbv34+LFy/igQceAKAd/ofH4+Hpp58e0O9jMOiK+H50X/YgV7ahWdUJR9EI/pUxBnRYaPhpWweAHmxhkgdbLFu2DKWlpcjMzIRMJsOePXvw4IMPori4GCEhIUhOTkZ7ezt++uknODo6orS0FE5OTvDz88MXX3yBWbNmoby8HC4uLrC3tx/Q72RQ2ABs2bKFBQQEMJFIxGJiYtixY8f6bL9r1y42duxYJhKJ2IQJE9i+fft0ln/xxRfsd7/7HXN3d2cAWGFhYY9ttLa2sr/85S/M3d2dOTo6sscff5zJ5XKDa1YoFAwAUygUBq/TbdLfvmcBK/ay4suNRq9rzVpbW1lpaSlrbW3VzlDdYCzNxTKT6sagPktdXR0DwIqLi1ltbS2zsbFhP/30E7c8Li6OrVixgjHG2JkzZxgAduLECW752bNnGQD297//3aD3A8D27NnDGGPsww8/ZG5ubuzGjZufYd++fYzP5zO5XM6uXbvGALDDhw/r3ZazszPbuXOnkZ+YsR07djCxWKwz79KlS0wgELArV67ozJ8+fTpLTU1ljDEWHh7O1qxZo3ebhw4dYgDY9evXja6HMT1/U7cw9Dtq9O5hVlYWUlJSkJaWhoKCAkRERCAhIaHXf4Vyc3MxZ84cLFq0CIWFhUhKSkJSUhJKSkq4Ns3NzZg6dSrWrVvX6/u++OKL+M9//oPdu3fjxx9/RHV1NR5//HFjyx8Q7uGtdGW81aAHW+hXXFwMtVqN0NBQnVEhfvzxR267zz//PN566y1MmTIFaWlpOH369IDfb0gYm5QxMTEsOTmZ+1mtVjOZTMbS09P1tp89ezZLTEzUmRcbG8uWLl3ao21FRYXenlZjYyOztbVlu3fv5uZ1/2uYl5dnUN2D6Wn9dXcRC1ixl/09u9zoda1Zj38VNRptj8cSk0ZjVO1jx45lM2bMYD/88AMrLS1lJSUlOr2fzz77jHl4eLD29nb21ltvsfDwcG7dr776itnY2DC1Wq2zTTc3twH1tF588UV2//336yxvbGxkANiPP/7IzSsoKGDvvPMOi4uLY05OTjp/2+Xl5ey9995jv/vd75hQKGRffvllvzXo62llZmYygUDAysrK2NmzZ3Wmq1evcu0qKyvZtm3b2GOPPcZsbW3Zpk2bGGNW2NNqb29Hfn4+4uPjuXl8Ph/x8fHIy8vTu05eXp5Oe0A77Edv7fXJz89HR0eHznbCwsLg7+/f63ZUKhWUSqXONFB0BrELjwcIHS0z0YMtTPJgi0mTJkGtVqO2thZjxozRmW59PqKfnx+eeeYZfPnll3jppZfw0UcfcdsEYPADM4aCUaFVX18PtVoNiUSiM18ikUAul+tdRy6XG9W+t20IhcIe15z0tZ309HSIxWJu8vPzM/j9bkdP5rEu9GALLX0PtggNDcXcuXMxf/58fPnll6ioqMDx48eRnp6Offv2AQBeeOEFfPfdd6ioqEBBQQEOHTrEvV9AQAB4PB727t2Luro6nTOg5nLHXvKQmpoKhULBTVVVVQPe1q1D1LCRMaS+VaMHW2j19mCLHTt2YP78+XjppZcwduxYJCUl4cSJE9zvQK1WIzk5GePGjcODDz6I0NBQfPDBBwC0Y3i98cYbePXVVyGRSLBs2TKjfx+DZsz+qEqlYgKBgNtX7zZ//nz2yCOP6F3Hz8+vx3GA1atXs4kTJ/Zo29sxrZycHL370f7+/uy9994zqPbBHNNSdahZcOo+FrBiL5Mreu6L36n6Ov4w0lRVVTEA7IcffrB0KVbN7Me0hEIhoqKikJOTw83TaDTIyclBXFyc3nXi4uJ02gNAdnZ2r+31iYqKgq2trc52ysvLUVlZadR2Bkpow4efm/Z6lPM0IOCIcPDgQXzzzTeoqKhAbm4unnzySXqwxTBh9O5hSkoKPvroI3zyySc4c+YMnn32WTQ3N2PhwoUAtBflpaamcu2XL1+OAwcO4N1330VZWRnWrFmDkydP6nQrGxoaUFRUhNLSUgDaQCoqKuKOV4nFYixatAgpKSk4dOgQ8vPzsXDhQsTFxeHee+8d1C/AUDQg4MhCD7YYvoy+vPuJJ55AXV0dVq9eDblcjsjISBw4cIA72F5ZWQk+/2YWTp48GRkZGVi5ciVee+01hISE4KuvvsKECRO4Nt988w0XegDw5JNPAgDS0tK4Ww/+/ve/g8/nY9asWVCpVEhISOD2s82Bu3F6pJ9BHCHowRbDFz2s1UCfHbuE1/eU4IGxXtixMGYIKhx+6GGtxNRM8bDWO/bsoakFe3Zdq0W7h4RYFIWWgUZ3XfZQ1dCC9k6NhasxrxHSGSdmYIq/JQotA3k5i+AoFEDDgMqGkdHb6j4+09JioZEdyB2n+29pMMf+RvA4K8bh8XgI9nJC8RUFztc1Y4y35R/WOdQEAgFcXV25m+EdHBwGdEU4IYwxtLS0oLa2Fq6urhAIBAPeFoWWEYK9HFF8RTGiziB234820LGkCLmVq6urzj2OA0GhZQTuYPwIusCUx+PBx8cH3t7e6OjosHQ5xIrZ2toOqofVjULLCEFeI/fGaYFAYJI/OEIGiw7EG4EGAyTE8ii0jNA92kNDczsaW9otXA0hIxOFlhEchDbwEWuv4j0/gg7GEzKcUGgZKXgEH9ciZDig0DLSzRunR84ZREKGEwotI9287IF6WoRYAoWWkbihl+upp0WIJVBoGWl012CAF6+1QK2hG4kJMTcKLSPJXO0htOGjvVOD6kb9A7ARQoYOhZaRBHweAj0cANB48YRYAoXWANDBeEIsh0JrAOhaLUIsh0JrALhrtegMIiFmR6E1AN2PE6PdQ0LMj0JrALrHi7+qaENLe6eFqyFkZKHQGgBXByHcHYUAqLdFiLlRaA1Q93EtOhhPiHlRaA1QMD1xmhCLoNAaIO5gPJ1BJMSsKLQGiLtxmnpahJgVhdYAjb7lAlN6AjMh5kOhNUB+7g7g84Abqk7UNaksXQ4hIwaF1gCJbATwc+++cZp2EQkxFwqtQQim23kIMbsBhdbWrVsRGBgIOzs7xMbG4vjx43223717N8LCwmBnZ4fw8HDs379fZzljDKtXr4aPjw/s7e0RHx+Ps2fP6rT59ddf8eijj8LT0xMuLi6YOnUqDh06NJDyTab7DGIF9bQIMRujQysrKwspKSlIS0tDQUEBIiIikJCQgNraWr3tc3NzMWfOHCxatAiFhYVISkpCUlISSkpKuDbr16/Hpk2bsH37dhw7dgyOjo5ISEhAW1sb12bmzJno7OzEwYMHkZ+fj4iICMycORNyuXwAH9s0gujhrYSYHzNSTEwMS05O5n5Wq9VMJpOx9PR0ve1nz57NEhMTdebFxsaypUuXMsYY02g0TCqVsg0bNnDLGxsbmUgkYp9//jljjLG6ujoGgP30009cG6VSyQCw7Oxsg+pWKBQMAFMoFIZ9UAMcOVfHAlbsZdPWHzTZNgkZqQz9jhrV02pvb0d+fj7i4+O5eXw+H/Hx8cjLy9O7Tl5enk57AEhISODaV1RUQC6X67QRi8WIjY3l2nh4eGDs2LH49NNP0dzcjM7OTvzzn/+Et7c3oqKijPkIJtU9XnzV9Va0d2osVgchI4mNMY3r6+uhVqshkUh05kskEpSVleldRy6X623fvVvX/dpXGx6Phx9++AFJSUlwdnYGn8+Ht7c3Dhw4ADc3N73vq1KpoFLdvBRBqVQa8UkN4+0sgqNQgOZ2NSobmjHG29nk70EI0WUVZw8ZY0hOToa3tzf++9//4vjx40hKSsLDDz+Mq1ev6l0nPT0dYrGYm/z8/ExeF4/HQxBdGU+IWRkVWp6enhAIBKipqdGZX1NTA6lUqncdqVTaZ/vu177aHDx4EHv37kVmZiamTJmCu+++Gx988AHs7e3xySef6H3f1NRUKBQKbqqqqjLmoxqMGy+eDsYTYhZGhZZQKERUVBRycnK4eRqNBjk5OYiLi9O7TlxcnE57AMjOzubaBwUFQSqV6rRRKpU4duwY16alpUVbLF+3XD6fD41G/7EkkUgEFxcXnWko3LwHka7VIsQsjD3Cn5mZyUQiEdu5cycrLS1lS5YsYa6urkwulzPGGJs3bx579dVXufZHjhxhNjY2bOPGjezMmTMsLS2N2drasuLiYq7N2rVrmaurK/v666/Z6dOn2aOPPsqCgoJYa2srY0x79tDDw4M9/vjjrKioiJWXl7OXX36Z2drasqKiIoPqHoqzh4wx9nXRFRawYi+b9cERk26XkJHG0O+o0aHFGGObN29m/v7+TCgUspiYGHb06FFu2bRp09iCBQt02u/atYuFhoYyoVDIxo8fz/bt26ezXKPRsFWrVjGJRMJEIhGbPn06Ky8v12lz4sQJNmPGDObu7s6cnZ3Zvffey/bv329wzUMVWsWXG1nAir3s7r99b9LtEjLSGPod5TE2MoYoUCqVEIvFUCgUJt1VbFZ1YnzadwCAU6tnQOxga7JtEzKSGPodtYqzh8OZo8gGUhc7AMB5ugeRkCFHoWUCNCAgIeZDoWUCN584TT0tQoYahZYJBHnSw1sJMRcKLROg3UNCzIdCywRGd/W0Kq41Q60ZESdjCbEYCi0T8HWzh1DAR3unBtWNrZYuh5A7GoWWCQj4PAR4aMeLp3sQCRlaFFomQvcgEmIeFFomwj1xmg7GEzKkKLRMhJ7MQ4h5UGiZCHeBKfW0CBlSFFom0j0YYLWiDS3tnRauhpA7F4WWibg5CuHWNcJDBZ1BJGTIUGiZEB2MJ2ToUWiZUPfDW6mnRcjQodAyIbpWi5ChR6FlQvRkHkKGHoWWCY2+ZbSHETKKNSFmR6FlQv4eDuDzgBuqTtTdUPW/AiHEaBRaJiSyEWCUW9eN03QGkZAhQaFlYjQgICFDi0LLxLiD8XQGkZAhQaFlYlxPi84gEjIkKLRMLJguMCVkSFFomVj3rTyVDS1o79RYuBpC7jwUWiYmcRHBUSiAWsNQ2dBi6XIIueNQaJkYj8dDEN3OQ8iQodAaAt0Pb6XjWoSYHoXWEOCGXqZrtQgxOQqtIXDzsgfaPSTE1Ci0hsBoGgyQkCEzoNDaunUrAgMDYWdnh9jYWBw/frzP9rt370ZYWBjs7OwQHh6O/fv36yxnjGH16tXw8fGBvb094uPjcfbs2R7b2bdvH2JjY2Fvbw83NzckJSUNpPwhF9i1e3ituR2Klg4LV0PIncXo0MrKykJKSgrS0tJQUFCAiIgIJCQkoLa2Vm/73NxczJkzB4sWLUJhYSGSkpKQlJSEkpISrs369euxadMmbN++HceOHYOjoyMSEhLQ1tbGtfniiy8wb948LFy4EKdOncKRI0fw1FNPDeAjDz0nkQ0kLiIAtItIiMkxI8XExLDk5GTuZ7VazWQyGUtPT9fbfvbs2SwxMVFnXmxsLFu6dCljjDGNRsOkUinbsGEDt7yxsZGJRCL2+eefM8YY6+joYL6+vuzjjz82tlyOQqFgAJhCoRjwNozx5D/zWMCKvezfJ6vM8n6EWDtDv6NG9bTa29uRn5+P+Ph4bh6fz0d8fDzy8vL0rpOXl6fTHgASEhK49hUVFZDL5TptxGIxYmNjuTYFBQW4cuUK+Hw+Jk2aBB8fHzz00EM6vbXbqVQqKJVKncmc6GA8IUPDqNCqr6+HWq2GRCLRmS+RSCCXy/WuI5fL+2zf/dpXmwsXLgAA1qxZg5UrV2Lv3r1wc3PD/fffj4aGBr3vm56eDrFYzE1+fn7GfNRB676dh67VIsS0rOLsoUajvYfv9ddfx6xZsxAVFYUdO3aAx+Nh9+7detdJTU2FQqHgpqqqKnOWTNdqETJEjAotT09PCAQC1NTU6MyvqamBVCrVu45UKu2zffdrX218fHwAAHfddRe3XCQSITg4GJWVlXrfVyQSwcXFRWcyp+7dw4r6Zmg0NF48IaZiVGgJhUJERUUhJyeHm6fRaJCTk4O4uDi968TFxem0B4Ds7GyufVBQEKRSqU4bpVKJY8eOcW2ioqIgEolQXl7Oteno6MDFixcREBBgzEcwm1FuDhAK+FB1anClsdXS5RBy5zD2CH9mZiYTiURs586drLS0lC1ZsoS5uroyuVzOGGNs3rx57NVXX+XaHzlyhNnY2LCNGzeyM2fOsLS0NGZra8uKi4u5NmvXrmWurq7s66+/ZqdPn2aPPvooCwoKYq2trVyb5cuXM19fX/bdd9+xsrIytmjRIubt7c0aGhoMqtvcZw8ZYyz+3cMsYMVedri81mzvSYi1MvQ7anRoMcbY5s2bmb+/PxMKhSwmJoYdPXqUWzZt2jS2YMECnfa7du1ioaGhTCgUsvHjx7N9+/bpLNdoNGzVqlVMIpEwkUjEpk+fzsrLy3XatLe3s5deeol5e3szZ2dnFh8fz0pKSgyu2RKhtfiTEyxgxV624+cLZntPQqyVod9RHmMj4wF9SqUSYrEYCoXCbMe31n5bhu0/nsf8uAD87dEJZnlPQqyVod9Rqzh7aK3oyTyEmB6F1hAaTYMBEmJyFFpDqHswwGpFG1rb1RauhpA7A4XWEHJ3FMLVwRYAXRlPiKlQaA0x7sp4ugeREJOg0BpiwTQgICEmRaE1xILo4a2EmBSF1hCjM4iEmBaF1hC7dfdwhFzHS8iQotAaYgEeDuDzgCZVJ+puqCxdDiFWj0JriIlsBBjl5gCADsYTYgoUWmZAB+MJMR0KLTMIpoPxhJgMhZYZ0LVahJgOhZYZjOauiqfQImSwKLTMIKhr97CyoQUdao2FqyHEulFomYHUxQ4OQgHUGobKhhZLl0OIVaPQMgMej8edQaTjWoQMDoWWmdw8GE9nEAkZDAotM6FrtQgxDQotMxlN48UTYhIUWmYS3DX0Mg0GSMjgUGiZSfdlD/U32qFo7bBwNYRYLwotM3ES2cDbWQSAjmsRMhgUWmZE9yASMngUWvoc+xCoP2fyzdI9iIQMHoXW7fI/Ab79K/DJTODaeZNump7MQ8jgUWjdbuzvAa9xQNNVYGeiSYNrNPW0CBk0Cq3bOXkBC/5zS3CZrsd16wWmGg2NF0/IQFBo6cMFVxjQVG2y4BrlZg9bAQ+qTg2qFa0mKJSQkYdCqzdOXsCCvTeD65OHgYYLg9qkjYCPAA9tb+vYhQZTVEnIiEOh1Zdbe1zKK9oe1yCD68HxUgDAmv/8gsprNEwNIcYaUGht3boVgYGBsLOzQ2xsLI4fP95n+927dyMsLAx2dnYIDw/H/v37dZYzxrB69Wr4+PjA3t4e8fHxOHv2rN5tqVQqREZGgsfjoaioaCDlG8fJWxtcnmO7guthoKFiwJt7fnoIJvm7oqmtE3/JyEdbh9qExRJy5zM6tLKyspCSkoK0tDQUFBQgIiICCQkJqK2t1ds+NzcXc+bMwaJFi1BYWIikpCQkJSWhpKSEa7N+/Xps2rQJ27dvx7Fjx+Do6IiEhAS0tbX12N4rr7wCmUxmbNmDwwVXKKC83NXjGlhwCW342PrU3XBzsEXJFSXe2ldq4mIJucMxI8XExLDk5GTuZ7VazWQyGUtPT9fbfvbs2SwxMVFnXmxsLFu6dCljjDGNRsOkUinbsGEDt7yxsZGJRCL2+eef66y3f/9+FhYWxn755RcGgBUWFhpct0KhYACYQqEweJ0elHLGNkczlubC2HvjGWuoGPCmDpXVsIAVe1nAir3sq8LLA6+JkDuEod9Ro3pa7e3tyM/PR3x8PDePz+cjPj4eeXl5etfJy8vTaQ8ACQkJXPuKigrI5XKdNmKxGLGxsTrbrKmpweLFi/F///d/cHBw6LdWlUoFpVKpMw2as0Tb4/IIARRV2h7X9YsD2tT9Y72x7IExAIDUL4txrrZp8PURMgIYFVr19fVQq9WQSCQ68yUSCeRyud515HJ5n+27X/tqwxjD008/jWeeeQbR0dEG1Zqeng6xWMxNfn5+Bq3XL2cp8PTeW4LrYeD6pQFt6sXfhSIu2AMt7Wo8+68CtLR3mqZGQu5gVnH2cPPmzWhqakJqaqrB66SmpkKhUHBTVVWV6QrigmsMoKjs6nEZH1wCPg//mBMJL2cRztbewMo9JWCMLjolpC9GhZanpycEAgFqamp05tfU1EAqlepdRyqV9tm++7WvNgcPHkReXh5EIhFsbGwwZox2tyo6OhoLFizQ+74ikQguLi46k0k5S7XXcXUH1yczgcZKozfj7WyHzXMmgc8Dviy8gswTJgxXQu5ARoWWUChEVFQUcnJyuHkajQY5OTmIi4vTu05cXJxOewDIzs7m2gcFBUEqleq0USqVOHbsGNdm06ZNOHXqFIqKilBUVMRdMpGVlYW3337bmI9gWi4+2uByH60NrJ2JAwque4M98HLCWABA2je/4JdqhakrJeTOYewR/szMTCYSidjOnTtZaWkpW7JkCXN1dWVyuZwxxti8efPYq6++yrU/cuQIs7GxYRs3bmRnzpxhaWlpzNbWlhUXF3Nt1q5dy1xdXdnXX3/NTp8+zR599FEWFBTEWltb9dZQUVFhmbOHvW78CmP/mKQ9q/j3cMauXzJ6E2q1hv3PjuMsYMVedt/6g0zR2m76OgkZxgz9jhodWowxtnnzZubv78+EQiGLiYlhR48e5ZZNmzaNLViwQKf9rl27WGhoKBMKhWz8+PFs3759Oss1Gg1btWoVk0gkTCQSsenTp7Py8vJe33/YhRZjeoKr0uhNXG9WscnpOSxgxV629NOTTKPRDEGhhAxPhn5HeYyNjCO/SqUSYrEYCoXC9Me3uDep1u4iNlwAXAOAp/cBrsadtSyqasQft+eiQ82wauZdWDQ1aGhqJWSYMfQ7ahVnD62Gi0x7jMstCGi8pD04r7hs1CYi/Vzx+u/HAQDS959B/qXrQ1EpIVaLQsvUxL7aHpZbkPbC052JRgfXgsmBSAz3QaeGYVlGARqa24emVkKsEIXWUBD7aq/jcgvsCq6ZgOKKwavzeDysnRWOIE9HXFW04cWsIho0kJAuFFpDRTyqq8cVCFyv6OpxGR5czna2+GDu3RDZ8PHjr3X44LDpH7RBiDWi0BpK4lHaY1yuAdrg+mSm9mC9gcb5uODNpAkAgPeyf0Xu+fqhqpQQq0GhNdRc/brOIgZozyruTDQquGZH++EPUaOgYcDznxehVtlzuB5CRhIKLXNw9dMe43L1vxlclccMXv3NRycgTOqM+hsqLPu8EJ1qzRAWS8jwRqFlLq7+XT2uruD63xlA1p8MeiisvVCArXPvhqNQgOMVDXg3+1czFEzI8EShZU6u/sCfc4C75wM8PnDmP8AHscC+l4EbdX2uOtrLCev+MBEAsO3weRwsq+mzPSF3Kgotc3PyBh7ZDDybC4Q+CGg6gRMfAZsigR83AO29P8h15kQZFsQFAABezDqFy9fpwRhk5KHQshTvccBTWdqRUH0igfYbwKG3gM1RQMGngEb/Ay9eSxyHiFFiKFo7kJxRiPZOOr5FRhYKLUsLug9YfAiY9f+0u49NV4FvngO2TQF+/Q647dZQkY0AW566G2J7W5yqasQ7+89YqHBCLINCazjg84HwPwDLTgIJ7wB2rkDdGSBjtvYhsVcKdJr7uTvgvdkRAICduRex7/RVCxRNiGVQaA0nNiIgLhlYXgRMfh4QiICL/wU+egD49yKdh2hMHyfBM9NGAwBWfHEaF+puWKZmQsyMQms4sncDZrwJPHcSmPikdl7Jv4Et9wAHXgNaGgAAL88IRUyQO26oOvGXzwrowa9kRKDQGs5c/YHH/wks/QkImgao24GjW7VnGo/8AzaadmyZMwmeTkKUyZuw+uuSfjdJiLWj0LIGPhHA/K+BP30BeI8H2hRA9mpgSzS8K77GpiciwOcBu05exu6T9GAMcmej0LIWPB4wJh545r/Aox8AzjLtcxf3LMHknMfxXpR2sMBVX5fgcHktmto6LFwwIUODhlu2Vh2twNFtwM9/B1Tap2eftovGK4o/oIz5AwB8xHYY4+2EMd5OCPF2RojECWO8nODmKLRk5YToZeh3lELL2jXXAz9tAE58DGg6oQEPp3hhyO8IRLEmCCUsCBeYD9gtnWpPJyEXZNpXJ4yROMHLSQQej2fBD0NGMgqt29yxodXt2nkg529A6Vc9Fqn4DqiwHY2izkDktvqjhAWhgkl1ggwAXOxsECJx1oZYdw9N4gyZ2I7CjAw5Cq3b3PGh1a3+HHD5OFBdCFQXAfJioLO1R7MOG0dctR+LMl4wjqn8cLjJFxc0PYMMAByFAozxdsJobyfc5eOCaaFeGOPtREFGTIpC6zYjJrRup+4E6su1AXa1qCvITgOdPQcTVNs64bp4HC7ahuA0C8JPTaNw5LoLOjQ9w8nf3QG/DfPG9HHeiAlyh8hGMOQfhdzZKLRuM2JDS58eQVbY1SPrGWRM5IxWjwmocQxDOX80Dil9sLdSiGb1zZByFApwX6gXfhvmjQfCvOHpJDLfZyF3DAqt21Bo9YMLssKbYdZbkIGHNnspqnkSlLS649d2T1Qxb1xi3qiCN/x9/TB9nATTx0kwzseZdiOJQSi0bkOhNQDqTqCu7OZu5dUioKYU6Oh9zC8AUDJ7VDFvVDJvXBP6wlk6BgFj7sK48REQeQQAAltzVE+sDIXWbSi0TIQx7WUW1y92TRXa14au16a+H9qhAR8t9j6w9QyGyCtY+4g19yDtq1ug9r5La8CYdtjsS7na6WoR4DUWiJwLBD8ACGwsXaHVodC6DYWWmXS0AY2XgOsX0VF/AfKLZ3BDfg6ipkrINHLY8fq5Ut/FF5BNAnzvBmR3a//b3tUspfeJMaCuHLh0pGvK1Y59po+TBJj4BBD5lHawR2IQCq3bUGhZFmMMZVeVOHq6FGfLiqGqOw8/Xi38u6ZAfi080ah33Q7XYDCfSNj4RYHvG6W9F1PoMLQFa9RAzS+6IdVyTbcN3xbwjQICp2hHn710BDi9C2htuNlGNknb+5owC3BwH9qarRyF1m0otIaX+hsqHC6vw8GyGvz0az1uqDrhiFaM513ERP4FRPDPYyLvAgL4tT3WVYOPSoE/LorG4qrjOFwTT0Cz61g4OzrAxd4WLnY2ENvbQmxvC5fuVztbCG36uNVW3QFcPaUNnotHgMqjgEqh28bGHvC7BwiYop1GRQO29rptOtuBs98DRRnA2e+0zwAAtAE39iFt72tMPB3X04NC6zYUWsNXe6cGJy424Kdf61DbpIKitQPK1g4oWjuAlgYEqMoRpjmHCP4FTOSfh4TX2GMbKmaLM8wfpzTBOK0ZjVMsGBeYDJpbLpZ1FtlA5moPXzd7BLjwESG4gLC205ApCuFUlw9+x20PChE6A/6xN0NKNgmwMeK+zeZ6oHi3NsDkp2/Od/QCwmdrA0w6wcjf1p1rSENr69at2LBhA+RyOSIiIrB582bExMT02n737t1YtWoVLl68iJCQEKxbtw6///3vueWMMaSlpeGjjz5CY2MjpkyZgm3btiEkJAQAcPHiRbz55ps4ePAg5HI5ZDIZ/vSnP+H111+HUGjYHxGFlnVTdaq7wqwTLfVV4FUXQFRbBKdrp+GuKIVdp7LHOi2wwy8IRkFnEE5rRkMJB0Tzy3Ev/wwieechuu342nXmhF9sxuOSUwSuecaA5zMBMjdnjHLTBp3UxQ42ggEOjCIvAU59DpzOAppveVycNFy7+xj+R8DRc2DbvkMMWWhlZWVh/vz52L59O2JjY/H+++9j9+7dKC8vh7e3d4/2ubm5uO+++5Ceno6ZM2ciIyMD69atQ0FBASZM0P4rs27dOqSnp+OTTz5BUFAQVq1aheLiYpSWlsLOzg4HDhxAVlYW5syZgzFjxqCkpASLFy/GvHnzsHHjRpP+QogV6j6TV12oHU+/ukC7q3d7z+k2SoEbim3Ckacei5yW0SjTjNJ7G1M3AZ8HqYsdfLt6a7e+dgdbv3cGqDuAczlA0WfArwe0AzsCAN8GCEkAIudoX43p0Rmjow24IQeUV7UnEpquakPU1V97fM57vMXOfA5ZaMXGxuKee+7Bli1bAAAajQZ+fn547rnn8Oqrr/Zo/8QTT6C5uRl79+7l5t17772IjIzE9u3bwRiDTCbDSy+9hJdffhkAoFAoIJFIsHPnTjz55JN669iwYQO2bduGCxcuGFQ3hdYIo1Frz/ZVF9wMstZGwC8GCJgMBEwFPEZrxykD0KHWQK5ow+XrrbjS2Ior11tx+XqL9r8bW1Hd2IoOdd9fFR4PkIntEejpgAAPRwR6dL86wt/dAfbC2wKtpQEo+UK7+1h9y8NL7N21Pa/Ip7QnHQy5OFej1u6ONlUDTXJA2fXK/dwVULeeJNDHxh6QRWoDrHty9TeshkEy9DtqVKS2t7cjPz8fqamp3Dw+n4/4+Hjk5eXpXScvLw8pKSk68xISEvDVV18BACoqKiCXyxEfH88tF4vFiI2NRV5eXq+hpVAo4O5OZ2NIL/gCQHKXdpr0p36b2wr48HN3gJ+7/rOSGg1D3Q2V/lDrmtfSruZC7si5az22IXER9QizAN/ZCJj4NJyV54FTGcCpLG1P6Pg/tZP3XdrwCpqmPXvZ3Tu6tafUJNdOzMBnBAhEgIsP4Nw1OXgA185qw12lBCrztFM3Ry/AN1obYKOitJeiWPAyFKNCq76+Hmq1GhKJRGe+RCJBWVmZ3nXkcrne9nK5nFvePa+3Nrc7d+4cNm/e3OeuoUqlgkql4n5WKnse8yDEUHw+DxIXO0hc7BAV0PMCWMYY6m+049K1Zly81qL7Wt8MZVsnapQq1ChVOF7Rs7fj6SREgMeDCBr1CKYKfkF047eQyQ+CX1sKfL/SsCJ5fMDRG3CWAi4y7atz1+utIWXvpr/npNEA184BV04CV/KByyeBmhLt7uOv32qnbh4h2rOn3b0xyYSh26W9jdVdtnvlyhU8+OCD+OMf/4jFixf32i49PR1vvPGGGSsjIxmPx4OXswheziJEB/bcA2hsab8lxLpDrRmXrrXgWnM76m9op/xLwL/hCWAeXPA4ZgqOYrbtfxHMu4pmW3e02UmgcZJC4OoLR49REEv8IXT11YaSo/fgjkfx+YBXqHaKfEo7r6NVew/q5a4gu3JSe+fDtbPa6dTn2nYCEeAzUbdH5hY0JLuVRn1CT09PCAQC1NTU6MyvqamBVCrVu45UKu2zffdrTU0NfHx8dNpERkbqrFddXY0HHngAkydPxocffthnrampqTq7pUqlEn5+fn1/QEKGiKuDEJEOQkT6ufZYpmzrQOW1Fi7EbvbSRMhQTkeGerq2YSsAJYDbLl3zcr6OUW5t8HOrgZ+7PUa5OcDPzQGj3Owhc7Xv+/q0/tjaa48D+t1ydUDzta4Ay7/ZK2u9Dlw+oZ262bt3BVg0EPGk9jYtEzAqtIRCIaKiopCTk4OkpCQA2gPxOTk5WLZsmd514uLikJOTgxdeeIGbl52djbi4OABAUFAQpFIpcnJyuJBSKpU4duwYnn32WW6dK1eu4IEHHkBUVBR27NgBPr/v/xEikQgiEQ2RQoY/FztbTPAVY4KvuMeylvZOVDa04NK1Fly+3oqqBu3r5estqGpoQXO7GnVNKtQ1qVBY2dhjfR4PkLrYcSE2yt0Bfm5dweY+wMs4HD2A0BnaCbh59rY7yC6f1F6X1toAnMvWTqOnWya0ACAlJQULFixAdHQ0YmJi8P7776O5uRkLFy4EAMyfPx++vr5IT08HACxfvhzTpk3Du+++i8TERGRmZuLkyZNcT4nH4+GFF17AW2+9hZCQEO6SB5lMxgXjlStXcP/99yMgIAAbN25EXd3N61x66+ERcidwENogTOqCMGnPs2mMMTS2dGjDrCvEuv+7O9jaOjS4qmjDVUUbjl/suX0bPg8+rnbwEdvDR2wHqdgOPi52kHb97ONqB09HEfj8PnbzeDztmViP0cDE2dp5ne1ATbH24P6VfO31aCZidGg98cQTqKurw+rVqyGXyxEZGYkDBw5wB9IrKyt1ekGTJ09GRkYGVq5ciddeew0hISH46quvuGu0AOCVV15Bc3MzlixZgsbGRkydOhUHDhyAnZ0dAG3P7Ny5czh37hxGjRqlU88IuaCfkB54PB7cHIVwcxQifFTPXlr3yYHuENOG2s3/vtJ1GUdVQyuqGnoOyd3NpuskBBdq4puh1v2zl5NIt8dmI7x5kB69H3se0Oem23gIGZk0GoaaJu21aVcVbZArul/buNfapjZoDEgIPg/wdr411HTDbZyPC5xEffeRhuQ6LULInYPP53XtFtr32qZTrUHdDdVtYaYbbjXKNnRqGOTKNsiVbSjS85Dzfz8Tp/es6kBQaBFCemUj4PcbbBoNQ32zSqeHphNuyjb4uPa+vtE1mWxLhJARic/nwdvZDt7Odpg4qv/2g36/oX8LQggxHQotQohVodAihFgVCi1CiFWh0CKEWBUKLUKIVaHQIoRYlRFznVb33Uo0GCAhw1P3d7O/OwtHTGg1NTUBAI2pRcgw19TUBLG45w3g3UbMDdMajQbV1dVwdnYGr5/RFLsHDKyqqrK6m6updsug2gePMYampibIZLI+x8sbMT0tPp/fY1ib/ri4uFjdH2A3qt0yqPbB6auH1Y0OxBNCrAqFFiHEqlBo6SESiZCWlmaVY8xT7ZZBtZvPiDkQTwi5M1BPixBiVSi0CCFWhUKLEGJVKLQIIVaFQus2W7duRWBgIOzs7BAbG4vjx49buqR+paen45577oGzszO8vb2RlJSE8vJyS5c1IGvXruUe4GsNrly5gj/96U/w8PCAvb09wsPDcfLkSUuX1S+1Wo1Vq1YhKCgI9vb2GD16NN58802reI4ohdYtsrKykJKSgrS0NBQUFCAiIgIJCQmora21dGl9+vHHH5GcnIyjR48iOzsbHR0dmDFjBpqbmy1dmlFOnDiBf/7zn5g4caKlSzHI9evXMWXKFNja2uLbb79FaWkp3n33Xbi5uVm6tH6tW7cO27Ztw5YtW3DmzBmsW7cO69evx+bNmy1dWv8Y4cTExLDk5GTuZ7VazWQyGUtPT7dgVcarra1lANiPP/5o6VIM1tTUxEJCQlh2djabNm0aW758uaVL6teKFSvY1KlTLV3GgCQmJrL/+Z//0Zn3+OOPs7lz51qoIsNRT6tLe3s78vPzER8fz83j8/mIj49HXl6eBSsznkKhAAC4u5vm4ZjmkJycjMTERJ3f/3D3zTffIDo6Gn/84x/h7e2NSZMm4aOPPrJ0WQaZPHkycnJy8OuvvwIATp06hZ9//hkPPfSQhSvr34i5Ybo/9fX1UKvVkEgkOvMlEgnKysosVJXxNBoNXnjhBUyZMgUTJkywdDkGyczMREFBAU6cOGHpUoxy4cIFbNu2DSkpKXjttddw4sQJPP/88xAKhViwYIGly+vTq6++CqVSibCwMAgEAqjVarz99tuYO3eupUvrF4XWHSY5ORklJSX4+eefLV2KQaqqqrB8+XJkZ2fDzs7O0uUYRaPRIDo6Gu+88w4AYNKkSSgpKcH27duHfWjt2rULn332GTIyMjB+/HgUFRXhhRdegEwmG/a10zGtLiqVigkEArZnzx6d+fPnz2ePPPKIZYoyUnJyMhs1ahS7cOGCpUsx2J49exgAJhAIuAkA4/F4TCAQsM7OTkuX2Ct/f3+2aNEinXkffPABk8lkFqrIcKNGjWJbtmzRmffmm2+ysWPHWqgiw9ExrS5CoRBRUVHIycnh5mk0GuTk5CAuLs6ClfWPMYZly5Zhz549OHjwIIKCgixdksGmT5+O4uJiFBUVcVN0dDTmzp2LoqIiCAQCS5fYqylTpvS4tOTXX39FQECAhSoyXEtLS4+B9gQCATQajYUqMoKlU3M4yczMZCKRiO3cuZOVlpayJUuWMFdXVyaXyy1dWp+effZZJhaL2eHDh9nVq1e5qaWlxdKlDYi1nD08fvw4s7GxYW+//TY7e/Ys++yzz5iDgwP717/+ZenS+rVgwQLm6+vL9u7dyyoqKtiXX37JPD092SuvvGLp0vpFoXWbzZs3M39/fyYUCllMTAw7evSopUvqFwC9044dOyxd2oBYS2gxxth//vMfNmHCBCYSiVhYWBj78MMPLV2SQZRKJVu+fDnz9/dndnZ2LDg4mL3++utMpVJZurR+0dA0hBCrQse0CCFWhUKLEGJVKLQIIVaFQosQYlUotAghVoVCixBiVSi0CCFWhUKLEGJVKLQIIVaFQosQYlUotAghVoVCixBiVf4/HJmFWULTX7AAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:45:02.039788Z", + "iopub.status.busy": "2024-03-24T03:45:02.039354Z", + "iopub.status.idle": "2024-03-24T03:49:10.158622Z", + "shell.execute_reply": "2024-03-24T03:49:10.157371Z" + }, + "papermill": { + "duration": 248.142335, + "end_time": "2024-03-24T03:49:10.161745", + "exception": false, + "start_time": "2024-03-24T03:45:02.019410", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:49:10.202994Z", + "iopub.status.busy": "2024-03-24T03:49:10.202021Z", + "iopub.status.idle": "2024-03-24T03:49:10.228075Z", + "shell.execute_reply": "2024-03-24T03:49:10.226971Z" + }, + "papermill": { + "duration": 0.049784, + "end_time": "2024-03-24T03:49:10.230545", + "exception": false, + "start_time": "2024-03-24T03:49:10.180761", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
realtabformer0.012350.0247040.0016762.6065610.2429557.0997750.4398750.0000037.1857560.0304650.0934370.0409450.1064770.0021519.792317
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "realtabformer 0.01235 0.024704 0.001676 2.606561 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss pred_duration \\\n", + "realtabformer 0.242955 7.099775 0.439875 0.000003 7.185756 \n", + "\n", + " pred_mae pred_mape pred_rmse pred_std std_loss \\\n", + "realtabformer 0.030465 0.093437 0.040945 0.106477 0.002151 \n", + "\n", + " total_duration \n", + "realtabformer 9.792317 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:49:10.271759Z", + "iopub.status.busy": "2024-03-24T03:49:10.271381Z", + "iopub.status.idle": "2024-03-24T03:49:10.664282Z", + "shell.execute_reply": "2024-03-24T03:49:10.663177Z" + }, + "papermill": { + "duration": 0.415434, + "end_time": "2024-03-24T03:49:10.666741", + "exception": false, + "start_time": "2024-03-24T03:49:10.251307", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:49:10.709397Z", + "iopub.status.busy": "2024-03-24T03:49:10.708310Z", + "iopub.status.idle": "2024-03-24T03:53:26.541630Z", + "shell.execute_reply": "2024-03-24T03:53:26.540526Z" + }, + "papermill": { + "duration": 255.858046, + "end_time": "2024-03-24T03:53:26.544418", + "exception": false, + "start_time": "2024-03-24T03:49:10.686372", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/realtabformer/all inf False\n", + "Caching in ../../../../contraceptive/_cache_bs_test/realtabformer/all inf False\n", + "Caching in ../../../../contraceptive/_cache_synth_test/realtabformer/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:53:26.586968Z", + "iopub.status.busy": "2024-03-24T03:53:26.586522Z", + "iopub.status.idle": "2024-03-24T03:53:26.619429Z", + "shell.execute_reply": "2024-03-24T03:53:26.618313Z" + }, + "papermill": { + "duration": 0.058489, + "end_time": "2024-03-24T03:53:26.622976", + "exception": false, + "start_time": "2024-03-24T03:53:26.564487", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:53:26.666784Z", + "iopub.status.busy": "2024-03-24T03:53:26.665830Z", + "iopub.status.idle": "2024-03-24T03:53:26.673429Z", + "shell.execute_reply": "2024-03-24T03:53:26.672285Z" + }, + "papermill": { + "duration": 0.029344, + "end_time": "2024-03-24T03:53:26.675476", + "exception": false, + "start_time": "2024-03-24T03:53:26.646132", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'realtabformer': 0.38464616315705435}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:53:26.716999Z", + "iopub.status.busy": "2024-03-24T03:53:26.716148Z", + "iopub.status.idle": "2024-03-24T03:53:27.259231Z", + "shell.execute_reply": "2024-03-24T03:53:27.258123Z" + }, + "papermill": { + "duration": 0.567621, + "end_time": "2024-03-24T03:53:27.261650", + "exception": false, + "start_time": "2024-03-24T03:53:26.694029", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABGfUlEQVR4nO3deXxU5b348c+Zfcks2RfIxq4iiCiIG7iDWqXWBbUFtIj16rWW8vopfd2i1FuxvZVKbcutVhOoC7dal7ZWxQ2tGyhubLJJQgiBkD2zb+f3x5kMDEkgy8ycZPK8X68Q5swz5zxnJvnm2R9JlmUZQRAEFWnUzoAgCIIIRIIgqE4EIkEQVCcCkSAIqhOBSBAE1YlAJAiC6kQgEgRBdSIQCYKgOhGIBEFQnQhEQrceeOABJElK6DkrKyuRJInPPvusR+n/53/+hxEjRqDVajnttNMSmhdh4BCBSOiVhx56iJdffjkl11q3bh3/7//9P8455xwqKip46KGHUnJdIfV0amdAGFweeughrr32WmbPnp30a73zzjtoNBqefPJJDAZD0q8nqEeUiNKE2+1WOwsJV19fj9lsTlgQkmUZr9ebkHP1h8fjUTsLA44IRINQR9vNtm3buOmmm8jMzOTcc88F4Omnn2by5MmYzWaysrKYM2cONTU1ca//97//zXXXXUdJSQlGo5Hi4mJ+8pOfnPCXVJIk3G43q1evRpIkJEli/vz5AFRXV/Mf//EfjB07FrPZTHZ2Ntdddx1VVVVdnsvj8XD77beTnZ2N3W5n7ty5NDc3x12roqICt9sdu1ZlZSUAoVCIBx98kJEjR2I0GikrK+NnP/sZfr8/7hplZWVceeWVvPHGG5xxxhmYzWb+9Kc/sX79eiRJ4q9//SvLli1j2LBh2Gw2rr32WlpbW/H7/dxzzz3k5eWRkZHBLbfc0uncPX2vZ8yYwfjx49m0aRPnn38+FouFn/3sZ8d9n4ciUTUbxK677jpGjx7NQw89hCzL/PKXv+TnP/85119/PQsWLODw4cM89thjnH/++XzxxRc4nU4Ann/+eTweD3fccQfZ2dls3LiRxx57jP379/P88893e72//OUvLFiwgClTprBw4UIARo4cCcCnn37KRx99xJw5cxg+fDhVVVWsWrWKGTNmsG3bNiwWS9y57rrrLpxOJw888AA7duxg1apVVFdXx4LEX/7yFx5//HE2btzIn//8ZwDOPvtsABYsWMDq1au59tpr+elPf8qGDRtYvnw527dv56WXXoq7zo4dO7jxxhu5/fbbue222xg7dmzsueXLl2M2m7nvvvvYvXs3jz32GHq9Ho1GQ3NzMw888ACffPIJlZWVlJeXs3Tp0thre/peAzQ2NjJr1izmzJnD97//ffLz83v5SQ8BsjDo3H///TIg33jjjbFjVVVVslarlX/5y1/Gpd28ebOs0+nijns8nk7nXL58uSxJklxdXd3pOkezWq3yvHnzOr2+q3N+/PHHMiCvWbMmdqyiokIG5MmTJ8uBQCB2/Ne//rUMyK+88krs2Lx582Sr1Rp3zi+//FIG5AULFsQdX7x4sQzI77zzTuxYaWmpDMivv/56XNp3331XBuTx48fH5eHGG2+UJUmSZ82aFZd+2rRpcmlpaexxb97r6dOny4D8v//7v53eH+EIUTUbxH70ox/F/v/iiy8SiUS4/vrraWhoiH0VFBQwevRo3n333Vhas9kc+7/b7aahoYGzzz4bWZb54osv+pSXo88ZDAZpbGxk1KhROJ1OPv/8807pFy5ciF6vjz2+44470Ol0/Otf/zrudTqeX7RoUdzxn/70pwC8+uqrccfLy8u57LLLujzX3Llz4/IwdepUZFnm1ltvjUs3depUampqCIVCQO/eawCj0cgtt9xy3Psa6kTVbBArLy+P/X/Xrl3Isszo0aO7THv0L9y+fftYunQpf//73+PaZQBaW1v7lBev18vy5cupqKigtrYW+aiFP7s657H5zMjIoLCwsNs2pQ7V1dVoNBpGjRoVd7ygoACn00l1dXXc8aPfo2OVlJTEPXY4HAAUFxd3Oh6JRGhtbSU7O7tX7zXAsGHDRK/fCYhANIgdXQqJRCJIksRrr72GVqvtlDYjIwOAcDjMJZdcQlNTE/feey/jxo3DarVSW1vL/PnziUQifcrLf/7nf1JRUcE999zDtGnTcDgcSJLEnDlz+nzO4+npQMuj36NjdfU+He94R3Dt6XvdkzwIChGI0sTIkSORZZny8nLGjBnTbbrNmzezc+dOVq9ezdy5c2PH33zzzR5dp7sA8MILLzBv3jweeeSR2DGfz0dLS0uX6Xft2sUFF1wQe+xyuairq+Pyyy8/7vVLS0uJRCLs2rWLk046KXb80KFDtLS0UFpa2qP76I+evtdCz4k2ojRxzTXXoNVqWbZsWVy1CJS/5I2NjcCRv/ZHp5FlmZUrV/boOlartcvgotVqO133scceIxwOd3mexx9/nGAwGHu8atUqQqEQs2bNOu71OwLVo48+Gnd8xYoVAFxxxRUnuoV+6+l7LfScKBGliZEjR/Lf//3fLFmyhKqqKmbPno3NZmPv3r289NJLLFy4kMWLFzNu3DhGjhzJ4sWLqa2txW6387e//a1TW1F3Jk+ezFtvvcWKFSsoKiqivLycqVOncuWVV/KXv/wFh8PBySefzMcff8xbb71FdnZ2l+cJBAJcdNFFXH/99ezYsYM//vGPnHvuuVx11VXHvf7EiROZN28ejz/+OC0tLUyfPp2NGzeyevVqZs+eHVfKSpaevtdCz4lAlEbuu+8+xowZw29/+1uWLVsGKA2vl156aewXXK/X849//IO7776b5cuXYzKZ+O53v8tdd93FxIkTT3iNFStWsHDhQv7rv/4Lr9fLvHnzmDp1KitXrkSr1fLMM8/g8/k455xzeOutt7rtsfr973/PM888w9KlSwkGg9x444387ne/61Hbz5///GdGjBhBZWUlL730EgUFBSxZsoT777+/F+9W//TkvRZ6TpKPLVsKgiCkmGgjEgRBdSIQCYKgOhGIBEFQnQhEgiCoTgQiQRBUJwKRIAiqG9TjiCKRCAcOHMBmsyV8kXdBEPpPlmXa29spKipCo+m+3DOoA9GBAwc6zZQWBGHgqampYfjw4d0+r3ogqq2t5d577+W1117D4/EwatQoKioqOOOMM074WpvNBig3abfbk51VQRB6qa2tjeLi4tjvandUDUTNzc2cc845XHDBBbz22mvk5uaya9cuMjMze/T6juqY3W4XgUgQBrATNZ2oGoh+9atfUVxcTEVFRezY8RayEgQhPanaa/b3v/+dM844g+uuu468vDwmTZrEE0880W16v99PW1tb3JcgCIOfqoHo22+/ZdWqVYwePZo33niDO+64g7vvvpvVq1d3mX758uU4HI7Yl2ioFoT0oOrse4PBwBlnnMFHH30UO3b33Xfz6aef8vHHH3dK7/f74/aX6mgIa21t7baNSJZlQqFQtwt0Cd3TarXodDoxNELos7a2NhwOx3F/R0HlNqLCwkJOPvnkuGMnnXQSf/vb37pMbzQaMRqNPT5/IBCgrq5O7KzZDxaLhcLCQrH4u5BUqgaic845hx07dsQd27lzZ0LWHY5EIuzduxetVktRUREGg0H8Ze8FWZYJBAIcPnyYvXv3Mnr06OMOSBOE/lA1EP3kJz/h7LPP5qGHHuL6669n48aNPP744zz++OP9PncgECASiVBcXNxpl1GhZ8xmM3q9nurqagKBACaTSe0sJZwsy2yqbub0kkw0GvGHSi2q/ok788wzeemll3juuecYP348Dz74II8++ig333xzwq4h/or3T7q/f4fa/Px7VwNbDvRtPzchMVQfWX3llVdy5ZVXqp0NYYg61OYD4KuaFiYMd6qbmSEsvf/cCUI36tt9BMMRxhbY0EgSDa4Ard7giV8oJIUIRENcWVlZpz3C0l0kIvPCpv2sWr8HTyBMgUPpia1pEr2rahGBSBhy6tv9+IMRdFoJp1lPgUPZEvqwy3+CVwrJIgJRGggEAmpnYVDpaBcqcpjRaCRyM5QS0eF2EYjUMiQDUSAU6fYrFI70OG2wB2n7YsaMGdx1113cddddOBwOcnJy+PnPfx7b3risrIwHH3yQuXPnYrfbWbhwIQAffPAB5513HmazmeLiYu6++27cbnfsvPX19XznO9/BbDZTXl7OM88806f8DXYdgSjPpgSgXNuRQCS2+VOH6r1mavjDu7u7fa48x8rsScNijx9/fw/BcNc/nMMzzVx3xpH5bk99uBdvIH4qyU8uGdOnPK5evZof/vCHbNy4kc8++4yFCxdSUlLCbbfdBsBvfvMbli5dGtvddM+ePcycOZP//u//5qmnnuLw4cOxYNaxusH8+fM5cOAA7777Lnq9nrvvvpv6+vo+5W8wq4+WfPLsSgDKshqYWOzAYdYTjsjotGI8UaoNyUA0GBQXF/Pb3/4WSZIYO3Ysmzdv5re//W0sEF144YX89Kc/jaVfsGABN998M/fccw8Ao0eP5ne/+x3Tp09n1apV7Nu3j9dee42NGzdy5plnAvDkk09y0kknpfze1BQKR2h0KVXZPLsyQFOrkbhwXL6a2RryhmQguvOCUd0+d+zg2oXnj+w27bEzRm49J3FrKZ111llxU1KmTZvGI488Epu8e+wKll999RVff/11XHVLluXYVJedO3ei0+mYPHly7Plx48bhdDoTlufBoMUbJCLLGHQabMYh+eM/IA3JT8Kg63nTWLLS9pfVao177HK5uP3227n77rs7pS0pKWHnzp2pytqAZtZrOX9MLuGIHBfoA6EILZ4AGo1ETkbPJ1YLiTEkA9FgsGHDhrjHn3zyCaNHj0ar1XaZ/vTTT2fbtm2MGtV1aW/cuHGEQiE2bdoUq5rt2LGDlpaWhOZ7oLMadUwu7bwU8ebaFt7f2cCYfBtXTChUIWdD25DsNRsM9u3bx6JFi9ixYwfPPfccjz32GD/+8Y+7TX/vvffy0Ucfcdddd/Hll1+ya9cuXnnlFe666y4Axo4dy8yZM7n99tvZsGEDmzZtYsGCBZjN5lTd0oBmN+kBcPnF6Go1iEA0QM2dOxev18uUKVO48847+fGPfxzrpu/KhAkTeO+999i5cyfnnXcekyZNYunSpRQVFcXSVFRUUFRUxPTp07nmmmtYuHAheXl5qbidAaO60U19u49wJL4nNMOkVA7afSE1sjXkiarZAKXX63n00UdZtWpVp+eqqqq6fM2ZZ57JunXruj1nQUEB//znP+OO/eAHP+hXPgcTWZZ5dXMd/mCEH0wrjWsLskYbrt3+MLIsi7WrUkyUiIQhwxMI4w9GkCRwmvVxz1kNOiQJIrKMJyCWFU41EYiEIaMlOrveZtKj08b/6Gs1ElaDUipy+UX1LNVE1WwAWr9+vdpZSEtt0UBkN3X9Y2816nD5Q7T7QuSL/TpTSgQiYcjoaIi2H1Mt6zBhuANfMINsq9goINVEIBKGjLZY1azrH/vxwxypzI5wFNFGJAwZ7f6OqlnXJSJBPaJEJAwZp5dkUpJlocjZ9SBOfyhMq0cJVh0TYoXUEIFIGDJKs62UZlu7fb6mycM/vqqj0GFizpSSFOZMEFUzQYgyR7vvxTii1BOBSBgS3P4Qu+tdNBxnXWqLXplQ7A2KQJRqIhAJQ0Jdq49/fHWAN7cd6jaN2aAEoq6WARaSSwQiYUho8524x8yo06CNrownSkWppWogeuCBB5AkKe5r3LhxybugLEMokPqvXizIvmbNGrKzs/H746sQs2fPHlITVBOtYzBjd2OIACRJwhItFR279riQXKr3mp1yyim89dZbscc6XRKzFA7Cvx9J3vm7c95PQdez0brXXXcdd999N3//+9+57rrrAGX3jVdfffW4M+uF42v3HX8wYwezQUu7LyQarFNM9UCk0+koKChQOxsDhtls5qabbqKioiIWiJ5++mlKSkqYMWOGupkbxNq8x5/e0WHCMCf+/DCZFjHoMZVUD0S7du2iqKgIk8nEtGnTWL58OSUlXY/h8Pv9cVWWtra23l1Mq1dKJ6mm7d0P9W233caZZ55JbW0tw4YNo7Kykvnz54s1cvqhpyWiU4eLaR5qUDUQTZ06lcrKSsaOHUtdXR3Lli3jvPPOY8uWLdhstk7ply9fzrJly/p+QUnqcRVJTZMmTWLixImsWbOGSy+9lK1bt/Lqq6+qna1BKxiOxKpaYnrHwCTJA2hry5aWFkpLS1mxYgU//OEPOz3fVYmouLiY1tZW7Pb4dRt8Ph979+6lvLwck2nwDddftWoVjz76KJdccgm7du3ijTfeUCUfg/19BGUvs28b3LT7Qpxe4jxuydIXDNPqDaLTSGSL3Tz6ra2tDYfD0eXv6NEGVPe90+lkzJgx7N7d9U6sRqMRu90e95WubrrpJvbv388TTzzBrbfeqnZ2BjWdVsOYfBuTSzNPWL3dcbCdZzfs46M9jSnKnQADLBC5XC727NlDYaHYzsXhcPC9732PjIwMZs+erXZ2hgxTdHS1T4wjSilVA9HixYt57733qKqq4qOPPuK73/0uWq2WG2+8Uc1sDRi1tbXcfPPNGI2iitAfB1q87K530eo98VZBJr3yK+ELiZHVqaRqY/X+/fu58cYbaWxsJDc3l3PPPZdPPvmE3NxcNbOluubmZtavX8/69ev54x//qHZ2Br2v97eyva6Nc0fncGZZ1nHTdpSI/KJElFKqBqK1a9eqefkBa9KkSTQ3N/OrX/2KsWPHqp2dQa+th133ACadqJqpQfVxREJn3e1bJvRNbK3qHnTdG6NVs2BYJhiOoNcOqGbUtCXeZSGtRSIyrh7MM+tg1GnQRHvW/KKdKGXSvkQ0gIZJDUqD/f1zBUJEZBmNdGTfsuORJImpI7LQaSR0GjGSPVXSNhDp9Uox3OPxYDZ3vUaxcGIejwc48n4ONkfPutf0MLCcNSI7mVkSupC2gUir1eJ0OqmvrwfAYrGIuVq9IMsyHo+H+vp6nE4nWq1W7Sz1SacthHxt8O274GuFYWdA/skq5k7okLaBCIjN6u8IRkLvOZ3OQb06wvBMM1dMKFQanUN++Oo58DQpT7bWKhOSc0bHvcblD+H2h7AYtNjE3LSUSOtAJEkShYWF5OXlEQyeeDCbEE+v1w/aklAHm0l/JJh8u14JQkYb2Ivg8A7YtQ6yRoDmyH1+sKuB7XVtnDc6hzNOMO5ISIy0DkQdtFrtoP+FEvop6IPaTcr/R1+qBJ/WPypVtfptUHBqLGlsdHVQ9Jqliui+F9LatgNt7K53Eajboizba81RqmJaHQw7XUl0aGvca8R8s9QTgUhIW7Is8+6Oev7x1QGCBzYrBwsmKOtSAeRFG6qbqyHgjr3OqFN+LcQ4otQRgUhIW/5QhEAogiHkwuSpUwJQ3klHEliywJYPcgQa98QOG6PTPPwhUSJKFRGIhLTV0XWfH65DK0lgKwDTMWtYZY1QvrdUxw51TPMQJaLUEYFISFtt0cGMBeE65UBH0Dmas1T53lwd2/YpVjUTbUQpMyR6zYShqd0XBDlCdvAAoIfM8s6JHMOVrnt/O3ibwZKFw6xnSnkWVqP49UgV8U4LaavNF8ISbMJMAHRWZezQsbR6yMiHtgPQXgeWLGwmPeeMykl9hocwUTUT0la7L4jNf0ipatmHxQ1ajNMRoNrqUpc5IY4oEQlp66wR2QTcAZztBiUQdccWncLSfiQQtXqC+EJhsq0GdGJNoqQT77CQtnIyjBRJzZj12q6rZR1s0c0aXAchovSUPbOxmmc37Is1eAvJJQKRkL4CbqUBGo5fIrJkKxtvhkPgaQDEWKJUE4FISEtuf4htO7bT7IlO69AfZ3NISQJrXvSFhwEwRLvwA2IsUUqIQCSkpQaXn63f7GRfk+f41bIO1mgvmbujRCQGNaaSCERCWmr3hbAGG5WAkpF/4hdYo1tYRUtERwY1ikCUCiIQCWmpzRvEEugIRHknfoElujysR9lqWrQRpZYIREJacrnbMYZdGHTaI+0/x9NRIvI2Qzgk5pulmBhHJKSlQOshAHTWzOM3VHcwWJV0QR94GinNsmLQahieKTZeSIUBUyJ6+OGHkSSJe+65R+2sCGkg3HYQAIOzh+ttSxJYOhqsDzMiN4NzRuVQmm1NUg6Fow2IQPTpp5/ypz/9iQkTJqidFSENhCMyGreyYYLJWdjzF3ZUz6LtRELqqB6IXC4XN998M0888QSZmZlqZ0dIAxJwfmGE0XkZmDJ7EYjM0Z8/bzOhcIQWT4AmdyApeRTiqR6I7rzzTq644gouvvjiE6b1+/20tbXFfQnCsTREcMht5GQYkXrSdd/BEt2xw9vMgRYfFR9W8erXB5KTSSGOqo3Va9eu5fPPP+fTTz/tUfrly5ezbNmyJOdKGPQ8TRAJK9M2TI6ev+6oEpFRp6xrLXrNUkO1ElFNTQ0//vGPeeaZZzCZetCrASxZsoTW1tbYV01NTZJzKQxGB+r2U9fqpU3jOLJQfk+YnEr6kB+j7ANEIEoV1UpEmzZtor6+ntNPPz12LBwO8/777/P73/8ev9/faS8yo9GI0WhMdVaFQaa2dj+hRg/ZTiv2Eyc/QqtTNl/0tWEMKdX+QChCJCKj0YjtypNJtUB00UUXsXnz5rhjt9xyC+PGjePee+8VGyIKfRZqV6Zp6O25vX+xORN8bRgCrYBSUg+EI5i6W1RNSAjVApHNZmP8+PFxx6xWK9nZ2Z2OC0JvyJ5GJMDs7MGI6mOZM6G5Gq2/Bb22kGBYxh+MxDZdFJJD9V4zQUgkORKJjQOyZvaix6xDXIO1mG+WKgNqisf69evVzoIwyLnampEiIZC0WO19WAD/qEB06nAH4YiMySBKQ8k2oAKRIPSXp0WZYyabnWj60s5ojo4l8jRx1uTsBOZMOB5RNRPSirdVmdoRW+istzrGHYX8ygRYISVEiUhIK8MMHrKH2QkMK+vbCXQGMFgg4MHvbsJryMGg02AxiF+VZBIlIiGt6H3N2Ix6snN7McfsWNFS0aZvqqj4sIqv97cmKHdCd0QgEtJLdBeO2IqLfWFUhkFaZDcgRlengihvCukj6GX/oQa0GolMg5OeTRzqQrREZA67APAHRfd9sokSkZA2Iq4Galu87GjREOjP31iTEwBjNBAFwqJElGwiEAlpw91aT0SGgMFJRn8al6MlImOoHRA7eaRCnwLRt99+m+h8CEK/eZuVMUSSNbt/k1SjgcgQDUSiRJR8fQpEo0aN4oILLuDpp5/G5xNjLYSBwd+mTHbV2fow2fVo0UCkj/jRRvyijSgF+hSIPv/8cyZMmMCiRYsoKCjg9ttvZ+PGjYnOmyD0SjA6697Yl1n3R9MZQG/GqNMwKU/DyUW9WFxN6JM+BaLTTjuNlStXcuDAAZ566inq6uo499xzGT9+PCtWrODw4cOJzqcgHF84RMjdDPRx1v2xTA6MOi3nFRuYUp7V//MJx9WvxmqdTsc111zD888/z69+9St2797N4sWLKS4uZu7cudTV1SUqn4JwfN4m/MEwYY0Bm83Z//N1TPXwicGMqdCvQPTZZ5/xH//xHxQWFrJixQoWL17Mnj17ePPNNzlw4ABXX311ovIpCMfnaWRcgY3xY0YyPMvS//NFA1HA1USrJ0g4Ivf/nEK3+tTHuWLFCioqKtixYweXX345a9as4fLLL0ejUeJaeXk5lZWVlJWVJTKvgtA9TyN6rQZndgEkYhGz6Fiid7/azbaaMuafXUam1dD/8wpd6lMgWrVqFbfeeivz58+nsLDrOT15eXk8+eST/cqcIPRYx6aI/ZnacbRoicgaiY6uFtM8kqpPgejNN9+kpKQkVgLqIMsyNTU1lJSUYDAYmDdvXkIyKQgn0tpQR3OjGwpNlCXihKbofLNoIAqIQJRUfWojGjlyJA0NDZ2ONzU1UV5e3u9MCUKvyDKulnrqWn3sdidol5eO0dUE0ESCYrnYJOtTIJLlrhvuXC5Xj/coE4SE8bXi9/uRJQ0We4KqZjoj6IxoNRLGsEtUzZKsV1WzRYsWASBJEkuXLsViOdI7EQ6H2bBhA6eddlpCMygIJ+RpxB+M4NU5yLckcN87kx2tpgFjqF0EoiTrVSD64osvAKVEtHnzZgyGI70IBoOBiRMnsnjx4sTmUBBOxNOILxTGp3fiMOsTd16TE51GwhhyiapZkvUqEL377ruAshHiypUrsdt7tY+mICRF2NWglIjMThyWBAYiox2bSc84m4zDLpockqlPvWYVFRWJzocg9Jmv9RAyEDRlYTMmcK0/k52cDCM5eRrIzUjceYVOevypXXPNNVRWVmK327nmmmuOm/bFF1/sd8YEoac6Zt3rbblIUgL3qI9N82hL3DmFLvU4EDkcjtiH7HCI2cjCABFwk6kPMbk0izGTxiX23EY7siwTcjfj94fISGRpS4jT43f26OpYoqpmq1atYtWqVVRVVQFwyimnsHTpUmbNmpWQ8wtDgKcRCQlDhpNsR4KrTyYHLd4g3+ytYa9+HzedNSKx5xdi+jSOyOv14vF4Yo+rq6t59NFHWbduXa/OM3z4cB5++GE2bdrEZ599xoUXXsjVV1/N1q1b+5ItYShyJ2DXju4YrGh1OkAm4m1P/PmFmD6VNa+++mquueYafvSjH9HS0sKUKVMwGAw0NDSwYsUK7rjjjh6d5zvf+U7c41/+8pesWrWKTz75hFNOOaVTer/fj9/vjz1uaxN19yHP08Sewy48Gh0j/CGsiaw+SRIaswNoRvKLn7Vk6vMKjeeddx4AL7zwAgUFBVRXV7NmzRp+97vf9Skj4XCYtWvX4na7mTZtWpdpli9fjsPhiH0VFxf36VpC+gi76jnc7ufrJh2JbKfuoDU7lf/42rqdUSD0X58CkcfjwWazAbBu3TquueYaNBoNZ511FtXV1b061+bNm8nIyMBoNPKjH/2Il156iZNPPrnLtEuWLKG1tTX2VVNT05fsC2nE21qPDITNWZgTsfzHMXQWJwD6UDvBsAhEydLnxfNffvllampqeOONN7j00ksBqK+v7/Ugx7Fjx/Lll1+yYcMG7rjjDubNm8e2bdu6TGs0GrHb7XFfwhAW8hNwKSsoGu15ie26j9JZ7EgoWwuJ3TySp0+BaOnSpSxevJiysjKmTp0aq0qtW7eOSZMm9epcBoOBUaNGMXnyZJYvX87EiRNZuXJlX7IlDDWeRnzBMEGtGXu0hJ5okskZnfjqFrt5JFGfWvauvfZazj33XOrq6pg4cWLs+EUXXcR3v/vdfmUoEonENUgLQrc8jXiDYby6LJyWJK2eaLKTYzNi1wfR68R+pMnS5y6GgoICCgoK4o5NmTKlV+dYsmQJs2bNoqSkhPb2dp599lnWr1/PG2+80ddsCUOJuwFfMIxXn0mpNYFzzI5mclCebVXqDmJAY9L06Z11u908/PDDvP3229TX1xOJxNede7oTbH19fWy3D4fDwYQJE3jjjTe45JJL+pItYajxNBIIK5NdneYklYiMdpAkiIQg6AGDNTnXGeL6FIgWLFjAe++9xw9+8AMKCwv73Ego1rQW+sXTyGnFTkafdCpmWwLXITqaRktEbyHsbSfiasKYJQJRMvQpEL322mu8+uqrnHPOOYnOjyD0TDgI3mYkJDIyC6A/e92fwNYmCVd9M7acA4zPEmPXkqFPrW+ZmZlkZYndLwUVuRtAlkFvBkNyl+iQogvpR7xis8Vk6VMgevDBB1m6dGncfDNBSCl3PXWtXr5oMrCnwZ3US0nR5UBkEYiSpk9Vs0ceeYQ9e/aQn59PWVkZen18j8Xnn3+ekMwJQrdch2nzhdirsVDuDSb1Usp8M5D9IhAlS58C0ezZsxOcDUHoJfdhvIEwHls2WUnegbVjvpnkE4EoWfoUiO6///5E50MQek6WCbcfxBcM49FnJT8QReebSX6xFEiy9HmoaEtLC3/+859ZsmQJTU1NgFIlq62tTVjmBKFLATd+jwsZibA5K+krJ+qs0apZ0AuhQFKvNVT16RP8+uuvufjii3E4HFRVVXHbbbeRlZXFiy++yL59+1izZk2i8ykIR7jr8QbD+HR2Mm3WpEx2PZrNmkGmw4ZFCoGvFTJyk3q9oahPJaJFixYxf/58du3aFbez6+WXX87777+fsMwJQpdch/EEwngMya+WAWRZDYwrHU5JlgXEAmlJ0adA9Omnn3L77bd3Oj5s2DAOHjzY70wJwnG564nIMj5DNtkZyQ9EwFE7eogG62ToU9XMaDR2uUzrzp07yc0VxVYhydoPUpplpXj8GUSynCm5ZMRgIxKJoPG29r1hVehWn97Tq666il/84hcEg8r4DUmS2LdvH/feey/f+973EppBQYgT8oOnEQCNvQCdNjVh4YVt7Xxa1YzP1ZyS6w01ffoUH3nkEVwuF7m5uXi9XqZPn86oUaOw2Wz88pe/THQeBeGI9oPK1A6THYwp3H3VqEzzCHtaUnfNIaRPVTOHw8Gbb77Jhx9+yFdffYXL5eL000/n4osvTnT+BCFe+0Ga3H62NGvR72lk2sgkbCPUBTkaiCKijSgpeh2IIpEIlZWVvPjii1RVVSFJEuXl5RQUFCDLctK7UoUhrr0Otz/MQTmTTH8oZZeVTE4Awt52iIRBk/iF+oeyXlXNZFnmqquuYsGCBdTW1nLqqadyyimnUF1dzfz58/u9TKwgnFD7QdyBEC5DLrnJWoOoC1qjlYikJRyJiC78JOhViaiyspL333+ft99+mwsuuCDuuXfeeYfZs2ezZs0a5s6dm9BMCgIAQS94m/EEwrgzc8hJVdc9YNBrCWgzCEdC4GsDc2bKrj0U9KpE9Nxzz/Gzn/2sUxACuPDCC7nvvvt45plnEpY5QYjTWksoEqFFshPSmsjJSF2JyKjT4NdlEI7IokSUBL0KRF9//TUzZ87s9vlZs2bx1Vdf9TtTgtCl1hrc/jDthnzsZj2mJGyo2J1Ch5mcnFzMeo0Y1JgEvaqaNTU1kZ+f3+3z+fn5NDeLcRaDWjgIe96F9gOQexIUTyEpezn3RWsNnkCIdmNBStuHAE4d7oBQGeytUapmQkL1KhCFw2F0uu5fotVqCYVS15MhJJgsw7ZXoGGX8ritTtm9omwArE0eDkL7QTSShC6zhLwUByIgNpZIlIgSr1eBSJZl5s+fj9HY9Q+B2BhxcGup3cHhbZ8jS1qKxk4ho2kLVH8IeSeBReU1ytvrIBImPzeHOdMmoMYu9BGDDTkioxVtRAnXq0A0b968E6YRPWaDkzcQ5vOP3sLsClBnG8/61pOZa3dhaquCfZ/AuMvVzWBzlfLdUQySRKori7vrXazb1Mj5TW2MH65XSo8DpcqaBnoViCoqKpKVD0Fln+2swtxejUmnwZ87Ebc/zEbzWM6nCg5thRHT1d1csHEPYVlGyixXZdKpUachoLUSiqBUVwPu1E4xSXNiIrGALMu49m8FZIpHjGHG6eMA+KIlA58pR/nFq/9GvQwG3NB+kENtPv68FT7c3ZDyLBh0GmRJi09jVg6I6llCqRqIli9fzplnnonNZiMvL4/Zs2ezY8cONbM0JEmSxMxCD+OH2ckuHU+hw8zEYgcXnZyPruhUJVH9NvUy2LRX+SZl4saMPkUz7o9m1CnX9EjRUqHoOUsoVQPRe++9x5133sknn3zCm2++STAY5NJLL8XtTu4+VcIxwkGklhpsRj1S1kgALhyXz/hhDnT5JyttIa371estatwNQA3K0JECu+l4qZPCqFPGLHkkK7Isi56zBEvuquMn8Prrr8c9rqysJC8vj02bNnH++ed3Su/3++N65rpanE3oHVmWoXU/UiQERhtYc+ITmOzgGA4tNdCwG4ZPTm0Gw0Fo3E0gHGG/thhJgjx76rvuDdESkV+XQVhuQieqZgk1oNqIWluVvzLdbWe9fPlyHA5H7Ku4WOxD3l91rT7+8e9P2dvgUgLOUT1BLn+IL/Y1822kQDkQLZmkVONuCAdpiVhwGfLIyTCmdER1B61GQq+V8GszCIVFiSjRBkwgikQi3HPPPZxzzjmMHz++yzRLliyhtbU19lVTU5PiXKafmiYPelcdwbCsBKJjnlu/4zCfuaNr/rTsU0ooqXRoKwC1hlKQJIZnmlN7/aOMzM1gWEG+EqtFIEooVatmR7vzzjvZsmULH3zwQbdpjEZjt4Mphb6pbfaQG6jHnqUH+7C458qyrUgS1Pos+M02jKF2aK6GnFGpyZyvNVYK2ymVATA805Kaa3dh1qmF4NLBp++JXrMEGxAlorvuuot//vOfvPvuuwwfPvzELxASQpZlWhsPoYv4ybCYICMv7nmzQUuezQSSxEF9kXIwldWz2s9BlpGdJYwoLaMsx8Iwp3olIkBpMwMI+pT1s4WEUDUQybLMXXfdxUsvvcQ777xDeXm5mtkZctq8IXSeQ2gksGQVdbnqYFm2UgLZGylUDjTtUUYVJ1vAAwc+B0AafiZnlmXx3UnDMRvUXRkxojEQ1kZL5aILP2FUDUR33nknTz/9NM8++yw2m42DBw9y8OBBvF6vmtkaMg62+bAEGrEYdGhsBV2mKYkGop3+TCKSVvnli+6ikVTVHynbO9vyIWd08q/XA29tO8TKt3ex36tXDoh2ooRRNRCtWrWK1tZWZsyYQWFhYezr//7v/9TM1pBxqM2HNdhEhlHbqVrWodBhxqDT4A5paDN2lIq+TW7GWvdD7WcAhMums7WuDU9A/VUddFqlR9GriQ5q9ItAlCiqNlbLqSjiC92ym3TYta1kmPSQ0fU6U1qNRHGWhaoGN82m4Th9+5VAVDwlOZnytcLWl5XqX8F4quV81m09QIZRx4LzylXdnKFj2IBXkwEcElWzBBowvWZC6p2Wr4N8gzJ2yNr9Dr0zxuZiOqUAgz8LNn6idOOHAqBL8JrRzVXwzavgbwdLNoy6hO3blYX2RudnqL5DTEcg8nSUiETVLGFEIBrKXPXKd3MWaPXdJrObos9ps5Q94H2tSjBKVDd+OAh734OaT6P5yYSJN+CV9ew57ALg5CJ7Yq7VDya90pLhIjqEQHThJ8yA6L4XUs8fChNqO6g86KZ9qBNJQs6K9mwmqp2o/RBsqjwShIomwRm3gsnBFzXNhCMyeXajMoxAZeZoiahdTHxNOFEiGqI272/l4IavOUXnpnxE9+uQd9hz2MUn3zYySnIyFRITiOq+hp1vKMuMGKww7grIVibduv0hvqxpAeDMMpVXh4zqqJq55GiJKOASmy0miAhEQ1SDK4Al0IjeIPW4RFTf5idkymKqRgveZvA09X0J2f2bYNc65f/Zo5QgZFB+wSMRmbe2H8IfjJBnNzIqd2AsQGYxaCnNtpBhsEGjTgmgfrHHWSKIQDRENbW7KQu2YjFYj9tQ3WF4phmNJNHkA7ejAKu7VikV9SEQhQ5up2HT32l0+anJmECj+UxydrUxPDPIKUUOJElZ6mNfo4dLTspHoxkYS7LaTHquOT068n+DXQnEvlYRiBJAtBENQZGIjK+1HgkZk8WqLP9xAkadlkKn0k5zQNP38URtTYfY/Paz7G1ws1M/lu3mydS3B9h2oI29Dco6VJIkMXVENnPPLiNPhbWHesTkVL57W9TMRdoQJaIhqM0XROdrRiOByZHf40Xgy7Kt1DZ7+Tacz2iAlmoIh0Dbsx+jSDjC5neeQ+Pz4rcWUTr1KqY4LHgDYRpcfrKs8cMBHObue/LUFInIYHIqf8W9TWpnJy2IQDQENbgCWIJNmPVaND3tMQNKsy18uBt2u81cbMxAG3BB894eT8HQHN7GxIx29gZNjJ85F5szO/bcqLyB0Q50Ims37qOu1cdNw6zKepFesaFoIoiq2RDU5A5gCTZjMWg7r8h4HHk2IzaTjkBY5pAx2o3f07Wsgz7Y8w4ZRh2nnn05NmfPrzuQaKKlR682Wp31iBJRIogS0RCUk2HAZvHi0Bh6FYgkSeLkIjutniC6zJOhfTM07DzhKOu6Vi/m6ndwBtxK4/bwJE0PSQFTdPa/W+tQDnhbxB5nCSAC0RA0IssIzgjIxh71mB3t7JHRwCXLUOM4snhZ/sldpg+EIrz72WZGVK9nXH4GzomX9rhNaSAyRdeudmsyQNIc6cI3OVTO2eAmqmZDkbtBCSR6c983TZQkyD9F+f+BL7pN9uHuw+TUvYdBK2EtHg9Zg3vNqY5Bjb6QDGanclBUz/pNBKIhxh8K09xwgIgs97o0dLRGl58vwqVKqaBl35F5a0epbfFSt/1jbP5DlOdnoh9zSX+yPiB0LMzmDYaVOXogGqwTQASiIWZ/s5d3Nm1j64G2PgciXzDMMxv2sb4qQKu1TDlY/VFcmmA4wvqvv6WkZQN5NiOZ4y85sszqINYx38wbCB8ZyCi68PtNBKIhpskdwBJoUmaSW7NP/IIumPRaRke72z8KjVWqafXbobU2lua9bw6RXfMmZilESdlIKDo9IflXm9OipzTbouytZukIRC2q5ikdiEA0xDS6AphDzcpf9n5Uzc4akY1GkvjGZaHOHB1HtP3vEHBT1+LGteVfOH37GVGQif7k74AmPX7UhmdauOb04UqjfUeJSLQR9Vt6/HQIPdba7sIYciltHZa+j+XJtBqYVOIE4O+usbTIFqVksPEJCr5Zw2R9FcMyLWROmg0ZfQ94A1pHG5GvBSIRVbMy2IlANITIsoy39RAAJqs9Ntu9r84ZlcMwpxlPxMBzvmlUeU0Q9CJ5WyjOzWT4OXO67dYf7MIRGdloU5YAiYTF+tX9NHgHdAi91u4PofM2IQFmZ9e7dvSGViMxe9Iw3t1Rz/Y6+Cb/esqG+ZX5Z85iJL3Ke5AlyZ///S3tvhDzzy4j05ypDIfwNIlZ+P0gAtEQ0uxW5piZejnH7HgMOg2XnVLA9DG5BMIRMA3MiaqJpI0uS+IJhsm05iiByN0QW9RN6D1RNRtC7CY9EzOD5NuNfe4x645Jrz2ytnWas3SMJQqEjrSzuQ+rmKPBT5SIhpBMq4FMqxd05n71mA11ZoPya+P2h4+8j54GFXM0+IkS0VAScIPfdcLtg4Tjs3RsKxQIH5k03DFtRugTVQPR+++/z3e+8x2KioqQJImXX35ZzeykvX379uILhpFNTtAZ1c7OoGUxdkzzCCkN1BqtsiWS2F6oz1QNRG63m4kTJ/KHP/xBzWwMCZ5AiI+/2s6XNS2ErYlpqB6qLNGqmScQ3cGjo7fMLapnfaVqG9GsWbOYNWuWmlkYMpSpHY0YdBp09v533Q9lmR3TPDr2WhM9Z/02qBqr/X4/fr8/9ritTRSFe6rJHcAabFB6fLrZ517omdJsK6XZRy2f0tFzJhqs+2xQNVYvX74ch8MR+youLlY7S4NGU7sbc7BVmWOWoDFEQpRVdOH316AKREuWLKG1tTX2VVNTo3aWBo32hjpAjk7tGBwL1Q90wXBE2dGjowfSfVjMOeujQVU1MxqNGI2it6e3ZFnG31wHgDmrUKyvnABrPq6i0RXg5qkl5GVkKcvfhkPKImkJHiw6FAyqEpHQNy5/CIOnDgmw5pSonZ20oIsua9LuDylLnHS0u7kOqpirwUvVEpHL5WL37t2xx3v37uXLL78kKyuLkhLxC5MoBp2Gs3K8hE0WtM7hamcnLWSYdBxqA7c/FD2QrywM137wyFreQo+pGog+++wzLrjggtjjRYsWATBv3jwqKytVylX6MUb85Gk94DCDrVDt7KSFjOigRpfvqEAEXa7dLZyYqoFoxowZyGJYfPK1K+1DWLL6vQaRoMgwKhN82ztKRLbo2CzXQbHPWR+INqIhYO+3O2nxBAhlFKmdlbSRYVT+hsdKRJYcZZR10Kfs9Sb0ighEaS4UjrB75za2H2zHbxEDGRPFZooGoo4SkVYHlmhvmeuQSrkavEQgSnONbS6s/kPotRKW/FFqZydt2M16SrIslGYfVdW1R0ucbbVdv0jo1qAaRyT0XlPtHiQ5giEjC0ksZZowDrOe700+pgfSPgwOfBm3rZLQM6JElObch5ThEfqcEaIBNdkc0cDUflAZ3Cj0mAhE6UyWiRzeBYC1YLTKmUlPgVAEXzCsPDBngt4MkZAY2NhLIhClMX/bISLuJmRJQ3bxOLWzk3be23mYP7y7m8/3NSsHJOlIqUhUz3pFBKI01lq9BRnw20qxWq0nTC/0Tsci+q2e4JGDHYGobb8KORq8RGN1upJlcjx7sA530Fp6htq5SUtOszKosdV7VCCyD1O+t9SIgY29IEpE6aq1Bo23CavZTNGo09TOTVpyWJRA1OQJHJkhYC8CrR6CXjGeqBdEIEpXB75QvuefIhbKT5IsiwGNJOEPRo4MbNRoIbNM+X/TXtXyNtiIQJSOPE24929hd72L3ToxiDFZdFoNzmipqNEVOPJEZrnyvVkEop4SgSgd7fuYVo+fXaFctraLSa7JlJ1hAKDRfWQtdbKigah1P4QCXbxKOJYIROmm7QAc3EyLJ8h+x2SGZ4pAlExl2VbGD3OQk3FU9decCWYnRMLQ9K1qeRtMRCBKJ+EQ7HiNUDjMt5pyXMZ8RuSIbvtkGj/MwSUn58fv6iFJkDtW+f/hb9TJ2CAjAlE62fMOuOppCeqock4h06In02pQO1dDU250AGnjbmUXWOG4RCBKFwe+gNpNAGxzTCeotVCeK3brSIVwROZQm49231EBx1YIJrsShET17IREIEoHDbtg5xsAhErOZrNP2WdrZK6olqXCG1sP8uyGfXxzsP3IQUmCvJOU/9d9pU7GBhERiAa75mrY9rIyirdwAq6Cs8i2GrCb9QxzmtXO3ZCQb1e2nj7Q4o1/ovA05XvTt+BtSWmeBhsxxWMwa66GzX9VGqmzR8GYmTg1WuZMKcEXDCOJ6QUpMTxTCfj7m72EIzJaTfR9t2Qpgxubq5Sq88gLuj3HUCdKRINVc9WRIJQ1Ak75rjKqN8qk13b/WiGh8mxGrEYtgVCE2uZjSkXDJivfazdBwJP6zA0SIhANRs1VsPn5aEloJIz/Hmh17DnswhsIq527IUeSJMqi3fd7Drvin8wZDbZ8pdG65hMVcjc4iEA02BzeAV8fFYROuQa0Ohpdfv75VR1PfvBt/GxwISVG59sA+OZgO6Fw5MgTkgRl5yv/378J3I0q5G7gE4FoMNm/Cba+pKwAmDM6FoRkWWb9jsNEZJmSbCuO6PIUQuqUZlmwmXT4gmEOtPjin8weqUz7iIRgx6vKiGshjghEg0E4CN+8CrvWKb1jRZNiQQhgU3Uz+5o86LUS543KUTmzQ5NGI3He6FzmTCmmJPuYaTWSBGNmgs6grNy4603lcxRiBkQg+sMf/kBZWRkmk4mpU6eyceNGtbM0cLTsg02VUPe18gM9YjqMuQw0yke39UArH+xuAOD8MbliJLWKxhbYKHR0M2TC7ISTrlI+wwNfwM7XRcnoKKoHov/7v/9j0aJF3H///Xz++edMnDiRyy67jPr6IbyHuCwrM7e3/A2+eAbcDWCwwsQ5UHo2SBKyLPPujnrWbT2ELMNpxU5OHeZQO+dCVG2Lly21x+z4mjNaKRlJkrLt0OerlT80ApKs8ubzU6dO5cwzz+T3v/89AJFIhOLiYv7zP/+T++6777ivbWtrw+Fw0Nrait1uT0V2Ey8SgaAHAi5wH4a2OmV+Use2xZKEXDCBQOl5uCNG7CYdOq3y9+PVr+vYeaidM8uyOGdUthg3NEC0eAI8s2EfgVCE8hwrpxTZKXCYyDDqlM/o8E6lrSgYbUvKyIXs0cq0EGuO8kdHa0iLZWZ7+juqaiAKBAJYLBZeeOEFZs+eHTs+b948WlpaeOWVV+LS+/1+/P4j6760tbVRXFx84kDUtBf2vN11vTzumNz9sbjjMnWtPmqbPcix10T/iT4cW5CB3aQHZA61+aludHdxLhltOMCovIxYA3ODy09Vo5uIpKPJOoqDjgm4tE6CYeW1104eTnGW0gbR5gvS5ApQJmbYDyiyLPPJt01s3NtE5KifJY0kYTZomFyaxeQCPVT9G2/Nl+ysa+l8DklLWKMnz2GlOCsDNFp8IZmv9seXsiwGDacOc/YsY4kObKder8ynO46eBiJVR1Y3NDQQDofJz4/fkz0/P59vvum8fMLy5ctZtmxZ7y8U8oPrcF+z2SXZ6yXo6X6AmuyXQKMEFzngI+x3d5kuDIRlwGABSw5us5UtEQutpmFEoq8nGoQMOg2t3iDF0dfaTfposBMGEkmSmDYym7EFNjbXtrKvyUOjy09ElnH7wwTDETBmwNhZuHLPYvN7H2D3H8AaaMIYakMrB1F+MgJEjGHwRduSgmEirpa4a0UMWnCrtJmjHDlxmh4aVFM8lixZwqJFi2KPO0pEJ+QsVtpX4Ki/Ckf9dYj7S3GC49FjGb4QJd7QkeTR48o3CaPVADotSBK2QIgR/hDSMedAkkBnwmKzg0EJKPnBMFdG1z/uOLVOo8Fs0GLQqd6kJ/RCltXA9DG5gDJD3xMI4Q2GsRiO/NrZ7XbOP/+i+BeGA0ghL1IkSIZeArMWIiG04Qglo/1xSfVawNbNmuTJruwYElcSVzUQ5eTkoNVqOXQofreDQ4cOUVBQ0Cm90WjEaOzDQvAG65HlOxPEboeetkpZo189YdJrxfSMNKTVSNhMemzHlGCNOm3n7n4sgLPTOfTAsOxk5VBdqv6JNRgMTJ48mbfffjt2LBKJ8PbbbzNt2jQVcyYIQiqpXjVbtGgR8+bN44wzzmDKlCk8+uijuN1ubrnlFrWzJghCiqgeiG644QYOHz7M0qVLOXjwIKeddhqvv/56pwZsQRDSl+rjiPojLcYRCUIa6+nvqOiGEQRBdSIQCYKgOhGIBEFQneqN1f3R0bzV1tamck4EQehKx+/miZqiB3Ugam9Xtm/p0ehqQRBU097ejsPR/eoQg7rXLBKJcODAAWw224Cced4xBaWmpiYte/XS+f7S+d4gdfcnyzLt7e0UFRWh0XTfEjSoS0QajYbhw4ernY0TstvtafnD3CGd7y+d7w1Sc3/HKwl1EI3VgiCoTgQiQRBUJwJREhmNRu6///6+rRgwCKTz/aXzvcHAu79B3VgtCEJ6ECUiQRBUJwKRIAiqE4FIEATViUAkCILqRCDqpd7uSvv8888zbtw4TCYTp556Kv/617/inpdlmaVLl1JYWIjZbObiiy9m165dybyFbvXm3p544gnOO+88MjMzyczM5OKLL+6Ufv78+UiSFPc1c+bMZN9Gt3pzf5WVlZ3ybjKZ4tIM1s9uxowZne5NkiSuuOKKWJqUf3ay0GNr166VDQaD/NRTT8lbt26Vb7vtNtnpdMqHDh3qMv2HH34oa7Va+de//rW8bds2+b/+679kvV4vb968OZbm4Ycflh0Oh/zyyy/LX331lXzVVVfJ5eXlstfrTdVtybLc+3u76aab5D/84Q/yF198IW/fvl2eP3++7HA45P3798fSzJs3T545c6ZcV1cX+2pqakrVLcXp7f1VVFTIdrs9Lu8HDx6MSzNYP7vGxsa4+9qyZYus1WrlioqKWJpUf3YiEPXClClT5DvvvDP2OBwOy0VFRfLy5cu7TH/99dfLV1xxRdyxqVOnyrfffrssy7IciUTkgoIC+X/+539iz7e0tMhGo1F+7rnnknAH3evtvR0rFArJNptNXr16dezYvHnz5KuvvjrRWe2T3t5fRUWF7HA4uj1fOn12v/3tb2WbzSa7XK7YsVR/dqJq1kOBQIBNmzZx8cUXx45pNBouvvhiPv744y5f8/HHH8elB7jsssti6ffu3cvBgwfj0jgcDqZOndrtOZOhL/d2LI/HQzAYJCsrK+74+vXrycvLY+zYsdxxxx00NjYmNO890df7c7lclJaWUlxczNVXX83WrVtjz6XTZ/fkk08yZ84crNb4Ta9S+dmJQNRDx9uV9uDBg12+5uDBg8dN3/G9N+dMhr7c27HuvfdeioqK4n4hZs6cyZo1a3j77bf51a9+xXvvvcesWbMIh8MJzf+J9OX+xo4dy1NPPcUrr7zC008/TSQS4eyzz2b//v1A+nx2GzduZMuWLSxYsCDueKo/u0E9+14YGB5++GHWrl3L+vXr4xp058yZE/v/qaeeyoQJExg5ciTr16/noosu6upUA8a0adPi9tY7++yzOemkk/jTn/7Egw8+qGLOEuvJJ5/k1FNPZcqUKXHHU/3ZiRJRD/V2V1qAgoKC46bv+N6bcyZDX+6tw29+8xsefvhh1q1bx4QJE46bdsSIEeTk5LB79+5+57k3+nN/HfR6PZMmTYrlPR0+O7fbzdq1a/nhD394wusk+7MTgaiH+rIr7bRp0+LSA7z55pux9OXl5RQUFMSlaWtrY8OGDSnd6bavO+7++te/5sEHH+T111/njDPOOOF19u/fT2NjI4WFhQnJd08lYkfhcDjM5s2bY3kf7J8dKENL/H4/3//+9094naR/dilrFk8Da9eulY1Go1xZWSlv27ZNXrhwoex0OmPduj/4wQ/k++67L5b+ww8/lHU6nfyb3/xG3r59u3z//fd32X3vdDrlV155Rf7666/lq6++WrUu4N7c28MPPywbDAb5hRdeiOvibW9vl2VZltvb2+XFixfLH3/8sbx37175rbfekk8//XR59OjRss/nS+m99eX+li1bJr/xxhvynj175E2bNslz5syRTSaTvHXr1liawfrZdTj33HPlG264odNxNT47EYh66bHHHpNLSkpkg8EgT5kyRf7kk09iz02fPl2eN29eXPq//vWv8pgxY2SDwSCfcsop8quvvhr3fCQSkX/+85/L+fn5stFolC+66CJ5x44dqbiVTnpzb6WlpTLQ6ev++++XZVmWPR6PfOmll8q5ubmyXq+XS0tL5dtuu63TWJxU6s393XPPPbG0+fn58uWXXy5//vnncecbrJ+dLMvyN998IwPyunXrOp1Ljc9OLAMiCILqRBuRIAiqE4FIEATViUAkCILqRCASBEF1IhAJgqA6EYgEQVCdCESCIKhOBCJBEFQnApEwqFRWVuJ0OmOPH3jgAU477bTY4/nz5zN79uyU50voHxGIhC51rFn8ox/9qNNzd955J5IkMX/+/Lj0iQ4AZWVlPProo3HHbrjhBnbu3Nnta1auXEllZWXs8YwZM7jnnnsSmi8h8UQgErpVXFzM2rVr8Xq9sWM+n49nn32WkpISVfJkNpvJy8vr9nmHwxFXYhIGBxGIhG6dfvrpFBcX8+KLL8aOvfjii5SUlDBp0qR+nburksrs2bNjpawZM2ZQXV3NT37yk9guEtC5anaso0tm8+fP57333mPlypWxc+zdu5dRo0bxm9/8Ju51X375JZIkpXytJEEhApFwXLfeeisVFRWxx0899RS33HJL0q/74osvMnz4cH7xi19QV1dHXV1dr8+xcuVKpk2bxm233RY7R0lJSad7AqioqOD8889n1KhRiboFoRdEIBKO6/vf/z4ffPAB1dXVVFdX8+GHH/ZoIa3+ysrKQqvVYrPZKCgo6NOqhw6HA4PBgMViiZ1Dq9Uyf/58duzYEdv7KxgM8uyzz3Lrrbcm+jaEHhJrVgvHlZubyxVXXEFlZSWyLHPFFVeQk5Ojdrb6paioiCuuuIKnnnqKKVOm8I9//AO/3891112ndtaGLFEiEk7o1ltvpbKyktWrVyes1KDRaDh2KaxgMJiQc/fEggULYg3xFRUV3HDDDVgslpRdX4gnApFwQjNnziQQCBAMBrnssssScs7c3Ny4dp9wOMyWLVvi0hgMhn5vX9PdOS6//HKsViurVq3i9ddfF9UylYmqmXBCWq2W7du3x/7fndbWVr788su4Y9nZ2RQXF3dKe+GFF7Jo0SJeffVVRo4cyYoVK2hpaYlLU1ZWxvvvv8+cOXMwGo19qhKWlZWxYcMGqqqqyMjIICsrC41GE2srWrJkCaNHj07pgvdCZ6JEJPSI3W7HbrcfN8369euZNGlS3NeyZcu6THvrrbcyb9485s6dy/Tp0xkxYgQXXHBBXJpf/OIXVFVVMXLkSHJzc/uU78WLF6PVajn55JPJzc1l3759sed++MMfEggEUtILKByfWLNaGLL+/e9/c9FFF1FTU9Npp1QhtUQgEoYcv9/P4cOHmTdvHgUFBTzzzDNqZ2nIE1UzYch57rnnKC0tpaWlhV//+tdqZ0dAlIgEQRgARIlIEATViUAkCILqRCASBEF1IhAJgqA6EYgEQVCdCESCIKhOBCJBEFQnApEgCKr7/+HIXI+JKe8ZAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:53:27.307638Z", + "iopub.status.busy": "2024-03-24T03:53:27.307085Z", + "iopub.status.idle": "2024-03-24T03:53:27.784030Z", + "shell.execute_reply": "2024-03-24T03:53:27.782911Z" + }, + "papermill": { + "duration": 0.504473, + "end_time": "2024-03-24T03:53:27.786324", + "exception": false, + "start_time": "2024-03-24T03:53:27.281851", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCeklEQVR4nO3deXxU9b34/9eZfcnMZN9ISNhE2XGBIu47WhW1VWtbQVyqxfYqX26Re1vc2oJWrdzqpdWrID8X2lqXtlZxBSsqKIiyL5GEAAkhe2bJrOf3x5kZMiQhyWRmDpl8no/HkMyZz5z5HGbmnc/+kWRZlhEEQVCRRu0MCIIgiEAkCILqRCASBEF1IhAJgqA6EYgEQVCdCESCIKhOBCJBEFQnApEgCKrTqZ2B/giFQhw6dAibzYYkSWpnRxCEY8iyTFtbG8XFxWg03Zd7BnQgOnToEKWlpWpnQxCEHlRXV1NSUtLt46oHooMHD7JgwQLefvtt3G43I0eOZPny5Zx++uk9PtdmswHKRdrt9mRnVRCEPmptbaW0tDT6Xe2OqoGoqamJ6dOnc/755/P222+Tl5fHnj17yMrK6tXzI9Uxu90uApEgnMB6ajpRNRA98sgjlJaWsnz58uixYcOGqZgjQRDUoGqv2d///ndOP/10vv/975Ofn8/kyZN59tlnu03v9XppbW2NuQmCMPCpGoi+/fZbli1bxqhRo1i9ejV33XUXP//5z3nhhRe6TL948WIcDkf0JhqqBSE9SGquR2QwGDj99NP59NNPo8d+/vOf88UXX/DZZ591Su/1evF6vdH7kYawlpYW0UYUJ1mWCQQCBINBtbMiDEBarRadTtdtG1BraysOh6PH76iqbURFRUWMGTMm5tgpp5zC3/72ty7TG41GjEZjKrI2KPh8PmpqanC73WpnRRjALBYLRUVFGAyGuM+haiCaPn06u3btijm2e/duysrKVMrR4BEKhdi3bx9arZbi4mIMBoMYFCr0iSzL+Hw+jhw5wr59+xg1atRxBy0ej6qB6N577+XMM8/kt7/9Lddffz0bNmzgmWee4ZlnnlEzW4OCz+cjFApRWlqKxWJROzuqkWUZty+IxaAVgTgOZrMZvV5PVVUVPp8Pk8kU13lUbaw+44wzeP3113nllVcYN24cDz/8ME8++SQ//OEP1czWoBLvX7B0EQjKOL0BPH7RRhavRHyGVB9Z/d3vfpfvfve7amdDGKT8oRAAHl8Qi0H1r8OgNbj/HAqDlj8YIiTLmHRaAAIhmUA4KAmpJwKRMOjIskyT28eRNi8hWUavldhfVYVeq2Xz5s39OndlZSWSJPV4nnXr1jF+/Hj0ej0zZ87s12umAxGIhEEnEJSRZZAk0Gok9NrOX4M1a9YgSRLNzc1JycO8efOYNGkS+/btY8WKFUl5jYFEBCJhwPP5fH1KH2kX0ms1SJKEToUG+4qKCi644AJKSkrIzMyM6xx9ve7+iAx8TRYRiIROfIFQt7dAMNTrtP5epu2r8847j7vvvpt77rmH3NxcLr30UrZu3cqMGTPIyMigoKCAH//4x9TX10ef884773DWWWeRmZlJSVEBP7r+WqorvwVApz3abS/LMpWVlZx//vkAZGVlIUkSs2fP7nSenJwcvvvd71JRUdEpjzt37uTMM8/EZDIxbtw41q5dCxytujU0NDBnzhwkSYqWiNauXcuUKVMwGo0UFRVx3333xXz5u7ruSMlt9erVTJ48GbPZzAUXXEBdXR1vv/02p5xyCna7nZtuuilm4GooFGLx4sUMGzYMs9nMxIkTefXVV6OPR8779ttvc9ppp2E0Gvnkk0/6/F71lugmEDp5+qO93T42LNfKzMlDovef+bgCf7DrWUIlWWa+f/rR+YDPr9uHx9e5m/zei0/qcx5feOEF7rrrLtatW0dzczMXXHABt912G7///e/xeDwsWLCA66+/ng8//BAAl8vFvHnzmDBhAgfqGln864f44Q3X8/XXm9FpJEz6o3+TS0tL+dvf/sZ1113Hrl27sNvtmM3mTudxOp0sWrSIa665hs2bN8d0Y//nf/4nTz75JGPGjOGJJ57gyiuvZN++fZSWllJTU8Po0aN56KGHuOGGG3A4HBw8eJDLL7+c2bNns3LlSnbu3Mntt9+OyWTigQce6PK6AWpqagB44IEHeOqpp7BYLFx//fVcf/31GI1GXn75ZZxOJ9dccw1/+MMfWLBgAaDM23zxxRf54x//yKhRo/j444/50Y9+RF5eHueee2709e677z4ee+wxhg8f3uvleeIhApEwII0aNYpHH30UgF//+tdMnjyZ3/72t9HHn3/+eUpLS9m9ezcnnXQS1113HaCUeGz5Jfz+6T8ydngp27dvZ9y4cdhMekBZN0er1ZKdnQ1Afn5+TNUpcp6Or5OXlxc9T8Tdd98dTbts2TLeeecdnnvuOX7xi19QWFiIJEk4HA4KCwsB+N///V9KS0t56qmnkCSJk08+mUOHDrFgwQIWLVoUDXIdrxuOBqJf//rXTJ8+HYBbb72VhQsXUlFRwfDhwwH43ve+x0cffcSCBQvwer389re/5f3332fatGkADB8+nE8++YQ//elPMYHooYce4uKLL+77G9RHIhAJncw9f2S3j2mOGXx8xzkjuk177EDlOdMTt9bUaaedFv3966+/5qOPPiIjI6NTuoqKCk466ST27NnDokWLWL9+PUfq6wmF24n2798fE0B60vE89cc5T+QLDqDT6Tj99NPZsWNHt+fdsWMH06ZNixndPX36dJxOJwcOHGDo0KGdrrujCRMmRH8vKCjAYrFEg1Dk2IYNGwDYu3cvbre7U4Dx+XxMnjw55lhvVkpNBBGIhE4Mut43HSYrbU+sVmv0d6fTyZVXXskjjzzSKV1RUREAV155JWVlZfzxT8+Qk19AKBRiyqmTog2+ofAiFMe2ax0rcp5nn32W4uJiQqEQ48aNS1nDccfr7kiv10d/lyQp5n7kWCRoOp1OAN566y2GDBkSk+7YSeXdvV6iiUAkDHinnnoqf/vb3ygvL0en6/yRbmhoYNeuXTz77LOcffbZAJ0aXtvDbVeRn5GZ5B2XR+nNeSI+//xzzjnnHAACgQAbN27k7rvv7vYaIqtOyLIcLRWtW7cOm8123EXn4zFmzBiMRiP79++PqYapSfSaCQPe3LlzaWxs5Ac/+AFffPEFFRUVrF69mltuuYVgMEhWVhY5OTk888wz7N27lw8//JB58+bFnEMbrnOGUEpGZWVlSJLEP//5T44cOYLT6ezVeSKefvppXn/9dXbu3MncuXNpampizpw53V7DT3/6U6qrq/nZz37Gzp07efPNN7n//vuZN29ewucD2mw25s+fz7333ssLL7xARUUFmzZt4g9/+EO3ixImmwhEwoBXXFzMunXrCAaDXHLJJYwfP5577rmHzMxMNBoNGo2GVatWsXHjRsaNG8c9994b0+ALRxd3j9TMhgwZwoMPPsh9991HQUEBd999d6fz3Hvvvfzud7/rMk9LlixhyZIlTJw4kU8++YS///3v5ObmdnsNQ4YM4V//+hcbNmxg4sSJ3Hnnndx666388pe/TMx/0jEefvhhfvWrX7F48WJOOeUULrvsMt566y3V1oxXdYXG/urt6m9CZ+3t7ezbt49hw4bFvXTDQCPLMkecXmQZsq2GmBHVwVCIeqfSzpNvM4olQfrgeJ+l3n5HRYlIGDRCMkT+7OqO6f7TdAg8oQH7p3ngEoFIGDSC4QijkaROJR5JkqLBKDRwKwkDlghEwqARCUTaYwdDhUUOB0WRKOVE970waERKOt0FIotBS0juXG0Tkk8EImHQiJaIuokzZrFCo2pE1UwYNII9lIgE9Yg/AcKgYTFoMWg1XS6EBkrVLVJq6i6NkBwiEAmDhlGnxXicT7wvEKLF40evlci2io08U0mEfUEIi3bfizX0U04EImFQCIZk2v3B486ujzQdiXFEqScCkTDgnHfeedxzzz19eo4/qFS7Wtv93aaJlIhk+haM4smPEEsEImFQONp1rwSbrtYP6jjYegBPwRyQVA1EDzzwAFJ4uH3kdvLJJ6uZpcFNliHgU+fWyy/+7NmzWbt2LUuXLo1+ZioqKrj11lujC8GPHj2apUuXxjzvjtvmMPum7/PE75ZQXFzM6NGjAfj000+ZNGkSJpOJM844g3fe+geFDjObvtocfe7xFubvKj+VlZUJeTsGE9V7zcaOHcv7778fvd/VwlZCigT98O/H1Xnts/8f6Aw9Jlu6dCm7d+9m3LhxPPTQQ4Cy00ZJSQl//etfycnJ4dNPP+WOO+6gqKiI66+/HlDi3L/XriHL4eC9994DlJnhV155JZdffjkvv/wyVVVV/Ozn/6GkD5egelqYv6v85OXlJfy/J92p/q3X6XTRBcQFoScOhwODwYDFYon53Dz44IPR34cNG8Znn33GX/7ylw6BSMZisfLHZ57FblV25PjjH/+IJEk8++yzmEwmxowZw7z/t5+5d92JJtxy/dRTT/W4MH9X+RH6RvVAtGfPHoqLizGZTEybNo3FixdHFwo/ltfrxev1Ru+3tramKpuDg1avlEzUeu1+ePrpp3n++efZv38/Ho8Hn8/HpEmToo/LwCljxmI2HR0ftGvXLiZMmBCzhs5ZZyqL3uvCAxp7szC/0H+qBqKpU6eyYsUKRo8eTU1NDQ8++CBnn302W7duxWazdUq/ePHimL98QoJJUq+qRyeaVatWMX/+fB5//HGmTZuGzWbjd7/7HevXrweUHjAZsFgt0cbq3urNwvxC/6kaiGbMmBH9fcKECUydOpWysjL+8pe/cOutt3ZKv3Dhwpg1gltbWyktLe2UTkhvBoMhZlH7devWceaZZ/LTn/40eqzj7qsSYNBq0Gk0MT1jo0eP5sUXX8Tr9UZ3r1i/XtlyJ7KjbU8L83eVH6HvTqju+8zMTE466ST27u16p1Gj0Yjdbo+5CYNPeXk569evp7Kykvr6ekaNGsWXX37J6tWr2b17N7/61a/44osvouklSUKrUW4dF0S76aabCIVC3HHHHezYsYPVq1fz+ONKY73HrwSinhbm7yo/ITE0u89OqEDkdDqpqKgQRV7huObPn49Wq2XMmDHk5eVx6aWXcu2113LDDTcwdepUGhoaYkpH3bHb7fzjH/9g8+bNTJo0if/+7/9m4X8ri9UbwiWknhbm7yo/+/fvT97FpylVF8+fP39+dMO6Q4cOcf/997N582a2b9/eqy5QsXh+/AbT4vm+QIiQLKPTSuh62Jpnxcr/jztuu5VvD9ZRkpeZmgwOcIlYPF/VNqIDBw7wgx/8gIaGBvLy8jjrrLP4/PPPxTgMIaE8/iDt/iAZRh06Y2wgWrlyJcOHD2fIkCF8/fXX/PK/FnLVNddhNJlVyu3gpGogWrVqlZovLwwS0UXzu1gQrba2lkWLFlFbW0tRURHXXfc97lm4SEzxSDHVxxEJQrIdO8+so1/84hf84he/iN4PhcJ7n6F0+2vE/mYpcUI1VgtCosmy3GHR/J7Tx058TVKmhE5EiWiQS/cqSMflPHpTupEkCatRh4Qy/kjoWSI+Q6JENEjp9cqUCrfbrXJOkiuyDpq2i00Vu5Nh1GE16rpsUxI6i3yGIp+peIgS0SCl1WrJzMykrq4OAIvFkpb7vbf7gvh9AWSNRHu7DKEg+FzKT70Z9Ok9dCGZZFnG7XZTV1dHZmYmWq027nOJQDSIRWaLR4JROorszCEBTRoJfE6QO4x81ls6TbgNyTKyrLQXicbqnmVmZvZ75QERiAYxSZIoKioiPz8fv7/7JVTTRvUGOPIVGKxgzYemfWDIgIk3guboX/N/7z5CxREnp5dnMW5Ipnr5HQD0en2/SkIRIhAJaLXahHyYTmj+dqj9AkI+GDkDsofD5/8L7XXQ+i0Ujo8m1RuNeEIe2kO6tB91fqIQjdVCWtt+qJW9dU58NVuVJWmtuZA7CrQ6GHKqkujwtpjnmPRKUG73ixn1qSICkZC2ZFnmo111/OPrQ/gPbVEOFk44Olgof4zys6lKacAOM+qUr4U3IGbRp4oIRELa8gZC+AIhDAEnJneNEoDyTzmawJINtgKl8brh6PpFRp02/HxRIkoVEYiEtNXqURrgC4I1yvQOWyGYjpkBnj1c+dlcFT1k1IsSUaqJQCSkrdb2AACFwRrlQCTodJRZpvxsqorO6YhWzUQbUcqIXjMhbbW1+0EOkeM/BOgha1jnRI4Speve2waeJrBk4zDrmTIsG6tRfD1SRfxPC2mrtT2Axd+IGR/orGAv7pxIq4eMAmg9BG01YMnGZtIzfWRu6jM8iImqmZC22tr92LyHlaqWfUjMoMUYkQDVWpO6zAkxRIlISFvfGZ6Dz+Ujs82gBKLu2MLTE9qOBqIWt5/2QJAcqyG6x5mQPOJ/WEhbuRlGiqUmzHpt19WyCFt4swZnLYR34HhpQxUvr98fbfAWkksEIiF9+VxKAzQcv0RkyVE2lgwGwF0PiLFEqSYCkZCWXN4A23ftoMkdntZxvOU+JEmZBAvgOgKAIdyF7xNjiVJCBCIhLdU7vWzbuZv9je7jV8sirOFeMlekRCQGNaaSCERCWmprD2D1NygBJaOg5ydYw1tYhUtERwc1ikCUCiIQCWmp1ePH4osEovyen2DJUX66GwDRRpRqIhAJacnpasMYdGLQaY+2/xxPpETkaYJgQMw3SzExjkhIS76WwwDorFm9W5faYFXS+dvB3UBZthWDVkNJltjxNRVOmBLRkiVLkCSJe+65R+2sCGkg2FoLgCGzl2spSxJYIg3WRxiel8H0kbmU5ViTlEOhoxMiEH3xxRf86U9/YsKECWpnRUgDwZCMxqVsCGDKLOr9EyPVs3A7kZA6qgcip9PJD3/4Q5599lmysrLUzo6QBiTgnKIQo/IzMGX1IRCZw58/TxOBYIhmt49Gly8peRRiqR6I5s6dyxVXXMFFF13UY1qv10tra2vMTRCOpSGEQ24lN8OI1Juu+whLtvLT08Sh5naWr6vkrW8OJSeTQgxVG6tXrVrFpk2b+OKLL3qVfvHixTz44INJzpUw4LkblQ0UdQYwOXr/vA4lIqNOWdda9Jqlhmolourqav7jP/6Dl156qddbtixcuJCWlpborbq6Osm5FAaiQzUHqGnx0KpxHF0ovzdMmUr6gBej3A6IQJQqqpWINm7cSF1dHaeeemr0WDAY5OOPP+app57C6/V22mvLaDRiNBpTnVVhgDl48ACBBjc5mVbsPSc/SqsDow3aWzEGlGq/LxAiFJLRaMSOr8mkWiC68MIL2bJlS8yxW265hZNPPpkFCxak/4Z/QtIE2pRpGnp7Xt+fbM6C9lYMvhZAKan7giFM3S2qJiSEaoHIZrMxbty4mGNWq5WcnJxOxwWhL2R3AxJgzuzFiOpjmbOgqQqttxm9tgh/UMbrD0U3XRSSQ/VeM0FIJDkUio4Dsmb1occsIqbBWsw3S5UTaorHmjVr1M6CMMA5W5uQQgGQtFjtcSyA3yEQjS9xEAzJmAyiNJRsJ1QgEoT+cjcrc8xkcyaaeNoZzeGxRO5GvnNaTgJzJhyPqJoJacXTokztiC501leRcUcBrzIBVkgJUSIS0soQg5ucIXZ8Q8rjO4HOAAYL+Nx4XY14DLkYdBosBvFVSSZRIhLSir69CZtRT05eH+aYHStcKtq4s5Ll6yr55kBLgnIndEcEIiG9hHfhiK64GA+jMgzSIrsAMbo6FUR5U0gffg8HDtej1UhkGTLp3cShLoRLROagEwCvX3TfJ5soEQlpI+Ss52Czh13NGnz9+RtrygTAGA5EvqAoESWbCERC2nC11BGSwWfIJKM/jcvhEpEx0AaInTxSIa5A9O233yY6H4LQb54mZQyRZM3p3yTVcCAyhAORKBElX1yBaOTIkZx//vm8+OKLtLeLsRbCicHbqkx21dnimOzaUTgQ6UNetCGvaCNKgbgC0aZNm5gwYQLz5s2jsLCQn/zkJ2zYsCHReROEPvGHZ90b45l135HOAHozRp2GyfkaxhT3YXE1IS5xBaJJkyaxdOlSDh06xPPPP09NTQ1nnXUW48aN44knnuDIkSOJzqcgHF8wQMDVBMQ56/5YJgdGnZazSw1MGZbd//MJx9WvxmqdTse1117LX//6Vx555BH27t3L/PnzKS0t5eabb6ampiZR+RSE4/M04vUHCWoM2GyZ/T9fZKpHuxjMmAr9CkRffvklP/3pTykqKuKJJ55g/vz5VFRU8N5773Ho0CGuvvrqROVTEI7P3cDJhTbGnTSCkmxL/88XDkQ+ZyMtbj/BkNz/cwrdiquP84knnmD58uXs2rWLyy+/nJUrV3L55Zej0ShxbdiwYaxYsYLy8vJE5lUQuuduQK/VkJlTCIlYxCw8luijr/eyvbqc2WeWk2U19P+8QpfiCkTLli1jzpw5zJ49m6Kiruf05Ofn89xzz/Urc4LQa5FNEfsztaOjcInIGgqPrhbTPJIqrkD03nvvMXTo0GgJKEKWZaqrqxk6dCgGg4FZs2YlJJOC0JOW+hqaGlxQZKI8ESc0heebhQORTwSipIqrjWjEiBHU19d3Ot7Y2MiwYcP6nSlB6BNZxtlcR01LO3tdCdrlJTK6Gh+akF8sF5tkcQUiWe664c7pdPZ6jzJBSJj2FrxeL7KkwWJPUNVMZwSdEa1Gwhh0iqpZkvWpajZv3jwAJEli0aJFWCxHeyeCwSDr169n0qRJCc2gIPTI3YDXH8Kjc1BgSeC+dyY7Wk09xkCbCERJ1qdA9NVXXwFKiWjLli0YDEd7EQwGAxMnTmT+/PmJzaEg9MTdQHsgSLs+E4dZn7jzmjLRaSSMAaeomiVZnwLRRx99BCgbIS5duhS7vU/7aApCUgSd9UqJyJyJw5LAQGS0YzPpOdkm47CLJodkiqvXbPny5YnOhyDErb3lMDLgN2VjMyZwrT+TndwMI7n5GsjLSNx5hU56/a5de+21rFixArvdzrXXXnvctK+99lq/MyYIvRWZda+35SFJCdyjPjrNozVx5xS61OtA5HA4om+ywyFmIwsnCJ+LLH2A08qyOWnyyYk9t9GOLMsEXE14vQEyElnaEmL0+n+2Y3UsUVWzZcuWsWzZMiorKwEYO3YsixYtYsaMGQk5vzAIuBuQkDBkZJLjSHD1yeSg2eNn575q9un3c9N3hif2/EJUXOOIPB4Pbrc7er+qqoonn3ySd999t0/nKSkpYcmSJWzcuJEvv/ySCy64gKuvvppt27bFky1hMHIlYNeO7hisaHU6QCbkaUv8+YWouMqaV199Nddeey133nknzc3NTJkyBYPBQH19PU888QR33XVXr85z5ZVXxtz/zW9+w7Jly/j8888ZO3Zsp/Rerxev1xu939oq6u6DnruRiiNO3Bodw70BrImsPkkSGrMDaELyis9aMsW9QuPZZ58NwKuvvkphYSFVVVWsXLmS//mf/4krI8FgkFWrVuFyuZg2bVqXaRYvXozD4YjeSktL43otIX0EnXUcafPyTaOORLZTR2jNmcov7a3dzigQ+i+uQOR2u7HZbAC8++67XHvttWg0Gr7zne9QVVXVp3Nt2bKFjIwMjEYjd955J6+//jpjxozpMu3ChQtpaWmJ3qqrq+PJvpBGPC11yEDQnI05Ect/HENnyQRAH2jDHxSBKFniXjz/jTfeoLq6mtWrV3PJJZcAUFdX1+dBjqNHj2bz5s2sX7+eu+66i1mzZrF9+/Yu0xqNRux2e8xNGMQCXnxOZQVFoz0/sV33YTqLHQllayGxm0fyxBWIFi1axPz58ykvL2fq1KnRqtS7777L5MmT+3Qug8HAyJEjOe2001i8eDETJ05k6dKl8WRLGGzcDbT7g/i1ZuzhEnqiSabM8MRXl9jNI4niatn73ve+x1lnnUVNTQ0TJ06MHr/wwgu55ppr+pWhUCgU0yAtCN1yN+DxB/Hossm0JGn1RJOdXJsRu96PXif2I02WuLsYCgsLKSwsjDk2ZcqUPp1j4cKFzJgxg6FDh9LW1sbLL7/MmjVrWL16dbzZEgYTVz3t/iAefRZl1gTOMevI5GBYjlWpO4gBjUkT1/+sy+ViyZIlfPDBB9TV1REKxdade7sTbF1dXXS3D4fDwYQJE1i9ejUXX3xxPNkSBht3A76gMtk105ykEpHRDpIEoQD43WCwJud1Brm4AtFtt93G2rVr+fGPf0xRUVHcjYRiTWuhX9wNTCrNZNQp4zHbErgOUUcaLSG9haCnjZCzEWO2CETJEFcgevvtt3nrrbeYPn16ovMjCL0T9IOnCQmJjKxC6M9e9z3Y1ijhrGvClnuIcdli7FoyxNX6lpWVRXa22P1SUJGrHmQZ9GYwJHeJDim8kH7IIzZbTJa4AtHDDz/MokWLYuabCUJKueqoafHwVaOBinpXUl9KCi8HIotAlDRxVc0ef/xxKioqKCgooLy8HL0+tsdi06ZNCcmcIHTLeYTW9gD7NBaGefxJfSllvhnIXhGIkiWuQDRz5swEZ0MQ+sh1BI8viNuWQ3aSd2CNzDeT2kUgSpa4AtH999+f6HwIQu/JMsG2Wtr9Qdz67OQHovB8M8krlgJJlriHijY3N/N///d/LFy4kMbGRkCpkh08eDBhmROELvlceN1OZCSC5uykr5yos4arZn4PBHxJfa3BKq538JtvvuGiiy7C4XBQWVnJ7bffTnZ2Nq+99hr79+9n5cqVic6nIBzlqsPjD9Kus5NlsyZlsmtHNmsGWQ4bFikA7S2QkZfU1xuM4ioRzZs3j9mzZ7Nnz56YnV0vv/xyPv7444RlThC65DyC2xfEbUh+tQwg22rg5LIShmZbQCyQlhRxBaIvvviCn/zkJ52ODxkyhNra2n5nShCOy1VHSJZpN+SQk5H8QAR02NFDNFgnQ1xVM6PR2OUyrbt37yYvTxRbhSRrq6Us20rpuNMJZWem5CVDBhuhUAiNpyX+hlWhW3H9n1511VU89NBD+P3K+A1Jkti/fz8LFizguuuuS2gGBSFGwAvuBgA09kJ02tSEhVe3t/FFZRPtzqaUvN5gE9e7+Pjjj+N0OsnLy8Pj8XDuuecycuRIbDYbv/nNbxKdR0E4qq1WmdphsoMxhbuvGpVpHkF3c+pecxCJq2rmcDh47733WLduHV9//TVOp5NTTz2Viy66KNH5E4RYbbU0urxsbdKir2hg2ogkbCPUBTkciEKijSgp+hyIQqEQK1as4LXXXqOyshJJkhg2bBiFhYXIspz0rlRhkGurweUNUitnkeUNpOxlJVMmAEFPG4SCoEn8Qv2DWZ+qZrIsc9VVV3Hbbbdx8OBBxo8fz9ixY6mqqmL27Nn9XiZWEHrUVovLF8BpyCMvWWsQdUFrtBKStARDIdGFnwR9KhGtWLGCjz/+mA8++IDzzz8/5rEPP/yQmTNnsnLlSm6++eaEZlIQAPB7wNOE2xfElZVLbqq67gGDXotPm0EwFID2VjBnpey1B4M+lYheeeUV/uu//qtTEAK44IILuO+++3jppZcSljlBiNFykEAoRLNkJ6A1kZuRuhKRUafBq8sgGJJFiSgJ+hSIvvnmGy677LJuH58xYwZff/11vzMlCF1qqcblDdJmKMBu1mNKwoaK3SlymMnNzcOs14hBjUnQp6pZY2MjBQUF3T5eUFBAU5MYZzGgBf1Q8RG0HYK8U6B0CknZyzkeLdW4fQHajIUpbR8CGF/igEA57KtWqmZCQvUpEAWDQXS67p+i1WoJBFLXkyEkmCzD9jehfo9yv7VG2b2i/ARYmzzoh7ZaNJKELmso+SkOREB0LJEoESVenwKRLMvMnj0bo7HrD4HYGHFgaz64iyPbNyFLWopHTyGjcStUrYP8U8Ci8hrlbTUQClKQl8uN0yagxi70IYMNOSSjFW1ECdenQDRr1qwe04ges4HJ4wuy6dP3MTt91NjGsaZlDDfbnZhaK2H/53Dy5epmsKlS+ekoBUki1ZXFvXVO3t3YwDmNrYwr0SulxxOlypoG+hSIli9fnqx8CCr7cncl5rYqTDoN3ryJuLxBNphHcw6VcHgbDD9X3c0FGyoIyjJS1jBVJp0adRp8WiuBEEp11edK7RSTNCcmEgvIsozzwDZApnT4SZx36skAfNWcQbspV/ni1e1UL4M+F7TVcri1nf/bBuv21qc8CwadBlnS0q4xKwdE9SyhVA1Eixcv5owzzsBms5Gfn8/MmTPZtWuXmlkalCRJ4rIiN+OG2MkpG0eRw8zEUgcXjilAVzxeSVS3Xb0MNu5TfkhZuDCjT9GM+46MOuU13VK4VCh6zhJK1UC0du1a5s6dy+eff857772H3+/nkksuweVK7j5VwjGCfqTmamxGPVL2CAAuOLmAcUMc6ArGKG0hLQfU6y1q2AtANcrQkUK76Xipk8KoU8YsuSUrsiyLnrMES+6q4z145513Yu6vWLGC/Px8Nm7cyDnnnNMpvdfrjemZ62pxNqFvZFmGlgNIoQAYbWDNjU1gsoOjBJqroX4vlJyW2gwG/dCwF18wxAFtKZIE+fbUd90bwiUiry6DoNyITlTNEuqEaiNqaVH+ynS3nfXixYtxOBzRW2mp2Ie8v2pa2vnHv79gX71TCTgdeoKc3gBf7W/i21ChciBcMkmphr0Q9NMcsuA05JObYUzpiOoIrUZCr5XwajMIBEWJKNFOmEAUCoW45557mD59OuPGjesyzcKFC2lpaYneqqurU5zL9FPd6EbvrMEflJVAdMxja3Yd4UtXeM2f5v1KCSWVDm8D4KChDCSJkixzal+/gxF5GQwpLFBitQhECaVq1ayjuXPnsnXrVj755JNu0xiNxm4HUwrxOdjkJs9Xhz1bD/YhMY+V51iRJDjYbsFrtmEMtEFTFeSOTE3m2luipbDdUjkAJVmW1Lx2F2aMLwKnDr5YK3rNEuyEKBHdfffd/POf/+Sjjz6ipKSk5ycICSHLMi0Nh9GFvGRYTJCRH/O42aAl32YCSaJWX6wcTGX17OAmkGXkzKEMLyunPNfCkEz1SkSA0mYG4G9X1s8WEkLVQCTLMnfffTevv/46H374IcOGDVMzO4NOqyeAzn0YjQSW7OIuVx0sz1FKIPtCRcqBxgplVHGy+dxwaBMAUskZnFGezTWTSzAb1F0ZMaQxENSGS+WiCz9hVA1Ec+fO5cUXX+Tll1/GZrNRW1tLbW0tHo9HzWwNGrWt7Vh8DVgMOjS2wi7TDA0Hot3eLEKSVvnyhXfRSKqqT5XtnW0FkDsq+a/XC+9vP8zSD/ZwwKNXDoh2ooRRNRAtW7aMlpYWzjvvPIqKiqK3P//5z2pma9A43NqO1d9IhlHbqVoWUeQwY9BpcAU0tBojpaJvk5uxlgNw8EsAguXnsq2mFbdP/VUddFqlR9GjCQ9q9IpAlCiqNlbLqSjiC92ym3TYtS1kmPSQ0fU6U1qNRGm2hcp6F02mEjLbDyiBqHRKcjLV3gLb3lCqf4XjqJILeHfbITKMOm47e5iqmzNEhg14NBnAYVE1S6ATptdMSL1JBTooMChjh6zd79B73ug8TGMLMXizYcPnSjd+wAe6BK8Z3VQJO98CbxtYcmDkxezYoSy0N6ogQ/UdYiKByB0pEYmqWcKIQDSYOeuUn+Zs0Oq7TWY3hR/TZit7wLe3KMEoUd34QT/sWwvVX4TzkwUTb8Aj66k44gRgTLE9Ma/VDya90pLhJDyEQHThJ8wJ0X0vpJ43ECTQWqvc6aZ9qBNJQs4O92wmqp2o7TBsXHE0CBVPhtPngMnBV9VNBEMy+XajMoxAZeZwiahNTHxNOFEiGqS2HGihdv03jNW5GDa8+3XIIyqOOPn82wZGSplMhcQEoppvYPdqZZkRgxVOvgJylEm3Lm+AzdXNAJxRrvLqkGGRqplTDpeIfE6x2WKCiEA0SNU7fVh8DegNUq9LRHWtXgKmbKZqtOBpAndj/EvIHtgIe95Vfs8ZqQQhg/IFD4Vk3t9xGK8/RL7dyMi8E2MBMotBS1mOhQyDDRp0SgD1ij3OEkEEokGqsc1Fub8Fi8F63IbqiJIsMxpJorEdXI5CrK6DSqkojkAUqN1B/ca/0+D0Up0xgQbzGeTuaaUky8/YYgeSpCz1sb/BzcWnFKDRnBhLstpMeq49NTzyf71dCcTtLSIQJYBoIxqEQiGZ9pY6JGRMFquy/EcPjDotRZlKO80hTfzjiVobD7Plg5fZV+9it340O8ynUdfmY/uhVvbVK+tQSZLE1OE53HxmOfkqrD3UK6ZM5aenWc1cpA1RIhqEWtv96Nqb0EhgchT0ehH48hwrB5s8fBssYBRAcxUEA6Dt3ccoFAyx5cNX0LR78FqLKZt6FVMcFjy+IPVOL9nW2OEADnP3PXlqCoVkMGUqf8U9jWpnJy2IQDQI1Tt9WPyNmPVaNL3tMQPKciys2wt7XWYuMmag9TmhaV+vp2BojmxnYkYb+/wmxl12M7bMnOhjI/NPjHagnqzasJ+alnZuGmJV1ov0iA1FE0FUzQahRpcPi78Ji0HbeUXG48i3GbGZdPiCMoeN4W783q5l7W+Hig/JMOoYf+bl2DJ7/7onEk249OjRhquzblEiSgRRIhqEcjMM2CweHBpDnwKRJEmMKbbT4vajyxoDbVugfnePo6xrWjyYqz4k0+dSGrdLkjQ9JAVM4dn/Lq1DOeBpFnucJYAIRIPQ8GwjZIZANvaqx6yjM0eEA5csQ7Xj6OJlBWO6TO8LhPjoyy0Mr1rDyQUZZE68pNdtSiciU3jtapcmAyTN0S58k0PlnA1somo2GLnqlUCiN8e/aaIkQcFY5fdDX3WbbN3eI+TWrMWglbCWjoPsgb3mVGRQY3tABnOmclBUz/pNBKJBxhsI0lR/iJAs97k01FGD08tXwTKlVNC8/+i8tQ4ONnuo2fEZNu9hhhVkoT/p4v5k/YQQWZjN4w8qc/RANFgngAhEg8yBJg8fbtzOtkOtcQeidn+Ql9bvZ02ljxZruXKw6tOYNP5giDXffMvQ5vXk24xkjbv46DKrA1hkvpnHFzw6kFF04febCESDTKPLh8XXqMwkt+b0/IQumPRaRoW72z8NjFaqaXU7oOVgNM3anYfJqX4PsxRgaPkIKD41IflXW6ZFT1mORdlbzRIJRM2q5ikdiEA0yDQ4fZgDTcpf9n5Uzb4zPAeNJLHTaaHGHB5HtOPv4HNR0+zCufVfZLYfYHhhFvoxV4ImPT5qJVkWrj21RGm0j5SIRBtRv6XHp0PotZY2J8aAU2nrsMQ/lifLamDy0EwA/u4cTbNsUUoGG56lcOdKTtNXMiTLQtbkmZARf8A7oUXaiNqbIRRSNSsDnQhEg4gsy3haDgNgstqjs93jNX1kLkMyzbhDBl5pn0alxwR+D5KnmdK8LEqm39htt/5AFwzJyEabsgRIKCjWr+6ngTugQ+izNm8AnacRCTBndr1rR19oNRIzJw/ho1117KiBnQXXUz7Eq8w/yyxF0qu8B1mS/N+/v6WtPcDsM8vJMmcpwyHcjWIWfj+IQDSINLmUOWamPs4xOx6DTsOlYws596Q8fMEQmE7MiaqJpA0vS+L2B8my5iqByFUfXdRN6DtRNRtE7CY9E7P8FNiNcfeYdcek1x5d2zrNWSJjiXyBo+1sriMq5mjgEyWiQSTLaiDL6gGduV89ZoOd2aB8bVze4NH/R3e9ijka+ESJaDDxucDr7HH7IOH4LJFthXzBo5OGI9NmhLiIQDSI7N+/j3Z/ENmUCTqj2tkZsCzGyDSPgNJArdEqWyKJ7YXipmog+vjjj7nyyispLi5GkiTeeOMNNbOT1ty+AJ99vYPN1c0ErYlpqB6sLOGqmdsX3sEj0lvmEtWzeKkaiFwuFxMnTuTpp59WMxuDgjK1owGDToPO3v+u+8EsKzLNI7LXWsfqmRAXVRurZ8yYwYwZM3qd3uv14vV6o/dbW0VRuLcaXT6s/nqlx6ebfe6F3inLsVKW02H5lEjPmWiwjtuAaiNavHgxDocjeistLVU7SwNGY5sLs79FmWOWoDFEQphVdOH314AKRAsXLqSlpSV6q66uVjtLA0ZbfQ0gh6d2DIyF6k90/mBI2dEj0gPpOiLmnMVpQI0jMhqNGI2it6evZFnG21QDgDm7SKyvnAArP6ukwenjh1OHkp+RrSx/Gwwoi6QleLDoYDCgSkRCfJzeAAZ3DRJgzR2qdnbSgi68rEmbN6AscRJpd3PWqpirgUsEokHAoNPwnVwPZTkWtJklamcnLWSYIqOrA+ED4UDUJgJRPFStmjmdTvbu3Ru9v2/fPjZv3kx2djZDh4q/3IliDHnJ17rBYQZbkdrZSQsZ4UGNzvZjAlEXa3cLPVM1EH355Zecf/750fvz5s0DYNasWaxYsUKlXKWhNqV9CEt2v9cgEhQZRmWCb1ukRGQLj81y1op9zuKgaiA677zzkMX8nKTb9+1ustw+MvKLB1bvxAksw6j8T0ZLRJZcZZS1v13Z6y2y1ZDQK6KNKM0FgiH27t7Ojto2vBYxkDFRbOE2ImekRKTVgSXcW+Y8rFKuBi4RiNJcQ6sTq/cweq2EpWCk2tlJG3aznqHZFspyOlR17cXKz9aDXT9J6JYoqae5xoMVSHIIQ0Y2kljKNGEcZj3XnXZMD6R9CBzaHLOtktA7okSU5lyHlV5Jfe5w0YCabI5wYGqrVQY3Cr0mAlE6k2VCR/YAYC0cpXJm0pMvEKLdH1TumLNAb4ZQQAxs7CMRiNKYt/UwIVcjsqQhp/RktbOTdtbuPsLTH+1l0/4m5YAkHS0ViepZn4hAlMZaqrYiA15bGVartcf0Qt9EFtFvcfuPHowEotYDKuRo4BKN1elKlsl1V2AtcdBSdrrauUlLmWZlUGOLp0Mgsg9RfjZXi4GNfSBKROmqpRqNpxGr2UzxyElq5yYtOSxKIGp0+44OzLUXg1YPfo8YT9QHIhClq0NfKT8LxoqF8pMk22JAI0l4/aGjAxs1WsgqV35v3Kda3gYaEYjSkbsR14Gt7K1zslcnBjEmi06rITNcKmpw+o4+kDVM+dkkAlFviUCUjvZ/Rovby55AHtvaxCTXZMrJMADQ4Dq6ljrZ4UDUcgACvi6eJRxLBKJ003oIarfQ7PZzwHEaJVkiECVTeY6VcUMc5GZ0qP6as5RJr6EgNH6rWt4GEhGI0kkwALveJhAM8q1mGE5jAcNzRbd9Mo0b4uDiMQWxu3pIEuSNVn4/slOdjA0wIhClk4oPwVlHs19HZeYUsix6sqwGtXM1OOWFB5A27FV2gRWOSwSidHHoKzi4EYDtjnPxay0MyxO7daRCMCRzuLWdtvYOAcdWBCa7EoRE9axHIhClg/o9sHs1AIGhZ7KlXdlna0SeqJalwupttby8fj87a9uOHpQkyD9F+b3ma3UyNoCIQDTQNVXB9jeUUbxFE3AWfoccqwG7Wc+QTLPauRsUCuzK1tOHmj2xDxRNUn42fgue5pTmaaARUzwGsqYq2PIXpZE6ZyScdBmZGi03ThlKuz+IJKYXpERJlhLwDzR5CIZktJrw/7slWxnc2FSpVJ1HnN/tOQY7USIaqJoqjwah7OEw9hplVG+YSa/t/rlCQuXbjFiNWnyBEAebjikVDTlN+XlwI/jcqc/cACEC0UDUVAlb/houCY2AcdeBVkfFESceX1Dt3A06kiRRHu6+rzjijH0wdxTYCpRG6+rPVcjdwCAC0UBzZBd80yEIjb0WtDoanF7++XUNz33ybexscCElRhXYANhZ20YgGDr6gCRB+TnK7wc2gqtBhdyd+EQgGkgObIRtrysrAOaOigYhWZZZs+sIIVlmaI4VR3h5CiF1yrIt2Ew62v1BDjW3xz6YM0KZ9hEKwK63lBHXQgwRiAaCoB92vgV73lV6x4onR4MQwMaqJvY3utFrJc4ematyZgcnjUbi7FF53DillKE5x0yrkSQ46TLQGZSVG/e8p7yPQtQJEYiefvppysvLMZlMTJ06lQ0bNqidpRNH837YuAJqvlE+0MPPhZMuBY3y1m071MIne+sBOOekPDGSWkWjC20UOboZMmHOhFOuUt7DQ1/B7ndEyagD1QPRn//8Z+bNm8f999/Ppk2bmDhxIpdeeil1dYN4D3FZVmZub/0bfPUSuOrBYIWJN0LZmSBJyLLMR7vqeHfbYWQZJpVmMn6IQ+2cC2EHmz1sPdgSezB3lFIykiRl26FNLyh/aAQkWeU9n6dOncoZZ5zBU089BUAoFKK0tJSf/exn3Hfffcd9bmtrKw6Hg5aWFux2eyqym3ihEPjd4HOC6wi01ijzk9rDH2JJQi6cgK/sbFwhI3aTDp1W+fvx1jc17D7cxhnl2UwfmSPGDZ0gmt0+Xlq/H18gxLBcK2OL7RQ6TGQYdcp7dGS30lbkD7clZeRBzihlWog1V/mjozWkxTKzvf2OqhqIfD4fFouFV199lZkzZ0aPz5o1i+bmZt58882Y9F6vF6/36Lovra2tlJaW9hyIGvdBxQdd18tjjsndH4s5LlPT0s7BJjdy9Dnhf8J3RxdmYDfpAZnDrV6qGlxdnEtGG/QxMj8j2sBc7/RS2eAiJOlotI6k1jEBpzYTf1B57vdOK6E0W2mDaG330+j0US5m2J9QZFnm828b2bCvkVCHz5JGkjAbNJxWls1phXqo/Dee6s3srmnufA5JS1CjJ99hpTQ7AzRa2gMyXx+ILWVZDBrGD8nsXcYSHdjGX6/MpzuO3gYiVUdW19fXEwwGKSiI3ZO9oKCAnTs7L5+wePFiHnzwwb6/UMALziPxZrNLsseD3939ADXZK4FGCS6yr52g19VluiAQlAGDBSy5uMxWtoYstJiGEAo/n3AQMug0tHj8lIafazfpw8FOOJFIksS0ETmMLrSx5WAL+xvdNDi9hGQZlzeIPxgCYwaMnoEz7ztsWfsJdu8hrL5GjIFWtLIf5ZPhI2QMQnu4LckfJORsjnmtkEELLpU2c5RDPafppQE1xWPhwoXMmzcvej9SIupRZqnSvgId/ip0+OsQ85eih+PhYxntAYZ6AkeTh48rPySMVgPotCBJ2HwBhnsDSMecA0kCnQmLzQ4GJaAU+IN8N7z+ceTUOo0Gs0GLQad6k57QB9lWA+eelAcoM/TdvgAefxCL4ejXzm63c845F8Y+MehDCniQQn4y9BKYtRAKoA2GGDrKG5NUrwVs3axJnuzKjiFxJXFVA1Fubi5arZbDh2N3Ozh8+DCFhYWd0huNRozGOBaCN1iPLt+ZIHY79LZVyhq+9YZJrxXTM9KQViNhM+mxHVOCNeq0nbv7sQCZnc6hB4bkJCuH6lL1T6zBYOC0007jgw8+iB4LhUJ88MEHTJs2TcWcCYKQSqpXzebNm8esWbM4/fTTmTJlCk8++SQul4tbbrlF7awJgpAiqgeiG264gSNHjrBo0SJqa2uZNGkS77zzTqcGbEEQ0pfq44j6Iy3GEQlCGuvtd1R0wwiCoDoRiARBUJ0IRIIgqE71xur+iDRvtba2qpwTQRC6Evlu9tQUPaADUVubsn1Lr0ZXC4Kgmra2NhyO7leHGNC9ZqFQiEOHDmGz2U7ImeeRKSjV1dVp2auXzteXztcGqbs+WZZpa2ujuLgYjab7lqABXSLSaDSUlJSonY0e2e32tPwwR6Tz9aXztUFqru94JaEI0VgtCILqRCASBEF1IhAlkdFo5P77749vxYABIJ2vL52vDU686xvQjdWCIKQHUSISBEF1IhAJgqA6EYgEQVCdCESCIKhOBKI+6uuutH/96185+eSTMZlMjB8/nn/9618xj8uyzKJFiygqKsJsNnPRRRexZ8+eZF5Ct/pybc8++yxnn302WVlZZGVlcdFFF3VKP3v2bCRJirlddtllyb6MbvXl+lasWNEp7yaTKSbNQH3vzjvvvE7XJkkSV1xxRTRNyt87Wei1VatWyQaDQX7++eflbdu2ybfffrucmZkpHz58uMv069atk7Varfzoo4/K27dvl3/5y1/Ker1e3rJlSzTNkiVLZIfDIb/xxhvy119/LV911VXysGHDZI/Hk6rLkmW579d20003yU8//bT81VdfyTt27JBnz54tOxwO+cCBA9E0s2bNki+77DK5pqYmemtsbEzVJcXo6/UtX75cttvtMXmvra2NSTNQ37uGhoaY69q6daus1Wrl5cuXR9Ok+r0TgagPpkyZIs+dOzd6PxgMysXFxfLixYu7TH/99dfLV1xxRcyxqVOnyj/5yU9kWZblUCgkFxYWyr/73e+ijzc3N8tGo1F+5ZVXknAF3evrtR0rEAjINptNfuGFF6LHZs2aJV999dWJzmpc+np9y5cvlx0OR7fnS6f37ve//71ss9lkp9MZPZbq905UzXrJ5/OxceNGLrroougxjUbDRRddxGeffdblcz777LOY9ACXXnppNP2+ffuora2NSeNwOJg6dWq350yGeK7tWG63G7/fT3Z2dszxNWvWkJ+fz+jRo7nrrrtoaGhIaN57I97rczqdlJWVUVpaytVXX822bduij6XTe/fcc89x4403YrXGbnqVyvdOBKJeOt6utLW1tV0+p7a29rjpIz/7cs5kiOfajrVgwQKKi4tjvhCXXXYZK1eu5IMPPuCRRx5h7dq1zJgxg2AwmND89ySe6xs9ejTPP/88b775Ji+++CKhUIgzzzyTAwcOAOnz3m3YsIGtW7dy2223xRxP9Xs3oGffCyeGJUuWsGrVKtasWRPToHvjjTdGfx8/fjwTJkxgxIgRrFmzhgsvvLCrU50wpk2bFrO33plnnskpp5zCn/70Jx5++GEVc5ZYzz33HOPHj2fKlCkxx1P93okSUS/1dVdagMLCwuOmj/zsyzmTIZ5ri3jsscdYsmQJ7777LhMmTDhu2uHDh5Obm8vevXv7nee+6M/1Rej1eiZPnhzNezq8dy6Xi1WrVnHrrbf2+DrJfu9EIOqleHalnTZtWkx6gPfeey+aftiwYRQWFsakaW1tZf369Snd6TbeHXcfffRRHn74Yd555x1OP/30Hl/nwIEDNDQ0UFRUlJB891YidhQOBoNs2bIlmveB/t6BMrTE6/Xyox/9qMfXSfp7l7Jm8TSwatUq2Wg0yitWrJC3b98u33HHHXJmZma0W/fHP/6xfN9990XTr1u3TtbpdPJjjz0m79ixQ77//vu77L7PzMyU33zzTfmbb76Rr776atW6gPtybUuWLJENBoP86quvxnTxtrW1ybIsy21tbfL8+fPlzz77TN63b5/8/vvvy6eeeqo8atQoub29PaXXFs/1Pfjgg/Lq1avliooKeePGjfKNN94om0wmedu2bdE0A/W9izjrrLPkG264odNxNd47EYj66A9/+IM8dOhQ2WAwyFOmTJE///zz6GPnnnuuPGvWrJj0f/nLX+STTjpJNhgM8tixY+W33nor5vFQKCT/6le/kgsKCmSj0ShfeOGF8q5du1JxKZ305drKyspkoNPt/vvvl2VZlt1ut3zJJZfIeXl5sl6vl8vKyuTbb7+901icVOrL9d1zzz3RtAUFBfLll18ub9q0KeZ8A/W9k2VZ3rlzpwzI7777bqdzqfHeiWVABEFQnWgjEgRBdSIQCYKgOhGIBEFQnQhEgiCoTgQiQRBUJwKRIAiqE4FIEATViUAkCILqRCASBpQVK1aQmZkZvf/AAw8wadKk6P3Zs2czc+bMlOdL6B8RiIQuRdYsvvPOOzs9NnfuXCRJYvbs2THpEx0AysvLefLJJ2OO3XDDDezevbvb5yxdupQVK1ZE75933nncc889Cc2XkHgiEAndKi0tZdWqVXg8nuix9vZ2Xn75ZYYOHapKnsxmM/n5+d0+7nA4YkpMwsAgApHQrVNPPZXS0lJee+216LHXXnuNoUOHMnny5H6du6uSysyZM6OlrPPOO4+qqiruvffe6C4S0LlqdqyOJbPZs2ezdu1ali5dGj3Hvn37GDlyJI899ljM8zZv3owkSSlfK0lQiEAkHNecOXNYvnx59P7zzz/PLbfckvTXfe211ygpKeGhhx6ipqaGmpqaPp9j6dKlTJs2jdtvvz16jqFDh3a6JoDly5dzzjnnMHLkyERdgtAHIhAJx/WjH/2ITz75hKqqKqqqqli3bl2vFtLqr+zsbLRaLTabjcLCwrhWPXQ4HBgMBiwWS/QcWq2W2bNns2vXrujeX36/n5dffpk5c+Yk+jKEXhJrVgvHlZeXxxVXXMGKFSuQZZkrrriC3NxctbPVL8XFxVxxxRU8//zzTJkyhX/84x94vV6+//3vq521QUuUiIQezZkzhxUrVvDCCy8krNSg0Wg4diksv9+fkHP3xm233RZtiF++fDk33HADFoslZa8vxBKBSOjRZZddhs/nw+/3c+mllybknHl5eTHtPsFgkK1bt8akMRgM/d6+prtzXH755VitVpYtW8Y777wjqmUqE1UzoUdarZYdO3ZEf+9OS0sLmzdvjjmWk5NDaWlpp7QXXHAB8+bN46233mLEiBE88cQTNDc3x6QpLy/n448/5sYbb8RoNMZVJSwvL2f9+vVUVlaSkZFBdnY2Go0m2la0cOFCRo0aldIF74XORIlI6BW73Y7dbj9umjVr1jB58uSY24MPPthl2jlz5jBr1ixuvvlmzj33XIYPH875558fk+ahhx6isrKSESNGkJeXF1e+58+fj1arZcyYMeTl5bF///7oY7feeis+ny8lvYDC8Yk1q4VB69///jcXXngh1dXVnXZKFVJLBCJh0PF6vRw5coRZs2ZRWFjISy+9pHaWBj1RNRMGnVdeeYWysjKam5t59NFH1c6OgCgRCYJwAhAlIkEQVCcCkSAIqhOBSBAE1YlAJAiC6kQgEgRBdSIQCYKgOhGIBEFQnQhEgiCo7v8HorWYTLhMsRAAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:53:27.829938Z", + "iopub.status.busy": "2024-03-24T03:53:27.828703Z", + "iopub.status.idle": "2024-03-24T03:53:28.083388Z", + "shell.execute_reply": "2024-03-24T03:53:28.082257Z" + }, + "papermill": { + "duration": 0.279131, + "end_time": "2024-03-24T03:53:28.085692", + "exception": false, + "start_time": "2024-03-24T03:53:27.806561", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEpCAYAAADCh6TFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAApG0lEQVR4nO3de1wTZ6I38F/CJYhcFBFERCN6LFpFFFbEtsfaCnTbeuxrt+W1FxC77q7KHmsOraW7hcPaU3S90e2hcuoWL223uvXSdqtFLcputV6helABF0TxEhBUQKCGkDzvH75MTRMgIBAm/L6fTz4wzzwz80wm/DLzMHmiEEIIEBHJjNLWDSAi6gyGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJks3DKyMjA2q1Gi4uLggPD8fx48fbrF9TU4PFixfDz88PKpUKY8aMwZ49e3qotUTUWzjacuPbtm2DRqNBZmYmwsPDkZ6ejujoaBQXF8PHx8esflNTEyIjI+Hj44Pt27fD398fly5dwoABA6zeptFoxLVr1+Du7g6FQtGFe0NEXUEIgdu3b2Po0KFQKts4vxI2NGXKFLF48WJp2mAwiKFDh4q0tDSL9devXy8CAwNFU1NTp7d5+fJlAYAPPvjo5Y/Lly+3+besEMI2o0o0NTXB1dUV27dvxzPPPCOVx8XFoaamBl988YXZMk8++SS8vLzg6uqKL774AoMHD8YLL7yAZcuWwcHBweJ2dDoddDqdNF1bW4vhw4ejrKwM7u7uXb5fvYVer8fBgwcxY8YMODk52bo51AX6yjG9ffs2Ro4ciZqaGnh6erZaz2aXjdXV1TAYDPD19TUp9/X1RVFRkcVlLly4gAMHDuDFF1/Enj17UFJSgkWLFkGv1yMlJcXiMmlpaUhNTTUrP3LkCFxdXe9/R3oxV1dXHDt2zNbNoC7UF45pY2MjALTbrWPTPq+OMhqN8PHxwQcffAAHBweEhobi6tWrWLVqVavhlZSUBI1GI03X1dUhICAAUVFR8PDw6Kmm9zi9Xo/9+/cjMjLSrt+l+5K+ckzr6uqsqmez8PL29oaDgwMqKytNyisrKzFkyBCLy/j5+cHJycnkEnHs2LGoqKhAU1MTnJ2dzZZRqVRQqVRm5U5OTnb9AmjRV/azL7H3Y2rtvtnsVglnZ2eEhoYiJydHKjMajcjJyUFERITFZR566CGUlJTAaDRKZefPn4efn5/F4CIi+2XT+7w0Gg02bNiAzZs3o7CwEAsXLkRDQwPi4+MBALGxsUhKSpLqL1y4EDdv3sSSJUtw/vx57N69G++88w4WL15sq10gIhuxaZ9XTEwMqqqqkJycjIqKCoSEhCA7O1vqxC8vLze5zyMgIAB79+7F0qVLERwcDH9/fyxZsgTLli2z1S4QkY3YvMM+ISEBCQkJFufl5uaalUVERODo0aPd3Coi6u1sHl5EZK6xsdHslqH6H3T4rqAUA71Pwq3fj/+ECgoKsvvbfixheBH1QkVFRQgNDbU4748/mc7Ly8PkyZO7v1G9DMOLqBcKCgpCXl6eSVmxtgaazwqw9rkJeMBvgEndvojhRdQLubq6mp1NKS/dgOrbHzB2/ESEjBhko5b1HjYfEoeIqDMYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkqVeEV0ZGBtRqNVxcXBAeHo7jx4+3WnfTpk1QKBQmDxcXlx5sLRH1Bo62bsC2bdug0WiQmZmJ8PBwpKenIzo6GsXFxfDx8bG4jIeHB4qLi6VphULRU83ttRobG1FUVCRN1/+gw3cFpRjofRJu/VQmdYOCguDq6trTTSTqUjYPr7Vr12LBggWIj48HAGRmZmL37t3IysrCG2+8YXEZhUKBIUOG9GQze72ioiKEhoaalf/RQt28vDxMnjy5+xtF1I1sGl5NTU3Iy8tDUlKSVKZUKjFz5kwcOXKk1eXq6+sxYsQIGI1GTJ48Ge+88w4efPBBi3V1Oh10Op00XVdXBwDQ6/XQ6/VdtCe2N2rUKBw7dkyaPq+txWu7zmHV/xmHMX6eZnXtad/7iubmZumnPR8/a/fNpuFVXV0Ng8EAX19fk3JfX1+TS6B7PfDAA8jKykJwcDBqa2uxevVqTJs2DWfPnsWwYcPM6qelpSE1NdWsfN++fXZ96VRTD6iGjEZNfSO02kaTeVqt1katovtxuR4AHHH06FFcPWPr1nSfxsbG9iuhF1w2dlRERAQiIiKk6WnTpmHs2LH4n//5HyxfvtysflJSEjQajTRdV1eHgIAAREVFwcPDo0fabAuny28CBScxdepUTBzuZevmUBfoK8e05eqoPTYNL29vbzg4OKCystKkvLKy0uo+LScnJ0yaNAklJSUW56tUKqhUKrNyJycnODk5dbzRMuHo6Cj9tOf97Ev6yjG1dt9sequEs7MzQkNDkZOTI5UZjUbk5OSYnF21xWAwoKCgAH5+ft3VTCLqhWx+2ajRaBAXF4ewsDBMmTIF6enpaGhokP77GBsbC39/f6SlpQEA/vCHP2Dq1KkYPXo0ampqsGrVKly6dAm//OUvbbkbRNTDbB5eMTExqKqqQnJyMioqKhASEoLs7GypE7+8vBxK5Y8niLdu3cKCBQtQUVGBgQMHIjQ0FN999x3GjRtnq10gIhuweXgBQEJCAhISEizOy83NNZlet24d1q1b1wOtIqLerFd8PIiIqKMYXkQkSwwvIpIlhhcRyVKv6LCnzimrbkCDrtnivNKqBulny82NlvRXOWKkd/9uaR9Rd2J4yVRZdQNmrM5tt95/bC9ot87BxEcZYL1AW29GgHVvSH3pzYjhJVMtL/L0mBCM9nEzn/+DDl/lHsHTj0agfz/zj0cBQMn1ery67VSbfzDUM6x9MwLaf0PqK29GDC+ZG+3jhvH+nmbler0eFYOBySMG2vXn4OxFe29GQPtvSH3tzYjhRdSLtPZmBPAN6af430YikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLHIxQpnSGO1C6XEVZXTGULuYjbzY3N+Na8zUU3ixsdbzzsrp6KF2uQme4A8DyAHhEvRXDS6auNVxC/5Hv4c3jbdd7P/v9Nuf3HwlcawhBKHy7sHVE3Y/hJVND+49AQ9lv8W5MCEZZGPO8ubkZhw8dxkMPP9TqmVfp9Xos2XYKQ2eM6O7mEnU5hpdMqRxcYLzjj5EeD2DcIMtfwFHmWIaxXmNbHe/ceKcWxjtVUDm4dHdzqR3tdQMA7XcF9LVuAIYXUS9gbTcA0HZXQF/qBmB4EfUC7XUDAO13BfS1bgCGF1Ev0F43ANB+V0Bf6wbo8H1eFy5c6I52EBF1SIfDa/To0ZgxYwY+/vhj3Llzp0sakZGRAbVaDRcXF4SHh+P4cSsu/AFs3boVCoUCzzzzTJe0g4jko8PhlZ+fj+DgYGg0GgwZMgS//vWvrQ4bS7Zt2waNRoOUlBTk5+dj4sSJiI6OxvXr19tc7uLFi0hMTMQjjzzS6W0TkXx1OLxCQkLw7rvv4tq1a8jKyoJWq8XDDz+M8ePHY+3ataiqqurQ+tauXYsFCxYgPj4e48aNQ2ZmJlxdXZGVldXqMgaDAS+++CJSU1MRGBjY0V0gIjvQ6Q57R0dHzJkzB0899RTef/99JCUlITExEW+++Saef/55rFy5En5+fm2uo6mpCXl5eUhKSpLKlEolZs6ciSNHjrS63B/+8Af4+PjglVdewbffftvmNnQ6HXQ6nTRdV1cH4G7np16vt2ZXe6Xm5mbpp6X9aClrax/bWwf1HGuORXvH1F6Op7Vt73R4nTx5EllZWdi6dSv69++PxMREvPLKK7hy5QpSU1Mxe/bsdi8nq6urYTAY4Otrek+Kr68vioqKLC5z6NAhfPjhhzh16pRV7UxLS0NqaqpZ+b59++Dq6mrVOnqjy/UA4IhDhw7hkuX/rAMA9u/ff9/roO7XkWPR2jG1l+PZ2NhoVb0Oh9fatWuxceNGFBcX48knn8SWLVvw5JNPQqm8ewU6cuRIbNq0CWq1uqOrbtft27fx8ssvY8OGDfD29rZqmaSkJGg0Gmm6rq4OAQEBiIqKgoeHR5e3saecvVaH1QVH8fDDD+PBoeb7odfrsX//fkRGRrZ6h31766CeY82xaO+Y2svxbLk6ak+Hw2v9+vWYP38+5s2b1+ploY+PDz788MN21+Xt7Q0HBwdUVlaalFdWVmLIkCFm9UtLS3Hx4kXMmjVLKjMajQDuXsYWFxdj1KhRJsuoVCqoVCqzdTk5ObX6Ry0HLTcpOjo6trkfbe2nteug7teRY9HaMbWX42lt2zscXvv378fw4cOlM60WQghcvnwZw4cPh7OzM+Li4tpdl7OzM0JDQ5GTkyPd7mA0GpGTk4OEhASz+kFBQSgoKDAp+/3vf4/bt2/j3XffRUBAQEd3h4hkqsPhNWrUKGi1Wvj4+JiU37x5EyNHjoTBYOjQ+jQaDeLi4hAWFoYpU6YgPT0dDQ0NiI+PBwDExsbC398faWlpcHFxwfjx402WHzBgAACYlRORfetweAkhLJbX19fDxaXjH0uIiYlBVVUVkpOTUVFRgZCQEGRnZ0ud+OXl5WZneUREVodXS6e3QqFAcnKyyX/qDAYDjh07hpCQkE41IiEhweJlIgDk5ua2ueymTZs6tU0ikjerw+v7778HcPfMq6CgAM7OztI8Z2dnTJw4EYmJiV3fQiIiC6wOr4MHDwIA4uPj8e6778r6NgMikr8O93lt3LixO9pBRNQhVoXXnDlzsGnTJnh4eGDOnDlt1t25c2eXNIyIqC1WhZenpycUCoX0OxGRrVkVXvdeKvKykYh6A95ARUSyZNWZ16RJk6TLxvbk5+ffV4OIiKxhVXhxmGUi6m2sCq+UlJTubgcRUYewz4uIZMmqMy8vLy+cP38e3t7eGDhwYJv9Xzdv3uyyxhERtcaq8Fq3bh3c3d2l363tvCci6i5Whde9AwvOmzevu9pCRGS1Dvd5OTg4WPxOxRs3bsDBwaFLGkVE1J4Oh1drgxHqdDqTYXKIiLqT1aNK/OlPfwJwdzDCP//5z3Bz+/G7lQwGA/7xj38gKCio61tIRGSB1eG1bt06AHfPvDIzM00uEZ2dnaFWq5GZmdn1LSQissDq8CorKwMAzJgxAzt37sTAgQO7rVFERO3p8GCELSOqEhHZUofDa/78+W3Oz8rK6nRjiIis1eHwunXrlsm0Xq/HmTNnUFNTg8cee6zLGkZE1JYOh9euXbvMyoxGIxYuXIhRo0Z1SaOIiNrTJR/MViqV0Gg00n8kiYi6W5eNKlFaWorm5uauWh0RUZs6fNnY8s3ZLYQQ0Gq12L17t8lnIImIulOHw6vlm7NbKJVKDB48GGvWrGn3P5FERF2F93kRkSxxJFUikiWGFxHJEsOLiGSJ4UVEstRl4XXlyhX86le/6qrVERG1qcvC68aNG/jwww87tWxGRgbUajVcXFwQHh6O48ePt1p3586dCAsLw4ABA9C/f3+EhITgo48+6myzZesHvQEAcOZqrcVH/qVbOFkF5F+61Wqdkuv1Nt4Los7r8K0SXW3btm3QaDTIzMxEeHg40tPTER0djeLiYvj4+JjV9/Lywu9+9zsEBQXB2dkZX331FeLj4+Hj44Po6Ggb7IFtlP7/4HljZ0EbtRzxUcmJdtfVX2XzlwFRh9n8Vbt27VosWLAA8fHxAIDMzEzs3r0bWVlZeOONN8zqP/rooybTS5YswebNm3Ho0KE+FV5RDw4BAIzycUM/J/MvPinW1uI/thdgzS8m4AE/z1bX01/liJHe/butnUTdxabh1dTUhLy8PCQlJUllSqUSM2fOxJEjR9pdXgiBAwcOoLi4GCtXrrRYR6fTQafTSdN1dXUA7g7lo9fr73MPbMfdWYFnJ/m1Ov/OnTsAgBEDVXjAx7XNdcn5ebAXLZ8Lbm5ubvV4tJS3Nt+adciBtW23OrzmzJnT5vyamhprVyWprq6GwWCAr6+vSbmvry+KiopaXa62thb+/v7Q6XRwcHDA+++/j8jISIt109LSkJqaala+b98+uLq2/UctZ5frAcARR48exdUztm4Ntae0DgAcsXXvYQzrb/kbuvRG4KYOuLBjP5ws9FZX/qAA4IBDhw7hkpv5fLlobGy0qp7V4eXp2fqlR8v82NhYa1d3X9zd3XHq1CnU19cjJycHGo0GgYGBZpeUAJCUlGTyYfK6ujoEBAQgKioKHh4ePdJeWzhdfhMoOImpU6di4nAvWzeH2vHXk1eAs+ew9cL9f/dp9OPToR4k366Alquj9lgdXhs3bux0Y1rj7e0NBwcHVFZWmpRXVlZiyJAhrS6nVCoxevRoAEBISAgKCwuRlpZmMbxUKhVUKpVZuZOTE5ycnO5vB3oxR0dH6ac976e9+HmwPxwcHFrtwwSs68e0hz5Ma1+vNu3zcnZ2RmhoKHJycvDMM88AuDsqa05ODhISEqxej9FoNOnXIpIbr/7O+L9ThrdZp6VPa9Tg/hjv3/aVUF9gdXhZO9xNR7+AQ6PRIC4uDmFhYZgyZQrS09PR0NAg/fcxNjYW/v7+SEtLA3C3DyssLAyjRo2CTqfDnj178NFHH2H9+vUd2i4RyZvV4bVp0yaMGDECkyZNghCWOxQ7IyYmBlVVVUhOTkZFRQVCQkKQnZ0tdeKXl5dDqfyxd7KhoQGLFi3ClStX0K9fPwQFBeHjjz9GTExMl7WJiHo/q8Nr4cKF+PTTT1FWVob4+Hi89NJL8PLqmo7ghISEVi8Tc3NzTabffvttvP32212yXSKSL6s/HpSRkQGtVovXX38df/vb3xAQEIDnn38ee/fu7dIzMSIia3Tos40qlQpz587F/v37ce7cOTz44INYtGgR1Go16uv5OTki6jmd/mC2UqmEQqGAEAIGg6Er20RE1K4OhZdOp8Onn36KyMhIjBkzBgUFBfjv//5vlJeXw81Nxrf0EpHsWN1hv2jRImzduhUBAQGYP38+Pv30U3h7e3dn24iIWmV1eGVmZmL48OEIDAzE3//+d/z973+3WG/nzp1d1jgiotZYHV6xsbFQKBTd2RYiIqt16CZVIqLegl/AQUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSy1CvCKyMjA2q1Gi4uLggPD8fx48dbrbthwwY88sgjGDhwIAYOHIiZM2e2WZ+I7JPNw2vbtm3QaDRISUlBfn4+Jk6ciOjoaFy/ft1i/dzcXMydOxcHDx7EkSNHEBAQgKioKFy9erWHW05EtmTz8Fq7di0WLFiA+Ph4jBs3DpmZmXB1dUVWVpbF+p988gkWLVqEkJAQBAUF4c9//jOMRiNycnJ6uOVEZEuOttx4U1MT8vLykJSUJJUplUrMnDkTR44csWodjY2N0Ov18PLysjhfp9NBp9NJ03V1dQAAvV4PvV5/H63v3Zqbm6Wf9ryffUlfOabW7ptNw6u6uhoGgwG+vr4m5b6+vigqKrJqHcuWLcPQoUMxc+ZMi/PT0tKQmppqVr5v3z64urp2vNEycbkeABxx9OhRXD1j69ZQV+grx7SxsdGqejYNr/u1YsUKbN26Fbm5uXBxcbFYJykpCRqNRpquq6uT+sk8PDx6qqk97nT5TaDgJKZOnYqJwy2flZK89JVj2nJ11B6bhpe3tzccHBxQWVlpUl5ZWYkhQ4a0uezq1auxYsUKfPPNNwgODm61nkqlgkqlMit3cnKCk5NT5xouA46OjtJPe97PvqSvHFNr982mHfbOzs4IDQ016Wxv6XyPiIhodbk//vGPWL58ObKzsxEWFtYTTSWiXsbml40ajQZxcXEICwvDlClTkJ6ejoaGBsTHxwMAYmNj4e/vj7S0NADAypUrkZycjL/85S9Qq9WoqKgAALi5ucHNzc1m+0FEPcvm4RUTE4OqqiokJyejoqICISEhyM7Oljrxy8vLoVT+eIK4fv16NDU14Re/+IXJelJSUvCf//mfPdl0IrIhm4cXACQkJCAhIcHivNzcXJPpixcvdn+DiKjXs/lNqkREncHwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSz1ijHs6f41NjaafMt4sbYGuooSFJ7pB+ONASZ1g4KC7PrbwqlvYHjZiaKiIoSGhpqVv7DZvG5eXh4mT57cA60i6j4MLzsRFBSEvLw8abr+Bx12HzyCp2ZEwK2fyqwukdwxvOyEq6urydmUXq/HrerriJgSZtdfDU99FzvsiUiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEs2D6+MjAyo1Wq4uLggPDwcx48fb7Xu2bNn8eyzz0KtVkOhUCA9Pb3nGkpEvYpNP9u4bds2aDQaZGZmIjw8HOnp6YiOjkZxcTF8fHzM6jc2NiIwMBDPPfccli5daoMWE/WMnw5xBLQ+zFGfHeJI2NCUKVPE4sWLpWmDwSCGDh0q0tLS2l12xIgRYt26dR3eZm1trQAgamtrO7ysnDQ1NYnPP/9cNDU12bop1Al5eXkCgFWPvLw8Wze3S1n7N2qzM6+mpibk5eUhKSlJKlMqlZg5cyaOHDnSZdvR6XTQ6XTSdF1dHYC7oy7o9fou205v07Jv9ryP9mzUqFE4duyYSVn9Dzrs/fYEoh/5mckwR6NGjbKr42ztvtgsvKqrq2EwGODr62tS7uvra3a6fD/S0tKQmppqVr5v374+caq9f/9+WzeButC0CaNwu+Ymbtf8WKbVam3Wnu7Q2NhoVT27H88rKSkJGo1Gmq6rq0NAQACioqLg4eFhw5Z1L71ej/379yMyMpLjedmJvnJMW66O2mOz8PL29oaDgwMqKytNyisrKzFkyJAu245KpYJKpTIrd3JysusXQIu+sp99ib0fU2v3zWa3Sjg7OyM0NBQ5OTlSmdFoRE5ODiIiImzVLCKSCZteNmo0GsTFxSEsLAxTpkxBeno6GhoaEB8fDwCIjY2Fv78/0tLSANzt5D937pz0+9WrV3Hq1Cm4ublh9OjRNtsPIup5Ng2vmJgYVFVVITk5GRUVFQgJCUF2drbUiV9eXg6l8seTw2vXrmHSpEnS9OrVq7F69WpMnz4dubm5Pd18IrIhm3fYJyQkICEhweK8nwaSWq2GEKIHWkVEvZ3NPx5ERNQZNj/z6mktZ27W/jtWrvR6PRobG1FXV2fX/5nqS/rKMW3522zvKqvPhdft27cBAAEBATZuCRG15fbt2/D09Gx1vkL0sU4ko9GIa9euwd3dHQqFwtbN6TYtN+NevnzZrm/G7Uv6yjEVQuD27dsYOnSoyT/sfqrPnXkplUoMGzbM1s3oMR4eHnb9Qu+L+sIxbeuMqwU77IlIlhheRCRLDC87pVKpkJKSYvFznSRPPKam+lyHPRHZB555EZEsMbyISJYYXkQkSwwvG7l48SIUCgVOnTrVI+s5fPgwJkyYACcnJzzzzDP3tU2i3oDh1Uvk5uZCoVCgpqamW9av0WgQEhKCsrIybNq0qVu2QeYeffRRvPrqq7ZuhqS3ted+MLw6oampydZN6LDS0lI89thjGDZsGAYMGNCpdfTkfgsh0Nzc3GPb683k+HrrEd37DWz2Yfr06WLx4sViyZIlYtCgQeLRRx8VBQUF4oknnhD9+/cXPj4+4qWXXhJVVVXSMl9//bV46KGHhKenp/Dy8hJPPfWUKCkpkeaXlZUJAOL777+Xfr/3ERcX16H1fPrppyIiIkKoVCrx4IMPitzcXJP59z42btwohBAiNzdX/OxnPxPOzs5iyJAhYtmyZUKv17e53wcPHhQARHZ2tggJCREuLi5ixowZorKyUuzZs0cEBQUJd3d3MXfuXNHQ0CCty2AwiHfeeUeo1Wrh4uIigoODxWeffSbNb1nvnj17xOTJk4WTk5M4ePBgVx7GHhcXF2f23JeUlIj58+dLz8OYMWNEenq62XKzZ88Wb7/9tvDz8xNqtVoIIcThw4fFxIkThUqlEqGhoWLXrl3Sa6hFW69LS+0pKyvrqaejyzG8rDB9+nTh5uYmXnvtNVFUVCSOHj0qBg8eLJKSkkRhYaHIz88XkZGRYsaMGdIy27dvFzt27BD//Oc/xffffy9mzZolJkyYIAwGgxDCNLyam5vFjh07BABRXFwstFqtqKmp6dB6hg0bJrZv3y7OnTsnfvnLXwp3d3dRXV0tmpubhVarFR4eHiI9PV1otVrR2Ngorly5IlxdXcWiRYtEYWGh2LVrl/D29hYpKSmt7ndRUZEUMlOnThWHDh0S+fn5YvTo0WL69OkiKipK5Ofni3/84x9i0KBBYsWKFdK63n77bREUFCSys7NFaWmp2Lhxo1CpVFLItqw3ODhY7Nu3T5SUlIgbN25096HtVjU1NSIiIkIsWLBAaLVaodVqxZ07d0RycrI4ceKEuHDhgvj444+Fq6ur2LZtm7RcXFyccHNzEy+//LI4c+aMOHPmjKitrRVeXl7ipZdeEmfPnhV79uwRY8aMMQmvW7dutfm6tNSe5uZmWzw1XYLhZYXp06eLSZMmSdPLly8XUVFRJnUuX74shY8lVVVVAoAoKCgQQpiGlxA//vHeunWrzba0tp57g0Kv14thw4aJlStXSmWenp7SGZcQQrz55pvigQceEEajUSrLyMgQbm5uUjD+dL/vbec333wjlaWlpQkAorS0VCr79a9/LaKjo4UQQty5c0e4urqK7777zmRdr7zyipg7d67Jej///PM2919upk+fLpYsWdJmncWLF4tnn31Wmo6LixO+vr5Cp9NJZevXrxeDBg0SP/zwg1S2YcMGk9eQNa9La9ojF+zzslJoaKj0++nTp3Hw4EG4ublJj6CgIAB3+5YA4J///Cfmzp2LwMBAeHh4QK1WA7g7Ln9HWLuee79xydHREWFhYSgsLGx1vYWFhYiIiDAZFuihhx5CfX09rly5YnG/7xUcHCz97uvrC1dXVwQGBpqUXb9+HQBQUlKCxsZGREZGmjxnW7ZskZ6vFmFhYa222V5kZGQgNDQUgwcPhpubGz744AOz4zlhwgQ4OztL08XFxQgODoaLi4tUNmXKFJNlrHld2pM+NyROZ/Xv31/6vb6+HrNmzcLKlSvN6vn5+QEAZs2ahREjRmDDhg0YOnQojEYjxo8f3+HO165aT2fdu9/3unckT4VCYTayp0KhgNFoBHD3+QKA3bt3w9/f36TeTz+n19r27MXWrVuRmJiINWvWICIiAu7u7li1ahWOHTtmUq8zz4M1r0t7wvDqhMmTJ2PHjh1Qq9VwdDR/Cm/cuIHi4mJs2LABjzzyCADg0KFDba6z5V3WYDB0aj1Hjx7Fv/7rvwIAmpubkZeX1+oXmwDA2LFjsWPHDgghpLOvw4cPw93dvcvHOxs3bhxUKhXKy8sxffr0Ll13b+fs7GxyTA8fPoxp06Zh0aJFUpk1Z0UPPPAAPv74Y+h0OinwT5w4YVKnvdelpfbIGS8bO2Hx4sW4efMm5s6dixMnTqC0tBR79+5FfHw8DAYDBg4ciEGDBuGDDz5ASUkJDhw4AI1G0+Y6R4wYAYVCga+++gpVVVWor6/v0HoyMjKwa9cuFBUVYfHixbh16xbmz5/f6vYWLVqEy5cv47e//S2KiorwxRdfICUlBRqNps3RKzvD3d0diYmJWLp0KTZv3ozS0lLk5+fjvffew+bNm7t0W72NWq3GsWPHcPHiRVRXV+Nf/uVfcPLkSezduxfnz5/HW2+9ZRZClrzwwgswGo341a9+hcLCQuzduxerV68GAOnNp73XpaX2tJwdyxHDqxOGDh2Kw4cPw2AwICoqChMmTMCrr76KAQMGQKlUQqlUYuvWrcjLy8P48eOxdOlSrFq1qs11+vv7IzU1FW+88QZ8fX2RkJDQofWsWLECK1aswMSJE3Ho0CF8+eWX8Pb2bnN7e/bswfHjxzFx4kT85je/wSuvvILf//739/XctGb58uV46623kJaWhrFjx+KJJ57A7t27MXLkyG7ZXm+RmJgIBwcHjBs3DoMHD0Z0dDTmzJmDmJgYhIeH48aNGyZnYa3x8PDA3/72N5w6dQohISH43e9+h+TkZACQ+sHae11aak9H+2B7Ew6JQyRTn3zyCeLj41FbW4t+/frZujk9jn1eRDKxZcsWBAYGwt/fH6dPn8ayZcvw/PPP98ngAhheRLJRUVGB5ORkVFRUwM/PD8899xz+67/+y9bNshleNhKRLLHDnohkieFFRLLE8CIiWWJ4EZEsMbyISJYYXtSt5s2bB4VCIX1429fXF5GRkcjKyurQR1M2bdrU6RFg78e8efM45n8vxfCibvfEE09Aq9Xi4sWL+PrrrzFjxgwsWbIETz/9NId6ps6z6WhiZPdahjT+qZycHAFAbNiwQQghxJo1a8T48eOFq6urGDZsmFi4cKG4ffu2EOLHgQrvfbSM+LplyxYRGhoq3NzchK+vr5g7d66orKyUtnPz5k3xwgsvCG9vb+Hi4iJGjx4tsrKypPnl5eXiueeeE56enmLgwIHi3/7t36ShkVNSUsy2K/ehqe0Jz7zIJh577DFMnDgRO3fuBAAolUr86U9/wtmzZ7F582YcOHAAr7/+OgBg2rRpSE9Ph4eHB7RaLbRaLRITEwEAer0ey5cvx+nTp/H555/j4sWLmDdvnrSdt956C+fOncPXX3+NwsJCrF+/XvrAul6vR3R0NNzd3fHtt9/i8OHDcHNzwxNPPIGmpiYkJibi+eefl84ctVotpk2b1rNPFLXO1ulJ9q21My8hhIiJiRFjx461OO+zzz4TgwYNkqY3btwoPD09293eiRMnBADprG3WrFkiPj7eYt2PPvrIbChsnU4n+vXrJ/bu3dtu+8m2eOZFNiPuGQjxm2++weOPPw5/f3+4u7vj5Zdfxo0bN9DY2NjmOvLy8jBr1iwMHz4c7u7u0mCHLUO9LFy4EFu3bkVISAhef/11fPfdd9Kyp0+fRklJCdzd3aVhk728vHDnzh27HDbZ3jC8yGYKCwsxcuRIXLx4EU8//TSCg4OxY8cO5OXlISMjA0Db31nY0NCA6OhoeHh44JNPPsGJEyewa9cuk+V+/vOf49KlS1i6dCmuXbuGxx9/XLrkrK+vR2hoKE6dOmXyOH/+PF544YVu3nu6XxxVgmziwIEDKCgowNKlS5GXlwej0Yg1a9ZIg+b99a9/NalvafjioqIi3LhxAytWrEBAQAAA4OTJk2bbGjx4MOLi4hAXF4dHHnkEr732GlavXo3Jkydj27Zt8PHxgYeHh8V22tOwyfaGZ17U7XQ6HSoqKnD16lXk5+fjnXfewezZs/H0008jNjYWo0ePhl6vx3vvvYcLFy7go48+QmZmpsk61Go16uvrkZOTg+rqajQ2NmL48OFwdnaWlvvyyy+xfPlyk+WSk5PxxRdfoKSkBGfPnsVXX32FsWPHAgBefPFFeHt7Y/bs2fj2229RVlaG3Nxc/Pu//7v0DUpqtRr/+7//i+LiYlRXV0Ov1/fMk0bts3WnG9m3e7+l2dHRUQwePFjMnDlTZGVlSd8PKYQQa9euFX5+fqJfv34iOjpabNmyxex7LH/zm9+IQYMGmdwq8Ze//EWo1WqhUqlERESE+PLLL82+y3Ds2LGiX79+wsvLS8yePVtcuHBBWqdWqxWxsbHC29tbqFQqERgYKBYsWCBqa2uFEEJcv35dREZGCjc3N94q0ctwPC8ikiVeNhKRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSpf8Hwogpq0/L8YQAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:53:28.129001Z", + "iopub.status.busy": "2024-03-24T03:53:28.128618Z", + "iopub.status.idle": "2024-03-24T03:53:28.419159Z", + "shell.execute_reply": "2024-03-24T03:53:28.418109Z" + }, + "papermill": { + "duration": 0.315295, + "end_time": "2024-03-24T03:53:28.421859", + "exception": false, + "start_time": "2024-03-24T03:53:28.106564", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATcAAAEmCAYAAAAHqNcMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABixklEQVR4nO2dd3wT5R/HP5fZPVgdUDaUXWahRVmigIjgAlFZAoKCChWkIFKWFpWlgEwZDmQvWQpl/dhQVhllFsrogO6Zeb8/0lzvsi9NmjR93q9XXm0uz909SS7f++6HommaBoFAILgYAkdPgEAgEOwBEW4EAsElIcKNQCC4JES4EQgEl4QINwKB4JIQ4UYgEFwSItwIBIJLQoQbgUBwSUSOnkB5o1ar8ezZM3h7e4OiKEdPh0Ag6EDTNPLy8hAcHAyBwHr9q9IJt2fPniEkJMTR0yAQCGZ4/PgxatWqZfX+lU64eXt7A9B8cD4+Pg6eDYFA0CU3NxchISHMb9VaKp1w05qiPj4+RLgRCE5MWd1GJKBAIBBcEiLcCASCS0KEG4FAcEkqnc+tokDTNJRKJVQqlaOnQiDYFKFQCJFIZPdULIcKt+XLl2P58uV4+PAhAKB58+aYMWMG+vTpY3D8+vXrMWLECM42qVSK4uJie0+1XJHL5UhJSUFhYaGjp0Ig2AUPDw8EBQVBIpHY7RwOFW61atXCvHnz0KhRI9A0jQ0bNqB///64fPkymjdvbnAfHx8f3L59m3nuaom4arUaSUlJEAqFCA4OhkQicbn3SKi80DQNuVyO58+fIykpCY0aNSpToq4pHCrc+vXrx3n+3XffYfny5Th79qxR4UZRFAIDA+0+t8TUXERtvoqqXhL8MbKj3c+nRS6XQ61WIyQkBB4eHuV2XgKhvHB3d4dYLMajR48gl8vh5uZml/M4TUBBpVJh06ZNKCgoQEREhNFx+fn5qFOnDkJCQtC/f3/cuHHD5HFlMhlyc3M5D4vmo6ZxMyUXt1PzeL0PW2GvuxmB4AyUx/Xt8F9QQkICvLy8IJVKMXbsWOzcuRPNmjUzODY0NBRr167F7t278eeff0KtViMyMhJPnjwxevzY2Fj4+voyD0tLr9zFQgBAkYI49AmEiojDhVtoaCiuXLmCc+fO4dNPP8WwYcNw8+ZNg2MjIiIwdOhQtG7dGl27dsWOHTtQvXp1rFy50ujxp06dipycHObx+PFji+blLikRbnIi3AiEiojDhZtEIkHDhg3Rrl07xMbGIiwsDD///LNF+4rFYrRp0wb37t0zOkYqlTKlVnxKrjzEGnekUk1DoVJbtA/B9jx8+BAUReHKlSvlcpxTp06hZcuWEIvFGDBgQJnOWVk5duwYKIpCdna2Q+fhcOGmi1qthkwms2isSqVCQkICgoKCbD4PN0npR0NMU+fB3j+cqKgotG7dGklJSVi/fr1dzkEoHxwaLZ06dSr69OmD2rVrIy8vDxs3bsSxY8fw77//AgCGDh2KmjVrIjY2FgAwe/ZsdOrUCQ0bNkR2djZ++uknPHr0CKNGjbL53CRCAQQUoKY1pqmPm9jm53B15HK5XfOY7MH9+/cxduzYMrXaKc/3TdM0VCoVRCLb/pQr4neni0M1t/T0dAwdOhShoaF45ZVXcOHCBfz777949dVXAQDJyclISUlhxmdlZWH06NFo2rQpXn/9deTm5uL06dNGAxBlgaIoeEg0F4yj/W40TaNQrnTIg6Zpi+fZrVs3jB8/HhMmTEC1atXQq1cvXL9+HX369IGXlxcCAgIwZMgQvHjxgtnn4MGDeOmll+Dn54eqVavijTfewP379w0e/+HDh+jevTsAwN/fHxRFYfjw4byOk5iYiMjISLi5uaFFixY4fvw4c2yKopCRkYGPP/4YFEUxmtvx48cRHh4OqVSKoKAgREdHQ6lUmnzfWg3z33//RZs2beDu7o4ePXogPT0dBw4cQNOmTeHj44MPPviAk6ytVqsRGxuLevXqwd3dHWFhYdi2bRvzuva4Bw4cQLt27SCVSnHy5Emz383cuXNRo0YNeHt7Y9SoUYiOjkbr1q2Z14cPH44BAwbgu+++Q3BwMEJDQwEAf/zxB9q3bw9vb28EBgbigw8+QHp6OufY+/fvR+PGjeHu7o7u3bszSfmOxqGa22+//Wby9WPHjnGeL1q0CIsWLbLjjLi4iYXIlykdbpYWKVRoNuNfh5z75uxejJC3hA0bNuDTTz/FqVOnkJ2djR49emDUqFFYtGgRioqKMGXKFAwcOBBHjhwBABQUFCAqKgqtWrVCfn4+ZsyYgbfeegtXrlzRSxcICQnB9u3b8c477+D27dvw8fGBu7s7r+NMnjwZixcvRrNmzbBw4UL069cPSUlJCAkJQUpKCkJDQzF79mwMGjQIvr6+ePr0KV5//XUMHz4cv//+OxITEzF69Gi4ublh5syZBt83AOamPHPmTCxduhQeHh4YOHAgBg4cCKlUio0bNyI/Px9vvfUWlixZgilTpgDQRPf//PNPrFixAo0aNcKJEyfw0UcfoXr16ujatStzvujoaMyfPx/169eHv7+/ye/kr7/+wnfffYdff/0VnTt3xqZNm7BgwQLUq1ePMy4uLg4+Pj44dOgQs02hUGDOnDkIDQ1Feno6oqKiMHz4cOzfvx+Api/i22+/jXHjxuGTTz7BxYsX8dVXX5m/UMoBUltqAo+SiGkhiZhaTKNGjfDjjz8C0GgLbdq0wffff8+8vnbtWoSEhODOnTto3Lgx3nnnHc7+a9euRfXq1XHz5k20aNGC85pQKESVKlUAADVq1ICfnx/zmqXHGT9+PDN2+fLlOHjwIH777Td8/fXXCAwMBEVR8PX1ZRLFf/31V4SEhGDp0qWgKApNmjTBs2fPMGXKFMyYMYMRnOz3DZQKt7lz56Jz584AgJEjR2Lq1Km4f/8+6tevDwB49913cfToUUyZMgUymQzff/89Dh8+zOR61q9fHydPnsTKlSs5wm327NmMhWOOJUuWYOTIkUzp4owZM/Dff/8hPz+fM87T0xNr1qzhmKMff/wx83/9+vXxyy+/oEOHDsjPz4eXlxeWL1+OBg0aYMGCBQA02Q8JCQn44YcfLJqbPSHCzQTaXLdiB2tu7mIhbs7u5bBz86Fdu3bM/1evXsXRo0fh5eWlN+7+/fto3Lgx7t69ixkzZuDcuXN48eIF1GpNZDo5OVlPuJnC0uOwE8RFIhHat2+PW7duGT3urVu3EBERwSmB69y5M/Lz8/HkyRPUrl1b732zadWqFfN/QEAAPDw8GMGm3Xb+/HkAwL1791BYWKgntORyOdq0acPZ1r59e6Nz1uX27dv47LPPONvCw8MZ7VlLy5Yt9fxs8fHxmDlzJq5evYqsrCzO59qsWTPcunULHTtyK3hMJeGXJ0S4mcDNSTQ3tv/P2fH09GT+z8/PR79+/QzexbUR7n79+qFOnTpYvXo1goODoVar0aJFC8jlcl7ntdVxrIX9vtmIxaWBKIqiOM+127QCQ6tJ7du3DzVr1uSMk0qlFp2vLOges6CgAL169UKvXr3w119/oXr16khOTkavXr3K7XMtCxXjF+MgPEiVQplo27Yttm/fjrp16xqM5mVkZOD27dtYvXo1Xn75ZQAw6xzXahbsVlB8jnP27Fl06dIFAKBUKhEfH4/x48cbPV/Tpk2xfft20DTNaG+nTp2Ct7d3mSKqhmjWrBmkUimSk5M5JmhZCQ0NxYULFzB06FBm24ULF8zul5iYiIyMDMybN4+p7Ll48SJnTNOmTbFnzx7OtrNnz9pg1mXH6fLcnAltlUIx8blZxbhx45CZmYnBgwfjwoULuH//Pv7991+MGDECKpUK/v7+qFq1KlatWoV79+7hyJEjiIqKMnnMOnXqgKIo7N27F8+fP0d+fj6v4yxbtgw7d+5EYmIixo0bh6ysLI5fSZfPPvsMjx8/xueff47ExETs3r0bMTExiIqKsnl9pLe3NyZNmoSJEydiw4YNuH//Pi5duoQlS5Zgw4YNVh/3888/x2+//YYNGzbg7t27mDt3Lq5du2a220zt2rUhkUiwZMkSPHjwAHv27MGcOXM4Y8aOHYu7d+9i8uTJuH37NjZu3Og0+YFEuJlA628qlCvNjCQYIjg4GKdOnYJKpcJrr72Gli1bYsKECfDz84NAIIBAIMCmTZsQHx+PFi1aYOLEifjpp59MHrNmzZqYNWsWoqOjERAQgPHjx/M6zrx58zBv3jyEhYXh5MmT2LNnD6pVq2byfPv378f58+cRFhaGsWPHYuTIkZg+fXqZPhtjzJkzB99++y1iY2PRtGlT9O7dG/v27dOLbPLhww8/xNSpUzFp0iS0bdsWSUlJGD58uNluHNWrV8f69euxdetWNGvWDPPmzcP8+fM5Y2rXro3t27dj165dCAsLw4oVKzgBJEdC0XwSmVyA3Nxc+Pr6Iicnx2wp1qStV7Et/gmm9G6CT7s1KJf5FRcXIykpCfXq1bNbKxgC4dVXX0VgYCD++OMPh5zf1HXO5zdqCuJzMwHpDEJwBQoLC7FixQr06tULQqEQf//9Nw4fPszJZ3NFiFlqAg+mMwgxSwnOg1rH2GrevDm8vLwMPv766y9QFIX9+/ejS5cuaNeuHf755x9s374dPXv2dNA7KB+I5mYCN6K5EZyMtNxipOUWo0F1L3hKNT/f/fv3Q6FQGBwfEBAAd3d3HD58uDyn6RQQ4WYCdyfJcyMQtKTlahZDepZThEY1vAFoIsgEfYhZagKtWeroCgUCgcAfItxMwJilRHMjECocRLiZoDTPjQg3AqGiQYSbCYhZSiBUXIhwMwHJcyMQKi5EuJmAREvLl7p162Lx4sXMc4qisGvXLofNx5kxXRVKAEgqiEnciVnqUFJSUsx2mSUQjEGEmwlIQMGxaLvhOhJ7LcBCsD8ONUuXL1+OVq1aMeuJRkRE4MCBAyb32bp1K5o0aQI3Nze0bNmS6eVuD5iFmRUqXgul2ByaBuQFjnnwfN95eXn48MMP4enpiaCgICxatAjdunXDhAkTeL9ttlmqXcBlx44d6N69Ozw8PBAWFoYzZ85w9jl58iRefvlluLu7IyQkBF988QUKCgqY180teGLtAiwE58Oht6NatWph3rx5aNSoEWiaxoYNG9C/f39cvnwZzZs31xt/+vRpDB48GLGxsXjjjTewceNGDBgwAJcuXeLVktpStJobTQMypZrJeyt3FIXA98GOOfe0Z4DE8q6vUVFROHXqFPbs2YOAgADMmDEDly5d4qy0VBa++eYbzJ8/H40aNcI333yDwYMH4969exCJRLh//z569+6NuXPnYu3atXj+/DnGjx+P8ePHY926dQDML3iihc8CLATnxKHCrV+/fpzn3333HZYvX46zZ88aFG4///wzevfujcmTJwPQ9L46dOgQli5dihUrVth8fuz1A4rkKscJtwpCXl4eNmzYgI0bN+KVV14BAKxbtw7BwbYTzJMmTULfvn0BALNmzULz5s1x7949NGnSBLGxsfjwww8ZLbFRo0b45Zdf0LVrVyxfvhxubm5mFzzRwmcBFoJz4jSOBJVKha1bt6KgoMDoAhNnzpzR67Daq1cvkxE1mUzGWcE+NzfX4jmJhAJIhALIVWoUKVRw2P1b7KHRoBx1bgt58OABFAoFwsPDmW2+vr7MGpi2gL3ginYdhvT0dDRp0gRXr17FtWvX8NdffzFjaJqGWq1GUlISmjZtanbBEy18FmAhOCcOF24JCQmIiIhAcXExvLy8sHPnTqOLLKempiIgIICzLSAgAKmpqUaPHxsbi1mzZlk9PzdxqXBzGBTFyzR0ZXQXXAHAWWBlzJgx+OKLL/T2q127Nq8FT+yxAIttIckg5nC4cAsNDcWVK1eQk5ODbdu2YdiwYTh+/LjNVpGfOnUqR9vLzc1lFruwBA+JCLnFSlJfagH169eHWCzGhQsXmCXvcnJycOfOHWZRFnvStm1b3Lx5Ew0bNjT4ekJCgtkFTwiug8OFm0QiYS7Gdu3a4cKFC/j555+xcuVKvbGBgYFIS0vjbEtLSzOZMiCVSvWWReMDO2JKMI23tzeGDRuGyZMno0qVKqhRowZiYmIgEAjMLkZiC6ZMmYJOnTph/PjxGDVqFDw9PXHz5k3GL8te8GTs2LG4fv263oInBNfB6SoU1Go1x0fGJiIiAnFxcZxthw4dsusisG4k140XCxcuREREBN544w307NkTnTt3RtOmTctlPYhWrVrh+PHjuHPnDl5++WW0adMGM2bMYAIalix4QnAhaAcSHR1NHz9+nE5KSqKvXbtGR0dH0xRF0f/99x9N0zQ9ZMgQOjo6mhl/6tQpWiQS0fPnz6dv3bpFx8TE0GKxmE5ISLD4nDk5OTQAOicnx6Lxb/96iq4zZS99ICGF35uzkqKiIvrmzZt0UVFRuZzP3uTn59O+vr70mjVrHD0Vl+Dq4yz66uMs+m5anqOnUiZMXed8f6PGcKhZmp6ejqFDhyIlJQW+vr5o1aoV/v33XyYEn5yczFkbMjIyEhs3bsT06dMxbdo0NGrUCLt27bJLjpsWbToIKcGyjMuXLyMxMRHh4eHIycnB7NmzAQD9+/d38MxcCxqVatE6q3CocPvtt99Mvn7s2DG9be+99x7ee+89O81IH1I8z5/58+fj9u3bkEgkaNeuHf73v//h1q1b6NOnj9F98vPzy3GGFZ8iuQq5RQr4uIvND66kODyg4OyQtkf8aNOmDeLj4/W2FxUV4cqVK+U/IRfmYUYBWtXyc/Q0nBYi3MxAzFLb4O7ubjRFg0CwB04XLXU2Ss3S8l27lHZkoT6BYGfK4/omws0MTJ6bXF0u59Nm4BcWFpbL+QgER6C9vtkVJ7aGmKVmKPW5lY/mJhQK4efnx7Th8fDwKJcEWELFgFaWlolRFIXi4mIHzoY/NE2jsLAQ6enp8PPzg1Bov2YURLiZwUNS/sv7aSsu2H3GCAQASM8qYv4XUIC40N2Bs7EePz8/uzcjJcLNDG4OiJZSFIWgoCDUqFEDCoWi3M5LcH5G7TjG/O8pFWHP+JccNxkrEYvFdtXYtBDhZgYPB+a5CYXCcrkICBWHp3ml12EVtbBcytoqKiSgYAaSCkJwVoQC4os1BRFuZnAjXUEIToqYCDeTEOFmBg/SFYTgpIiE5OdrCvLpmIFZu5QIN4KTISKam0mIcDMDs3YpMUsJToZISISbKYhwM4O7A/LcCARLEArIz9cU5NMxg1ZzkynVUKtJvSfBeSAuN9OQj8cMHpLSVEASMSU4E+ryKXeusBDhZgapqPQjIsKN4EyoSecYkxDhZgaBgIKbWPMxEb8bwVYcvJ6CX4/dK9MxiGwzTZnKr/Lz85kFcbX4+PiUaULOiIdEhGKFnGhuBJsxfdcNvMiXoW/LINSpat0C0ERzMw1vzS0pKQl9+/aFp6cnfH194e/vD39/f/j5+cHf35/XsWJjY9GhQwd4e3ujRo0aGDBgAG7fvm1yn/Xr14OiKM7D3vV1TNsjorkRbERukaYhQr7M+lZaRLiZhrfm9tFHH4GmaaxduxYBAQFl6jV2/PhxjBs3Dh06dIBSqcS0adPw2muv4ebNm/D0NH438/Hx4QhBe/c705qlpEqBYAuUKjXkKo3FI1daHxUgss00vIXb1atXER8fj9DQ0DKf/ODBg5zn69evR40aNRAfH48uXboY3Y+iKLv3gmKjjZiS4nmCLWC7N8oi3IjmZhreZmmHDh3w+PFje8wFOTk5AIAqVaqYHJefn486deogJCQE/fv3x40bN4yOlclkyM3N5Tz4QlbAItgStntDobJeQJG0S9Pw1tzWrFmDsWPH4unTp2jRooVeD/RWrVpZNRG1Wo0JEyagc+fOJhdZDg0Nxdq1a9GqVSvk5ORg/vz5iIyMxI0bN1CrVi298bGxsZg1a5ZVc9JC1i4l2BKO5qay/poimptpeAu358+f4/79+xgxYgSzjaIo0DQNiqKgsvLLGjduHK5fv46TJ0+aHBcREYGIiAjmeWRkJJo2bYqVK1dizpw5euOnTp2KqKgo5nlubi5CQkJ4zY1obgRbwr5JEp+b/eAt3D7++GO0adMGf//9d5kDClrGjx+PvXv34sSJEwa1L1OIxWK0adMG9+4ZzhmSSqWQSqVlmh/pDEKwJeybpIz43OwGb+H26NEj7NmzxyYL7NI0jc8//xw7d+7EsWPHUK9ePd7HUKlUSEhIwOuvv17m+RiDmKUEW2I7nxsRbqbgHVDo0aMHrl69apOTjxs3Dn/++Sc2btwIb29vpKamIjU1FUVFpSv8DB06FFOnTmWez549G//99x8ePHiAS5cu4aOPPsKjR48watQom8zJEMQsJdiSIhuZpSpSW2oS3ppbv379MHHiRCQkJKBly5Z6AYU333zT4mMtX74cANCtWzfO9nXr1mH48OEAgOTkZAhYrV2ysrIwevRopKamwt/fH+3atcPp06fRrFkzvm/FYkqTeMt31XmCa1LISQWx/oZZHqu2V2R4C7exY8cC0GhQuvANKFjy5Rw7dozzfNGiRVi0aJHF57AF7mQdBYINKSZmabnAW7jp1pJWBkrN0sr33gm2p5BlAcjLYFuSPDfT8PK5KRQKiEQiXL9+3V7zcUpKV50nZimh7BSSaGm5wEu4icVi1K5d2+pctooKMUsJtoRrlpI8N3vBO1r6zTffYNq0acjMzLTHfJwSN9IVhGBDbJXESzQ30/D2uS1duhT37t1DcHAw6tSpo9e949KlSzabnLPgQfLcCDaEFM6XD7yF24ABA+wwDedGG1AgXUEItqDIRmYpCSiYhrdwi4mJscc8nBpSoUCwJbbS3Eiem2msbjMeHx+PW7duAQCaN2+ONm3a2GxSzgapUCDYEvZNUqZSQ6lSY/zGy2hT2w9jujaw+DhEczMNb+GWnp6O999/H8eOHYOfnx8AIDs7G927d8emTZtQvXp1W8/R4TCF8waEm0pNQyggK38TLId9k1Qo1Th8Kw0Hb6Ti4I1UnsKNSDdT8I6Wfv7558jLy8ONGzeQmZmJzMxMXL9+Hbm5ufjiiy/sMUeH4yHW3AMUKprjI/n7fDJazvwX5x5klOn4mQVy/O/uc7LocyWBU1uqUutZBGm5xdifkAKVmeuByDbT8BZuBw8exK+//oqmTZsy25o1a4Zly5bhwIEDNp2cs+AmMbx26al7L1AoV+Hqk+wyHb/X4hMY8tt5bLv0pEzHIVQMdH1uAp22YXP23sRnf13C8Tvp5T01l4K3cFOr1XrF8oAmwddVS7MkQgG0lic7AVPrOynrHfR5ngwA8N+NtLIdiFAh0I2W6gq3rEI5gNLrgmAdVrU8+vLLL/Hs2TNm29OnTzFx4kS88sorNp2cs0BRFLNITCFHuGnKsYh1QOADp7ZUqdbz2WpvlmUpzSJYIdyWLl2K3Nxc1K1bFw0aNECDBg1Qr1495ObmYsmSJfaYo1PgZiBiaqpi4XFmIYavO4/T91/YfW6EioVuJ1625qZS04xwK0uaCMGKaGlISAguXbqEw4cPIzExEQDQtGlT9OzZ0+aTcyYMVSmYMkv3JaTg2O3n8PeQILJBtXKZI8H5UatpFLO6y8hVXM1NoVIzUVCiuZUNq/LcKIrCq6++ildffdXW83FaDFUpMMLNgGGaVaDxmyhJBJTAolinOaVCpYaQZT8p1aVXk4zkVZYJq4RbXFwc4uLikJ6erhdEWLt2rU0m5my4SfSL5wtNtEDKLlQA4JeLZIO1dghOjm6Vi260VKFUM05cSzQ3tZqGgORZGoS3cJs1axZmz56N9u3bIygoyCarX1UEPEo0t0JDmpsB+aWNeJESGQIbXT+trl9NoVYzloAlwk1F0xCgcvwG+cJbuK1YsQLr16/HkCFDynzy2NhY7NixA4mJiXB3d0dkZCR++OEHhIaGmtxv69at+Pbbb/Hw4UM0atQIP/zwg11XvwL0l/dTqWmTF192UYnmRtwmBBa6CbsKFc25OSpVNFNWZZHmRm6eRuEdLZXL5YiMjLTJyY8fP45x48bh7NmzOHToEBQKBV577TUUFBQY3ef06dMYPHgwRo4cicuXL2PAgAEYMGCA3bsD69aXmjJJASC7RHMjFx+BjVZz01qScqWaU4mgVNGMti+zYPEYcvM0Dm/hNmrUKGzcuNEmJz948CCGDx+O5s2bIywsDOvXr0dycjLi4+ON7vPzzz+jd+/emDx5Mpo2bYo5c+agbdu2WLp0qU3mZAzdziBs88KQ6Vnqc7PrtAgVDO314+OuSYSXq9RQsa4fuUpdGlCw0CwlGIa3WVpcXIxVq1bh8OHDaNWqlV61wsKFC62eTE5ODgCgSpUqRsecOXMGUVFRnG29evXCrl27rD6vJehrbmzhxh1L07RVAQWC61Ok0Gj8vu5i5hphCzGlWs0rz41cX8bhLdyuXbuG1q1bA4CeKViW4IJarcaECRPQuXNntGjRwui41NRUBAQEcLYFBAQgNTXV4HiZTAaZrLSMJTc316r56S4SY6q3W5FCxaxqRC4+Apsiuea68HUvVQrY6UVKFc1LcyPNFozDW7gdPXrUHvPAuHHjcP36dZw8edKmx42NjcWsWbPKfBzdCgW2z0338soquSMDxCwlcNFeN2zhxs5nU6jUpT43C/LcyPVlHN4+N3swfvx47N27F0ePHkWtWrVMjg0MDERaGrfAPC0tDYGBgQbHT506FTk5Oczj8ePHVs2RWQGr5M5ryizVBhM0r/HIc7NqZoSKhFZL85KKSpsxsCoW2NFTi3xuRLoZxaHCjaZpjB8/Hjt37sSRI0dQr149s/tEREQgLi6Os+3QoUOIiIgwOF4qlcLHx4fzsAbGLFXom6W6FQrZHM2NXHyEUrTXjbtYCIlI8/PjmqWleW7E51Y2rG4zbgvGjRuHjRs3Yvfu3fD29mb8Zr6+vnB3dwcADB06FDVr1kRsbCwA4Msvv0TXrl2xYMEC9O3bF5s2bcLFixexatUqu85Vd3k/rZAzBEe4kVA9gYXWreEuEUIiFKBYwW1WqVCzNTdLzFIi3IzhUM1t+fLlyMnJQbdu3RAUFMQ8Nm/ezIxJTk5GSkoK8zwyMhIbN27EqlWrEBYWhm3btmHXrl0mgxC2QDdaWiAzbpZmscxScvER2BQZ1NxY0VKVmlcSLzFLjcNbcztx4gQiIyMhEnF3VSqVOH36NLp06WLxsSzxRx07dkxv23vvvYf33nvP4vPYAg+d2lJOnpvO2JyiUs2NyDYCG+3N0aNEcwO4xfScgIJF0VI7TNJF4K25de/e3eBq8zk5OejevbtNJuWMmMpz00XbEQQghfMELtrrxk3C0tw4nXlLrxfLoqXk7mkM3sKNpmmD+WwZGRl6q8+7EroVCoVsn5vOBZZdRAIKBOBOWh7+ufqMY6EwPje2WcrS3DhJvBYs2KxboXD9aQ62XHxMGjaAh1n69ttvA9Ak6g4fPhxSqZR5TaVS4dq1azarOXVGdJf3K5QZN0uzOT4368/5KKMA4zdeRsLTHMx/LwzvtjOdJkNwLl5bdAIA4OchxsuNNEteKkpMTbFQALFQ3+emUNKcZpXGlAktukm8X2+7hpspuWhZ0xdNg6zLDHAVLBZuvr6+ADSam7e3NxPNBACJRIJOnTph9OjRtp+hk+CuEy01ZZayo6VluYOO/fMSbqVoKiombb1KhFsF5XZqHiPctFcDRcFgKoim5ZEGmtaYqRKRCeGmc3lllrhE8opNN3aoDFgs3NatWwcAqFu3LiZNmuTSJqghGLNUoQJN05xUENPRUuvP+eB5vvU7E5wGbRoRUHqtUKCYgEKRbvkV64KSKVWMEDSEbrRUeyxillrhc4uJial0gg0o1dxoWmMumErizbGRz43461wDtnDT6m5czY1doaDmXE3mEnl1rxGtFuh0V07qdeDwzHJNH+At3NLS0jBkyBAEBwdDJBJBKBRyHq6KO+sCLVaojJql7I4gQNk0N5LDVHFha05uYgFru+avgAKjuXFrS2mOZDKXDsIWbjRd2kDVqe6LifuA314DTi4C4teX22l557kNHz4cycnJ+PbbbytVm3GRUACJUAC5SqO1cQrnWRdSvkzJWRSmLOYBkW0VF3ZKh5uo9MaoFUYUKKPlV2qOWWpauLFvgOyxhhYtKndoWiPQ4mYDoIH63YDmA8rt9LyF28mTJ/G///2PaXtUmXATa4RbkY7mxr6M2FobYN60/GZngi2nSHAS2H40KVtz0/5DgYmW6pVfsY5jrgSLfX1x1mdwtGxTFAP/fAlc26R53mE00DsWEIpN72dDrFq3tLI6Kz0kIuQWK1EkVxldkFlXuJkzLf86l2yz+RGcB7apKRbqm6UUDPvclCo1xxIw73Mr/Z+dL+fQX2h+OrDpQ+DJeYASAn1+AMLLP5OCt89t8eLFiI6OxsOHD+0wHeeGaXukq7mxriR2pFT3NXNQpOmRy8AWWOxvVXs5CCjKYBKvgtUVBOBnlrLP6TD9IzUBWNVdI9jcfIGPtjtEsAFWaG6DBg1CYWEhGjRoAA8PD70244ZKs1wFdmcQbrPK0itJW50gFlJQqGgS8ayk6C6+rEVr9VCsgAL7ElGoaE69qExheUCB7btziM/t1l5gxyeAogCo2hAYvBmo1rD851ECb+G2ePFiO0yjYqAtns8tVnAcxuzrSFud4O8hQXqejAQFKinFRupCGbOUlQrCRqlTCW/W58Yazvbdles9laaBkwuBuDlgAgfvrQfc/ctxEvrwFm7Dhg2zxzwqBNp0kMwCudExWp9bFU+tcCPSrTLCMRFZ27UaFTuJl41uEq85n5vKqOZWTiiKgX++AK6VtClzQODAGFb1c7t//z6mT5+OwYMHIz09HQBw4MAB3Lhxw6aTcza0ZmlGvo5fjfW/1udWxVOieY3ItkqJ7uLLWsxpbnKdJF4+eW4yjs+tHC68/HRgwxsawUYJgb4LgL7znUKwAVYIt+PHj6Nly5Y4d+4cduzYgfx8TYnQ1atXERMTY/MJOhNaszSjQMbZzr6QcliaG2BdlUGBTIndV55aO02CE2DeLKU4UVQtSp0V6M2bpfodR4By0NxSrpUEDi6UBg46jLL3WXnBW7hFR0dj7ty5OHToECQSCbO9R48eOHv2rE0n52y4G9Hc2OhqbtYIt6+3X8OXm67wnyDBaTAm3EqTeI373KxN4uWc057S7dZeYG0vIPeJJnAw6gjQwPl6OfIWbgkJCXjrrbf0tteoUQMvXrywyaScFXdGczOe7qGNlpYKN/7n2XctxfwggtPxIl+Go7fToVSpjUY5DXUFYaNQ0TxrS0v/5/r57CDdaBr43wJg84eAolATOBh12KERUVPwDij4+fkhJSVFb6Wqy5cvo2bNmjabmDPCCLd8HbOU9b82oFCV8bnZtxOvUqXG0+wi1Kla+ZoZOBvR26/h8K10vNuuFlrV8mW2cy4BpraUgkSo/4UrdJJ4+fjciu0ZLdUNHIR/AvSKBYQOXWPKJLw1t/fffx9TpkxBamoqKIqCWq3GqVOnMGnSJAwdOpTXsU6cOIF+/fohODgYFEVh165dJscfO3YMFEXpPYytNm9rGLPUZLRUa5ZqmnnaOxVk7J/x6PrTMey5+sy+JyKY5fAtTXBtW/wT4z43mDFLVTTYt0tzrcZVRnxuNiUvTT9w8PpPTi3YACuE2/fff48mTZogJCQE+fn5aNasGbp06YLIyEhMnz6d17EKCgoQFhaGZcuW8drv9u3bSElJYR41atTgtb+1aAMKuiVW2rukWk0z7Y78PTURI3ungmh/UGtPJtn1PATLEVBcE5GNVhZpknj1u+goWKtfAXyjpXbQ3FKuAat7lAQO/IAhO5wucGAM3qJXIpFg9erV+Pbbb3H9+nXk5+ejTZs2aNSoEe+T9+nTB3369OG9X40aNeDn58d7v7LC7ctVivZunFesZC5MxudWTlm8laQ5S4VAKKBMpIIwXjeDHXaVat1mlTzMUqXh3DqrufVPScVBoSZw8MEWoGoDWxy5XLBar6xduzZq165ty7lYTOvWrSGTydCiRQvMnDkTnTt3LpfzajU3Y2gjpR4SIdPmprzy3Ihscx4EFGXCLNVAUZoSPV10m1Waj5aW/s/1uZXhwtMGDo7M0Tyv3x14b53DKw74YpFwi4qKwpw5c+Dp6YmoqCiTYxcuXGiTiRkiKCgIK1asQPv27SGTybBmzRp069YN586dQ9u2bQ3uI5PJIJOVBgByc3OtPr+7Mc2t5DrSRkr9PSQQlKhSfMzSslyPAqK6OQ1CAWU0clnarJIyHi3lked2Jy0Pz/NkqO4tNbmWrsUoioE9nwMJWzTPK0DgwBgWzfjy5ctQKBTM/8awd+PK0NBQhIaGMs8jIyNx//59LFq0CH/88YfBfWJjYzFr1iybnN/NQs3N113MmIl8rFLdZdp02Rb/BG1q+6FBdS+914hscx4EFGU0EFBqlAJSgwEFfnluq048wKoTD/BwXl+uWWqNdMtL06R5PLmgCRy8/hPQYaQVB3IOLBJuR48eNfi/MxAeHo6TJ08afX3q1KkcbTM3NxchISFWncvDiOamRVud4O8phkDAX3NTmlmnctLWqwCAh/P66r1WWToiVwQElGVdQQxWKKhpbptxM11B2HBNYZ7SLeUa8PdgTWKumx8w8Hegfld+x3AyKp6uqcOVK1cQFBRk9HWpVMpZY7UsuOtobm5iAYoVauaC1Wpufu4SCBjNjYdwK0PwgYg2x8Jeq1agY5aefZCJyAbVAFhQW6rUWSDGgoWZtVid58YJHDQCPthcoQIHxrBIuGkXZLaEHTt2WDw2Pz8f9+7dY54nJSXhypUrqFKlCmrXro2pU6fi6dOn+P333wFo2i3Vq1cPzZs3R3FxMdasWYMjR47gv//+s/icZUHX5+YpEaFYIWcuRm2KiJ+HmOVzs/z4ZVkQhihujuW9FWeY/4U6AYVf4u4i6tXGAFh5bpSRriBqNTdayiN3jXdXEIOBg/WAu5/F53RmLBJu2gWZAY1avXPnTvj6+qJ9+/YAgPj4eGRnZ/MSggBw8eJFdO9eWpOmNR+HDRuG9evXIyUlBcnJpW245XI5vvrqKzx9+hQeHh5o1aoVDh8+zDmGPdHV3DykQmQUlD7X3r39PNg+t/LR3EhAwbHcTS9dY5aiTKWClIyBEbNUpbuGAh/NjYfPTS9wMAbo9X2FDBwYw6J3ol2QGQCmTJmCgQMHYsWKFcxSfiqVCp999hl8fHx4nbxbt24mQ9br16/nPP/666/x9ddf8zqHLTGkuQGmo6U0rbkhWOITM+dzMwWRbc6DUGBJEi9lMKAgNxFQMPVbUatpyzvx5qUBmz4Anl50icCBMXhXKKxduxaTJk3irFEqFAoRFRWFtWvX2nRyzoaHRKTzvCSXDVqfm0a4+bqLOZqUpcpbWczSU/cysO4UqVJwBoSmoqXmuoLopILIlZb50RRqtWWdeFOuaioOnl4sqTjY6ZKCDbBCuCmVSiQmJuptT0xMhFptveZREdC903pKuZpbDqvFuIClSVlqmpbFLAWAWf/c5KztQOCS8CQHB6/bvw5ZE1Aw7SszluemVPNL4mX2U9FGu/8y3NwDrO1d0qqoETD6SIWPiJqCt4E9YsQIjBw5Evfv30d4eDgA4Ny5c5g3bx5GjBhh8wk6EwIBxURIAf2KhSxWQIFthloqs2yxwjxnbQcCh35LNSlDB758GU2D+LlQ+CCgKBQaTQXR/DWWCqJQ0WB7P2QWllQpVGqd2lLWaJoG/jcfODJX87xBD+DddS4TODAGb+E2f/58BAYGYsGCBUhJ0fQdCwoKwuTJk/HVV1/ZfILOhkdJhBRg+dxKXisNKFinuSnK4HPTcvLuC/RtZTw1hgA8yiiws3Az0YmX1RVEJKBAUfomJPsGZVRg6aBQ0YZz6xRFJYGDrZrnLhg4MAbvdygQCBjHvraUiW8goSLDDip4SEvrR5UqNXKLNSYhOxVE+7ol2EJzG7fxEn6J88aHnWpjaETdMh/PFbF3wrPAROE88xVTpekgpkxPdp6bqaujWKHiCEWaBjdwIBBpAgftP+bxTio2Vi0Qo8XHx6dSCTZAk7irxZMVYNAKNgDw0wkolJfPTcvttDzM2O3ai/WUBWNpM/sTUjBx8xWz/jJddKPcajVt9IZWGlDQzMFQrhsbmVKN+EdZmL4rgWmnZYgCHV+rT/ZNYHV3buCgEgk2wMoKhW3btmHLli1ITk6GXM5t3Hjp0iWbTMxZYUdMS/PeaKY6wVsqgkgogFJd+gOxVLjZQnMjmEdgRHH77C/Ntds4wBufdrM8Q79QRxia8ntqX9HOQSISADKjw0HTwDvLTwMwXYqVx7q59hKcR5eTKwFVkUtVHPCFt+b2yy+/YMSIEQgICMDly5cRHh6OqlWr4sGDB1b1ZqtoaM1SiUgAkaA0l43xt5U0qRQK+AcUbKW5EUxjLuH5eZ4JaWMAdjcOwEyEk5XnBhhOBzHGQ3bGuA55xQoANMYJd2GlZDFEqiJN4GDU4Uop2AArhNuvv/6KVatWYcmSJZBIJPj6669x6NAhfPHFF8jJybHHHJ0KrbbmIRFyfDdM6ZW7pkkl1+dmWGjpbieaWzmhI9vuP8/HwkN3mOd8F1cplOtqbsaFG7P6VckcDEVMjSESGB9bUJCPxeJlmCzWVBzcrfcR8MFWl4+ImoK3WZqcnIzIyEgAgLu7O/Ly8gAAQ4YMQadOnbB06VLbztDJ0GpubH8bTXPTQADoREsNH0s/Sma/PEFLqyQqA7qa26sLj3O+I77tgnRzC02tWMVueQTw09zERsZWRxYiTgxFNeF1KGghYpTD0ab5RDSqBBFRU/DW3AIDA5GZmQlA041Xu1ZpUlJS+axy7WC0mhu7zpQGzUkDAaCT52b4c9HdrrRTjtqyo/fQ8fs4PM4stMvxKxq6Pjfdmw/f61jXLDXVyYPWNUt5aG5iA87C5tRD7JZ+i2o515FNe2KoIhobVa/Yf1HmCgBv4dajRw/s2bMHgCahd+LEiXj11VcxaNAgg+uZuhpcs7R0u9Ys9S/R3ACYbXuk+6Oyl+b207+3kZ4nw0//3rbL8SsalJkGUXwFg65Zasi9oBWYpV1BNNtNaW4iHWEm0mlL3ktwHlslsxBMZSLDvS76y+fgjLp5yQl5vQWXhLfeumrVKqbMaty4cahatSpOnz6NN998E2PGjLH5BB1K/Aag7ksch6zWLHUXC5kfCU0D2UXaXm5s4UZBTRtPC9DT3HQWB7E15jr9ujLsz9VYtLR0LL9j6wo3Q8iUariJhZyuIIBpzc1NLES+rNTkFTFjaYwX7sIksSYx97iqFU43/BGPLmQyY+2yKHMFg5dwUyqV+P777/Hxxx+jVq1aADTrmL7//vt2mZxDuX1Aswitu7+mK2m9LgBYPjcpy+cGts9NwmzX+HZoo4ECQz8iUj5lH9iftTnfI1/BUKQwX88rU+gINwuipVKRAOz1vyVCAaSQ40fxKvQXatJD1il7Ya7yI/RXuXPfA7mM+JmlIpEIP/74I5TKSlCcHdwGqNkOKMoC/ngLuKhp+8T2ubF/Izk6AQUAZnu6GfoR2TOoUJlv5uzvwFxcha9gKJCZ19y0pVFaDVJggVmqu5SkvyoDmyVz0F94GgpaiKmKkZilHAYVhJw8N6BSf9UMvH1ur7zyCo4fP26PuTgX3oHA8H1Ai3cBtRLYOwHY/zXCgr0gFFBoW9ufMS000dLSjiBa2D3dDGFIobNrxLQSX/Lsz9pcnhvfT0k3oGBqTGm0VDMH9vJ+XlKuIcWuhmlOJeGLpLFoLbiPLNoLQxXR+Fv1CvN6vq5wq7xfNQNvn1ufPn0QHR2NhIQEtGvXDp6enpzX33zzTZtNzuGI3YF31gA1mmg6KpxfiZcy7uJ69G9w96mClcfvA9BGS0t6ufEKKOhv59Mzny+V+YJXO9jnVqq5aZ5r5auI5XPz9xRzfGxaza234DwWiX+Fu0KOe+pgjFRMwiM6kHP8PBm3NKsy38i08BZun332GQDD65NSFAWVil9dntNDUUCXyUC1UGDnGOD+EbhveA34YHOpecOqUDCkuRnNczMgxyxJB9HmrPENPhy4norMAjmqeErMD3YxuD43c2N5JvFa4HPTam766T+lF0EVTykeZxYxz91EAowX7mQCB/d8OuKt9NHIg4fe8Ynmpg9vs1StVht9uJxgY9PsTeDjg4BPTSDjLrC6B2pmXgCg0bYKSi5edrTUnM/NoOZmQXNCNQ3kFisQs4d/cfznf1/C+aRMpOYU8963IsP1uZkxS3kKBkvMUm0PQMYsLZkCO4Dk41aqa0ghR1Tej4xgW6vsje1NFhoUbACIz80AZeoKUlZOnDiBfv36ITg4GBRFYdeuXWb3OXbsGNq2bQupVIqGDRvqrbNgV4LCgNFHgZrtgeJs9Ln6GT4SHmK6NVAU4MNOBWFqT40FFPSxpPNqRoEMrWb+h9/PPOL9Fk7dy8DAlWfQKTaO974VGa5Zalq48VnUR6WmrTJLtXNg+1ilIo0ZWh1Z2CyZjc5Fx5jAwWzlUIhEYhhDV7gR1Y2HWVpUVIS4uDi88cYbADSLHctkpXFqoVCIOXPmwM3NzeKTFxQUICwsDB9//LFFK2clJSWhb9++GDt2LP766y/ExcVh1KhRCAoKQq9evSw+b5nwDtAEGvZ8DkHCFswVr8M/6ek4jffh5ebGKZg3Z5Ya+hEZ6wPG5sz9DOvmrsOLfBmqedlmTVdn4/id54h/mIkJPRtDIKA434G5IjRLxcLp+y8wcv1Fi76zYkYAcpN4OcJNLEBzKglrJAsQRGUiX+CDUcVf4Ky6GQBuMwZddH21RLTxEG4bNmzAvn37GOG2dOlSNG/eHO7umvyaxMREBAcHY+LEiRafvE+fPrw6iaxYsQL16tXDggULAABNmzbFyZMnsWjRovITbgAgdgPeXoXzhQFof28p+sn2wU/8CD+4R3OGWRpQEFBAkK87nmYXWdRLzFa5cIUyFeBlk0M5HcPWngcANAzwxpthwTpJvLYxS8f8Hm+RYAMMBBSg1dxKT9ah8H+YL5kLd0oTONjS8CecvW5dPTBR3HiYpX/99Rc++eQTzraNGzfi6NGjOHr0KH766Sds2bLF5hNkc+bMGfTs2ZOzrVevXjhz5oyRPQCZTIbc3FzOwyZQFK7V/RhjFRNQSEvxsvA6VsqmAC/usYaUaG5GLE22iaJNCbDkx1KWJQDZVIaKhZRsjYOeo7mZCyhYqPcIhZYLHq3PTbcriEZzo/G5cAeGP5kBd0qOY6owvCWfjTyPOpxj8GkaUxnqvM1hsXC7d+8eWrZsyTx3c3ODgNWCJTw8HDdv3rTt7HRITU1FQEAAZ1tAQAByc3NRVFRkcJ/Y2Fj4+voyj5CQEJvNh6Io/KfugHfkM/GUroqaqqfAmh7A/aMALNfcKKo0JcCSFcZt1feNj2+poqI15fi81/Rcy/q56dZ+mkI3z027K6Usxi/ipfhKvA0A8JuyD0YqJiEPHpw8N4CfwHL9b9Y8Fgu37Oxsjo/t+fPnqFu3LvNcrVZzXncWpk6dipycHObx+PFjmx1be2nfouugv2wuktybA8U5wJ/vAOdXm03iZZfiaPt6lafmpnbB/nGXk7OQnFHa/aTU72n5ez1574VF40z5wHTRNUsBCshNwQ+50XhTeAYKWogD9aZijnIIVCitX2bD5z1UgvuWWSwWbrVq1cL169eNvn7t2jWm3tReBAYGIi0tjbMtLS0NPj4+jO9PF6lUyqz1YOs1H9jmzQv44s/QZUDYYIBWAfsnYbJyFURQQk3TOJqYjs7zjuA064fD9rlJSkwcYyuVs7GV5uZqZunDFwV469fT6PLTUWabVv7Y462aah6pS7FchZvPcpnIuvRFArC6B5qo7yKL9sJfjX9GYjA3qKZbfmXJe9DV9iozFn8Sr7/+OmbMmIHiYv38qKKiIsyaNQt9+/a16eR0iYiIQFwcN4Xh0KFDiIiIsOt5LcXd3QMYsBzoOQsAhbdVB7Fe/AOo4iyMWH8BT7OLMPr3i8x4rs+tRHOzIK3AVgEFV1tDOzE1T2+bNWappei2IDLFhYdZzLqpfQTnUHPHACDvmSY5fHQchn3wEaQ6gkl3EXBL3oFW23Ot25Z1WBwtnTZtGrZs2YLQ0FCMHz8ejRs3BgDcvn0bS5cuhVKpxLRp03idPD8/H/fulTrgk5KScOXKFVSpUgW1a9fG1KlT8fTpU/z+++8AgLFjx2Lp0qX4+uuv8fHHH+PIkSPYsmUL9u3bx+u8tkL30naXCDXq3EsTgGqNUbhpBF4S3kDxngFoQI3Dfbomk+wLsDU3ivmhWGKWqmwklVzN52YoUCBghJvtz8fHLD3zIAMAjS+EOxEl3gYoATTsCby7Fv5uvgBK89y06Ao3S74vd7EQWVCQgAJ4CLeAgACcPn0an376KaKjo0uXKKMovPrqq/j111/1nP3muHjxIrp37848j4qKAgAMGzYM69evR0pKCpKTk5nX69Wrh3379mHixIn4+eefUatWLaxZs6Z800BY6Ga6c3wkTV7H5+4/YFbhHNTKe4idkhiMU3yBk3QrZoj2B0ehtJc+CShYj6EUDyETsbaD5sZDuEkhx0/ilXhTqIns57YeDZ9+8ziLI7OFmYACpFaYpewO0ZUdXrWl9erVw8GDB5GZmcloXA0bNkSVKlWsOnm3bt1M3mEMVR9069YNly9ftup89kb3wkoS1UV/2Rwcq/0bfNIvYr34B3yvHgbQrwOs2lCKKm1aWGxBhYKt2pG72oI0hkSNoYCC9l+Vmsb8/6zvTiy00OdWA1lYJVmA1oIHUNBCTFd+jHEvz4aPULcLSOn1o12wmY0l2pj2GnSx+5ZVWOV9rFKlCsLDwxEeHm61YHMFdBUFDx3hJqAoZMAX8V3XY5uqC4QUjW+F64G9EwGVgtHcBAJ+PjdLNbe4W2kmX3c1zY2vWbrz8lMsP3bf6vNZorm1oB5gt/RbtBY8QBbthSGKqdis6m5wrmzNjQL0fHCWfO2lPjfX+m6tgYRWyoCez02sK9w0f5/mqzFJMQbfKT6AmqaA+HXAH2+BKsosGcfP52ZpKsjIDRdNvu5iipths7TkCjckyMu6YI65gMLrgrPYKpmNICoTD6la6C+fw5RSmRVulL4PziKfW8mqbC5237IKItxsiK5Zqv2xpWQXA6CwWvUGRim+Ai3xBB7+DyHb30AD6ikolJqlFgk3G0mlmN038MjEQr8VDgMC478bGu2VbdJptRpLhMXZBxkY+0e8wS4qxjU3Gl8Id+BXyS9wp+Q4qgqDcsR/SBMGlU7VgHRj+9goUPrRUos0N0HJDAhEuJUFnQtU1yzVXsDPckqrJ46o2yLl3X8Av9qQ5D7CTskMRNJXeAUUbNWt92ZKLgatPGuTYzkDhkTNgeupALha6g8HE5GcUWiRcHt/1VkcvJGKr7Ze0XvNULRUCjmWiJdoIqIA1ij7YHuTBWhYuybn5mdorrqaG581TbUwZimRbkS4lQXdC1Q36VJ77Ws0t1JS3eoDo4+iIDAcPlQRFqu/R5esrQBoi5J4bRkISM0tnVtGvgx1o/fhs7/ibXZ8Z4EtyE7dy8C7K07zMssTUzQ5dM+yixgtUDeJtwaysEUyG/2EZ6GghZiiGI25yiGgKf2KA8vMUn6pIEIBxZTxEZ8bEW42xUPCjX4xZmkOt+41M18OeFbDw9c3YrOyG4RQo+/TXxArWgOF3HwTSVuZpboMWqXR4vYnpNrl+PbGVLcP3dTA9DwZr/SQ3GIFdl5+gsh5RzDvYCIArubWgnqAPdLpCBM8QCbthY/k07BZ1Z1zDHcds1MXNz2zlJ/PzU0k4KzrUdkhwq0M6P6WjAUUUnT8NS9K1mtTCySYohyNX4TDoYYAg0VH8U3mNPjDdOcSPrWl8w4kWjz2Xnq+xWMrGoYEA59osUJF45udmvLDlccfAChd3EUbOAiksnBHXRP95XNwkWqmdwy28DLkrtPT3HhGS93EQrMdTyoTRLiVAd27r25AQetz03bX1TaGfJylidJpflwUNov7Y0eT+cij3dFKeR27JDPQkHpi9LwKHhrHiuPWpzoAFat1zqx/DLddv5uWZ1CT4eu61O24SwH4UridEzh4Rz4Lj+kAvbQOQKfu05Bw4wg//gEFN85C4RXne7MXRLjZEGOam5ZO9TU5gQ9faIQbu5/+k2ov4235LDxFAOoI0rFTEoNuAsPJyqpyWrg5/lEmWs8+hG3xxgWtM3H/ueHI76uLTpRZc9NDUYRP0udiong7AE3gYKRiMrPGga5gAqATUDAQjNDZRzegYE5guYkFjOZGZBsRbmWCbQKIBJTexajrA+pYvyoAIOmF5kfIri0VCwW4S9fCEEEszqmbwJsqwm/i+Rgp3AfdwL7SDhXvCwxk6o/98xJyihSYtPWqzc9X3vAVbqYESQ1kAev6oEOBZo2Dr0sCB2rWz0m32gAA3ESWBxTkKrWez80izU0r3EwPrRQQ4VYG2NenrtYGcIWbgAI61PUHADzKKABN05zVx7X+mxSFJz6ST8Pfyu6aigbxX/hBtBpilC4AYquuIGyWHLmnt60i3P3/u5GKQzdNV2IAhv1VpoSbMcu/ZUngAM8uI1/gg4/k07BFJ3AAGNbc3HTMTv19Sl+XK9UGuoKY09yEAGOWmhxaKeC9binBMIYKltnXbzUvKepV84SAAgrkKjzPl5WWX7FaHhUrVaAhwlTlKNyla+Eb0Z8YJDqGuoJUfCqfgEz42EVzq4jkFCnwyR+atJX61T1NjjWkif15NtnASA0qNa2Xx9ZXcBbzxSvgTsmB6k0wz306zt3RfG9iIcW56eimBQHcAIEhv79Yp+JBPxXE6HQBaG6wpZobkW5EcysDbOGlm8ALcO/Ogb5ukIqECPbTNNV8lFFYmopAlXYFYXdqXavS+HFyaXd0FCRit+RbNKYe26xwvqKTVSBn/n9gxN+mhW/2DFero/GlcDuWsQIH9Mj/kKSuzozQFYSGAwqmzVLdqgXd4nmzqSBikgrChgi3MsB2Chu6U7NzPAN8NEse1q2q0TCSXhSw+ulTendtLcfUrfGWfDYeqgMQIniOHZIYhBXZtqrAmH/J2dMKLFkvVAvf4IE2UdoNMiwVL9ELHIzffh+n7pUusSjU+bB02xUBOj43swsMlhyHJSTNvQUp8blxIMKtLPDR3LTCrZommvYoo4DR3AQszc0Q9+maGCCfjTOqZvCiihGdPQujhXthq0u4ot7lLV1WD+Av3JRqGsh9hs2SOXhDeBZyncDBvoQUzniTmlvJSxyz1MJfHnsfc9FSd1YqSIX9Um0IEW42wpDPTdcsBUo1t4cvCvHtbk1S6J20fJPCDQCy4Y0himhsVPaAADS+EW/Ej6JVkEBR5rlX1J+BJe2htPD9rVPPLgOre3AqDgwFDrTofn8GfW4GTFVzsIMM5pN4BURzY0GEWxngRkv1YzPsm7nWLK1XTSPczj7I4ORlmRNuAKCECNOUI7HcfTRUNIWBouP4U/I9qiLHujdQgjGNoECmNLjdGUjOKMTvZx5aPJ6P5tZXcBbef/cD8lKYioPzdFOT+5jS3LQYSg8xBzu9yHz5lZD43FgQ4VYG2BeoWc2tRLjVKdHcMljOcEA/UmbirNgmegPDFVOQS3sgXHAbu6XfIpQyHvkzh7FaVT4+rfKmy09H8Z8FKSBaLAso0Jgg2oZlkl9AKYuBRq/h7ZKKA3OM696Q89yQ5uZmgeamm/jtJS29aRp7C9rWS5o8t5JUEKK7EeFmKzwMXMwUxyzVlF7VruJhsK7QEs1Ni1JN43/qVnhLPgtJ6gDUol5gu2QmXhFY183jSGK6VftVJMxqPSWBgwmiHQCA/LZjgMGbkF9ScWCOt9rWxKw3mzPPrYmWAvpVCV/3DmX+P377ucF9tNcO+wZLNDcnEW7Lli1D3bp14ebmho4dO+L8+fNGx65fvx4URXEebm5u5TjbUjhmqUHNrfR/rVkqEQlQ019/jVVewq0kFUQTaJiDU6rm8KKKsVq8EGOE/4CvxyW/WN/8tFXPOGfBlDM+AJnYIpnNCRxkvTQTEFi+2IpYIECdqqWC0JAJyg4OGOtgortuwsuNqmN6X41JnK/jJuhQ1x+D2ocwWr9URHxubBwu3DZv3oyoqCjExMTg0qVLCAsLQ69evZCeblyb8PHxQUpKCvN49OhROc64FPb1acos9ZQI4e0mZraP6dIAXRpXx/++7o6329TEio/aQiKyPO+CncSbAy8MU0zBn8pXIKBoTBX/jQXiFbwCDYbM0i0XH1u8f3mTW8w/iGIs71lbcdBKkMQJHPCNroqFFEdTN+hzE5kXlhIDY4ZH1sWmTzrhr1Ed0a6OP7N969hI/PBuK2YfbuE8r+nblXyZEnfT8nD8znM9AW1PHF6hsHDhQowePRojRowAAKxYsQL79u3D2rVrER0dbXAfiqIQGBhYntM0iyGzVJvnFuDL1Sw/6lQHH3WqAwBYOKg1AODq42yLz6WbxKuECNOVI3GbDkGM6He8I/wf6lKpGCOPwgv4mj3etJ0JetueZhUZGOkc7L7yjPc+p+6/0Nv2huAM5otXwI1S4I66JkYqJjH+Nb4NQYUCiqOpG/S5WWCWGhKKIqEAnUrqko8kpiP+URbndYmQ7XPTbCsvn1uxQoWUnGKkZBfhGftvThFSsovxLKcIeSzLYPunkRwBbU8cKtzkcjni4+MxdepUZptAIEDPnj1x5swZo/vl5+ejTp06UKvVaNu2Lb7//ns0b97c4FiZTAaZTMY8z8013SuND+Y0N+2dPMjXvNnMxyw1ZjL+oXoNSXQQlol/RjvBXeySfovR8q9wi65j8bErAtasQbru1EPmfwpqfCnawfjXjqha4wvFeI5/TaWmsfDQHYuPT1EUx9Q0JKS4fjgjZqkVrcXFJfto8tw0aDW33Vee4uD1VLzWPAB9WgQZFLrGUKjUSM0p1givnCI8y+b+TckpRqZOYMwY3m4iBPu68+pFWFYcKtxevHgBlUqlt5hzQEAAEhMNN1kMDQ3F2rVr0apVK+Tk5GD+/PmIjIzEjRs3UKtWLb3xsbGxmDVrll3mz75ATZmlWn+bKfiYpaa0ipPqlhggn4PfxD+hviAV2yQzMUExDofU7S0+PuDcPpuytCpygwzzxSvwhvAcAGCVsi/mKQdzOnoAgIqm8UvcXV7HpoxobtrNUjPNKgF9n5slaPdxEwtQ3VsTuErJKYZaTWPWPzeRWSDHgeupiNl9A2+1qYn3w2ujaZAPaJrGzZRcPHxRyBVeJRrY83yZhYvSCBHk54ZgX3cE+bohyM8dwTp/2VHf8sLhZilfIiIiEBERwTyPjIxE06ZNsXLlSsyZM0dv/NSpU5mV7AGN5hYSEmLzeRnuCqL5G2iBcOOluZnRXJLoIAyQz8Yy8S94WXgdK8WLMF85EL+q3oSl6aNpuebbnZcHFx9mok5VT+ZHC1i/hkQAMrFasgCtBEmQ00J8oxyJrapuBsfyOceol+oBAD/NzVhAwYzmZmivOlU9cDc9H3WqejIpLw+e5+PGs1xkFsjhJhagqqcUT7OLsOHMI2w48whhIX6o6ikxGymXCAUI9HVDkK8bgv10hJevO4L93ODrLrY4b688cahwq1atGoRCIdLSuPlKaWlpFvvUxGIx2rRpg3v39Fv2AIBUKoVUKjX4WlkxVzivTdgNC/Ezeyx+0VLzqn0uvDBC8TW+pf/AMNEhfC3ejIaCp5iqGAUZJGb333HpqcXzsRen77/AB6vPQUABD2L7MtutUdxaUfexWrIAAVQ2MmkvjJVPNJmYy6fxivY6YAs3jvlXsrmGj1R3kx7mhJuht75kcFuk5hajXjVPZi5JLwpw/I5GcL3cqDpWfNQOJ++9wKbzyTh0M43x8YoEFMJC/LjCq0RoBfm6o6qnhFnYuqLhUOEmkUjQrl07xMXFYcCAAQAAtVqNuLg4jB8/3qJjqFQqJCQk4PXXX7fjTM1jyJfxRY9GeKdtLYRUMZ8rZUjzM4alSoUSIsQoR+AuXQszRRvwtvAkE2h4Dj+Lz6fL7itPIVOqMbC97TVgNifuaIIAuu+Xr1nKDhzcVtfCSMUkPKFrmNxH2wqeD2wZYEhzq+Hthl8Gt4GbSGBUYFiezF2Ku0TI3EhrV/GASEChUK5iOih3aVQNQgGFro2ro2vj6nieJ8OOS08Q/ygLY7rWR7s6VXifsyLgcLM0KioKw4YNQ/v27REeHo7FixejoKCAiZ4OHToUNWvWRGxsLABg9uzZ6NSpExo2bIjs7Gz89NNPePToEUaNGuXIt6G38hUACASURYINMOyzsxV/ql7FAzoIv4p/RlvBPSbQcJOua/Ex0vOKUc1Tii0XHyN6hya6+kqTGqjqZR+tGDC+/oOlwp2CGhNEO/BlSeAgTtUGXyrGWZSYG7PH8HoMBs9Toi4Z87mxeTMs2OSxDKWC8EEsFKB2FQ88eFGAhxkaAd2lcXXOmOreUozp2qBM56kIOFy4DRo0CM+fP8eMGTOQmpqK1q1b4+DBg0yQITk5GQJW76CsrCyMHj0aqamp8Pf3R7t27XD69Gk0a6a/2pC94ZRf8dC8DCEVCSCg+Pcds5TT6hYYIJ+N38Tz0UCQgm2SWZio+BT/qsMt2j/8uzi9bVmFclTxlJS7v8USzc0NMiwQL0dfoSYh3FjgwBjP82Sc5x4SodlyNGN5bpa2N9Ldz1rqV/fCg5JW9nWqejAlf5UNhyfxAsD48ePx6NEjyGQynDt3Dh07dmReO3bsGNavX888X7RoETM2NTUV+/btQ5s2bRwwa/MVCryORVHwNKD92ZKHdBDeks/GCVVLeFAyrJQsxjjhLlgbG+258ASG/Ga8msRemEsF0VYc9BWeh5wWYrLiE3yv/NBiwWYIIUXhdHQPvNtOPyKvvQ44AQWxdeeyJqCgSwNWV+KXG1Wzah6ugFMIN1fAFmalPU1TLbnwxAjF11in7AUAmCzegsXiZZDCsnwlXU7e00+OtTemZFsr6j5TcZBBe+ND+TdGI6J8EAgoBPu5G0xA1bYX5/rczCfsGkJqRSqILuyW610aVTcx0rUhwq0MsM0jQxUKfDEUcbUHKggxSzkM0xQjoaCFGCA8jc2SOaiOLPM7Owh2hNiYWdpPcBpbJLMRQGXjtroW+svn4ALdxCbn17Y0MvQdaZtmcqOl1v20+ETNjVG/uhcATSQ0okHVMh+vokKEWxkoZnWCtY3mVr4u0I2qVzBUEY1s2hOtBfexW/otmlNJ5ToHS2n4zQG9JRG1UFBjomgblkiWwo1SIE7VBu/IZ5qNiPJBK7gM+VaL5JryIsqI5sYHayoUdGkT4od32tbCpF6hnJrmygYRbmWA3QnWFo5gU5qbu1iIQXZIvTijbo7+8jm4pw5GMJWJbZJZ6C3g50crr4Tf7vOPIWrLFc4yhJpWRb8wEdGVyr4YrfjK4lZFllK3pOOHoai4VnNjBw7s5XOzBJFQgAUDwzC2EkRETUGEWxkoUpSaSraIGJoSboeiukBkRQ6UJTyiA/GWfDaOqcLgTsmxQrIYnwt3wNJAQ8fv49Bz4XHEP8os0zxomsbt1DzIlWqk5Bgu3GcnFwciA1slsziBg9gyBg6MoRU6hjR0bRSVXdVgqPzKEoZGaOqA+7RwrsYQFRGHp4JUZIp5LFBiCabSSWr5e9jEH2OMPHhgpGISptEbMVJ0AF+Jt6GR4CkmK8ZYVNFwLz0fPx68jc1jIsyONcb+hFSM23gJA9vXgr+H6XOGUfewSrIQAVQ2MmhvjJVPtJl/zRCdG2qijoZuQNrrQK4quyZfp6onbs7uVebUIgLR3MqErYWbp5niYt3s9YY1vGx6fhWEmKMcgmjFKChoId4UnsFmyWzUsDDQ8DS7bG2SNpf0kNty8QlWnnhgdFw/gSYAEkBlI1EdYtPAgSHmvxeGT7rUB2BYuGk1N5myVJNnF8Dz6cShOYfIqCXQo2kNo/MgcCHCrQzwWVrOEswFJXQ1N5Gdav42qXrgI/k0ZNJeaC3QNHNsQRkXNlrKapnnFpluQqkJHGzlBA7elcfYNHBgiHfb1TLYyluL9jpgrzhPURRi+jVDk0BvfNmzkc3mEtmgGnaP64zT0T1sdkxXhQi3MiAS2PbjM9cgUqQj3Oxppp6jm6K/fA7uqGsikMrCVslsvC4wvRg0n0x8Q+iuIMXGHcVYJv4FX4p2ArBf4MAcBgMKJZpbWC1NY1BvN82YEZ3r4eCELqhm4xK1sBA/+Jkx2wlEuJWJz7o3QFiIH+YOaGGT43UP1SRcGusiItExS9vUNjzOVjymA/C2fBaOlgQafpX8gi+F22Eq0GBu4WBT6K7ariUQGdgimY3XSwIHkxRj7BY4MIfBVJASzc3PQ4JL376KC9/0LO9pEQxAhFsZqOYlxe5xnZmW4WXl/fDaWD20PbaM6cTZvvOzSABcTc1dLMTUPtyWPdpIG6ARfLZIHcmHB0YqJmO1UtN1ZaJ4O5aKl8ANMr2xyZmFqDd1P+/1Tp9mF0GmVBk088Ooe9gj/RYtBQ+RQXvjA/k32Kbqat2bsZDezY1HKg1pl+ya0yqeEt4+NoJ9INFSJ8JNLMSrzbhdiZsF+aBNbU3JD9ssDQ301vP/zOzXHEMj6qBBdS9QFIXh62xT96mGAN8pP8JduibmitbiDeFZ1KbSMFr+FdKg3y7nn6vP8H54bYuOvevyU0zYfMXga/0Ep/GTeCXcKAUS1SEYpfjK7v41ABgWWRcHb6QC0CzOokv9ap5MYToAyJWutVKYq0A0Nydlw8fhCK9bBUs/KG0KwDZLmwb5AABT69i5YVUIBBQa1vBmIm3JGfx7kplii6o7PpJPQwbtjVaCJE0NJ6XflsjYsnWGMCTYKKgRJdrCBA4O26HiwBQtS3xnwb5umPmm/tocBya8jGszX8OYrpoIanQf+0VqCdZDNDcnRdtYkA1bc2sW5A0AWDWkHfZcfYYBrWtafOxmQT64mWLdQjnnSwINv4nnI1TwBFskszFJMRZ71aX5bZZ0br2UnIVfj+oLRncUY4F4BV4vaVW0QvkGflS+b3f/2qZPOuGzvy5hdv/m8JKKkDDzNaPVAlKREFKRENG9m+DD8DoIqaK/Di3B8RDNrQLB9rlpNbeqXlKM6FwP/p760bOf3gtDlZLtrzQp1XrqVS9bf68ndA28I5+JOFUbuFEKLJUswUTRVlDQmGeWZKi8/etpHL7FbS+vqTjgBg7mKT+wuWBrEqi5MTSo7omH8/ri4by+6FS/KuKn98QbrTTNJL3dxGbrQymKQu2qHk65fgCBaG4VinzWYsRNSoSbKdrV8celb19FsUIFqUiA3CIl1DSN3GIF9l1LKdtc4IHRiq8whf4bY0T78KVoJxpRT/GVYiyOJKZj77UUTOjZCK1q+Vl0vNYlFQc1SioOxsgn4qKNE3N7NKmBI4npWP5RO8iVaj2Niwgp14JobhWIrMJS4cZnqTTNYr0UfD3E8PeUoE5VT+z9/KUyz0cNAWKVH2KSYgzktBCvC89ji2Q2Ll67jiOJ6Xhz6SmD++XoJOu+WVJxUIOpOJhbJsH2aTfDBeNrhrZHwszXUK+aJ0IDvQ3mrBFcB4ouS2JSBSQ3Nxe+vr7IycmBj4957ceZSM0pxgdrzmJg+xCbdHyIf5SFAB8pXvrhaJmP1Z5KxErJIlSl8pBO++ETeRSu0A1xaGIXNArwZsYlvShA9/nHAJS2KvpCtAsAcEjVFhMU41AAy3xYo16qh5caVcPwdReYbfWreeLIpG7IKVTgwYt8/Hk2GZ92q48gX3ez5W0E58BWv1Ei3Ah4nFmIRYfv4MSd56jp78Es+8aXWtRzrBHPRxPBY8hoMSYrPoGg1XtY/H5pxDf2wC2sPP7AQOCgH35UDrLYvyYWUrgztw8oikKhXIlmM/4FAJyO7oFgP+Lgr8jY6jfqFGbpsmXLULduXbi5uaFjx444f950ftbWrVvRpEkTuLm5oWXLlti/f385zdQ1CanigYUDW+Pi9Fexe1xnZvsrTWrg4by+Jvbk8oSujnfkM3FI1RZSSoFfJMvQ4PpinLv/nBmz6fxjBBkMHFi2eIu7WIgVH7VD4pw+jI/MQyLCqJfqYUTnukSwERgcLtw2b96MqKgoxMTE4NKlSwgLC0OvXr2Qnm54JezTp09j8ODBGDlyJC5fvowBAwZgwIABuH79ejnP3HXZPa4zVg9tj9+GdwBQ6sNyFwtxaGIX9GhiPN+sAO4Yo4jCCmU/AMDnol3IXD8Yb/98CDRNo17xLeyWfosWgod4QftgsHy6wYqDAa2D8ZmO7yyiflXcmtMbvVsE6lUKTH+jGWL66eekESovDjdLO3bsiA4dOmDp0qUANIsyh4SE4PPPP0d0dLTe+EGDBqGgoAB79+5ltnXq1AmtW7fGihUrzJ6PmKX8oWkaD14UoH41T0Zbomkah2+l48D1FIgEFLZcfKK33zuCE/hevAZSSokb6jrYrOqGb0QbIaUUuKUOwWjFJDyh9RcwEQoo3J3bB0o1jcbTDwAAlgxugzdaBZGIZiXAJXxucrkcHh4e2LZtG7PiPAAMGzYM2dnZ2L17t94+tWvXRlRUFCZMmMBsi4mJwa5du3D16lW98TKZDDJZaR1kbm4uQkJCiHCzMbnFCuy89FRvMeO21B2slCxEdao0adhY4GBYRB3U8HHDZ90acIRooVxFggGVCFsJN4deMS9evIBKpWIWYNYSEBCAxMREg/ukpqYaHJ+ammpwfGxsLGbNmmWbCROM4uMmxrDIuhjSqQ6eZBUhu0iOd1ecwSVlYwyQzcEayQI0FSRzAgcd61XB2uEdTAouiqKIYCNYhctfNVOnTkVUVBTzXKu5EeyDQKDJ2q8ND9yZ24fZnpX7Hh5nPMHYeqEY68D5ESoPDhVu1apVg1AoRFoatwwnLS0NgYGG284EBgbyGi+VSiGV2rZZIIE//j6e8PcJdfQ0CJUIh0ZLJRIJ2rVrh7i4OGabWq1GXFwcIiIMLzQSERHBGQ8Ahw4dMjqeQCBUThxulkZFRWHYsGFo3749wsPDsXjxYhQUFGDEiBEAgKFDh6JmzZqIjY0FAHz55Zfo2rUrFixYgL59+2LTpk24ePEiVq1a5ci3QSAQnAyHC7dBgwbh+fPnmDFjBlJTU9G6dWscPHiQCRokJydDwFqrIDIyEhs3bsT06dMxbdo0NGrUCLt27UKLFrZp9U0gEFwDh+e5lTckz41AcG5cqvyKQCAQbA0RbgQCwSUhwo1AILgkDg8olDdaF2NurnVrCBAIBPui/W2WNRxQ6YRbXl4eAJAqBQLBycnLy4Ovr6/V+1e6aKlarcazZ8/g7e1t9w4T2lKvx48fV/rILPksuJDPgwv78/D29kZeXh6Cg4M5aWB8qXSam0AgQK1atcr1nD4+PuQCLoF8FlzI58FF+3mURWPTQgIKBALBJSHCjUAguCREuNkRqVSKmJgY0pUE5LPQhXweXOzxeVS6gAKBQKgcEM2NQCC4JES4EQgEl4QINwKB4JIQ4UYgEFwSItxsSGZmJj788EP4+PjAz88PI0eORH5+vsl9Vq1ahW7dusHHxwcURSE7O7t8JmsHli1bhrp168LNzQ0dO3bE+fPnTY7funUrmjRpAjc3N7Rs2RL79+8vp5mWD3w+jxs3buCdd95B3bp1QVEUFi9eXH4TLSf4fB6rV6/Gyy+/DH9/f/j7+6Nnz55mryddiHCzIR9++CFu3LiBQ4cOYe/evThx4gQ++eQTk/sUFhaid+/emDZtWjnN0j5s3rwZUVFRiImJwaVLlxAWFoZevXohPT3d4PjTp09j8ODBGDlyJC5fvowBAwZgwIABuH79ejnP3D7w/TwKCwtRv359zJs3z+hiRxUZvp/HsWPHMHjwYBw9ehRnzpxBSEgIXnvtNTx9+tTyk9IEm3Dz5k0aAH3hwgVm24EDB2iKouinT5+a3f/o0aM0ADorK8uOs7Qf4eHh9Lhx45jnKpWKDg4OpmNjYw2OHzhwIN23b1/Oto4dO9Jjxoyx6zzLC76fB5s6derQixYtsuPsyp+yfB40TdNKpZL29vamN2zYYPE5ieZmI86cOQM/Pz+0b9+e2dazZ08IBAKcO3fOgTOzP3K5HPHx8ejZsyezTSAQoGfPnjhz5ozBfc6cOcMZDwC9evUyOr4iYc3n4crY4vMoLCyEQqFAlSpVLD4vEW42IjU1FTVq1OBsE4lEqFKlClJTUx00q/LhxYsXUKlUzKI+WgICAoy+99TUVF7jKxLWfB6ujC0+jylTpiA4OFjvhmgKItzMEB0dDYqiTD4SExMdPU0CwWWZN28eNm3ahJ07d8LNzc3i/SpdyyO+fPXVVxg+fLjJMfXr10dgYKCec1SpVCIzM9MlHcRsqlWrBqFQiLS0NM72tLQ0o+89MDCQ1/iKhDWfhytTls9j/vz5mDdvHg4fPoxWrVrxOi/R3MxQvXp1NGnSxORDIpEgIiIC2dnZiI+PZ/Y9cuQI1Go1Onbs6MB3YH8kEgnatWuHuLg4ZptarUZcXBwiIiIM7hMREcEZDwCHDh0yOr4iYc3n4cpY+3n8+OOPmDNnDg4ePMjxZVsM77AHwSi9e/em27RpQ587d44+efIk3ahRI3rw4MHM60+ePKFDQ0Ppc+fOMdtSUlLoy5cv06tXr6YB0CdOnKAvX75MZ2RkOOItWM2mTZtoqVRKr1+/nr558yb9ySef0H5+fnRqaipN0zQ9ZMgQOjo6mhl/6tQpWiQS0fPnz6dv3bpFx8TE0GKxmE5ISHDUW7ApfD8PmUxGX758mb58+TIdFBRET5o0ib58+TJ99+5dR70Fm8L385g3bx4tkUjobdu20SkpKcwjLy/P4nMS4WZDMjIy6MGDB9NeXl60j48PPWLECM6XkZSURAOgjx49ymyLiYmhAeg91q1bV/5voIwsWbKErl27Ni2RSOjw8HD67NmzzGtdu3alhw0bxhm/ZcsWunHjxrREIqGbN29O79u3r5xnbF/4fB7aa0P30bVr1/KfuJ3g83nUqVPH4OcRExNj8flIyyMCgeCSEJ8bgUBwSYhwIxAILgkRbgQCwSUhwo1AILgkRLgRCASXhAg3AoHgkhDhRiAQXBIi3AgEgktChBvBKRg+fLjBjiu9e/d29NQIFRTSFYTgNPTu3Rvr1q3jbDO2ArlCoYBYLOZsk8vlkEgkvM9r7X4E54ZobgSnQSqVIjAwkPPw9/cHAFAUheXLl+PNN9+Ep6cnvvvuO8ycOROtW7fGmjVrUK9ePabXV3JyMvr37w8vLy/4+Phg4MCBnHY7xvYjuBZEuBEqDDNnzsRbb72FhIQEfPzxxwCAe/fuYfv27dixYweuXLkCtVqN/v37IzMzE8ePH8ehQ4fw4MEDDBo0iHMs3f0IrgcxSwlOw969e+Hl5cXZNm3aNGZlsA8++AAjRozgvC6Xy/H777+jevXqADQ94RISEpCUlISQkBAAwO+//47mzZvjwoUL6NChg8H9CK4HEW4Ep6F79+5Yvnw5Zxt7QRBDDQvr1KnDEVC3bt1CSEgII9gAoFmzZvDz88OtW7cY4aa7H8H1IMKN4DR4enqiYcOGJl+3ZJul5yK4NsTnRnApmjZtisePH+Px48fMtps3byI7OxvNmjVz4MwI5Q3R3AhOg0wm01vqTSQSoVq1ahYfo2fPnmjZsiU+/PBDLF68GEqlEp999hm6du1qXR9+QoWFaG4Ep+HgwYMICgriPF566SVex6AoCrt374a/vz+6dOmCnj17on79+ti8ebOdZk1wVkibcQKB4JIQzY1AILgkRLgRCASXhAg3AoHgkhDhRiAQXBIi3AgEgktChBuBQHBJiHAjEAguCRFuBALBJSHCjUAguCREuBEIBJeECDcCgeCSEOFGIBBckv8DBbO6TI/6fC4AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.021, + "end_time": "2024-03-24T03:53:28.463475", + "exception": false, + "start_time": "2024-03-24T03:53:28.442475", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4490.212381, + "end_time": "2024-03-24T03:53:31.210295", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/realtabformer/3/mlu-eval.ipynb", + "output_path": "eval/contraceptive/realtabformer/3/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/contraceptive/realtabformer/3", + "path_prefix": "../../../../", + "random_seed": 3, + "single_model": "realtabformer" + }, + "start_time": "2024-03-24T02:38:40.997914", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/realtabformer/model.pt b/contraceptive/realtabformer/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..d37e72b8bff9ac081e26d20de7ed98f7e96bcbb7 --- /dev/null +++ b/contraceptive/realtabformer/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5bdcb53e3984f2a795219f33e05a754079f00b6950a1045621ac9a8b9831cf3 +size 34761390 diff --git a/contraceptive/realtabformer/params.json b/contraceptive/realtabformer/params.json new file mode 100644 index 0000000000000000000000000000000000000000..356b42b334f31447c4f03cb255b78a486258a11e --- /dev/null +++ b/contraceptive/realtabformer/params.json @@ -0,0 +1 @@ +{"gradient_penalty_mode": "ALL", "loss_balancer_beta": 0.7999999999999999, "loss_balancer_r": 0.96, "tf_pma_low_exp_2": 4, "grad_loss_fn": "mae", "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.7999999999999999, "bias_weight_decay": 0.05, "head_activation": "relu6", "tf_activation": "leakyhardtanh", "dataset_size_exp_2": 11, "batch_size_exp_2": 2, "epochs": 100, "lr_mul": 0.09, "n_warmup_steps": 80, "Optim": "amsgradw", "fixed_role_model": "realtabformer", "mse_mag_target": 1.0, "g_loss_mul": 0.1, "d_model_exp_2": 8, "attn_activation": "leakyhardtanh", "tf_d_inner_exp_2": 9, "tf_n_layers_enc": 3, "tf_n_head_exp_2": 5, "tf_activation_final": "leakyhardtanh", "tf_num_inds_exp_2": 4, "ada_d_hid_exp_2": 9, "ada_n_layers": 8, "ada_activation": "softsign", "ada_activation_final": "leakyhardsigmoid", "head_d_hid_exp_2": 9, "head_n_layers": 9, "head_n_head_exp_2": 6, "head_activation_final": "leakyhardsigmoid", "mse_mag": true, "mse_mag_multiply": true, "single_model": true, "models": ["realtabformer"], "max_seconds": 3600} \ No newline at end of file diff --git a/contraceptive/tab_ddpm_concat/eval.csv b/contraceptive/tab_ddpm_concat/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..7e7dfc4684f6a3a95184352c90d292be55c0fda7 --- /dev/null +++ b/contraceptive/tab_ddpm_concat/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tab_ddpm_concat,0.0237087070193229,0.015246206644986842,0.0019968665969887904,2.455820083618164,0.031993232667446136,0.9314482808113098,0.04156145825982094,3.5965997540188255e-06,2.924487829208374,0.034377582371234894,0.09727923572063446,0.04468631371855736,0.08744335174560547,0.016910480335354805,5.380307912826538 diff --git a/contraceptive/tab_ddpm_concat/history.csv b/contraceptive/tab_ddpm_concat/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..582b5d24b8b96aba4a48b1a32e303432e110783f --- /dev/null +++ b/contraceptive/tab_ddpm_concat/history.csv @@ -0,0 +1,11 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.02739265464660194,1.1173570523565528,0.0028521220052700534,0.025119311889219616,0.0,0.0,0.0,0.0,0.032262347015138304,900,225,242.43868017196655,1.0775052452087401,0.26937631130218503,0.1347517560606098,0.010731746522295806,0.1493522447505341,0.00033917589715015434,0.0,0.0,0.0,0.0,0.0,0.010731746522295806,450,113,95.67721843719482,0.8467010481167684,0.21261604097154405,0.11410295747352385 +1,0.006208910436317739,0.8280145235490416,0.00010301458273136867,0.0001026008653247522,0.0,0.0,0.0,0.0,0.008540319710607744,900,225,238.80838418006897,1.0613705963558622,0.26534264908896554,0.099707847174464,0.0031002509969726413,1.9949667974594831,8.847060396063784e-06,0.0,0.0,0.0,0.0,0.0,0.0031002509969726413,450,113,99.60530400276184,0.8814628672810783,0.22134512000613743,0.055952131002317225 +2,0.0029732521030948395,0.5794144581668509,1.9065153760119242e-05,0.0,0.0,0.0,0.0,0.0,0.0029732521030948395,900,225,252.49774837493896,1.1222122149997287,0.28055305374993217,0.10085780898264299,0.0026095288946655477,1.492227238601099,6.204995277141776e-06,0.0,0.0,0.0,0.0,0.0,0.0026095288946655477,450,113,102.85050177574158,0.9101814316437308,0.22855667061275906,0.04310003738537167 +3,0.0027397574804975495,0.5239720547694948,8.14726122460083e-06,0.0,0.0,0.0,0.0,0.0,0.0027397574804975495,900,225,244.73538613319397,1.0877128272586398,0.27192820681465996,0.10142784766697635,0.003070051697626089,1.5696541906561932,1.166988729526114e-05,0.0,0.0,0.0,0.0,0.0,0.003070051697626089,450,113,101.8385443687439,0.9012260563605654,0.22630787637498645,0.06228261488949287 +4,0.0023581975257548037,0.3410672404423733,1.1994138098866438e-05,1.4866391300327249e-05,0.0,0.0,0.0,0.0,0.00252653114331123,900,225,247.05686163902283,1.0980304961734348,0.2745076240433587,0.09989041606171263,0.0022714212175924333,1.8832387516627003,7.2981029367872995e-06,0.0,0.0,0.0,0.0,0.0,0.0022714212175924333,450,113,100.17966938018799,0.8865457467273273,0.22262148751152885,0.052650303883782994 +5,0.0024742192417842386,0.3507677028266865,6.229266246524888e-06,2.6323258255918822e-06,0.0,0.0,0.0,0.0,0.0028022190412189552,900,225,247.09229588508606,1.0981879817114937,0.2745469954278734,0.1015876992026137,0.002520995658188541,1.5546536559230526,5.602417538170796e-06,0.0,0.0,0.0,0.0,0.0,0.002520995658188541,450,113,99.56677508354187,0.8811219033941758,0.2212595001856486,0.048813573702906086 +6,0.002148849349759985,0.3859976332026933,5.054510944888613e-06,0.0,0.0,0.0,0.0,0.0,0.002148849349759985,900,225,248.287282705307,1.1034990342458089,0.2758747585614522,0.10126116779943307,0.0025426524340744234,1.8411945350910226,1.1723544970543994e-05,0.0,0.0,0.0,0.0,0.0,0.0025426524340744234,450,113,100.16489577293396,0.8864150068401235,0.22258865727318658,0.05908973131790361 +7,0.0020464518228416433,0.2880907567315516,5.413586949876549e-06,0.0,0.0,0.0,0.0,0.0,0.0020464518228416433,900,225,248.08354210853577,1.1025935204823811,0.2756483801205953,0.10204729804148277,0.002012234483457481,1.4328002845302086,5.459719962973194e-06,0.0,0.0,0.0,0.0,0.0,0.002012234483457481,450,113,100.88333654403687,0.8927728897702377,0.2241851923200819,0.05221234420335036 +8,0.001945750925806351,0.2846149470687626,3.628518232128718e-06,0.0,0.0,0.0,0.0,0.0,0.001945750925806351,900,225,249.2025444507599,1.1075668642255996,0.2768917160563999,0.10608977859425876,0.0024451360522329602,1.5600430507726564,1.1627008480942101e-05,0.0,0.0,0.0,0.0,0.0,0.0024451360522329602,450,113,100.29779386520386,0.8875910961522465,0.22288398636711967,0.04840205141458562 +9,0.0018871993097127416,0.3353491520646711,5.283127970706369e-06,4.544905904266569e-08,0.0,0.0,0.0,0.0,0.002344711327880052,900,225,247.01215744018555,1.0978318108452692,0.2744579527113173,0.10287626234814524,0.001859568822482187,1.4678312607579032,4.946820349966716e-06,0.0,0.0,0.0,0.0,0.0,0.001859568822482187,450,113,100.66262602806091,0.8908196993633709,0.22369472450680203,0.05059397140254093 diff --git a/contraceptive/tab_ddpm_concat/mlu-eval.ipynb b/contraceptive/tab_ddpm_concat/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..ae1436dca41649a585f1b021720e01e2564a879e --- /dev/null +++ b/contraceptive/tab_ddpm_concat/mlu-eval.ipynb @@ -0,0 +1,2313 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:35:02.995339Z", + "iopub.status.busy": "2024-03-26T07:35:02.994416Z", + "iopub.status.idle": "2024-03-26T07:35:03.027714Z", + "shell.execute_reply": "2024-03-26T07:35:03.027003Z" + }, + "papermill": { + "duration": 0.048104, + "end_time": "2024-03-26T07:35:03.029650", + "exception": false, + "start_time": "2024-03-26T07:35:02.981546", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:35:03.055960Z", + "iopub.status.busy": "2024-03-26T07:35:03.055176Z", + "iopub.status.idle": "2024-03-26T07:35:03.062623Z", + "shell.execute_reply": "2024-03-26T07:35:03.061845Z" + }, + "papermill": { + "duration": 0.022772, + "end_time": "2024-03-26T07:35:03.064554", + "exception": false, + "start_time": "2024-03-26T07:35:03.041782", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:35:03.087890Z", + "iopub.status.busy": "2024-03-26T07:35:03.087631Z", + "iopub.status.idle": "2024-03-26T07:35:03.091817Z", + "shell.execute_reply": "2024-03-26T07:35:03.090938Z" + }, + "papermill": { + "duration": 0.018301, + "end_time": "2024-03-26T07:35:03.093817", + "exception": false, + "start_time": "2024-03-26T07:35:03.075516", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:35:03.118179Z", + "iopub.status.busy": "2024-03-26T07:35:03.117885Z", + "iopub.status.idle": "2024-03-26T07:35:03.121725Z", + "shell.execute_reply": "2024-03-26T07:35:03.120894Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018653, + "end_time": "2024-03-26T07:35:03.123548", + "exception": false, + "start_time": "2024-03-26T07:35:03.104895", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:35:03.147229Z", + "iopub.status.busy": "2024-03-26T07:35:03.146680Z", + "iopub.status.idle": "2024-03-26T07:35:03.152343Z", + "shell.execute_reply": "2024-03-26T07:35:03.151495Z" + }, + "papermill": { + "duration": 0.019628, + "end_time": "2024-03-26T07:35:03.154323", + "exception": false, + "start_time": "2024-03-26T07:35:03.134695", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "adaee8e7", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:35:03.179799Z", + "iopub.status.busy": "2024-03-26T07:35:03.179435Z", + "iopub.status.idle": "2024-03-26T07:35:03.184808Z", + "shell.execute_reply": "2024-03-26T07:35:03.184010Z" + }, + "papermill": { + "duration": 0.020621, + "end_time": "2024-03-26T07:35:03.186624", + "exception": false, + "start_time": "2024-03-26T07:35:03.166003", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"tab_ddpm_concat\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 1\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/tab_ddpm_concat/1\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.010991, + "end_time": "2024-03-26T07:35:03.208708", + "exception": false, + "start_time": "2024-03-26T07:35:03.197717", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:35:03.232116Z", + "iopub.status.busy": "2024-03-26T07:35:03.231793Z", + "iopub.status.idle": "2024-03-26T07:35:03.241260Z", + "shell.execute_reply": "2024-03-26T07:35:03.240402Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023482, + "end_time": "2024-03-26T07:35:03.243190", + "exception": false, + "start_time": "2024-03-26T07:35:03.219708", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/tab_ddpm_concat/1\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:35:03.266996Z", + "iopub.status.busy": "2024-03-26T07:35:03.266687Z", + "iopub.status.idle": "2024-03-26T07:35:05.265679Z", + "shell.execute_reply": "2024-03-26T07:35:05.264724Z" + }, + "papermill": { + "duration": 2.013942, + "end_time": "2024-03-26T07:35:05.268393", + "exception": false, + "start_time": "2024-03-26T07:35:03.254451", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:35:05.300607Z", + "iopub.status.busy": "2024-03-26T07:35:05.299496Z", + "iopub.status.idle": "2024-03-26T07:35:05.312893Z", + "shell.execute_reply": "2024-03-26T07:35:05.312104Z" + }, + "papermill": { + "duration": 0.029904, + "end_time": "2024-03-26T07:35:05.314826", + "exception": false, + "start_time": "2024-03-26T07:35:05.284922", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:35:05.339428Z", + "iopub.status.busy": "2024-03-26T07:35:05.339167Z", + "iopub.status.idle": "2024-03-26T07:35:05.349826Z", + "shell.execute_reply": "2024-03-26T07:35:05.348900Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.02534, + "end_time": "2024-03-26T07:35:05.351893", + "exception": false, + "start_time": "2024-03-26T07:35:05.326553", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:35:05.375758Z", + "iopub.status.busy": "2024-03-26T07:35:05.375488Z", + "iopub.status.idle": "2024-03-26T07:35:05.868120Z", + "shell.execute_reply": "2024-03-26T07:35:05.866917Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.507742, + "end_time": "2024-03-26T07:35:05.870935", + "exception": false, + "start_time": "2024-03-26T07:35:05.363193", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:35:05.898221Z", + "iopub.status.busy": "2024-03-26T07:35:05.897740Z", + "iopub.status.idle": "2024-03-26T07:35:19.187741Z", + "shell.execute_reply": "2024-03-26T07:35:19.186849Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 13.3063, + "end_time": "2024-03-26T07:35:19.190224", + "exception": false, + "start_time": "2024-03-26T07:35:05.883924", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-26 07:35:10.386642: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-26 07:35:10.386743: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-26 07:35:10.519405: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:35:19.218014Z", + "iopub.status.busy": "2024-03-26T07:35:19.216822Z", + "iopub.status.idle": "2024-03-26T07:35:19.226877Z", + "shell.execute_reply": "2024-03-26T07:35:19.226183Z" + }, + "papermill": { + "duration": 0.026024, + "end_time": "2024-03-26T07:35:19.228924", + "exception": false, + "start_time": "2024-03-26T07:35:19.202900", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:35:19.254876Z", + "iopub.status.busy": "2024-03-26T07:35:19.254580Z", + "iopub.status.idle": "2024-03-26T07:35:28.746044Z", + "shell.execute_reply": "2024-03-26T07:35:28.744936Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 9.507866, + "end_time": "2024-03-26T07:35:28.748938", + "exception": false, + "start_time": "2024-03-26T07:35:19.241072", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-26T07:35:28.777730Z", + "iopub.status.busy": "2024-03-26T07:35:28.776930Z", + "iopub.status.idle": "2024-03-26T07:35:28.783689Z", + "shell.execute_reply": "2024-03-26T07:35:28.782890Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.023315, + "end_time": "2024-03-26T07:35:28.785971", + "exception": false, + "start_time": "2024-03-26T07:35:28.762656", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 46,\n", + " 'realtabformer': (24, 72, Embedding(72, 672), True),\n", + " 'lct_gan': 40,\n", + " 'tab_ddpm_concat': 10}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:35:28.810552Z", + "iopub.status.busy": "2024-03-26T07:35:28.810301Z", + "iopub.status.idle": "2024-03-26T07:35:28.815012Z", + "shell.execute_reply": "2024-03-26T07:35:28.814118Z" + }, + "papermill": { + "duration": 0.019136, + "end_time": "2024-03-26T07:35:28.816851", + "exception": false, + "start_time": "2024-03-26T07:35:28.797715", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:35:28.842295Z", + "iopub.status.busy": "2024-03-26T07:35:28.841737Z", + "iopub.status.idle": "2024-03-26T07:35:49.690743Z", + "shell.execute_reply": "2024-03-26T07:35:49.689743Z" + }, + "papermill": { + "duration": 20.864112, + "end_time": "2024-03-26T07:35:49.692830", + "exception": false, + "start_time": "2024-03-26T07:35:28.828718", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/tab_ddpm_concat/all inf False\n", + "../../../../ml-utility-loss/aug_test/contraceptive 0\n", + "Caching in ../../../../contraceptive/_cache_bs_test/tab_ddpm_concat/all inf False\n", + "../../../../ml-utility-loss/bs_test/contraceptive 0\n", + "Caching in ../../../../contraceptive/_cache_synth_test/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/contraceptive 600\n", + "600\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:35:49.721298Z", + "iopub.status.busy": "2024-03-26T07:35:49.720938Z", + "iopub.status.idle": "2024-03-26T07:35:50.346184Z", + "shell.execute_reply": "2024-03-26T07:35:50.345173Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.643036, + "end_time": "2024-03-26T07:35:50.348714", + "exception": false, + "start_time": "2024-03-26T07:35:49.705678", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 4,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7494458230986923,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'head_activation': torch.nn.modules.activation.ReLU6,\n", + " 'loss_balancer_beta': 0.6806661100374879,\n", + " 'loss_balancer_r': 0.9427716710925113,\n", + " 'tf_activation': torch.nn.modules.activation.ReLU6,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.07424782199493057,\n", + " 'n_warmup_steps': 104,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'fixed_role_model': 'tab_ddpm_concat',\n", + " 'd_model': 128,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 8,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 16,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tab_ddpm_concat'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 16,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.20359405820922769,\n", + " 'multiply': True,\n", + " 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "# params = {\n", + "# **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "# }\n", + "params = getattr(PARAMS, dataset_name).BEST_DICT[gp][gp_multiply][single_model]\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).DEFAULTS,\n", + " **params,\n", + "}\n", + "if isinstance(params, (list, tuple)):\n", + " params = params[param_index]\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:35:50.377743Z", + "iopub.status.busy": "2024-03-26T07:35:50.377384Z", + "iopub.status.idle": "2024-03-26T07:36:37.247201Z", + "shell.execute_reply": "2024-03-26T07:36:37.246033Z" + }, + "papermill": { + "duration": 46.898103, + "end_time": "2024-03-26T07:36:37.260942", + "exception": false, + "start_time": "2024-03-26T07:35:50.362839", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_train/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/contraceptive [400, 0]\n", + "Caching in ../../../../contraceptive/_cache_aug_val/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/contraceptive [0, 200]\n", + "Caching in ../../../../contraceptive/_cache_bs_train/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/contraceptive [100, 0]\n", + "Caching in ../../../../contraceptive/_cache_bs_val/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/contraceptive [0, 50]\n", + "Caching in ../../../../contraceptive/_cache_synth/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/contraceptive [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-26T07:36:37.289564Z", + "iopub.status.busy": "2024-03-26T07:36:37.289232Z", + "iopub.status.idle": "2024-03-26T07:36:37.730723Z", + "shell.execute_reply": "2024-03-26T07:36:37.729782Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.458557, + "end_time": "2024-03-26T07:36:37.732807", + "exception": false, + "start_time": "2024-03-26T07:36:37.274250", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['tab_ddpm_concat'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:36:37.763346Z", + "iopub.status.busy": "2024-03-26T07:36:37.763000Z", + "iopub.status.idle": "2024-03-26T07:36:37.767427Z", + "shell.execute_reply": "2024-03-26T07:36:37.766569Z" + }, + "papermill": { + "duration": 0.022401, + "end_time": "2024-03-26T07:36:37.769485", + "exception": false, + "start_time": "2024-03-26T07:36:37.747084", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:36:37.798816Z", + "iopub.status.busy": "2024-03-26T07:36:37.798462Z", + "iopub.status.idle": "2024-03-26T07:36:37.806558Z", + "shell.execute_reply": "2024-03-26T07:36:37.805620Z" + }, + "papermill": { + "duration": 0.0258, + "end_time": "2024-03-26T07:36:37.808874", + "exception": false, + "start_time": "2024-03-26T07:36:37.783074", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "7827841" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:36:37.840768Z", + "iopub.status.busy": "2024-03-26T07:36:37.840422Z", + "iopub.status.idle": "2024-03-26T07:36:37.924244Z", + "shell.execute_reply": "2024-03-26T07:36:37.923277Z" + }, + "papermill": { + "duration": 0.102431, + "end_time": "2024-03-26T07:36:37.926641", + "exception": false, + "start_time": "2024-03-26T07:36:37.824210", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 10] --\n", + "├─Adapter: 1-1 [2, 1179, 10] --\n", + "│ └─Sequential: 2-1 [2, 1179, 128] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 1024] 11,264\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 128] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 128] 131,200\n", + "│ │ │ └─LeakyHardsigmoid: 4-16 [2, 1179, 128] --\n", + "├─Adapter: 1-2 [2, 294, 10] (recursive)\n", + "│ └─Sequential: 2-2 [2, 294, 128] (recursive)\n", + "│ │ └─FeedForward: 3-9 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-18 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 128] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 128] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-32 [2, 294, 128] --\n", + "├─TwinEncoder: 1-3 [2, 512] --\n", + "│ └─Encoder: 2-3 [2, 4, 128] --\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 1179, 128] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 128] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 16, 128] 2,048\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 16, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 16, 128] 16,384\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 128] 16,384\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 128] 16,384\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 16, 128] 16,512\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-6 [2, 16, 128] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 128] 16,384\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 16, 128] 16,384\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 16, 128] 16,384\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 16] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 128] 16,512\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-12 [2, 1179, 128] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 128] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 66,048\n", + "│ │ │ │ │ └─ReLU6: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 128] 65,664\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 1179, 128] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 128] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 16, 128] 2,048\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 16, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 16, 128] 16,384\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 128] 16,384\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 128] 16,384\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 16, 128] 16,512\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-18 [2, 16, 128] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 128] 16,384\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 16, 128] 16,384\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 16, 128] 16,384\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 16] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 128] 16,512\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-24 [2, 1179, 128] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 128] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 66,048\n", + "│ │ │ │ │ └─ReLU6: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 128] 65,664\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 4, 128] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 128] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 16, 128] 2,048\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 16, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 16, 128] 16,384\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 128] 16,384\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 128] 16,384\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 16, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 16, 128] 16,512\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-30 [2, 16, 128] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 128] 16,384\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 16, 128] 16,384\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 16, 128] 16,384\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 16] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 128] 16,512\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-36 [2, 1179, 128] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 128] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 66,048\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 128] 65,664\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 4, 128] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 4, 128] 512\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 4, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 4, 128] 16,384\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 128] 16,384\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 128] 16,384\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 4, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 4, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 4, 128] 16,512\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-42 [2, 4, 128] --\n", + "│ └─Encoder: 2-4 [2, 4, 128] (recursive)\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 294, 128] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-8 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-21 [2, 16, 128] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-22 [2, 16, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 16, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 16, 128] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-48 [2, 16, 128] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-23 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 16, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 16, 128] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 16] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-54 [2, 294, 128] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-9 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─ReLU6: 6-25 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-26 [2, 294, 128] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 294, 128] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 16, 128] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 16, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 16, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 16, 128] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-60 [2, 16, 128] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 16, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 16, 128] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 16] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-66 [2, 294, 128] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─ReLU6: 6-31 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 294, 128] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 4, 128] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 16, 128] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 16, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 16, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 16, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 16, 128] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-72 [2, 16, 128] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 16, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 16, 128] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 16] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-78 [2, 294, 128] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 294, 128] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-14 [2, 4, 128] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 4, 128] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-40 [2, 4, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 4, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 128] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 4, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 4, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 4, 128] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-84 [2, 4, 128] --\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-19 [2, 256] --\n", + "│ │ │ └─Linear: 4-39 [2, 256] 131,328\n", + "│ │ │ └─ReLU6: 4-40 [2, 256] --\n", + "│ │ └─FeedForward: 3-20 [2, 256] --\n", + "│ │ │ └─Linear: 4-41 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-42 [2, 256] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-25 [2, 256] --\n", + "│ │ │ └─Linear: 4-51 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-52 [2, 256] --\n", + "│ │ └─FeedForward: 3-26 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 7,827,841\n", + "Trainable params: 7,827,841\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 30.23\n", + "========================================================================================================================\n", + "Input size (MB): 0.12\n", + "Forward/backward pass size (MB): 260.50\n", + "Params size (MB): 31.31\n", + "Estimated Total Size (MB): 291.93\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T07:36:37.959402Z", + "iopub.status.busy": "2024-03-26T07:36:37.958471Z", + "iopub.status.idle": "2024-03-26T08:41:08.861629Z", + "shell.execute_reply": "2024-03-26T08:41:08.860610Z" + }, + "papermill": { + "duration": 3870.937668, + "end_time": "2024-03-26T08:41:08.879443", + "exception": false, + "start_time": "2024-03-26T07:36:37.941775", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 600]\n", + "Creating model of type \n", + "[*] Embedding False True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.02739265464660194, 'avg_role_model_std_loss': 1.1173570523565528, 'avg_role_model_mean_pred_loss': 0.0028521220052700534, 'avg_role_model_g_mag_loss': 0.025119311889219616, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.032262347015138304, 'n_size': 900, 'n_batch': 225, 'duration': 242.43868017196655, 'duration_batch': 1.0775052452087401, 'duration_size': 0.26937631130218503, 'avg_pred_std': 0.1347517560606098}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.010731746522295806, 'avg_role_model_std_loss': 0.1493522447505341, 'avg_role_model_mean_pred_loss': 0.00033917589715015434, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010731746522295806, 'n_size': 450, 'n_batch': 113, 'duration': 95.67721843719482, 'duration_batch': 0.8467010481167684, 'duration_size': 0.21261604097154405, 'avg_pred_std': 0.11410295747352385}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006208910436317739, 'avg_role_model_std_loss': 0.8280145235490416, 'avg_role_model_mean_pred_loss': 0.00010301458273136867, 'avg_role_model_g_mag_loss': 0.0001026008653247522, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008540319710607744, 'n_size': 900, 'n_batch': 225, 'duration': 238.80838418006897, 'duration_batch': 1.0613705963558622, 'duration_size': 0.26534264908896554, 'avg_pred_std': 0.099707847174464}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0031002509969726413, 'avg_role_model_std_loss': 1.9949667974594831, 'avg_role_model_mean_pred_loss': 8.847060396063784e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0031002509969726413, 'n_size': 450, 'n_batch': 113, 'duration': 99.60530400276184, 'duration_batch': 0.8814628672810783, 'duration_size': 0.22134512000613743, 'avg_pred_std': 0.055952131002317225}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0029732521030948395, 'avg_role_model_std_loss': 0.5794144581668509, 'avg_role_model_mean_pred_loss': 1.9065153760119242e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0029732521030948395, 'n_size': 900, 'n_batch': 225, 'duration': 252.49774837493896, 'duration_batch': 1.1222122149997287, 'duration_size': 0.28055305374993217, 'avg_pred_std': 0.10085780898264299}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0026095288946655477, 'avg_role_model_std_loss': 1.492227238601099, 'avg_role_model_mean_pred_loss': 6.204995277141776e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0026095288946655477, 'n_size': 450, 'n_batch': 113, 'duration': 102.85050177574158, 'duration_batch': 0.9101814316437308, 'duration_size': 0.22855667061275906, 'avg_pred_std': 0.04310003738537167}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0027397574804975495, 'avg_role_model_std_loss': 0.5239720547694948, 'avg_role_model_mean_pred_loss': 8.14726122460083e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0027397574804975495, 'n_size': 900, 'n_batch': 225, 'duration': 244.73538613319397, 'duration_batch': 1.0877128272586398, 'duration_size': 0.27192820681465996, 'avg_pred_std': 0.10142784766697635}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003070051697626089, 'avg_role_model_std_loss': 1.5696541906561932, 'avg_role_model_mean_pred_loss': 1.166988729526114e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003070051697626089, 'n_size': 450, 'n_batch': 113, 'duration': 101.8385443687439, 'duration_batch': 0.9012260563605654, 'duration_size': 0.22630787637498645, 'avg_pred_std': 0.06228261488949287}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0023581975257548037, 'avg_role_model_std_loss': 0.3410672404423733, 'avg_role_model_mean_pred_loss': 1.1994138098866438e-05, 'avg_role_model_g_mag_loss': 1.4866391300327249e-05, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00252653114331123, 'n_size': 900, 'n_batch': 225, 'duration': 247.05686163902283, 'duration_batch': 1.0980304961734348, 'duration_size': 0.2745076240433587, 'avg_pred_std': 0.09989041606171263}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022714212175924333, 'avg_role_model_std_loss': 1.8832387516627003, 'avg_role_model_mean_pred_loss': 7.2981029367872995e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022714212175924333, 'n_size': 450, 'n_batch': 113, 'duration': 100.17966938018799, 'duration_batch': 0.8865457467273273, 'duration_size': 0.22262148751152885, 'avg_pred_std': 0.052650303883782994}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0024742192417842386, 'avg_role_model_std_loss': 0.3507677028266865, 'avg_role_model_mean_pred_loss': 6.229266246524888e-06, 'avg_role_model_g_mag_loss': 2.6323258255918822e-06, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0028022190412189552, 'n_size': 900, 'n_batch': 225, 'duration': 247.09229588508606, 'duration_batch': 1.0981879817114937, 'duration_size': 0.2745469954278734, 'avg_pred_std': 0.1015876992026137}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002520995658188541, 'avg_role_model_std_loss': 1.5546536559230526, 'avg_role_model_mean_pred_loss': 5.602417538170796e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002520995658188541, 'n_size': 450, 'n_batch': 113, 'duration': 99.56677508354187, 'duration_batch': 0.8811219033941758, 'duration_size': 0.2212595001856486, 'avg_pred_std': 0.048813573702906086}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002148849349759985, 'avg_role_model_std_loss': 0.3859976332026933, 'avg_role_model_mean_pred_loss': 5.054510944888613e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002148849349759985, 'n_size': 900, 'n_batch': 225, 'duration': 248.287282705307, 'duration_batch': 1.1034990342458089, 'duration_size': 0.2758747585614522, 'avg_pred_std': 0.10126116779943307}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0025426524340744234, 'avg_role_model_std_loss': 1.8411945350910226, 'avg_role_model_mean_pred_loss': 1.1723544970543994e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0025426524340744234, 'n_size': 450, 'n_batch': 113, 'duration': 100.16489577293396, 'duration_batch': 0.8864150068401235, 'duration_size': 0.22258865727318658, 'avg_pred_std': 0.05908973131790361}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0020464518228416433, 'avg_role_model_std_loss': 0.2880907567315516, 'avg_role_model_mean_pred_loss': 5.413586949876549e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0020464518228416433, 'n_size': 900, 'n_batch': 225, 'duration': 248.08354210853577, 'duration_batch': 1.1025935204823811, 'duration_size': 0.2756483801205953, 'avg_pred_std': 0.10204729804148277}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002012234483457481, 'avg_role_model_std_loss': 1.4328002845302086, 'avg_role_model_mean_pred_loss': 5.459719962973194e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002012234483457481, 'n_size': 450, 'n_batch': 113, 'duration': 100.88333654403687, 'duration_batch': 0.8927728897702377, 'duration_size': 0.2241851923200819, 'avg_pred_std': 0.05221234420335036}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001945750925806351, 'avg_role_model_std_loss': 0.2846149470687626, 'avg_role_model_mean_pred_loss': 3.628518232128718e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001945750925806351, 'n_size': 900, 'n_batch': 225, 'duration': 249.2025444507599, 'duration_batch': 1.1075668642255996, 'duration_size': 0.2768917160563999, 'avg_pred_std': 0.10608977859425876}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0024451360522329602, 'avg_role_model_std_loss': 1.5600430507726564, 'avg_role_model_mean_pred_loss': 1.1627008480942101e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0024451360522329602, 'n_size': 450, 'n_batch': 113, 'duration': 100.29779386520386, 'duration_batch': 0.8875910961522465, 'duration_size': 0.22288398636711967, 'avg_pred_std': 0.04840205141458562}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0018871993097127416, 'avg_role_model_std_loss': 0.3353491520646711, 'avg_role_model_mean_pred_loss': 5.283127970706369e-06, 'avg_role_model_g_mag_loss': 4.544905904266569e-08, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002344711327880052, 'n_size': 900, 'n_batch': 225, 'duration': 247.01215744018555, 'duration_batch': 1.0978318108452692, 'duration_size': 0.2744579527113173, 'avg_pred_std': 0.10287626234814524}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001859568822482187, 'avg_role_model_std_loss': 1.4678312607579032, 'avg_role_model_mean_pred_loss': 4.946820349966716e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001859568822482187, 'n_size': 450, 'n_batch': 113, 'duration': 100.66262602806091, 'duration_batch': 0.8908196993633709, 'duration_size': 0.22369472450680203, 'avg_pred_std': 0.05059397140254093}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0018023989783654947, 'avg_role_model_std_loss': 0.23040446630229616, 'avg_role_model_mean_pred_loss': 4.0523389028073935e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0018023989783654947, 'n_size': 900, 'n_batch': 225, 'duration': 246.93518829345703, 'duration_batch': 1.097489725748698, 'duration_size': 0.2743724314371745, 'avg_pred_std': 0.10276419790254698}\n", + "Time out: 3726.1000921726227/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tab_ddpm_concat', 'n_size': 600, 'n_batch': 150, 'role_model_metrics': {'avg_loss': 0.0019968665854927774, 'avg_g_mag_loss': 0.01371617328345046, 'avg_g_cos_loss': 0.022173850040417165, 'pred_duration': 2.9101011753082275, 'grad_duration': 2.418388843536377, 'total_duration': 5.3284900188446045, 'pred_std': 0.08744335174560547, 'std_loss': 0.016910480335354805, 'mean_pred_loss': 3.5965992992714746e-06, 'pred_rmse': 0.04468631371855736, 'pred_mae': 0.034377578645944595, 'pred_mape': 0.09727925807237625, 'grad_rmse': 0.04156146198511124, 'grad_mae': 0.031993232667446136, 'grad_mape': 0.9314482808113098}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0019968665854927774, 'avg_g_mag_loss': 0.01371617328345046, 'avg_g_cos_loss': 0.022173850040417165, 'avg_pred_duration': 2.9101011753082275, 'avg_grad_duration': 2.418388843536377, 'avg_total_duration': 5.3284900188446045, 'avg_pred_std': 0.08744335174560547, 'avg_std_loss': 0.016910480335354805, 'avg_mean_pred_loss': 3.5965992992714746e-06}, 'min_metrics': {'avg_loss': 0.0019968665854927774, 'avg_g_mag_loss': 0.01371617328345046, 'avg_g_cos_loss': 0.022173850040417165, 'pred_duration': 2.9101011753082275, 'grad_duration': 2.418388843536377, 'total_duration': 5.3284900188446045, 'pred_std': 0.08744335174560547, 'std_loss': 0.016910480335354805, 'mean_pred_loss': 3.5965992992714746e-06, 'pred_rmse': 0.04468631371855736, 'pred_mae': 0.034377578645944595, 'pred_mape': 0.09727925807237625, 'grad_rmse': 0.04156146198511124, 'grad_mae': 0.031993232667446136, 'grad_mape': 0.9314482808113098}, 'model_metrics': {'tab_ddpm_concat': {'avg_loss': 0.0019968665854927774, 'avg_g_mag_loss': 0.01371617328345046, 'avg_g_cos_loss': 0.022173850040417165, 'pred_duration': 2.9101011753082275, 'grad_duration': 2.418388843536377, 'total_duration': 5.3284900188446045, 'pred_std': 0.08744335174560547, 'std_loss': 0.016910480335354805, 'mean_pred_loss': 3.5965992992714746e-06, 'pred_rmse': 0.04468631371855736, 'pred_mae': 0.034377578645944595, 'pred_mape': 0.09727925807237625, 'grad_rmse': 0.04156146198511124, 'grad_mae': 0.031993232667446136, 'grad_mape': 0.9314482808113098}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T08:41:08.913434Z", + "iopub.status.busy": "2024-03-26T08:41:08.913073Z", + "iopub.status.idle": "2024-03-26T08:41:08.917724Z", + "shell.execute_reply": "2024-03-26T08:41:08.916933Z" + }, + "papermill": { + "duration": 0.024161, + "end_time": "2024-03-26T08:41:08.919711", + "exception": false, + "start_time": "2024-03-26T08:41:08.895550", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T08:41:08.952210Z", + "iopub.status.busy": "2024-03-26T08:41:08.951891Z", + "iopub.status.idle": "2024-03-26T08:41:09.022114Z", + "shell.execute_reply": "2024-03-26T08:41:09.021265Z" + }, + "papermill": { + "duration": 0.089496, + "end_time": "2024-03-26T08:41:09.024569", + "exception": false, + "start_time": "2024-03-26T08:41:08.935073", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T08:41:09.060795Z", + "iopub.status.busy": "2024-03-26T08:41:09.060444Z", + "iopub.status.idle": "2024-03-26T08:41:09.366330Z", + "shell.execute_reply": "2024-03-26T08:41:09.365392Z" + }, + "papermill": { + "duration": 0.326389, + "end_time": "2024-03-26T08:41:09.368482", + "exception": false, + "start_time": "2024-03-26T08:41:09.042093", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA6oklEQVR4nO3de1xUdf4/8NeZYS6AMMNFLoPAkEoYopkEYRe3ZKViM8qNYlklv6a2i2WxbWaldPkmdnFrU7eydu23uxnqfrVt1doQtTIGUS6moeQFAeUmIjPcZ5j5/P4Y5ujgAHODEXk/H4/zmOGcz/nMe8aZt+d8zud8PhxjjIEQQkYIgasDIIQQW1DSIoSMKJS0CCEjCiUtQsiIQkmLEDKiUNIihIwolLQIISOKm6sDGC4GgwG1tbXw8vICx3GuDocQ0gdjDK2trVAoFBAI+j+eGjVJq7a2FqGhoa4OgxAyiJqaGowbN67f7aMmaXl5eQEwfiDe3t4ujoYQ0pdGo0FoaCj/W+3PqElaplNCb29vSlqEXMMGa76hhnhCyIhCSYsQMqJQ0iKEjCijpk2LOEav10On07k6DDKCiUQiCIVCh+uhpEUGxBhDfX09WlpaXB0KuQ7I5XIEBQU51FeSkhYZkClhBQQEwMPDgzrmErswxtDR0YHGxkYAQHBwsN11UdLqo7VLh+N1rdAbGBLG+7k6HJfS6/V8wvLzG92fBXGcu7s7AKCxsREBAQF2nypSQ3wfhWeakfqRCm/sLnd1KC5nasPy8PBwcSTkemH6LjnSPkpJq48If+OHerapAzR8vhGdEhJnccZ3iZJWH6G+HuA4oK27Bxfbta4OhxDSByWtPiRuQihkxnPvs03tLo6GENIXJS0LIvw9AQCVlLSInTiOwxdffOHqMJzqlVdewc033+zqMChpWaLsbdequtjh4kgIsd+nn34KuVzutPqee+455OfnO60+e1GXBwuUfr1HWhfpSItc/7RaLcRi8aDlxowZgzFjxgxDRAOjIy0LTEmL2rTMMcbQoe1xyWLrldyvv/4ad9xxB+RyOfz8/PCrX/0Kp0+fBgDMmDEDy5cvNyt/4cIFiEQifPfddwCAuro6JCcnw93dHREREdi8eTOUSiXee+89uz67o0eP4p577oG7uzv8/PywePFitLW18dv379+PuLg4eHp6Qi6X4/bbb0dVVRUA4MiRI7j77rvh5eUFb29vTJ8+HYcPHx7w9fbv348FCxZArVaD4zhwHIdXXnkFAKBUKvH6669j/vz58Pb2xuLFiwEAy5cvR2RkJDw8PHDDDTdg5cqVZl0T+p4ePv7440hJScE777yD4OBg+Pn5ITMzc8hv96IjLQuU/peTFmOMLvn36tTpcdOq/7rktctfS4KH2Pqva3t7O7KysjBlyhS0tbVh1apVeOihh1BWVob09HS89dZbWLNmDf9vu2XLFigUCtx5550AgPnz56OpqQn79++HSCRCVlYW35vbVu3t7UhKSkJCQgIOHTqExsZGPPHEE1i6dCk+/fRT9PT0ICUlBYsWLcLnn38OrVaLoqIiPrb09HRMmzYNH3zwAYRCIcrKyiASiQZ8zRkzZuC9997DqlWrUFFRAQBmR0nvvPMOVq1ahezsbH6dl5cXPv30UygUChw9ehSLFi2Cl5cXnn/++X5fZ9++fQgODsa+fftw6tQpPProo7j55puxaNEiuz4ra1DSsiDU1x0CDmjX6tHUpsVYL4mrQyI2mjt3rtnff/vb3zB27FiUl5cjNTUVzzzzDA4cOMAnqc2bNyMtLQ0cx+HEiRPYs2cPDh06hNjYWADAJ598gokTJ9oVy+bNm9HV1YW///3v8PQ0/oe4fv16PPDAA3jzzTchEomgVqvxq1/9CuPHjwcATJo0id+/uroaf/zjHxEVFQUAVsUhFoshk8nAcRyCgoKu2n7PPffgD3/4g9m6l19+mX+uVCrx3HPPITc3d8Ck5ePjg/Xr10MoFCIqKgrJycnIz8+npDXcJG5CKOTuOHepE2cvtlPS6uUuEqL8tSSXvbYtTp48iVWrVuHgwYNoamqCwWAAYEwAkydPxuzZs/HZZ5/hzjvvRGVlJVQqFT766CMAQEVFBdzc3HDLLbfw9U2YMAE+Pj52xX78+HFMnTqVT1gAcPvtt8NgMKCiogJ33XUXHn/8cSQlJeGXv/wlEhMTkZqayt+fl5WVhSeeeAL/+Mc/kJiYiEceeYRPbvYyJeMrbdmyBe+//z5Onz6NtrY29PT0DDrKb3R0tNntOMHBwTh69KhDsQ2G2rT6Qd0ersZxHDzEbi5ZbD1Ff+CBB9Dc3IyPP/4YBw8exMGDBwEYG50B4ynXv/71L+h0OmzevBkxMTGIiYlx+mdmrU2bNkGlUmHGjBnYsmULIiMjUVhYCMDYlvTTTz8hOTkZe/fuxU033YQdO3Y49HpXJlAAUKlUSE9Px/3334+dO3eitLQUL730Ev959afvaSrHcfx/EEOFklY/TI3xVXQFccS5ePEiKioq8PLLL2PWrFmYNGkSLl26ZFbmwQcfRFdXF77++mts3rwZ6enp/LYbb7wRPT09KC0t5dedOnXqqjqsNWnSJBw5cgTt7Ze/Sz/88AMEAgFuvPFGft20adOwYsUKFBQUYPLkydi8eTO/LTIyEs8++yy++eYbPPzww9i0adOgrysWi6HX662KsaCgAOHh4XjppZcQGxuLiRMn8hcCrjV2Ja0NGzZAqVRCKpUiPj4eRUVFA5bftm0boqKiIJVKERMTg927d5ttf+WVVxAVFQVPT0/4+PggMTGR/5/RpLm5Genp6fD29oZcLsfChQvNrr44W7jf5XsQycji4+MDPz8/bNy4EadOncLevXuRlZVlVsbT0xMpKSlYuXIljh8/jrS0NH5bVFQUEhMTsXjxYhQVFaG0tBSLFy+Gu7u7XRdl0tPTIZVKkZGRgWPHjmHfvn146qmnMG/ePAQGBqKyshIrVqyASqVCVVUVvvnmG5w8eRKTJk1CZ2cnli5div3796Oqqgo//PADDh06ZNbm1R+lUom2tjbk5+ejqakJHR39f5cnTpyI6upq5Obm4vTp03j//fcdPpobKjYnrS1btiArKwvZ2dkoKSnB1KlTkZSU1O+VlYKCAqSlpWHhwoUoLS1FSkoKUlJScOzYMb5MZGQk1q9fj6NHj+LAgQNQKpWYPXs2Lly4wJdJT0/HTz/9hLy8POzcuRPfffcdf6l2KNDp4cglEAiQm5uL4uJiTJ48Gc8++yzefvvtq8qlp6fjyJEjuPPOOxEWFma27e9//zsCAwNx11134aGHHuKvpEmlUpvj8fDwwH//+180Nzfj1ltvxa9//WvMmjUL69ev57efOHECc+fORWRkJBYvXozMzEwsWbIEQqEQFy9exPz58xEZGYnU1FTcd999ePXVVwd93RkzZuDJJ5/Eo48+irFjx+Ktt97qt+ycOXPw7LPPYunSpbj55ptRUFCAlStX2vxehwWzUVxcHMvMzOT/1uv1TKFQsJycHIvlU1NTWXJystm6+Ph4tmTJkn5fQ61WMwBsz549jDHGysvLGQB26NAhvsxXX33FOI5j58+ft1hHV1cXU6vV/FJTU8MAMLVabdX7PNXYysKX72Q3rfyKGQwGq/a53nR2drLy8nLW2dnp6lBczvT9MX0niX0G+k6ZfveD/UZtOtLSarUoLi5GYmIiv04gECAxMREqlcriPiqVyqw8ACQlJfVbXqvVYuPGjZDJZJg6dSpfh1wuN7vikZiYCIFAcNVppElOTg5kMhm/2Dq7dKiPB9/t4UJbt037kpFv7969+PLLL1FZWYmCggI89thjUCqVuOuuu1wd2qhnU9JqamqCXq9HYGCg2frAwEDU19db3Ke+vt6q8jt37sSYMWMglUrx7rvvIi8vD/7+/nwdAQEBZuXd3Nzg6+vb7+uuWLECarWaX2pqamx5qxC7CRDiYxrtgdq1RhudTocXX3wR0dHReOihhzB27Fi+o+lnn33G39LSd4mOjh62GO+7775+41i9evWwxTHcrpl+WnfffTfKysrQ1NSEjz/+GKmpqTh48OBVycpaEokEEolj/auUfp6oaTb21YqL8HWoLjKyJCUlISnJcp+0OXPmID4+3uK2wXqqO9Mnn3yCzs5Oi9t8fa/f76tNScvf3x9CoRANDQ1m6xsaGiz2ugWAoKAgq8p7enpiwoQJmDBhAm677TZMnDgRf/3rX7FixQoEBQVd1dDf09OD5ubmfl/XGZR+nvj+ZBPdg0jMeHl5wcvLy9VhICQkxNUhuIRNp4disRjTp083G57CYDAgPz8fCQkJFvdJSEi4ajiLvLy8fstfWW93dzdfR0tLC4qLi/nte/fuhcFg6Pd/PGfg70GkvlqEXDNsPj3MyspCRkYGYmNjERcXh/feew/t7e1YsGABAOONpiEhIcjJyQEALFu2DDNnzsTatWuRnJyM3NxcHD58GBs3bgRgvJn0jTfewJw5cxAcHIympiZs2LAB58+fxyOPPALA2Dnv3nvvxaJFi/Dhhx9Cp9Nh6dKleOyxx6BQKJz1WVzFNF58JbVpEXLNsDlpPfroo7hw4QJWrVqF+vp63Hzzzfj666/5xvbq6moIBJcP4GbMmIHNmzfj5ZdfxosvvoiJEyfiiy++wOTJkwEAQqEQJ06cwP/7f/8PTU1N8PPzw6233orvv//erFHzs88+w9KlSzFr1iwIBALMnTsX77//vqPvf0DhV/SKZzTaAyHXBI6x0THljEajgUwmg1qtHvQmUBNtjwFRK7+CgQFFL85CgLftHQtHsq6uLlRWViIiIsKuTpWE9DXQd8ra3yjdezgAsZsA43xMp4jUrkXItYCS1iBMjfE0XjyxxfU4scW1gpLWIJS9N07TePFkpHH2xBaAcRhnjuPQ0tLi1HptQUlrEDRePCHXFkpag4jg+2rR6SEYA7TtrlloYgunTWzR3d2N5557DiEhIfD09ER8fDz279/P71tVVYUHHngAPj4+8PT0RHR0NHbv3o2zZ8/i7rvvBmAc/ofjODz++ON2fR6OuGZu47lWmcbVom4PAHQdwOqh6xc3oBdrAbHn4OV60cQW/U9ssXTpUpSXlyM3NxcKhQI7duzAvffei6NHj2LixInIzMyEVqvFd999B09PT5SXl2PMmDEIDQ3F//3f/2Hu3LmoqKiAt7c33N3d7fpMHEFJaxChvh4QCjh0aPVobO1G4Cjr9jBS0cQWlie2qK6uxqZNm1BdXc13zH7uuefw9ddfY9OmTVi9ejWqq6sxd+5cfvjpG264gd/fdE9jQECA09vLrEVJaxAioQDjfNxRdbEDlU3toztpiTyMRzyuem0b0MQWlh09ehR6vR6RkZFm67u7u+Hn5wcAePrpp/G73/0O33zzDRITEzF37lxMmTLFrtcbCtSmZYVwGi/eiOOMp2iuWGhiC6dMbNHW1gahUIji4mKUlZXxy/Hjx/HnP/8ZAPDEE0/gzJkzmDdvHo4ePYrY2FisW7fOae/VUZS0rBDhR/cgjiQ0sYWRpYktpk2bBr1ej8bGRn5UFdNy5WlkaGgonnzySWzfvh1/+MMf8PHHH/N1ArB6woyhQEnLClfOOE2ufTSxhZGliS0iIyORnp6O+fPnY/v27aisrERRURFycnKwa9cuAMAzzzyD//73v6isrERJSQn27dvHv154eDg4jsPOnTtx4cKFIZ1cpl9DMQ70tcja8act2Xu8gYUv38mS3v12CCK7do3kMeLz8vLYpEmTmEQiYVOmTGH79+9nANiOHTv4Mrt372YA2F133XXV/rW1tey+++5jEomEhYeHs82bN7OAgAD24YcfWvX6fV/rxx9/ZHfffTeTSqXM19eXLVq0iLW2tjLGGKuvr2cpKSksODiYicViFh4ezlatWsX0ej3r7u5mjz32GAsNDWVisZgpFAq2dOlSq/9NnnzySebn58cAsOzsbMYYY1qtlq1atYoplUomEolYcHAwe+ihh9iPP/7IGGNs6dKlbPz48UwikbCxY8eyefPmsaamJr7O1157jQUFBTGO41hGRoZVcZg4Y4x4umHaCpVN7bj7nf38DMujpdsD3TB92blz5xAaGoo9e/Zg1qxZrg5nxHLGDdN09dAK43zcIRRw6NTp0aDpRpBsdP+AR4O9e/eira0NMTExqKurw/PPP08TW1wjqE3LCqZuDwCNYjpa0MQW1y460rKS0s8TVRc7cLapHbfd4OfqcMgQo4ktrl2UtKwU4e+Jb3++QKM9EJrYwsXo9NBKpnsQR2O3h1FyrYYMA2d8lyhpWWk0DgZoOtXp6Bg975kMLdN3yZHTaDo9tFKEaVyti+0wGBgEguu/24NQKIRcLudHN/Dw8Bg13T2IczHG0NHRgcbGRsjlcgiFQrvroqRlpXE+7nATcOjSGdDQ2oVg2fAPyeEKpls77B2WhZAryeVyhydYpqRlJbfebg9nL3bgbFPHqElaHMchODgYAQEB0Ol0rg6HjGAikcihIywTSlo2UPp7GpPWxXYkjB9d3R6EQqFTvnCEOMquhvgNGzZAqVRCKpUiPj4eRUVFA5bftm0boqKiIJVKERMTg927d/PbdDodli9fjpiYGHh6ekKhUGD+/PmorTUft0mpVPLDxpqWNWvW2BO+3Wi8eEJcz+aktWXLFmRlZSE7OxslJSWYOnUqkpKS+m3zKCgoQFpaGhYuXIjS0lKkpKQgJSUFx44dA2C8mlBSUoKVK1eipKQE27dvR0VFBebMmXNVXa+99hrq6ur45amnnrI1fIeYZuahXvGEuJBNt2gzxuLi4lhmZib/t16vZwqFguXk5Fgsn5qaypKTk83WxcfHsyVLlvT7GkVFRQwAq6qq4teFh4ezd99919ZweY6M8mCy74RxtIfZfxpdoz0QMhys/Y3adKSl1WpRXFyMxMREfp1AIEBiYiJUKpXFfVQqlVl5wHiLRH/lAfAziPQdg3rNmjXw8/PDtGnT8Pbbb6Onp6ffOrq7u6HRaMwWR12emcfY7YEQMvxsaohvamqCXq9HYGCg2frAwECcOHHC4j719fUWy9fX11ss39XVheXLlyMtLc1seIqnn34at9xyC3x9fVFQUIAVK1agrq4Of/rTnyzWk5OTg1dffdWWtzeoELmx20N3z+jq9kDIteSaunqo0+mQmpoKxhg++OADs21Xjjw5ZcoUiMViLFmyBDk5OZBIJFfVtWLFCrN9NBoNQkNDHYrPTShAqK8HKpvaUdnUTkmLEBew6fTQ398fQqEQDQ0NZusbGhr67TAWFBRkVXlTwqqqqkJeXt6gA/XFx8ejp6cHZ8+etbhdIpHA29vbbHEGvjGexosnxCVsSlpisRjTp09Hfn4+v85gMCA/Px8JCQkW90lISDArDwB5eXlm5U0J6+TJk9izZw8/ldFAysrKIBAIEBAQYMtbcFj4FbfzEEKGn82nh1lZWcjIyEBsbCzi4uLw3nvvob29HQsWLABgnJk3JCQEOTk5AIBly5Zh5syZWLt2LZKTk5Gbm4vDhw9j48aNAIwJ69e//jVKSkqwc+dO6PV6vr3L19cXYrEYKpUKBw8e5KcGV6lUePbZZ/Hb3/7W7rno7BVBk1wQ4lr2XJpct24dCwsLY2KxmMXFxbHCwkJ+28yZM68a7H7r1q0sMjKSicViFh0dzXbt2sVvq6ysZAAsLvv27WOMMVZcXMzi4+OZTCZjUqmUTZo0ia1evZp1dXVZHbMzujwwxtj+ikYWvnwn++Wf9jtUDyHEHE1s0YcjE1tcqepiO2a+vR8SNwGOv3bvqBjtgZDhYO1vlMbTstGV3R7qNV2uDoeQUYeSlo3chAKE+Y7eUUwJcTVKWnYwjWJK48UTMvwoadnBNF78aBp6mZBrBSUtO5i6PVTS6SEhw46Slh1oXC1CXIeSlh1MSauquYNGeyBkmFHSsoNCLoVIyEHbY0AddXsgZFhR0rKDabQHgE4RCRlulLTsZDpFpMZ4QoYXJS078e1a1FeLkGFFSctOEf7G08NKGleLkGFFSctONK4WIa5BSctOpg6m1Rc7oKduD4QMG0padlLI3Y3dHvQG1Kk7XR0OIaMGJS07CQXcFd0eqF2LkOFCScsBEdSuRciwo6TlACWNF0/IsKOk5QB+OjE60iJk2FDScoCShqghZNhR0nKAqVd8TXMndXsgZJhQ0nKAQu4OsVAArd6A2hbq9kDIcKCk5QBjtwd3ANSuRchwoaTlIH7GaRovnpBhYVfS2rBhA5RKJaRSKeLj41FUVDRg+W3btiEqKgpSqRQxMTHYvXs3v02n02H58uWIiYmBp6cnFAoF5s+fj9raWrM6mpubkZ6eDm9vb8jlcixcuBBtbW32hO9UNPQyIcPL5qS1ZcsWZGVlITs7GyUlJZg6dSqSkpLQ2NhosXxBQQHS0tKwcOFClJaWIiUlBSkpKTh27BgAoKOjAyUlJVi5ciVKSkqwfft2VFRUYM6cOWb1pKen46effkJeXh527tyJ7777DosXL7bjLTtXOPXVImR4MRvFxcWxzMxM/m+9Xs8UCgXLycmxWD41NZUlJyebrYuPj2dLlizp9zWKiooYAFZVVcUYY6y8vJwBYIcOHeLLfPXVV4zjOHb+/HmLdXR1dTG1Ws0vNTU1DABTq9VWv1drfP/zBRa+fCe75519Tq2XkNFGrVZb9Ru16UhLq9WiuLgYiYmJ/DqBQIDExESoVCqL+6hUKrPyAJCUlNRveQBQq9XgOA5yuZyvQy6XIzY2li+TmJgIgUCAgwcPWqwjJycHMpmMX0JDQ619mzZR9o6rRd0eCBkeNiWtpqYm6PV6BAYGmq0PDAxEfX29xX3q6+ttKt/V1YXly5cjLS0N3t7efB0BAQFm5dzc3ODr69tvPStWrIBareaXmpoaq96jrYJl1O2BkOHk5uoArqTT6ZCamgrGGD744AOH6pJIJJBIJE6KrH9CAYcwPw+camxDZVM7P/IDIWRo2HSk5e/vD6FQiIaGBrP1DQ0NCAoKsrhPUFCQVeVNCauqqgp5eXn8UZapjr4N/T09PWhubu73dYcTjRdPyPCxKWmJxWJMnz4d+fn5/DqDwYD8/HwkJCRY3CchIcGsPADk5eWZlTclrJMnT2LPnj3w8/O7qo6WlhYUFxfz6/bu3QuDwYD4+Hhb3sKQMN04TePFEzIMbG3hz83NZRKJhH366aesvLycLV68mMnlclZfX88YY2zevHnshRde4Mv/8MMPzM3Njb3zzjvs+PHjLDs7m4lEInb06FHGGGNarZbNmTOHjRs3jpWVlbG6ujp+6e7u5uu599572bRp09jBgwfZgQMH2MSJE1laWprVcVt7ZcIe/1CdZeHLd7IFm4qcXjcho4W1v1GbkxZjjK1bt46FhYUxsVjM4uLiWGFhIb9t5syZLCMjw6z81q1bWWRkJBOLxSw6Oprt2rWL31ZZWckAWFz27dvHl7t48SJLS0tjY8aMYd7e3mzBggWstbXV6piHMmkdOGns9nA3dXsgxG7W/kY5xtiouE6v0Wggk8mgVqvN2suc4dylDtzx5j6IhByOv3Yv3IR0dxQhtrL2N0q/LidQyNwhdhNAp2eobelydTiEXNcoaTmBQMAh3JdGMSVkOFDSchJ+vHhKWoQMKUpaTnK52wMlLUKGEiUtJ6GZeQgZHpS0nCSC7xVPHUwJGUqUtJzENK5WdXMHevQGF0dDyPWLkpaTBHtLIXEToMfAcJ5GeyBkyFDSchKBgEM4P3krnSISMlQoaTlROI0XT8iQo6TlRBE04zQhQ46SlhPRuFqEDD1KWk6kpDYtQoYcJS0nMnUwraFuD4QMGUpaThRE3R4IGXKUtJzoym4P1BhPyNCgpOVkSur2QMiQoqTlZBH8EDXUGE/IUKCk5WR8B1Pq9kDIkKCk5WRK/95uD3R6SMiQoKTlZKbTw5pLndBRtwdCnI6SlpMFekkhFQmgNzCcv0TdHghxNkpaTmac5KL3HkRq1yLE6exKWhs2bIBSqYRUKkV8fDyKiooGLL9t2zZERUVBKpUiJiYGu3fvNtu+fft2zJ49G35+fuA4DmVlZVfV8Ytf/AIcx5ktTz75pD3hDzlq1yJk6NictLZs2YKsrCxkZ2ejpKQEU6dORVJSEhobGy2WLygoQFpaGhYuXIjS0lKkpKQgJSUFx44d48u0t7fjjjvuwJtvvjngay9atAh1dXX88tZbb9ka/rAw3c5DQy8TMgRsnbo6Li6OZWZm8n/r9XqmUChYTk6OxfKpqaksOTnZbF18fDxbsmTJVWUrKysZAFZaWnrVtpkzZ7Jly5bZGi7P2im3nWHzwSoWvnwnm//Xg0P+WoRcL6z9jdp0pKXValFcXIzExER+nUAgQGJiIlQqlcV9VCqVWXkASEpK6rf8QD777DP4+/tj8uTJWLFiBTo6+j+S6e7uhkajMVuGi5L6ahEyZNxsKdzU1AS9Xo/AwECz9YGBgThx4oTFferr6y2Wr6+vtynQ3/zmNwgPD4dCocCPP/6I5cuXo6KiAtu3b7dYPicnB6+++qpNr+Espm4P53q7PYiEdL2DEGexKWm50uLFi/nnMTExCA4OxqxZs3D69GmMHz/+qvIrVqxAVlYW/7dGo0FoaOiwxBrgJYFUJECXzoBzlzr5JEYIcZxNhwD+/v4QCoVoaGgwW9/Q0ICgoCCL+wQFBdlU3lrx8fEAgFOnTlncLpFI4O3tbbYMF4GAoxunCRkiNiUtsViM6dOnIz8/n19nMBiQn5+PhIQEi/skJCSYlQeAvLy8fstby9QtIjg42KF6hoopadEQNYQ4l82nh1lZWcjIyEBsbCzi4uLw3nvvob29HQsWLAAAzJ8/HyEhIcjJyQEALFu2DDNnzsTatWuRnJyM3NxcHD58GBs3buTrbG5uRnV1NWprawEAFRUVAIxHaUFBQTh9+jQ2b96M+++/H35+fvjxxx/x7LPP4q677sKUKVMc/hCGQnhvXy0aL54QJ7Pn0uS6detYWFgYE4vFLC4ujhUWFvLbZs6cyTIyMszKb926lUVGRjKxWMyio6PZrl27zLZv2rSJAbhqyc7OZowxVl1dze666y7m6+vLJBIJmzBhAvvjH/9oU/eF4ezywBhjn/d2e5hH3R4IsYq1v1GOMcZcmDOHjUajgUwmg1qtHpb2rcIzF/HYxkKE+Xrgu+fvHvLXI2Sks/Y3Stfih4ipTevcpQ5oe2i0B0KchZLWEAn0lsBdJISBGRMXIcQ5KGkNEY67PMkF9YwnxHkoaQ0hfrz4JjrSIsRZKGkNIRovnhDno6Q1hCL8aQ5EQpyNktYQotEeCHE+SlpDyDQY4PlLndTtgRAnoaQ1hAK8JPAQG7s91FC3B0KcgpLWEDJ2e6DRHghxJkpaQ0zJ99WiIy1CnIGS1hBT+tORFiHORElriEXQFURCnIqS1hCjW3kIcS5KWkMsgro9EOJUlLSG2Ngruj1UN1NjPCGOoqQ1xKjbAyHORUlrGJjuQaR2LUIcR0lrGNA9iIQ4DyWtYXB5DkRq0yLEUZS0hgHfwZSOtAhxGCWtvlrrgfzXgcIPnValsrdNq7alE909eqfVS8hoREmrr8rvge/fAb5fC/R0O6XKsWMk8DSN9kDdHghxCCWtvqJTAC8F0N4IHP2XU6o07/ZASYsQR9iVtDZs2AClUgmpVIr4+HgUFRUNWH7btm2IioqCVCpFTEwMdu/ebbZ9+/btmD17Nvz8/MBxHMrKyq6qo6urC5mZmfDz88OYMWMwd+5cNDQ02BP+wIQiIH6J8blqA+CkuWwjqF2LEKewOWlt2bIFWVlZyM7ORklJCaZOnYqkpCQ0NjZaLF9QUIC0tDQsXLgQpaWlSElJQUpKCo4dO8aXaW9vxx133IE333yz39d99tln8Z///Afbtm3Dt99+i9raWjz88MO2hm+d6RmAyBNo/Ak4s98pVSppvHhCnIPZKC4ujmVmZvJ/6/V6plAoWE5OjsXyqampLDk52WxdfHw8W7JkyVVlKysrGQBWWlpqtr6lpYWJRCK2bds2ft3x48cZAKZSqayKW61WMwBMrVZbVZ7t+iNj2d6M/WOudeUHseVQNQtfvpP95mPr4iVktLH2N2rTkZZWq0VxcTESExP5dQKBAImJiVCpVBb3UalUZuUBICkpqd/ylhQXF0On05nVExUVhbCwsH7r6e7uhkajMVtsctuTADjgVB7QeMK2fS2gORAJcQ6bklZTUxP0ej0CAwPN1gcGBqK+vt7iPvX19TaV768OsVgMuVxudT05OTmQyWT8EhoaavXrAQB8bwCiko3PC/9i274WmDqY1qo70aWjbg+E2Ou6vXq4YsUKqNVqfqmpqbG9koSlxscjuUB7k0Px+I8Rw1MsBKNuD4Q4xKak5e/vD6FQeNVVu4aGBgQFBVncJygoyKby/dWh1WrR0tJidT0SiQTe3t5mi83CbgNCpgP6buDQX23f/wocx13RM56SFiH2silpicViTJ8+Hfn5+fw6g8GA/Px8JCQkWNwnISHBrDwA5OXl9VvekunTp0MkEpnVU1FRgerqapvqsRnHAQmZxueHPgZ0XQ5VR+PFE+I4N1t3yMrKQkZGBmJjYxEXF4f33nsP7e3tWLBgAQBg/vz5CAkJQU5ODgBg2bJlmDlzJtauXYvk5GTk5ubi8OHD2LhxI19nc3MzqqurUVtbC8CYkADjEVZQUBBkMhkWLlyIrKws+Pr6wtvbG0899RQSEhJw2223OfwhDGjSg4AsFFDXAEe3AbfMs7sq08w8ldRXixD72XNpct26dSwsLIyJxWIWFxfHCgsL+W0zZ85kGRkZZuW3bt3KIiMjmVgsZtHR0WzXrl1m2zdt2sQAXLVkZ2fzZTo7O9nvf/975uPjwzw8PNhDDz3E6urqrI7Z5i4PV/rhfWP3h/XxjBkMtu/fayt1eyCkX9b+RjnGnNTl+xqn0Wggk8mgVqttb9/qUgN/ugnQtgG//T9gQuLg+1hw+Gwzfv2hCiFyd/zwwj121UHI9cra3+h1e/XQqaQy4Jb5xueqDXZXE07dHghxGCUta8UvATgBcHov0FBuVxX+Y8QYI3Gjbg+EOICSlrV8lMCkB4zPC+072uI4ju8Z//1Jx/p9ETJaUdKyhamz6Y9bgTbLN4gPJjV2HADg3T0/40Krc8brImQ0oaRli9A4YNytgF4LHPrErip+Ex+OmBAZWrt6sHr3cScHSMj1j5KWrfjOpp8Auk6bdxcKOPxvymRwHLCj9DxUpy86OUBCrm+UtGwV9QAgCwM6LgI/brGriqmhcqTHhwEAVv77GLQ9BmdGSMh1jZKWrYRuvcPWAFD9BTDYl3D+ODsK/mPEONXYhk8OnHFigIRc3yhp2WPaPEDsBTRVAKfzBy9vgcxDhBfvnwQAeD//JM5doi4QhFiDkpY9pN7GIZkBQLXe7moemhaCuAhfdOkMePU/9vX9ImS0oaRlr/glACc0jiFff2zQ4pZwnLFR3k3AIa+8AXvKh2CiDkKuM5S07CUPA2560PjcgZFNIwO98MSdNwAAsr/8CZ1aur2HkIFQ0nLElZ1NW60fPrqvp2dNgEImxfmWTqzfd9JJwRFyfaKk5Yhx04HQ2wCDzu7OpgDgIXZD9pxoAMDG787gVGOrsyIk5LpDSctRfGfTvwJa+68Azr4pELOiAqDTM6z84ieMkhGDCLEZJS1HRSUD8nCgsxk48rnd1XAch1fmREMqEkB15iL+XVbrxCAJuX5Q0nKUQAjc9nvj80L7O5sCQKivB566ZyIA4H93HYe6U+eMCAm5rlDScoZp6YBEBlw8BZz8xqGqnrgzAjeM9URTWzf+9E2FkwIk5PpBScsZJF5O6WwKABI3If73wckAgH8UVuHoObWj0RFyXaGk5SymzqZnvwfqjjhU1YwJ/njwZgUMDHj5i6PQG6hRnhATSlrOIhsHRD9kfK6yv7OpyUv3T4KXxA1HzqnxeVG1w/URcr2gpOVMpu4Px/4FaBy7+hfgLcUfZkcCAN76+gSNckpIL0pazhRyCxA2AzD0AEUfO1zdvAQlohXe0HT1IOcrGuWUEMDOpLVhwwYolUpIpVLEx8ejqKhowPLbtm1DVFQUpFIpYmJisHv3brPtjDGsWrUKwcHBcHd3R2JiIk6eNL+dRalUguM4s2XNmjX2hD+0TEdbh/8GaB2bSfrKUU63l5xH4Rka5ZQQm5PWli1bkJWVhezsbJSUlGDq1KlISkpCY6PliR4KCgqQlpaGhQsXorS0FCkpKUhJScGxY5dHRnjrrbfw/vvv48MPP8TBgwfh6emJpKQkdHV1mdX12muvoa6ujl+eeuopW8MfejfeB/hEAF0tQNlmh6ubFuaDtLjeUU6/oFFOCYGtU1fHxcWxzMxM/m+9Xs8UCgXLycmxWD41NZUlJyebrYuPj2dLlixhjDFmMBhYUFAQe/vtt/ntLS0tTCKRsM8//5xfFx4ezt59911bw+VZO+W2UxR+xFi2N2N/vpkxvd7h6lrateyW175h4ct3sg/2n3JCgIRce6z9jdp0pKXValFcXIzExMvTwgsEAiQmJkKlUlncR6VSmZUHgKSkJL58ZWUl6uvrzcrIZDLEx8dfVeeaNWvg5+eHadOm4e2330ZPT0+/sXZ3d0Oj0Zgtw+bm3xhnpW4+A/z8tcPVyTxEWNE7yumf95zE+RbbJ9Qg5HphU9JqamqCXq9HYGCg2frAwEDU11semqW+vn7A8qbHwep8+umnkZubi3379mHJkiVYvXo1nn/++X5jzcnJgUwm45fQ0FDr36ijJGOA2P8xPlfZN7FrX3NvCUGc0hedOj1e/fInp9RJyEg0Yq4eZmVl4Re/+AWmTJmCJ598EmvXrsW6devQ3W25K8CKFSugVqv5paamZngDjlsMCNyAqgNAbanD1XEch9d7Rzn9prwB+cdplFMyOtmUtPz9/SEUCtHQYP6DaWhoQFBQkMV9goKCBixverSlTgCIj49HT08Pzp49a3G7RCKBt7e32TKsvBXA5LnG50462roxyAsL74gAQKOcktHLpqQlFosxffp05OdfnoHGYDAgPz8fCQkJFvdJSEgwKw8AeXl5fPmIiAgEBQWZldFoNDh48GC/dQJAWVkZBAIBAgICbHkLw8s0+sNPOwD1OadU+fSsiVDIpDh3qRMb9p1ySp2EjCi2tvDn5uYyiUTCPv30U1ZeXs4WL17M5HI5q6+vZ4wxNm/ePPbCCy/w5X/44Qfm5ubG3nnnHXb8+HGWnZ3NRCIRO3r0KF9mzZo1TC6Xs3//+9/sxx9/ZA8++CCLiIhgnZ2djDHGCgoK2LvvvsvKysrY6dOn2T//+U82duxYNn/+fKvjHtarh1falGy8kvjNSqdV+dXROha+fCeb8OIudqqx1Wn1EuJK1v5GbU5ajDG2bt06FhYWxsRiMYuLi2OFhYX8tpkzZ7KMjAyz8lu3bmWRkZFMLBaz6OhotmvXLrPtBoOBrVy5kgUGBjKJRMJmzZrFKioq+O3FxcUsPj6eyWQyJpVK2aRJk9jq1atZV1eX1TG7LGmd2G1MWqtDGetyToIxGAzs8b8dZOHLd7LffKxiBoPBKfUS4krW/kY5xkbHuL4ajQYymQxqtXp427cMBmB9LNB8GrjvLeNoEE5QfbEDv3z3W3T3GPDnx27GgzeHOKVeQlzF2t/oiLl6OGIJBEDClSObOqfxPMzPA0vvngDAOMqppotGOSWjAyWt4TA1DXD3AS6dBSp2D1rcWotn3oAIf09caO3Gn7752Wn1EnIto6Q1HMSeTu9sChhHOX29d5TTv6vO4th5GuWUXP8oaQ2XWxcBAhFQrQLOFTut2jsm+uOBqcZRTl/64hgMNMopuc5R0hou3sFAzK+Nzwudd7QFAC8nT8IYiRuO1LQg99Aw9/wnZJi5uTqAUeW23xvnRvzpC0CvA/zGA77jLz+OCQA4zuZqA72lyPplJF7bWY43vz6B2dGB8B8jcTxefQ+grjG2xV2qBJorgZ4uICQWCLsNkIfZFS8hjqAuD8Ptn3OBU3ssbxOPAXxvuDqZ+Y0HPPwGTBA9egPmrP8B5XUaeEvd4O8lgdxdBJm7CHIPMWT8c+Ni/FsMuZsWvt3n4NVxDm7qqsvJ6dJZY8Iy9D+SBrwUxuQVlmB8DIw2zgNJiB2s/Y1S0hpu2nbg7AHg4mlj3y3TY0sNgAH+KSQywDein4TmCwAorb6ERz8qhFZ/5UCBDGPRgnCuAeFcI8IEjQjjGhDONSCUa8RYbuAhe3ScGGpJMNo8QtHtFQaRSAT/S0fg1XwMHOuT0CTeQGjc5UQWMh0Qudv3OdnAYGBo6dShub0bze3Gx4vtWjS3adHcoUVzu3FhDIgK8kJ0iDeiFTLc4O8JNyG1kFwrKGn1cc0krf70dBuPbvoms4tnAM0g9y1K5XwS6/IKQ3trC7hLZyHSVMG9rQZuhq4Bd29mY1DNAlDNAlHFAo3PDYGoYgFogA+YhaZPKbpxs+A0YrkK3CqowHTBSYzhzMf56oEb6jxuRL1sGpr9b0F7YCyksgB4S0XwdnfrfRTBS+oG0RXJQ9tj4BNNc7sWF9u70dyuxaV2rTEZ9T5e6n1+qUMLe64/SEUCRAV5I1phTGLRCm/cGOQFqYiOFl2BklYf13zSGoiu00JCO2N8bLVi1h9OYJzizCfCeLTmozR7rhd7Q9Opg7pThxbTY4cW6k4d1B1XrtNB3alFa1cPNJ06aLp60NZtPNoSQo8orgaxAmMSu1VwAoFcy1WhnDSE4JAhEocNN+IQuxE1LAAABw+xEGMkbujU6tHaPcApqRkGT3TBh2uDHK1QSDoxTtKJIFEnAtw64C9sgxzt8EYrxug1EPa0Q6vrgbZHj54ePcAM4MAg4Bg4GBcBGEQCDiIBIBIwuAk4uPVuB2MAMwDofWSGK9bBeLHFP7J3mXj5uefYa7Ptr7sNaKk2zhzlFWiM1c0JbaF2oqTVx4hOWgPRthvboEzJ7NJZ44zXVyYneRggFA3Jy/foDWjr7oGmsweaLl1vMtNB06GDoaUKssbDGNtSinGaIwjSnr1q/wYmxyHDjThsuBGHDZHQQgQftMFH0IYQSSeCxZ0IELbDX9gOOdcGb9YGT70a7j0aiHVqCAwj4E4AqexyAvObcPm5b8SQ/bsAAHRdvRdSqoCW3uVSlTFRtVQBHX0mSuGExiP2gJt6l0nGR98Iq9oqGWO41KFDbUsnzrd0orZ3Od/SidcfnAy/QS4OUdLq47pNWiNJRzNQcxCoKgCqC42DIzoj6QglxnY9d9/eR5/Lj/w6X2My5wRXLJzxERzAcWAchwttOpxu6sCpCx04daEdpxo7UN/aDQYOht7jsd7jM/h6ijEx0BuRQd6IDPBEILuAMa2V8Gw9DXf1aYhbTkOorjYepVkicDP+p9L3yMx/gjH2weh1xiGP+iajlmrj322WRxM2I5Ubx37TnAe6+umc7CYF/COhHzsJaq8JqJfcgDOCMJzukqNW3YVa9eUk1aWzPPHKvzNvx9RQ+YChUNLqg5LWNUjXCZwvMXa4rS4Ezh8GwF2dgNx9AQ8fC+t6y4k9hjTMS+1a/FSrwU+1av7xTFM7rPnlSKBFONeA8VwtxnO1mCiswwRBLSJQCw/039aoFvrggjgMTVIlLnmEQyfyhl9PA3y0dZBr6+DdVQuPrgYIMPDsTAaRJ/SyMDBZGDifcAh8wiH0VQI+4cYjcKkMAMAMBlxqrIH67BF01x6D4MJxjFGfhH/nGYiZ5dGBNcwdP7NQ/GwYhwoWip/ZOFQYQiH0GguF3B0hcikUMnco5O64PyYYQTLpgLFS0uqDkhZxpg5tD47XtaK8Vo1j5zU4Ua9Ba3cPunUGaPUGdOv06O4xoLvfKd8YgtCM8YJaPqGN52oxXlCHYK7Z6ji6mAjn2FicY2NRc8VjDQvAOeaPS/ACYN6eJhRwkLgJIHYTQOImgJtAgKa2bouxcjAglLuAG7kaRHLncJPwHCa5nUeY4Rzc0M/N/55je08toy+fYgZGD/qfCyWtPihpEVdgjEGnZ+juMSYxbW8i6+7RX5HgjH+btum7NJCqz8BTcwZj2irh3VYJUU8bmkWBaHILwgW3IDQIAlHPBeACk6Fbj6vq1/aY19lj5eXVAC9J71GSOxRyKRRy9yv+doePhwgcxwE9WmM7asNPQOPx3qXc2KZq6XR4/pfADTMHfG1rf6PUI56QIcRxHMRuHMRuAnhZvVcIgElXrY1wIA69gV2VyK5MdH6eEgTKJJC4Wdndw03cexTVJ05tO3Ch4nISMyW0vuUcQEmLkFFAKODgLhbCXTzEfdDEnkDILcZliFB3YELIiEJJixAyolDSIoSMKJS0CCEjCiUtQsiIQkmLEDKiUNIihIwoo6aflqnjv0Yz8KB3hBDXMP02B7tJZ9QkrdbWVgBAaGioiyMhhAyktbUVMpms3+2j5t5Dg8GA2tpaeHl5Ge+dGoBGo0FoaChqampG3H2KFLtrUOyOY4yhtbUVCoUCAkH/LVej5khLIBBg3LhxNu3j7e094r6AJhS7a1DsjhnoCMuEGuIJISMKJS1CyIhCScsCiUSC7OxsSCSuG+TfXhS7a1Dsw2fUNMQTQq4PdKRFCBlRKGkRQkYUSlqEkBGFkhYhZEShpEUIGVEoafWxYcMGKJVKSKVSxMfHo6ioyNUhDSonJwe33norvLy8EBAQgJSUFFRUVLg6LLusWbMGHMfhmWeecXUoVjl//jx++9vfws/PD+7u7oiJicHhw4ddHdag9Ho9Vq5ciYiICLi7u2P8+PF4/fXXB71Z+VpASesKW7ZsQVZWFrKzs1FSUoKpU6ciKSkJjY2Nrg5tQN9++y0yMzNRWFiIvLw86HQ6zJ49G+3t7a4OzSaHDh3CRx99hClTprg6FKtcunQJt99+O0QiEb766iuUl5dj7dq18PGxYkp7F3vzzTfxwQcfYP369Th+/DjefPNNvPXWW1i3bp2rQxscI7y4uDiWmZnJ/63X65lCoWA5OTkujMp2jY2NDAD79ttvXR2K1VpbW9nEiRNZXl4emzlzJlu2bJmrQxrU8uXL2R133OHqMOySnJzM/ud//sds3cMPP8zS09NdFJH16Eirl1arRXFxMRITE/l1AoEAiYmJUKlULozMdmq1GgDg6+vr4kisl5mZieTkZLPP/1r35ZdfIjY2Fo888ggCAgIwbdo0fPzxx64OyyozZsxAfn4+fv75ZwDAkSNHcODAAdx3330ujmxwo2aUh8E0NTVBr9cjMDDQbH1gYCBOnDjhoqhsZzAY8Mwzz+D222/H5MmTXR2OVXJzc1FSUoJDhw65OhSbnDlzBh988AGysrLw4osv4tChQ3j66achFouRkZHh6vAG9MILL0Cj0SAqKgpCoRB6vR5vvPEG0tPTXR3aoChpXWcyMzNx7NgxHDhwwNWhWKWmpgbLli1DXl4epFKpq8OxicFgQGxsLFavXg0AmDZtGo4dO4YPP/zwmk9aW7duxWeffYbNmzcjOjoaZWVleOaZZ6BQKK752KlNq1d3dzcTCoVsx44dZuvnz5/P5syZ45qgbJSZmcnGjRvHzpw54+pQrLZjxw4GgAmFQn4BwDiOY0KhkPX09Lg6xH6FhYWxhQsXmq37y1/+whQKhYsist64cePY+vXrzda9/vrr7MYbb3RRRNajNq1eYrEY06dPR35+Pr/OYDAgPz8fCQkJLoxscIwxLF26FDt27MDevXsRERHh6pCsNmvWLBw9ehRlZWX8Ehsbi/T0dJSVlUEoFLo6xH7dfvvtV3Ut+fnnnxEeHu6iiKzX0dFx1eigQqEQBoPBRRHZwNVZ81qSm5vLJBIJ+/TTT1l5eTlbvHgxk8vlrL6+3tWhDeh3v/sdk8lkbP/+/ayuro5fOjo6XB2aXUbK1cOioiLm5ubG3njjDXby5En22WefMQ8PD/bPf/7T1aENKiMjg4WEhLCdO3eyyspKtn37dubv78+ef/55V4c2KEpafaxbt46FhYUxsVjM4uLiWGFhoatDGhQAi8umTZtcHZpdRkrSYoyx//znP2zy5MlMIpGwqKgotnHjRleHZBWNRsOWLVvGwsLCmFQqZTfccAN76aWXWHd3t6tDGxSNp0UIGVGoTYsQMqJQ0iKEjCiUtAghIwolLULIiEJJixAyolDSIoSMKJS0CCEjCiUtQsiIQkmLEDKiUNIihIwolLQIISPK/weK36q1YVR5hgAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T08:41:09.403812Z", + "iopub.status.busy": "2024-03-26T08:41:09.403488Z", + "iopub.status.idle": "2024-03-26T08:43:35.721493Z", + "shell.execute_reply": "2024-03-26T08:43:35.720641Z" + }, + "papermill": { + "duration": 146.338301, + "end_time": "2024-03-26T08:43:35.723993", + "exception": false, + "start_time": "2024-03-26T08:41:09.385692", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T08:43:35.760009Z", + "iopub.status.busy": "2024-03-26T08:43:35.759656Z", + "iopub.status.idle": "2024-03-26T08:43:35.780792Z", + "shell.execute_reply": "2024-03-26T08:43:35.779964Z" + }, + "papermill": { + "duration": 0.041366, + "end_time": "2024-03-26T08:43:35.782769", + "exception": false, + "start_time": "2024-03-26T08:43:35.741403", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tab_ddpm_concat0.0237090.0152460.0019972.455820.0319930.9314480.0415610.0000042.9244880.0343780.0972790.0446860.0874430.016915.380308
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "tab_ddpm_concat 0.023709 0.015246 0.001997 2.45582 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss \\\n", + "tab_ddpm_concat 0.031993 0.931448 0.041561 0.000004 \n", + "\n", + " pred_duration pred_mae pred_mape pred_rmse pred_std \\\n", + "tab_ddpm_concat 2.924488 0.034378 0.097279 0.044686 0.087443 \n", + "\n", + " std_loss total_duration \n", + "tab_ddpm_concat 0.01691 5.380308 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T08:43:35.816375Z", + "iopub.status.busy": "2024-03-26T08:43:35.816090Z", + "iopub.status.idle": "2024-03-26T08:43:36.199071Z", + "shell.execute_reply": "2024-03-26T08:43:36.198183Z" + }, + "papermill": { + "duration": 0.402447, + "end_time": "2024-03-26T08:43:36.201191", + "exception": false, + "start_time": "2024-03-26T08:43:35.798744", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T08:43:36.238415Z", + "iopub.status.busy": "2024-03-26T08:43:36.238062Z", + "iopub.status.idle": "2024-03-26T08:46:05.123316Z", + "shell.execute_reply": "2024-03-26T08:46:05.122429Z" + }, + "papermill": { + "duration": 148.907419, + "end_time": "2024-03-26T08:46:05.125776", + "exception": false, + "start_time": "2024-03-26T08:43:36.218357", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/tab_ddpm_concat/all inf False\n", + "Caching in ../../../../contraceptive/_cache_bs_test/tab_ddpm_concat/all inf False\n", + "Caching in ../../../../contraceptive/_cache_synth_test/tab_ddpm_concat/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T08:46:05.166746Z", + "iopub.status.busy": "2024-03-26T08:46:05.165537Z", + "iopub.status.idle": "2024-03-26T08:46:05.187747Z", + "shell.execute_reply": "2024-03-26T08:46:05.186770Z" + }, + "papermill": { + "duration": 0.044373, + "end_time": "2024-03-26T08:46:05.190001", + "exception": false, + "start_time": "2024-03-26T08:46:05.145628", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T08:46:05.226990Z", + "iopub.status.busy": "2024-03-26T08:46:05.226208Z", + "iopub.status.idle": "2024-03-26T08:46:05.231946Z", + "shell.execute_reply": "2024-03-26T08:46:05.231004Z" + }, + "papermill": { + "duration": 0.026219, + "end_time": "2024-03-26T08:46:05.233979", + "exception": false, + "start_time": "2024-03-26T08:46:05.207760", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tab_ddpm_concat': 0.4075574319064617}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T08:46:05.269156Z", + "iopub.status.busy": "2024-03-26T08:46:05.268758Z", + "iopub.status.idle": "2024-03-26T08:46:05.681822Z", + "shell.execute_reply": "2024-03-26T08:46:05.680840Z" + }, + "papermill": { + "duration": 0.433186, + "end_time": "2024-03-26T08:46:05.683950", + "exception": false, + "start_time": "2024-03-26T08:46:05.250764", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABDRklEQVR4nO3deXhTVf4/8PfNnqZJ2tK9dGNr2SlQEBABZQBFBXUcEdRWRJkRBpUv8wjz/Qki4wCOsriAjo+0MiKMOjg6Ogp8UUBQWWVHlgKllJa20D1ptnt+f9wmEJq2SZrkNunn9Tx9mtyc3Jzb23569sMxxhgIIUREErEzQAghFIgIIaKjQEQIER0FIkKI6CgQEUJER4GIECI6CkSEENFRICKEiI4CESFEdBSIgkxaWhruvfdev34Gx3F4+eWXW0338ssvg+M4v+aFdAwUiALsxx9/xMsvv4yqqiqxs0KCwJo1a5Cfny92NvyOAlGA/fjjj1i8eDEFIuIWCkSEEBIgFIgC6OWXX8af/vQnAEB6ejo4jgPHcbh48SLy8vJw5513IjY2FkqlEr169cLatWubPdfWrVsxYMAAqFQq9OrVC5s3b/Y4PyaTCS+88AJiYmKg1Wpx//334/Llyy7T7t69G9nZ2VCpVOjatSvee+89l+k4jsPs2bOxYcMGZGRkQKVSYdCgQdi1a1eTnwXHcThz5gwee+wx6PV6xMTE4KWXXgJjDEVFRZg0aRJ0Oh3i4+PxxhtveHx9APDNN99g1KhR0Gq10Ol0yM7Oxscff+yU5tNPP8WgQYOgVqsRHR2Nxx57DMXFxU5pcnNzER4ejuLiYkyePBnh4eGIiYnBvHnzYLPZnNLyPI/Vq1ejb9++UKlUiImJwYQJE3DgwAFHGnfud1paGk6cOIGdO3c6fldGjx7t1c+h3WMkYI4cOcIeffRRBoCtXLmS/eMf/2D/+Mc/WF1dHcvOzma5ubls5cqV7K233mLjxo1jANjbb7/tdI7U1FTWo0cPFhERwebPn89WrFjB+vbtyyQSCdu6datH+XnssccYADZ16lT29ttvswcffJD169ePAWCLFi1ypDt69ChTq9UsJSWFLV26lC1ZsoTFxcU50t4MAOvTpw+Ljo5mr7zyClu+fDlLTU1larWaHTt2zJFu0aJFDAAbMGAAe/TRR9maNWvYxIkTGQC2YsUKlpGRwf7whz+wNWvWsBEjRjAAbOfOnR5dX15eHuM4jvXp04e9+uqr7J133mEzZsxgjz/+uFMaACw7O5utXLmSzZ8/n6nVapaWlsYqKysd6XJycphKpWK9e/dm06dPZ2vXrmUPPfQQA8DWrFnj9Lm5ubkMALv77rvZqlWr2Ouvv84mTZrE3nrrLUcad+73559/zjp37swyMzMdvyue3uNgQYEowP72t78xAOzChQtOxw0GQ5O048ePZ126dHE6lpqaygCwf/3rX45j1dXVLCEhgWVlZbmdj8OHDzMA7Nlnn3U6PnXq1CaBaPLkyUylUrHCwkLHsZMnTzKpVOoyEAFgBw4ccBwrLCxkKpWKPfDAA45j9kD0zDPPOI5ZrVbWuXNnxnEcW7ZsmeN4ZWUlU6vVLCcnx+3rq6qqYlqtlg0dOpQZjUan13ieZ4wxZjabWWxsLOvTp49Tmq+++ooBYAsXLnQcy8nJYQDYK6+84nSurKwsNmjQIMfz7777jgFgc+bMaZIn++cy5v797t27Nxs1apQbVxzcqGrWTqjVasfj6upqVFRUYNSoUTh//jyqq6ud0iYmJuKBBx5wPNfpdHjiiSfwyy+/oLS01K3P++9//wsAmDNnjtPx559/3um5zWbDli1bMHnyZKSkpDiO9+zZE+PHj3d57mHDhmHQoEGO5ykpKZg0aRK2bNnSpBozY8YMx2OpVIrBgweDMYannnrKcTwiIgIZGRk4f/68W9cGANu2bUNtbS3mz58PlUrl9Jp9yMGBAwdQVlaGZ5991inNxIkTkZmZia+//rrJeX//+987PR85cqRTvv71r3+B4zgsWrSoyXtvHurgyf3uCCgQtRN79uzB2LFjodFoEBERgZiYGPz5z38GgCa/mN26dWsyfqdHjx4AgIsXL7r1eYWFhZBIJOjatavT8YyMDKfn5eXlMBqN6N69e5Nz3JrWzlXaHj16wGAwoLy83On4zcENAPR6PVQqFaKjo5scr6ysbP6CblFQUAAA6NOnT7NpCgsLAbi+jszMTMfrdvb2nptFRkY65augoACJiYmIiopqMX+e3O+OQCZ2Bojwy3vXXXchMzMTK1asQHJyMhQKBf773/9i5cqV4Hle7Cz6jVQqdesYADCRVzVuLl+e6sj3uzkUiALM1Ujk//znPzCZTPjyyy+dSgjff/+9y3OcO3cOjDGnc505cwaA0NPijtTUVPA8j4KCAqcSwenTp53SxcTEQK1W4+zZs03OcWtaO1dpz5w5g7CwsCYlCn+xl/SOHz+Obt26uUyTmpoKQLiOO++80+m106dPO1739HO3bNmC69evN1sq8uR+d5SR61Q1CzCNRgMATgMa7f9pb/6PX11djby8PJfnuHLlCj7//HPH85qaGqxfvx4DBgxAfHy8W/m4++67AQBvvvmm0/FVq1Y5PZdKpRg/fjz+/e9/49KlS47jp06dwpYtW1ye+6effsKhQ4ccz4uKivDFF19g3LhxPitVtGbcuHHQarVYunQpGhoanF6z/5wHDx6M2NhYvPvuuzCZTI7Xv/nmG5w6dQoTJ070+HMfeughMMawePHiJq/ZP9eT+63RaDrE4FcqEQWYvRH3f//3fzFlyhTI5XLccccdUCgUuO+++zBz5kzU1dXh/fffR2xsLEpKSpqco0ePHnjqqaewf/9+xMXFYd26dbh69WqzgcuVAQMG4NFHH8WaNWtQXV2N4cOHY/v27Th37lyTtIsXL8a3336LkSNH4tlnn4XVasVbb72F3r174+jRo03S9+nTB+PHj8ecOXOgVCqxZs0ax3kCRafTYeXKlZgxYways7MxdepUREZG4siRIzAYDPjwww8hl8uxfPlyPPnkkxg1ahQeffRRXL16FatXr0ZaWhpeeOEFjz93zJgxePzxx/Hmm2/i7NmzmDBhAniexw8//IAxY8Zg9uzZGDdunNv3e9CgQVi7di3+8pe/oFu3boiNjW1SegsJ4nXYdVxLlixhSUlJTCKROLryv/zyS9avXz+mUqlYWloaW758OVu3bl2Trv7U1FQ2ceJEtmXLFtavXz+mVCpZZmYm+/TTTz3Oh9FoZHPmzGGdOnViGo2G3XfffayoqKhJ9z1jjO3cuZMNGjSIKRQK1qVLF/buu+86uuBvBoDNmjWLffTRR6x79+5MqVSyrKws9v333zuls7+3vLzc6XhOTg7TaDRN8jpq1CjWu3dvj6/xyy+/ZMOHD2dqtZrpdDo2ZMgQtnHjRqc0//znP1lWVhZTKpUsKiqKTZs2jV2+fNmtfLn6GVitVva3v/2NZWZmMoVCwWJiYtjdd9/NDh486JQvd+53aWkpmzhxItNqtQxAyHblc4zRvmbEdziOw6xZs/D222+LnRUSRKiNiBAiOmojCkGtDWpUq9XQ6/UByo3vlZeXNxkYeTOFQtHqOB7SvlAgCkEJCQktvp6TkxPUS0tkZ2c3GWx4s1GjRmHHjh2ByxBpMwpEIWjbtm0tvp6YmOi3zw5Ek+OGDRtgNBqbfT0yMtLveSC+RY3VhBDRUWM1IUR0QV0143keV65cgVar7TBD4QkJJowx1NbWIjExERJJ8+WeoA5EV65cQXJystjZIIS0oqioCJ07d2729aAORFqtFoBwkTqdTuTcEEJuVVNTg+TkZMffanOCOhDZq2M6nY4CESHtWGtNJ9RYTQgRHQUiQojoKBARQkQX1G1E7mCMwWq1tjg3ibgmlUohk8loaATxu5AORGazGSUlJTAYDGJnJWiFhYUhISEBCoVC7KyQEBaygYjneVy4cAFSqRSJiYlQKBT0n90DjDGYzWaUl5fjwoUL6N69e4sD0ghpi5ANRGazGTzPIzk5GWFhYWJnJyip1WrI5XIUFhbCbDY32R8sGB0srMT58jqM7B6DeH3wX0+oCPl/cfRfvG1C6edXVtuAXWfKUdNghbUDbtnTnoXObxkhrTh5pQYAkBShRudIKiW3JxSISIdRXCWsYZQerRE5J+RWFIg6uLS0tCZ7mYUik9WG8lph77LECBUaLDZU1JlaeRcJFApEpEO4Xm8GY4BWJUNNgxVrdxTgqyNXxM4WaUSBKASYzWaxs9DuXa8XfkaRYQpoVUJncbXRCp6nBUrbgw4ZiMxWvtkvq413O63FjbTeGD16NGbPno3Zs2dDr9cjOjoaL730kmM96LS0NCxZsgRPPPEEdDodnnnmGQDA7t27MXLkSKjVaiQnJ2POnDmor693nLesrAz33Xcf1Go10tPTsWHDBq/yF4wkHIdorRIxWiW0ShlkEg48Y6hpsIidNYIQHkfUkne+b7qtsl16tAaTs5Icz/++qwAWm+v/mp0j1Xh48I2F2dbtuQCj2XkqyQu/6eFVHj/88EM89dRT2LdvHw4cOIBnnnkGKSkpePrppwEAr7/+OhYuXIhFixYBAAoKCjBhwgT85S9/wbp161BeXu4IZvatqHNzc3HlyhV8//33kMvlmDNnDsrKyrzKX7DpmaBDz4QbS8VEhMlRUWdGlcGCiDAaNS62DhmIgkFycjJWrlwJjuOQkZGBY8eOYeXKlY5AdOedd+J//ud/HOlnzJiBadOm4fnnnwcAdO/eHW+++SZGjRqFtWvX4tKlS/jmm2+wb98+ZGdnAwA++OAD9OzZM+DX1h7owxRCIDJSiag96JCBaNaYbs2+JrllFsgzd3RtNu2tM0amj0hvS7ac3HbbbU5TUoYNG4Y33njDMXl38ODBTumPHDmCo0ePOlW3GGOOqS5nzpyBTCbDoEGDHK9nZmYiIiLCZ3kOJvZ2oroGq8g5IUAHDUQKmftNY/5K21YajfNYmLq6OsycORNz5sxpkjYlJQVnzpwJVNbaHYPZirw9F6FTyTBtaCokEg7hysZAZKJA1B50yEAUDPbu3ev0/Oeff0b37t0hlUpdph84cCBOnjyJbt1cl/YyMzNhtVpx8OBBR9Xs9OnTqKqq8mm+26PaBivMVh5Giw2SxiJvgl6FASkRSNSrRc4dATpor1kwuHTpEubOnYvTp09j48aNeOutt/Dcc881m/7FF1/Ejz/+iNmzZ+Pw4cM4e/YsvvjiC8yePRsAkJGRgQkTJmDmzJnYu3cvDh48iBkzZkCtDv0/xNrG6pdWJXcc6xwZhjEZsciIb3lRdxIYogYim82Gl156Cenp6VCr1ejatSuWLFkSkG2L27snnngCRqMRQ4YMwaxZs/Dcc885uuld6devH3bu3IkzZ85g5MiRyMrKwsKFC522l87Ly0NiYiJGjRqFBx98EM888wxiY2MDcTmiqm3sordXx0j7I+qdWb58OdauXYsPP/wQvXv3xoEDB/Dkk09Cr9e7bOvoSORyOVatWoW1a9c2ee3ixYsu35OdnY2tW7c2e874+Hh89dVXTscef/zxNuUzGNjbgewN1HYGsxV1Jis6aZSQ3tpLQQJK1ED0448/YtKkSZg4cSIAYaDexo0bsW/fPjGzRULMjaqZ86973p6LMFt55AxPQ5SGxhKJSdSq2fDhw7F9+3ZHj86RI0ewe/du3H333S7Tm0wm1NTUOH0R0po6F21EABCmEBr+DWbqORObqCWi+fPno6amBpmZmZBKpbDZbHj11Vcxbdo0l+mXLl2KxYsXBziXgbdjxw6xsxBSIsLkMNt46FwEoiqDBQ0W2lhBbKIGok8++QQbNmzAxx9/jN69e+Pw4cN4/vnnkZiYiJycnCbpFyxYgLlz5zqe27ezJaQl43rHuzyukgslIqOZVmsUm6iB6E9/+hPmz5+PKVOmAAD69u2LwsJCLF261GUgUiqVUCqVgc4mCVFqeyCiEpHoRG0jMhgMTdZElkql4Gk9YRIAamojajdELRHdd999ePXVV5GSkoLevXvjl19+wYoVKzB9+nQxs0VCSGl1Azb/chmxWhV+O6iz02v2EhG1EYlP1ED01ltv4aWXXsKzzz6LsrIyJCYmYubMmVi4cKGY2SIhpN5shcnSdO0oAIjXq5CVEoEEmuYhOlEDkVarxapVqzrEmslEHAaTUNqxd9XfrHNkGO3m0U7QXDMS0uztP2EKmt7RnlEgIiHNYGm+RMQYQ53J6tjdg4iHAhEJafaqmdpFIDJZeby/6zw++rnQZRsSCZyOFYgYA6zmwH95sJrA+vXr0alTJ5hMzv+lJ0+e3CEmqPqavWqmcVE1U8okkDSugkk9Z+LqWBVnmwX44Y3Af+7I/wFk7k2qfPjhhzFnzhx8+eWXePjhhwEIu298/fXXLc6sJ65FaRQw2/gmE14BgOM4KOUSGM02NFh4aFUiZJAA6GgloiCgVqsxdepUx84bAPDRRx8hJSUFo0ePFi9jQequnnGYNjQViRGuu+hVjcv7mqxUIhJTxyoRSeVC6USMz/XA008/jezsbBQXFyMpKQn5+fnIzc11Wkyf+IZSLgVggcnLPeiIb3SsQMRxbleRxJSVlYX+/ftj/fr1GDduHE6cOIGvv/5a7GyFJGVjiYjaiMTVsQJREJkxYwZWrVqF4uJijB07llYZ8MK1OhP+eaAIEWoFpg5NcZnGPgOfSkTiojaidmrq1Km4fPky3n//fZp75yWjxQaThW+x/Se1UxiyUiIQE06rOoiJAlE7pdfr8dBDDyE8PByTJ08WOztBqcEilHLsk1td6Z2ox+iMWCRH0VQPMVEgaseKi4sxbdo0WoPJS/Z2H1eDGUn7Qm1E7VBlZSV27NiBHTt2YM2aNWJnJ2jZFzxTypoPRDaewWC2ggFNlpIlgUOBqB3KyspCZWUlli9fjoyMDLGzE7TcKREVlNfh66MlSIpU43eDqUNALBSI2qHm9i0jnjGahUBkH7ToiqqxtGSi7ntRURsRCVlalRxxOhX0Yc1XuZRy+8hq6r4XU8iXiGj76rYJ5p/fsK6dMKxrpxbTKGUUiNqDkC0RyeXCf0GDwSByToKb/edn/3mGGntDttnKw8YHb9ANdiFbIpJKpYiIiEBZWRkAICwsjOZqeYAxBoPBgLKyMkREREAqDc0ucOVN7Ucmq41WchRJSP/U4+OFjfXswYh4LiIiwvFzDCaMMfx913nIpRJMGZLcbICRSDgoZBKYrTxMFh5h7X8qYkgK6UDEcRwSEhIQGxsLi8UidnaCjlwuD9qSkMnKw2C2AbBBLm25BaJPkh6MMchb6F0j/hXSgchOKpUG7R8U8Y6pcXqHXMq1GohG9YgJRJZIC+hfAAlJ9lHVqhbmmZH2o0OUiEjH0+BBILLYeJisPORSrsXpIMR/qEREQpK9RNTSzHu7rSeu4v1d53GqpNbf2SLNoEBEQpInVTNFYyO1mQY1ioYCEQlJKpkUcToVIjWtD8SkQCQ+aiMiIalXog69EnVupVU09qqZbTTxVSxUIiIdHpWIxEeBiHR4NPFVfFQ1IyHp0wNFqG2wYnyfeCQ1s7minYICkegoEJGQVG20oLbBCokb85wjwuTonahDlIYmmomFAhEJSfbSjTvjiGK1KozrHXwTe0MJtRGRkGO18Y6GZ5riERwoEJGQYy8NcZzzekPNYYzBbOVRZ7L6O2ukGRSISMhpuGkbIXcWwzPbeLzz/Tm8v+s8rDZqsBYDBSISchoc1TL3fr0VNy0TYqZAJAoKRCTkcADidCpEu7mfPcdxNKhRZKIHouLiYjz22GPo1KkT1Go1+vbtiwMHDoidLRLEEiPUmDo0Bff1T3T7PUoKRKIStfu+srISI0aMwJgxY/DNN98gJiYGZ8+eRWRkpJjZIh0QDWoUl6iBaPny5UhOTkZeXp7jWHp6erPpTSYTTCaT43lNTY1f80c6Dns7EQUicYhaNfvyyy8xePBgPPzww4iNjUVWVhbef//9ZtMvXboUer3e8ZWcTHuVk6Z2nSnHut0XcPRyldvvoTYicYkaiM6fP4+1a9eie/fu2LJlC/7whz9gzpw5+PDDD12mX7BgAaqrqx1fRUVFAc4xCQa1DVZUGy0ebZiY2ikMvRJ10KlpsoEYRP2p8zyPwYMH469//SsAICsrC8ePH8e7776LnJycJumVSiWUSvd6QkjH5cl61XaDUqP8lR3iBlFLRAkJCejVq5fTsZ49e+LSpUsi5YiEggYr7eARbEQtEY0YMQKnT592OnbmzBmkpqaKlCMSChosng1oBIRpHhYbAwOjnTxEIGqJ6IUXXsDPP/+Mv/71rzh37hw+/vhj/P3vf8esWbPEzBYJco6qmQcB5dClKrzz/Tl8d4q2JxeDqIEoOzsbn3/+OTZu3Ig+ffpgyZIlWLVqFaZNmyZmtkgQs/HMq5n3N9atpl4zMYjeRXDvvffi3nvvFTsbJERYbDzidCo0WGxuzby3o+57cYkeiAjxJZVciqlDUzx+nyMQUYlIFKLPNSOkPaASkbgoEBGCm9qIKBCJgqpmJKScLq3FnnMVSI/WYExmrNvvoxKRuCgQkZBSbxamd9i78N2lkkvQLTYcCpkEjDG3VnYkvkOBiIQUb6Z3AMKysp6sX0R8i9qISEgxNY6qVnowqpqIj+4WCSnelogA+zQPHrwHs/aJb1DVjIQUoxfTO+zW/1SI6/Vm/C47udVtqolveVUiOn/+vK/zQYhPeDPh1U5OXfii8SoQdevWDWPGjMFHH32EhoYGX+eJEK+Fq2TQq+UIU7go7DMGXNoL7F4F7FkNFB90elkuFXrKKBAFnleB6NChQ+jXrx/mzp2L+Ph4zJw5E/v27fN13gjx2P39EzH99nTE61VNX7y8Hyj4DrAYAbMBOLMVKDnieJnGEonHq0A0YMAArF69GleuXMG6detQUlKC22+/HX369MGKFStQXl7u63wS0jaG68D5ncLj9DuAlKHC43P/JwQl3LSlkM2zMUik7drUayaTyfDggw/i008/xfLly3Hu3DnMmzcPycnJeOKJJ1BSUuKrfBLSNoU/ArwViEwDUocDXcYA4bGA1SyUlHBziYh6zQKtTYHowIEDePbZZ5GQkIAVK1Zg3rx5KCgowLZt23DlyhVMmjTJV/kkpFVVBjPW7b6ATw/csqmCqRYoOyk8Tr8D4DjhK3W4cKzkCMDboJAKPW00Az/wvOq+X7FiBfLy8nD69Gncc889WL9+Pe655x5IJEJcS09PR35+PtLS0nyZV0JaZLTYUG20oEl55uoJgLcB+iThyy66B6DQAOZ64FoBorXx6Bobjk4aRSCzTeBlIFq7di2mT5+O3NxcJCQkuEwTGxuLDz74oE2ZI8QTzXbdXz0hfI/v63xcIgViewlVs4rTyOzZA5nxugDklNzKq0C0bds2pKSkOEpAdowxFBUVISUlBQqFwuWWQIT4i8u1qusrgLoyIejEZDZ9U0wPIRBdOyeUmiS0cL4YvGoj6tq1KyoqKpocv379eotbRhPiTy6nd5Q37hITmQ7IXYyW1nUGFGGApQGoKQZjzKONGYlveBWIGHN9o+rq6qBSuRi/QUgAuKyaXW+cBdCpq+s3SSRAhLB9VdmlM3hz+zl8vLfQn9kkLnhUNZs7dy4AgOM4LFy4EGFhYY7XbDYb9u7diwEDBvg0g4S4q8nGipYGoOaK8DiqS/NvjEwFyk5BWVcEnqXARAMaA86jQPTLL78AEEpEx44dg0Jxo3dBoVCgf//+mDdvnm9zSIiblFIJIsLkCFc2/lpXXgQYD4R1AtQRzb+xsUQkry+FhLfAbKNFKQLNo0D0/fffAwCefPJJrF69Gjod9TCQ9mN4t2gM7xZ944CjWtZCaQgA1JGASgdZfRW05quolSbTKo0B5lXoz8vLoyBE2r/qxoGNEWktp+M4QJcEKcch3FQOnjFYqcE6oNwuET344IPIz8+HTqfDgw8+2GLazZs3tzljhLSJqU6YXwY4D2Jsji4RkrKT0FqELafNVt6xLAjxP7cDkV6vdxRV9Xq93zJEiLf+8XMhpByHe/snQFdbLBzURLvutr+VLhEcOOgt5QATtq3WKP2bX3KD24EoLy/P5WNC2gMbz1BRawIAyCUSoMpeLXNz19fwOICTIEZpRUYkg4TahwLKq5HVRqMRjDFH931hYSE+//xz9OrVC+PGjfNpBglxh30wI8c1Ludhbx/Sd3bvBFI5EB6L7oxH9xQeCJP7KafEFa8qwZMmTcL69esBAFVVVRgyZAjeeOMNTJo0CWvXrvVpBglxh/GmUdUS3iJM6wDcD0QAoGvcTqiWlq8JNK9XaBw5ciQA4LPPPkN8fDwKCwuxfv16vPnmmz7NICHuMJqFQKSWS4GaYmH8kEovfLkrXNgZltWVNTt7gPiHV4HIYDBAq9UCALZu3YoHH3wQEokEt912GwoLaXg8CTx71Uwtl94o0eg83DAxPA7nyuuw58hJHCmq8m0GSYu8Xjz/3//+N4qKirBlyxZHu1BZWRmNLyKicFTNFDcFIq3rJWqapYkBwEFqNcLaUOvbDJIWeRWIFi5ciHnz5iEtLQ1Dhw7FsGHDAAilo6ysLJ9mkBB3SDgOEWFyaFUyoLZUOKiN9+wkUjl4dZTwuPaqbzNIWuRVr9lvf/tb3H777SgpKUH//v0dx++66y488MADPsscIe7qk6RHnyS9sNrilRrhoKeBCADTxAC4Aq6eAlEgeb3Ta3x8POLjnW/0kCFD2pwhQtrEXhoKiwJkno9IZI0N1hJD0/W2iP94FYjq6+uxbNkybN++HWVlZeB552UTaCdYIhpvq2WNOI3wPqmBtsQKJK8C0YwZM7Bz5048/vjjSEhIoFnKRHRfHC5GncmKe7hLiAQ8b6huxGmFEpG0oRKwWYSBjsTvvApE33zzDb7++muMGDHCZxlZtmwZFixYgOeeew6rVq3y2XlJx3CtzoxqowUS6VWAg9clIo1WD1N4ODRSqzBpVhvn24wSl7zqNYuMjERUVJTPMrF//36899576Nevn8/OSToWo8UGuc0AhbVOmOcR7l0ASYoMQ9/uXZHWSQNQO1HAeBWIlixZgoULF8JgMLQ5A3V1dZg2bRref/99REZGtvl8pOOx2nhhtry5AnIpJ6zI6EVDtYOmcXG1emonChSvqmZvvPEGCgoKEBcXh7S0NMjlzvXoQ4cOuX2uWbNmYeLEiRg7diz+8pe/tJjWZDLBZDI5ntfU1HiWcRKSGhrXmNZaKiCVeF8actDECN/rqUQUKF4FosmTJ/vkwzdt2oRDhw5h//79bqVfunQpFi9e7JPPJqHDPs8swnYNHDivG6oBYarIv47Xo9uVa8hWR7ZtT3biNq8C0aJFi9r8wUVFRXjuueewbds2t7cgWrBggWMnEUAoESUnJ7c5LyS42eeZ6W3XAMi9bqgGALlUgkqmA88AW/11SGxWQOr1cDviJq9/wlVVVfjss89QUFCAP/3pT4iKisKhQ4cQFxeHpKTWl+Y8ePAgysrKMHDgQMcxm82GXbt24e2334bJZIJU6rzrplKphFJJy+YRZ4wB0QoztDACnKJNVTOphAOTa2CTKITxccbrjln5xH+8CkRHjx7F2LFjodfrcfHiRTz99NOIiorC5s2bcenSJcdaRS256667cOzYMadjTz75JDIzM/Hiiy82CUKENCelUxge760CeF1jQ7Wi9Te1QCGXwiCPhI1vEBqsKRD5nVeBaO7cucjNzcVrr73mWA4EAO655x5MnTrVrXNotVr06dPH6ZhGo0GnTp2aHCekVY4Z995Xy+wUMgmM8kjY+CvUYB0gXrXF7d+/HzNnzmxyPCkpCaWlpW3OFCEec0zt8L6h2k4hkzSWiBiNJQoQr0pESqXSZdf5mTNnEBMT43VmduzY4fV7Scf13a9XoT95CulahigflIjkUgnqZJGwMQbUX/NBDklrvCoR3X///XjllVdgsVgAABzH4dKlS3jxxRfx0EMP+TSDhLSmqqoS5voa8AxtH0MEIFarRGRMgjAmyVgJ2KxtzyRpkVeB6I033kBdXR1iYmJgNBoxatQodOvWDVqtFq+++qqv80hIi7jGapkkPNYnk1RHZ8TigdsyEaENF9a+Nla2+ZykZV5VzfR6PbZt24Y9e/bgyJEjqKurw8CBAzF27Fhf54+QVnF1QkO1TN/2atmNk3JAWDRQc0VoJwr3vsmBtM7jQMTzPPLz87F582ZcvHgRHMchPT0d8fHxYIzRkiAkoGw8g9IgbB2kiHRja2lPaBoDEfWc+Z1HVTPGGO6//37MmDEDxcXF6Nu3L3r37o3CwkLk5ubSMrEk4AwmCzTmCnAAlJEe7trRjMNFVXhvZwEOljf+UzVQg7W/eVQiys/Px65du7B9+3aMGTPG6bXvvvsOkydPxvr16/HEE0/4NJOENMdYVwU5b4RMJgXng4ZqQPiHazDbUBsWIRygLny/86hEtHHjRvz5z39uEoQA4M4778T8+fOxYcMGn2WOkFbVlkAtl4ALj/HZaooKmfBnUSdr3JzRUAncshwy8S2PAtHRo0cxYcKEZl+/++67ceTIkTZnihB3xaISA5IjMah3L5+dU9kYiAxcOCCRAbwVaKjy2flJUx4FouvXryMurvnib1xcHCorqauTBFAbF8t3RS4V/ixMPITdQABqJ/IzjwKRzWaDTNZ8s5JUKoXVSoO/SIAw5v2uri2wV83MVv6m1RqpncifPGqsZowhNze32aU4bl49kRC/a6jCxdJrqDbxCMtUobuPdjtXSG8KRGGdhIPUYO1XHgWinJycVtNQjxkJmNpSGM02lPN6JDLfLRujkksRq1NCJZMKgxoBqpr5mUeBKC8vz1/5IMRztSWw2HjUKWIQpvBdINIoZZg2NFV4Yq+S1VcIVUEasOsXtCQvCV61pTDbeNQrYqBR+mk5V3UkwEmEzRZNtFmDv1AgIsGJMfA1JbDYGOoU0Qj3VyCSSIVgBFD1zI8oEJHgVF8Bs8kIGyeHSdkJKrlvf5U/2V+E93YWoKymAdA0NljT2kR+Q4GIBKeay7BYedQrY6BRKXw+2brBaoPBbIPJyt/UYE09Z/5CgYgEp+piYSE0XRIiw3wzteNm9i58k1MXPpWI/IU2bCLBqeYK9Go5xvcdBER39vnpVXKhF67BYgN0NKjR36hERIKP2XCjdKLzzdIft7LPN3OUiDgOsBgBc71fPq+jo0BEgk/NFeF7WBSgCPPLR9hLRCaLTZjVr2wctk3VM7+gQESCT00xAOBAZRg27C1E0XWDzz/CqUQE0JwzP6NARIJP1SUAQAmiUVbjn/mNOrUcMVrljRHb1GDtV9RYTYKL1eyYcX9VImwF7Y9R1X2S9OiTpL9xgAKRX1GJiASXmssAb4NVEY5ahAMANErfzTNrFlXN/IoCEQkulYUAAENYZ4DjoJJLoZQFIBDZBzWaagErLXfjaxSISHBpbB+qUQoLoWlV/mldqDKYkb/nAtb/dFE4IFcBSqEERtUz36NARIKH1eRYGva6Uhg/pFP7flQ1AEgkHCoNFlQZLGCMCQft7URUPfM5CkQkeFy/IGwBHRYFiUqPKI0CUWEKv3yUqrG6Z+MZLDZ7IKI5Z/5CvWYkeFw7J3zv1LVpr5aPyaUcJBwHnjGYrDZhHWv7LHzDdb99bkdFJSISHBgDrhcIjzt18/vHcRznWFqkwdI4qJGqZn5DgYgEh9oSYY6ZTAHokwPykTdGV9uEA/aqWUOVsGIj8RkKRCQ4lJ0Svkd1gZVxeHdnATbsLRR22vCTGzPwGz9DoRF6zxij6pmPUSAi7R9jNwJRbC9UGy0wmm2oMlggl/pvMftIjQIxWiVkksbP4DgaYe0n1FhN2r/qImEgoUwBRHVF5bUGAEBkmO9XZrzZ+N4udo8Niwaqi6nnzMeoRETav9LjwveYTEAqQ5XBDAB+WZmxVTTVwy8oEJH2zdIAlJ0QHsf3BQBcrxcCUYSfxhC1iKpmfiFqIFq6dCmys7Oh1WoRGxuLyZMn4/Tp02JmibQ3pUcBm1UoiTT2llUZhB6rSI1/S0S/ltYgf88FfPfr1RsH7YHIWAnw/mso72hEDUQ7d+7ErFmz8PPPP2Pbtm2wWCwYN24c6utpOU4CIQAV7RMeJw1y7LJa2Vg189eoajueh2Oah4NKD0hlAG8TghHxCVEbq7/99lun5/n5+YiNjcXBgwdxxx13iJQr0m5c+UVopFZqgfh+AIQpF/F6Fa7Xm6H3cxuRunFRNKPFduMgxwkN1rWlQH35jdHWpE3aVa9ZdXU1ACAqKsrl6yaTCSbTjSUYampoC+CQZaoFLv4gPE4dLpRCAEglHCYNSApIFtSN44iMZpvzC+FxQiCquwrEZgYkL6Gu3TRW8zyP559/HiNGjECfPn1cplm6dCn0er3jKzk5MCNsSYAxBpzZIsy218YDCQNEyYajRGS23ZiBDwiBCADqykTIVWhqN4Fo1qxZOH78ODZt2tRsmgULFqC6utrxVVRUFMAckoBgDDi/A6g4K+w7n3EPILnxa2qy3hIU/MheIrLePAMfAMKFJWpRd9XFu4g32kXVbPbs2fjqq6+wa9cudO7c/GZ5SqUSSqUygDkjAWVpAM5/D1w5LDzvMQHQxjkl+dfBYtQ0WHBvvwR0jvTPVkJ2cikHmYSDlWcwmhtn4AM3ApGpVpj/5qctjToSUQMRYwx//OMf8fnnn2PHjh1IT08XMztELIwBV08ABd/d2MCw+2+AhH5OyWw8w7U6E6w8Q7gfFsy/FcdxiNEqYWMMtptLYTIloI4AjFVAfRmgSPN7XkKdqIFo1qxZ+Pjjj/HFF19Aq9WitFRYfU+v10OtVouZNRIo9RVCe1DjErAIixKCUFSXJknLa4UgpJJLoffTyoy3mjIkxfUL4bFCIKorAyLTApKXUCZqIFq7di0AYPTo0U7H8/LykJubG/gMkcDheaDoZ+DibmFMjlQGpI4AOg9x9JDdqrhK2EgxMULl1zlmbgmPA8rPUDuRj4heNSMdkNUMnPpSaJAGhIXOuv9GqO604HKlEQDQObIdlJY11GDtS+2isZp0IDYrcPwzYVsgiQzoMU4YrNhKCcdq428KRIFrHD50qRJHi6qQmaDDbV1uGrxob7A2XBdKdJIAbGkUwtpN9z3pABgDfv1KCEJSOdB/CpDQv9UgBACXrhtgtvLQqmSI1Qau59Ri5VFpsKDGeMuKjCq90GjN22gmvg9QICKBU3JYWOBMIgX6PASmdx6qsff8NWw/dRUNFluTt0aGKTAkPQr9kyMC2j4UphAqDcZb88RxNw1sLA1YfkIVBSISGA01Qvc8AHQZjUplEv65vwh1JisAgOcZjhVX4+jlavzjp0IUXTc4vT1So8CIbtHITnM9/cdf1ArhT6TJNA9AGPUNADUlAcxRaKJARPzPMWXDDOgSYYgdgM9/KUZJdQN+OFMOQChgjO8dj8gwOepMVvzr0GXsPlsBk9XmsoQUKPYSUb2rQKQTNnlE7ZUA5ig0USAi/nf1hLAnmUQKlnEPvj1RhmqjBXq1HHf0iAEgDB5MjgrD1KGp6JOkB2PA/ovXseb7AmzadwnVt7bRBIimceCkwWRt2surFba9Rl057erRRhSIiH+Z6oBz24THqSNw+LoMhdcMkEk4TBqQ6PhDt1PIJPhNrzjc2y8BGqXQE2W28Y7lYQNNo7gx38x0644hKr0wvYPx1I3fRtR9T/zr3DZhDll4LCqisrB7fzEA4I4eMegU3nzvV/c4LbrFhqPOZIVGIYNEIs4ARplUguhwBaQSCcw23rHFEAChPqlLEsZD1ZQA+ubnSZKWUSAi/lN+Gij7FeAksPW4B9+eLIeVZ0iP1qBf59a3i+Y4DlqVCAvk3+LxYWnNv6hNEAIRtRO1CVXNiH9YjEIDNQCk3AaDMhoMwqaFY3vFiT9Fw1d0je1E1HPWJlQiIv5xbrswk14TDaSOgFYqw9QhKbhWbwrIzPmAsTdYGytpSZA2oBIR8b2Ks0DpMYDjwHpMcFrmNVarEjlznjt2uRp5ey5gZ+NQAydytbBiAADUUPXMWxSIyA2MAfXXhCU5zF7upGI2AKe/ER53zsbOq0r8WFABng/eCc48Y6gyWJofQtC4zRGqLwUuUyEmhMrIpE1qS4UAUnvTdIXo7kDXO2/8x28NY8Dp/zqqZKfDBuKXE0IpIrWTBkkR7WDWvBfsQwzqG0eBNxGRDJQcubGmEvEYBSICVF4Ejn0qzIyXSIXte4xVQhWr8oIQjBIHtj459eJux1rTFcnj8H+/CruhZqdFBW0QAuBo06praC4QNS6eVntVWPBfRssZe4oCUUdnuA4c+0wIQlHpQM/7AIVGmFF+dpsQpM5sFb5n3CO0ibhy5RchEAGoTRmLzWetMFt5JEeFYXjX4N77S6duDEQmK6w2HjLpLS0aKr3w1VANVF8GOnUVIZfBjdqIOjLeBpz8tzA9ISIZ6PNbIQgBQm9X/ylAt7FCKan8DHBgHVB1y84pPA9c+AE4LWyWWRMzEP+8HIF6kw3RWiXu7Zcg2mBEX1HLpY6F82taKxVV084y3qASUUd2eT9qKy7DyBTQ95gI5a1LtHIckJwtjBg++YXQRf3LR8J//IgUoRpS/qtQqgJg6zwEn5SkodZkRadwBR7ISnIeiRykOI6DTi1HRa0J1UYLojQutrqOSBZ6CqmdyCsUiDoqUy1wcTeqDBbskAxE7f5ruK0LMKBzRNMSjC4BGPykUFW7ehy4ViB82cnVQLexkMb3wTBNNY4XV+P+/kmODQpDQaxWCZmEQ7OFO3uJqKZEmNIiD75hCmKiQNTBlNU2QCmTQn/he8BmgSwyGTZ5L5iMVuw8XY7TpbUY2zMOMbeugihTAj3vBVKGARWngfpy1FmAXw06RKT2Rbf4aABArwQdesbrgr46dqvxveNbTqCOBMI6AYZrQnsabUXtEQpEHUiDxYYvD1+BylCKB3AEGqUcCQPvwxPh8ThxpQY/nCtHaXUDPt57CdnpkRiSFtWkYdamjsIFdT+cqKzGhYp6MAZoC2qQGhcFuVQCjuPcWfk1NHXqIgSi6wUUiDxEgagD2X22ArVGC3pW/wxlJwkQ3xfQJUACoG9nPdJjNPju1zIUlNVh7/nrsPEMI7sL6wVdrKjHr6U1uFBhcFqoLD1agwHJEZCFWAmoOYyx5ufJRXUFivYL1VbG3FqLmwgoEHUQpdUNOFZcjSjjBfTV1kEmVwNpI53ShCtluK9fAs6V1eHnC9cxOPXGQMbzFXU4VVLrSJcRr0WfJL3rhtsQVNNgwb8OXobJymPmHV1cB6OIFGFTAHO9MDDUPiGWtIoCUQfx0/kKcMyGIfxR6FRyIHkooNI1ScdxnGMtoJv/2JIiwqCSSZEcFYakCHXItQG1RqOQocZoBc8Yak1W4Wd4K4lUGItVfgaoOEOByAM0jqgDKK1uwMUKAxLqTiFdYxbGCiXf1uJ7bv2PnxGvxfBu0UiOCutwQQgQJuxGaYTgc62uhdUiYxrbhspOCdUz4hYKRB3A3gvXoLDWIYs/JozrSR8JyDpGlcqX7CtKXqsztZCou7DagLESqKU1itxFgSjE8TyDRiFD1+qfkKyTCjtPxPcXO1tBqVNje1hFSyUimUIIRgBQdjIAuQoNFIhCnETCYWxUGcbGVEGtkAMZdwMSuu3esJeIKloqEQFAXG/h+9WTwjQa0ir6jQx1deXAmW8hk0iAtBE39mwnHovV2atmZphv3dHjZlFdhHY4c72wbjdpFQWiEHb4zEXU7f9ImFkfmQakDBc7S0FNq5Shc6QaPRO0sNhaCEQSKZCYJTwuPhiYzAU5CkQh6lrpJVT/uA7HzxfDqIgEek2iKlkbcRyHhwcnY1zv+Cb7sTWROADgJMKyILSEbKvoNzPUMAZcPoCKXR9AYTMgPCoe6kFTaVH3QFNqb7QVXfhB3LwEAQpEoaT6MnBoPepPfIPrtfWoUnVG/JinXQ5cJN7jeYbiKmPL1TNAaJPjJMD180BlYWAyF6QoEIUCYyVw4nPg0D/AaopxodKCC5EjwPf5HaIjWt/IkHjmnweK8Mn+IhReM7ScUB0JJDQOlTi7VWirIy5RIApmlgZh/7B97zfuqMqhRJ2BHfoHcE3fB7c3TlglvhWvF9YaOldW13ri9DuEanF9BVC4x885C14UiIIRbwMuHwT2vgsU7ROeR6ahof8T+Mo8ABaZBoNSI6EPE3+75lCUGa8FAJy9WguDuZVSjiIM6PYb4fGln4TNBUgTFIiCCWPCL/L+D4SivsUorC3d73dA/ylQRiRgWJdoxOtVyE53cwsg4rF4nQpxOhWsPMPhoqrW3xDXC0gaKNy/k18IC6cRJ+0iEL3zzjtIS0uDSqXC0KFDsW/fPrGz1L4wJqxx88s/hB03DNeE/7Q9xgODnxLWkOY4cByHvp31mJKdDPmtO00Qn+E4DoPTIgEABy9WosrQwpQPu25jhYGONgtw9FNh1xOaFOsg+m/rP//5T8ydOxeLFi3CoUOH0L9/f4wfPx5lZWViZ018FqMwIO5gHnD0E6C6GJDIgNRhwNDfA0kDUW2y4dvjJahtuLELabMLdxGf6R4bjuSoMFh5hi+PXGl5pDUgDHLs85CwaSVvFXY9ObxB6E2jgASOMXF/CkOHDkV2djbefvttAADP80hOTsYf//hHzJ8/v8X31tTUQK/Xo7q6GjpdEHdR8zbAYhCmBBiuAfXlwm4QNSUAa/wFl8iApCxYE7NRjzBcrW3A+fJ6nL1aCyvPkBShxsODO1MQCqCaBgs+2V+EOJ0K9/ZLAMdxsNp4cBwHaXNLpTAGFO0Vxhbxje1L6khhHSNdkrCrrkoPyNQhMQDV3b9RURdGM5vNOHjwIBYsWOA4JpFIMHbsWPz0009N0ptMJphMNyYc1tTUuPdB1y8ABdtd/+dxOsZcHHd1rOW0v1yqgo3nnV5nja9plbLGxk7h+eFL18FbTE1PBYYwhQw9u3cVZsvH9cZHh8pRfuFqk0voHKnGnZmxFIQCTKeS46GBnaFWSB0/+4LyenxzvAQquRQyiRCQbg5KYzJikZxyGxDbE6VHt6Pk7CFI+CoAF5qcPzkmAlFaDSCRodJgQVmdGRnxOgDcjWVoxbznfX/nszFqogaiiooK2Gw2xMXFOR2Pi4vDr7/+2iT90qVLsXjxYs8/yGoSJn8GiMVYAxvvuqBpgwyw3PixW80NsNgYAA4WiQoNcj2M8gjUKWIhj0lHz+y+jrSmxuK/TMIhUqNAUqQaPeN1iNMpKQiJJPKWpXLrTBYwBhjNrmfd2+8hVHpUJo/FrsoMRDRchtZUgnBzBZTWGihswvgk3mICGpufmNEEvt4EGNrRWCTWSnXUA0G1VOyCBQswd+5cx/OamhokJye3/saIZGHXUuCm/yA3/eE6/RG7Ou5Z2qRuxiZp7btbyKUSIPzGnlepvcywydTgZCqhwfmmU8tvKZo/NDAJSpkUKrmEAk87NTAlEj0TdDCYbeB5BivPnP4pRYff2KYpJSoMD2anA0gHcFNpmLeBs5mgVgGQ2gBmg9psQYzJCmgUjQGAid+2ZN8V2AdEDUTR0dGQSqW4etW5unH16lXExzfdR0qpVEKpVDY53iqFRqiDB0iS1v20MR7cy4gwWlWxveM4DmEKGcIUrf9paZSyFibPOv8SqRu/QpWorWEKhQKDBg3C9u3bHcd4nsf27dsxbNgwEXNGCAkk0atmc+fORU5ODgYPHowhQ4Zg1apVqK+vx5NPPil21gghASJ6IHrkkUdQXl6OhQsXorS0FAMGDMC3337bpAGbEBK6RB9H1BYhM46IkBDl7t9o8I+YIoQEPQpEhBDRUSAihIhO9MbqtrA3b7k91YMQElD2v83WmqKDOhDV1tYCgHujqwkhoqmtrYVe3/yyxUHda8bzPK5cuQKtVtsupzzYp6AUFRWFXK8eXVtwCvS1McZQW1uLxMRESFpYTSCoS0QSiQSdO3cWOxut0ul0IfcLbUfXFpwCeW0tlYTsqLGaECI6CkSEENFRIPIjpVKJRYsWebdiQDtH1xac2uu1BXVjNSEkNFCJiBAiOgpEhBDRUSAihIiOAhEhRHQUiDzk6a60n376KTIzM6FSqdC3b1/897//dXqdMYaFCxciISEBarUaY8eOxdmz4uyP7sm1vf/++xg5ciQiIyMRGRmJsWPHNkmfm5vbuGnAja8JEyb4+zJc8uTa8vPzm+RbpVI5pQnW+zZ69Ogm18ZxHCZOnOhII8p9Y8RtmzZtYgqFgq1bt46dOHGCPf300ywiIoJdvXrVZfo9e/YwqVTKXnvtNXby5En2//7f/2NyuZwdO3bMkWbZsmVMr9ezf//73+zIkSPs/vvvZ+np6cxoNAbqshhjnl/b1KlT2TvvvMN++eUXdurUKZabm8v0ej27fPmyI01OTg6bMGECKykpcXxdv349UJfk4Om15eXlMZ1O55Tv0tJSpzTBet+uXbvmdF3Hjx9nUqmU5eXlOdKIcd8oEHlgyJAhbNasWY7nNpuNJSYmsqVLl7pM/7vf/Y5NnDjR6djQoUPZzJkzGWOM8TzP4uPj2d/+9jfH61VVVUypVLKNGzf64Qqa5+m13cpqtTKtVss+/PBDx7GcnBw2adIkX2fVY55eW15eHtPr9c2eL5Tu28qVK5lWq2V1dXWOY2LcN6qaucm+K+3YsWMdx1ralRYAfvrpJ6f0ADB+/HhH+gsXLqC0tNQpjV6vx9ChQ5s9pz94c223MhgMsFgsiIqKcjq+Y8cOxMbGIiMjA3/4wx9w7do1n+a9Nd5eW11dHVJTU5GcnIxJkybhxIkTjtdC6b598MEHmDJlCjQa532tAn3fKBC5qaVdaUtLS12+p7S0tMX09u+enNMfvLm2W7344otITEx0+qOYMGEC1q9fj+3bt2P58uXYuXMn7r77bthsrndB9Qdvri0jIwPr1q3DF198gY8++gg8z2P48OG4fPkygNC5b/v27cPx48cxY8YMp+Ni3Legnn1P2odly5Zh06ZN2LFjh1Oj7pQpUxyP+/bti379+qFr167YsWMH7rrrLjGy6pZhw4Y57as3fPhw9OzZE++99x6WLFkiYs5864MPPkDfvn0xZMgQp+Ni3DcqEbnJ011pASA+Pr7F9PbvnpzTH7y5NrvXX38dy5Ytw9atW9GvX78W03bp0gXR0dE4d+5cm/PsrrZcm51cLkdWVpYj36Fw3+rr67Fp0yY89dRTrX5OIO4bBSI3ebMr7bBhw5zSA8C2bdsc6dPT0xEfH++UpqamBnv37g3oTrfe7rj72muvYcmSJfj2228xePDgVj/n8uXLuHbtGhISEnySb3f4Yjdhm82GY8eOOfId7PcNEIaVmEwmPPbYY61+TkDuW0CbxoPcpk2bmFKpZPn5+ezkyZPsmWeeYREREY6u3ccff5zNnz/fkX7Pnj1MJpOx119/nZ06dYotWrTIZfd9REQE++KLL9jRo0fZpEmTROsG9uTali1bxhQKBfvss8+cunlra2sZY4zV1tayefPmsZ9++olduHCB/d///R8bOHAg6969O2toaGjX17Z48WK2ZcsWVlBQwA4ePMimTJnCVCoVO3HihNP1B+N9s7v99tvZI4880uS4WPeNApGH3nrrLZaSksIUCgUbMmQI+/nnnx2vjRo1iuXk5Dil/+STT1iPHj2YQqFgvXv3Zl9//bXT6zzPs5deeonFxcUxpVLJ7rrrLnb69OlAXEoTnlxbamoqA9Dka9GiRYwxxgwGAxs3bhyLiYlhcrmcpaamsqeffrrJeJxA8eTann/+eUfauLg4ds8997BDhw45nS9Y7xtjjP36668MANu6dWuTc4l132gZEEKI6KiNiBAiOgpEhBDRUSAihIiOAhEhRHQUiAghoqNARAgRHQUiQojoKBARQkRHgYgElfz8fERERDiev/zyyxgwYIDjeW5uLiZPnhzwfJG2oUBEXLKvW/z73/++yWuzZs0Cx3HIzc11Su/rAJCWloZVq1Y5HXvkkUdw5syZZt+zevVq5OfnO56PHj0azz//vE/zRXyPAhFpVnJyMjZt2gSj0eg41tDQgI8//hgpKSmi5EmtViM2NrbZ1/V6vVOJiQQHCkSkWQMHDkRycjI2b97sOLZ582akpKQgKyurTed2VVKZPHmyo5Q1evRoFBYW4oUXXnDsJAE0rZrd6uaSWW5uLnbu3InVq1c7znHhwgV069YNr7/+utP7Dh8+DI7jArpWErmBAhFp0fTp05GXl+d4vm7dOjz55JN+/9zNmzejc+fOeOWVV1BSUoKSkhKPz7F69WoMGzYMTz/9tOMcKSkpTa4JAPLy8nDHHXegW7duvroE4gEKRKRFjz32GHbv3o3CwkIUFhZiz549bi2m1VZRUVGQSqXQarWIj4/3auVDvV4PhUKBsLAwxzmkUilyc3Nx+vRpx/5fFosFH3/8MaZPn+7ryyBuojWrSYtiYmIwceJE5OfngzGGiRMnIjo6WuxstUliYiImTpyIdevWYciQIfjPf/4Dk8mEhx9+WOysdVhUIiKtmj59OvLz8/Hhhx/6rNQgkUhw61JYFovFJ+d2x4wZMxwN8Xl5eXjkkUcQFhYWsM8nzigQkVZNmDABZrMZFosF48eP98k5Y2JinNp9bDYbjh8/7pRGoVC0eQub5s5xzz33QKPRYO3atfj222+pWiYyqpqRVkmlUpw6dcrxuDnV1dU4fPiw07FOnTohOTm5Sdo777wTc+fOxddff42uXbtixYoVqKqqckqTlpaGXbt2YcqUKVAqlV5VCdPS0rB3715cvHgR4eHhiIqKgkQicbQVLViwAN27dw/oovekKSoREbfodDrodLoW0+zYsQNZWVlOX4sXL3aZdvr06cjJycETTzyBUaNGoUuXLhgzZoxTmldeeQUXL15E165dERMT41W+582bB6lUil69eiEmJgaXLl1yvPbUU0/BbDYHpBeQtIzWrCYd1g8//IC77roLRUVFTXZLJYFFgYh0OCaTCeXl5cjJyUF8fDw2bNggdpY6PKqakQ5n48aNSE1NRVVVFV577TWxs0NAJSJCSDtAJSJCiOgoEBFCREeBiBAiOgpEhBDRUSAihIiOAhEhRHQUiAghoqNARAgR3f8HZ5uWifYdSoUAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T08:46:05.719383Z", + "iopub.status.busy": "2024-03-26T08:46:05.719014Z", + "iopub.status.idle": "2024-03-26T08:46:06.042463Z", + "shell.execute_reply": "2024-03-26T08:46:06.041356Z" + }, + "papermill": { + "duration": 0.343484, + "end_time": "2024-03-26T08:46:06.044610", + "exception": false, + "start_time": "2024-03-26T08:46:05.701126", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABAQUlEQVR4nO3deXxTVf4//tfNnjRLF7o3XYCy77IIuMsoiwqjH4GPGwXEjXHG4YcL4wiig+AKzOiPmWGkHVRAcJ3RzwgMCiqyySZ7pZa2QEv3Jm3TbPd8/7hNSromaZLbtO/n49FHm5uTm5Omefecc895H44xxkAIISKSiF0BQgihQEQIER0FIkKI6CgQEUJER4GIECI6CkSEENFRICKEiI4CESFEdDKxK9AZPM/j8uXL0Ol04DhO7OoQQpphjMFsNiMpKQkSSdvtnrAORJcvX4bRaBS7GoSQDhQVFSElJaXN+8M6EOl0OgDCi9Tr9SLXhhDSnMlkgtFodH9W2xLWgcjVHdPr9RSICOnCOho6ocFqQojoKBARQkRHgYgQIrqwHiPqrhhjcDgccDqdYleFkHZJpVLIZLJOT5+hQNTF2Gw2FBcXo76+XuyqEOIVjUaDxMREKBQKv89BgagL4Xke+fn5kEqlSEpKgkKhoImapMtijMFms6GsrAz5+fnIzMxsd9JieygQdSE2mw08z8NoNEKj0YhdnW6pzuqA1cFDp5RBLqMh0s5Sq9WQy+UoKCiAzWaDSqXy6zwUiLogf/+rkPbZnTxqrQ5IOA4MlKo9UALx90p/8aTHaLALg/8KmQQKmVTk2pCrUSAiPYbNwQMAlNQl63LoHSGiycrKwowZMwJ6zptuuglPPfVUi+M8Y3DwQndsQGYfrF69GnYnH9DnJv6jQEQCoq0A0FU4nEIQkjRehay1OlBjsYtZpS7vxRdfxIgRI0LyXBSISI/g5IXWj0zCXXWMgfYX7RooEIUJm4Nv88vRrIvRXtnm3ZHWyvgqKysLe/bswdq1a8FxHDiOQ15eHubPn4+MjAyo1Wr0798fa9eubfXxy5cvR2xsLPR6PR577DHYbDavnreurg4PPfQQtFotEhMT8eabb7YoU1paijvvvBPRBh3GDhuAj7du9rjfyTNwHId169ZhypQpUKvV6N27Nz766CN3mQsXLoDjOGzduhXXX3891Go1xowZg9zcXBw6dAijR4+GVqvFlClTUFZW5vXvbcOGDRg8eDCUSiUSExPxm9/8xn1fYWEhpk+fDq1WC71ej5kzZ+LKlSvu+12tlffeew/p6ekwGAyYPXs2zGazuwzP83jttdfQt29fKJVKpKamYsWKFe77n332WfTr1w8ajQa9e/fGCy+8ALtdaCXm5ORg+fLlOH78uPs9zcnJ8fq1+You34eJd7453+Z9Gb0iMGNksvv237/Ng93Z+n/6lCg17h3dlExuw958WGyeS0l+/6t+PtVt7dq1yM3NxZAhQ/DSSy8BAKKiopCSkoJt27YhJiYGP/zwAx555BEkJiZi5syZ7sfu2rULKpUKu3fvxoULFzB37lzExMR4fGDa8vTTT2PPnj34/PPPERcXhz/84Q84cuSIR3ciKysLly9fxjfffAO5XI7f/va3KC0tdXfRnI3jRi+88AJWrVqFtWvX4r333sPs2bNx4sQJDBw40H2uZcuWYc2aNUhNTcW8efNw3333QafTYe3atdBoNJg5cyaWLl2KdevWdVj3devWYdGiRVi1ahWmTJmCmpoa7N27F4AQQFxBaM+ePXA4HFi4cCFmzZqF3bt3u8+Rl5eHzz77DF988QWqqqowc+ZMrFq1yv27W7JkCdavX4/Vq1fjuuuuQ3FxMc6ePet+vE6nQ05ODpKSknDixAksWLAAOp0OzzzzDGbNmoWTJ0/iq6++wn//+18AgMFg6PB1+Y2FsZqaGgaA1dTUiF2VgLBYLOz06dPMYrG0uO+tHefa/Pr0yEWPsn/Zldtm2a2HCj3Krtt9vkUZf9x4443sd7/7XbtlFi5cyO655x737Tlz5rDo6GhWV1fXVJ9165hWq2VOp7Pdc5nNZqZQKNjWrVvdxyoqKpharXbX49y5cwwAO3jwoLvMmTNnGAD2yquvs5IaC6uz2hkA9thjj3mcf9y4cezxxx9njDGWn5/PALB//OMf7vs3b97MALBdu3a5j61cuZL179+/3Xq7JCUlseeff77V+3bs2MGkUikrLGx6r06dOuXxWpYtW8Y0Gg0zmUzuMk8//TQbN24cY4wxk8nElEolW79+vVf1YYyx119/nV1zzTXu28uWLWPDhw/v8HHt/d16+xmlFlGYWHhz3zbvkzRbBfLIDX3aLNt8xci8iRmdqVa73nnnHWzYsAGFhYWwWCyw2WwtBj+HDx/uMYt8/PjxqK2tRVFREdLS0to8d15eHmw2G8aNG+c+Fh0djf79+7tvnzlzBjKZDNdcc4372IABAxAZGeleOuNqEY0fP97j/OPHj8exY8c8jg0bNsz9c3x8PABg6NChHsdKS0vbrLNLaWkpLl++jFtvvbXV+8+cOQOj0eiRBnnQoEGIjIzEmTNnMGbMGABAenq6R+bDxMRE9/OfOXMGVqu1zecAgA8//BB//vOfkZeXh9raWjgcDtESDNIYUZgQJuG1/iWTSrwuK/eibCBs2bIFixcvxvz587Fjxw4cO3YMc+fO9Xr8J5CcPEOpqQEVtVb34LQrePM+jFXL5XL3z65A1vwYz3c8xqZWq71/Ui/r0/z5O3qOffv24f7778fUqVPxxRdf4OjRo3j++edFeX8ACkQkQBQKhUfakr1792LChAl44oknMHLkSPTt2xd5eXktHnf8+HFYLBb37f3790Or1Xa4KUKfPn0gl8tx4MAB97Gqqirk5ua6bw8YMAAOhwM//vgjGIS5RLm5uaiuroaE46BWSN2TG/fv3+9x/v3793uMDwWSTqdDeno6du3a1er9AwcORFFREYqKitzHTp8+jerqagwaNMir58jMzIRarW7zOX744QekpaXh+eefx+jRo5GZmYmCggKPMs3f02CirhkJiPT0dBw4cAAXLlyAVqtFZmYmNm7ciO3btyMjIwPvvfceDh06hIwMz66gzWbD/Pnz8cc//hEXLlzAsmXL8Jvf/KbD9UtarRbz58/H008/jZiYGMTFxeH555/3eFz//v0xefJkPPH443jlzTVQKOR4cckzUKvVkEkl0KuaWhTbtm3D6NGjcd111+GDDz7AwYMH8e677wb2l3SVF198EY899hji4uIwZcoUmM1m7N27F08++SQmTZqEoUOH4v7778eaNWvgcDjwxBNP4MYbb8To0aO9Or9KpcKzzz6LZ555BgqFAhMnTkRZWRlOnTqF+fPnIzMzE4WFhdiyZQvGjBmDL7/8Ep9++qnHOdLT05Gfn49jx44hJSUFOp0OSqUyGL8OcQerHQ4H++Mf/8jS09OZSqVivXv3Zi+99BLjed6rx/ekwequ7ty5c+zaa69larWaAWBnz55lWVlZzGAwsMjISPb444+z5557zmPwc86cOWz69Ols6dKlLCYmhmm1WrZgwQLW0NDg1XOazWb2wAMPMI1Gw+Lj49lrr73WYtC8uLiYTZ4ylSmVSpZiTGUbN25kaWlpbPXq1e4yANg777zDfvWrXzGlUsnS09PZhx9+6L7fNVh99OhR97FvvvmGAWBVVVXuY9nZ2cxgMHj9O/vrX//K+vfvz+RyOUtMTGRPPvmk+76CggJ21113sYiICKbT6di9997LSkpK3Pe3NpC8evVqlpaW5r7tdDrZn/70J5aWlsbkcjlLTU1lr7zyivv+p59+2v17nzVrFlu9erVH/RsaGtg999zDIiMjGQCWnZ3d6usIxGC1qIFoxYoVLCYmhn3xxRcsPz+fbdu2jWm1WrZ27VqvHk+BiHjDZLGxkhoLM1lsHscdTp7ZhCnX7NNPPxWnct1A2F81++GHHzB9+nRMmzYNgNAU3Lx5Mw4ePChmtUg3w/OeyztcKmqtlAykixB1sHrChAnYtWuXe4Dx+PHj+P777zFlypRWy1utVphMJo8v0j0VFhZCq9W2+VVYWOj1uVxzO6XN5jlIms97CKD26v7dd98F7XnDlagtoueeew4mkwkDBgyAVCqF0+nEihUrcP/997dafuXKlVi+fHmIa0nEkJSU1GIeT/P7vSWVcGCMg7RZi0jCAU4AFpsDKnlg8xO1V/fk5OQ27+upRA1EW7duxQcffIBNmzZh8ODBOHbsGJ566ikkJSVhzpw5LcovWbIEixYtct92bWdLuh+ZTIa+fduexOkLg1re6nGhq8bAB2Hha6Dq3lOIGoiefvppPPfcc5g9ezYAYZZqQUEBVq5c2WogUiqVwbt8SHoc16REWoAvPlHHiOrr61vMF5FKpV7NTiWks5pmV1MkEpuoLaI777wTK1asQGpqKgYPHoyjR4/irbfewrx588SsFulG7A4eVRYbZBIJoiM8991yXUWjQCQ+UQPRX/7yF7zwwgt44oknUFpaiqSkJDz66KNYunSpmNUi3YiTMTCGVnftkEs5aBTSFuvvSOiJGoh0Oh3WrFmDNWvWiFkN0o25WjvNr5gBgEImpd08ugj6V0C6Ndfq+uaTGUnXQoGIBERXS57vqg/fLO3H1RhjcPKMdvPoAigQkS4jGLlwXMs7uFZaRIwB5bVWVNbZaMBaZBSIujrGAIct9F8+fDD9TZ7v2tdsxYoVSEpKcmdX/OGHHzBixAioVCqMHj0an332GTiO85itfPLkSUyZMgVarRbx8fF48MEHUV5e3qI+URFKJBjUuFjomWsH8MxWySgQiYryEXV1TjvwXcvdKYLu+v8PkCk6LofOJ8/X6/XYuXMnAGG2/J133ompU6di06ZNKCgoaNHlq66uxi233IKHH34Yq1evhsViwbPPPouZM2fi66+/9qjP00tegJ3nkZrSckmIEDSFmMszgIatxUOBiHSawWCAQqGARqNBQkKC+/jV6wIzMjKwb98+bN261SMQRURE4B//+AcUCiHo/fWvfwXHcVi/fj1UKhUGDRqES5cuYcGCBe7HvP322xg5ciReeeUV97ENGzbAaDQiNzcX/fr1c9cnM6P9JUASjmu8xE8tIjFRIOrqpHKhdSLG83aSN8nzhw4d6g5CAHDu3DkMGzYMKpXKfWzs2LEejzl+/Di++eYbaLXaFs+Zl5eHfv283w6J8yN3NQk8CkRdHcd53UXqSlzJ8998802MHz8eOp0Or7/+ukeOaUBoEfmqtrYWd955J1599dUW9yUmJvp0LgmEha/UIhIXBSISEO0lz3dpLXl+c/3798f7778Pq9XqXuB86NAhjzKjRo3Cxx9/jPT0dMhkrf8JKxQK2B0OlJobIOU4xGhbXyztz24eJPDoqhkJiKuT55eXlyMzMxM//vgjtm/fjtzcXLzwwgstAkpr7rvvPvA8j0ceeQRnzpzB9u3b8cYbbwBougS/cOFCVFZW4n//939x6NAh5OXlYfv27Zg7d647GKanp+PggQMouFCA8vLyNhdSK2SSxmUeNOFRTBSISEAsXrwYUqkUgwYNQmxsLG6//XbcfffdmDVrFsaNG4eKigqP1lFb9Ho9/v3vf+PYsWMYMWIEnn/+effaQ9e4UVJSEvbu3Qun04nbbrsNQ4cOxVNPPYXIyEh3NofFixdDIpXixnEjMSAjpc2MjmqFDDqVnJZ6iIxjYdw5NplMMBgMqKmpEW2HykBqaGhAfn4+MjIyPAZre7oPPvgAc+fORU1NjU+bE1psDpgaHFDKJIjUhN84W7ho7+/W288ojRGRLmfjxo3o3bs3kpOTcfz4cfccIV93SHWN+7Q2q9qFsaYMjdIO9lIjwUOBiHQ5JSUlWLp0KUpKSpCYmIh7770XK1as8Pk87a0zc7E6eNRY7JBLW+YrIqFDgYh0Oc888wyeeeaZTp/HmxaR664wHqHoFqgtSrotKcdBJuFabCN0taYsjaGqFWkNtYi6IPrvHBhalQzaDv7EXSGKfuf+C8TvjlpEXYhcLiyrqK+vF7kmPYerRcRAwchfrr9X19+vP6hF1IVIpVJERkaitLQUAKDRaNod3yCdxxiDvTEPUr2FtduNI54YY6ivr0dpaSkiIyMhlfo/F4sCURfjWr3uCkbEf+YGOzhw0Cil7aaKNTfYwRhQo5QFdRvq7ioyMtIj64I/KBB1MRzHITExEXFxcbDb7WJXJ2xZ7U5sPijMpr7/2rR2d+o4mF8JgKFvsgERCvpI+EIul3eqJeRCv/UuSiqVBuQN7qmsvB0WXlhDpovQtFv2hoEtk6aR0KLBatItWezC4leVnIJ5OKAWEemWGnwIRHYnD6uDh1zKQUmLX0VBLSLSLblaRGovAtGOU1ew/ttfcKbYHOxqkTZQICLdki9dM4VM+BjYHLS/mVgoEJFuSSWTIl6vQlREx5PsKBCJj8aISLc0KEmPQUne5ahSNF7at12V6paEFrWISI9HLSLxUSAiPZ6yMRBZKRCJhrpmpFva9mMRzA0O3D4kAcmR7Wd2VFAgEh0FItIt1VjsMDc42s3O6BKpkWNwkp4yNIqIAhHpllytG2/mEcXpVLhtcOcWbZLOoTEi0u04nLx74JmWeIQHCkSk23G1hjiuaSC6PYwx2Bw8aq2OYFeNtIECEel2XOvMlDKpV4nlbE4e73xzHuu//QUOJw1Yi4ECEel2GtzdMu/+vBVX5SqyUSASheiB6NKlS3jggQcQExMDtVqNoUOH4scffxS7WiSMcQDi9Sr00iq9K89xNKlRZKJeNauqqsLEiRNx88034z//+Q9iY2Px888/IyoqSsxqkTCXFKnGfeNSfXqMUiaBzcFTIBKJqIHo1VdfhdFoRHZ2tvtYRkaGiDUiPRVNahSXqF2zf/3rXxg9ejTuvfdexMXFYeTIkVi/fn2b5a1WK0wmk8cXIYHgGieiQCQOUQPRL7/8gnXr1iEzMxPbt2/H448/jt/+9rf45z//2Wr5lStXwmAwuL+MRmOIa0zCwbe5ZdjwfT5+uljt9WNojEhcHBNxVzmFQoHRo0fjhx9+cB/77W9/i0OHDmHfvn0tylutVlitVvdtk8kEo9GImpoa6PXepXwg3d+XPxUj94oZN/WPxchU78YbDxdUorzWhsFJeqREtZ9sn3jPZDLBYDB0+BkVdYwoMTERgwYN8jg2cOBAfPzxx62WVyqVUCq9uxJCei5f8lW7XJMWHazqEC+I2jWbOHEizp0753EsNzcXaWlpItWIdAcNDtrBI9yIGoh+//vfY//+/XjllVdw/vx5bNq0CX//+9+xcOFCMatFwlyD3bcJjUDTMg+rg7I0ikHUQDRmzBh8+umn2Lx5M4YMGYKXX34Za9aswf333y9mtUiYc3fNfNga6EhhNd755jy+PkNbfYtB9DQgd9xxB+644w6xq0G6CSfP/Fp535S3mq6aiUH0QERIINmdPOL1KjTYnV6tvHehy/fiokBEuhWVXOrz8g7gqkBELSJRiL7olZCugFpE4qJARAiuGiOiQCQK6pqRbuVciRl7z5cjo1cEbh4Q5/XjqEUkLgpEpFupszlQY7G7L+F7SyWXoG+cFgqZBIwxrzI7ksChQES6FX+WdwBCWtk7hycFo0rECzRGRLoVa+OsaqUPs6qJ+OjdIt2Kvy0iQFjmYXfy4HnRElL0WNQ1I92KxY/lHS4b9xWgss6GmWOMHW5TTQLLrxbRL7/8Euh6EBIQ/ix4dZHTJXzR+BWI+vbti5tvvhnvv/8+GhoaAl0nQvymVclgUMuhUbTS2GcMKDwAfL8G2LsWuHTY4265VLhSRoEo9PwKREeOHMGwYcOwaNEiJCQk4NFHH8XBgwcDXTdCfHbX8CTMuy4DCQZVyzsvHgLyvgbsFsBWD+TuAIqPu++muUTi8SsQjRgxAmvXrsXly5exYcMGFBcX47rrrsOQIUPw1ltvoaysLND1JKRz6iuBX/YIP2fcAKSOE34+/18hKKFpe2qbk3IShVqnrprJZDLcfffd2LZtG1599VWcP38eixcvhtFoxEMPPYTi4uJA1ZOQzin4AeAdQFQ6kDYB6H0zoI0DHDahpYSrW0R01SzUOhWIfvzxRzzxxBNITEzEW2+9hcWLFyMvLw87d+7E5cuXMX369EDVk5AOVdfbsOH7fGz7scjzDqsZKD0t/JxxA8BxwlfaBOFY8XGAd0IhFa600Qr80PPr8v1bb72F7OxsnDt3DlOnTsXGjRsxdepUSCRCXMvIyEBOTg7S09MDWVdC2mWxO1FjsaNFe+bKKYB3AoZk4culVz9AEQHY6oCKPPTSJaBPnBYxEYpQVpvAz0C0bt06zJs3D1lZWUhMTGy1TFxcHN59991OVY4QX7R56f7KKeF7wlDP4xIpEDdI6JqVn8OAgf0wIIG2pRKDX4Fo586dSE1NdbeAXBhjKCoqQmpqKhQKBebMmROQShLijVZzVdeVA7WlQtCJHdDyQbH9hEBUcV5oNUlo5w8x+DVG1KdPH5SXl7c4XllZSXvXE9G0uryjrHG7qqgMQN7KbGl9CqDQAPYGwHQJjDE4aYlHyPkViNraHLa2thYqVSvzNwgJgVa7ZpWNqwBi+rT+IIkEiBT20SstzMWfd53HpgMFwawmaYVPXbNFixYBADiOw9KlS6HRNG3N63Q6ceDAAYwYMSKgFSTEWy02VrQ3AKbLws/Rvdt+YFQaUHoGytoi8CwVVprQGHI+BaKjR48CEFpEJ06cgELRdHVBoVBg+PDhWLx4cWBrSIiXlFIJIjVyaJWNf9ZVFwDGA5oYQB3Z9gMbW0TyuhJIeDtsTkpKEWo+BaJvvvkGADB37lysXbsWej1dYSBdx4S+vTChb6+mA+5uWTutIQBQRwEqPWR11dDZrsAsNVKWxhDzK/RnZ2dTECJdX03jxMbI9PbLcRygT4aU46C1loFnDA4asA4pr1tEd999N3JycqDX63H33Xe3W/aTTz7pdMUI6RRrrbC+DPCcxNgWfRIkpaehswtbTtscvDstCAk+rwORwWBwN1UNBkPQKkSIv97bXwApx+GO4YnQmy8JByN6tX7Zvjl9EjhwMNjLACZsWx2hDG59SROvA1F2dnarPxPSFTh5hnKzFQAgl0iAale3zMtdX7XxACdBrNKB/lEMEhofCim/ZlZbLBYwxtyX7wsKCvDpp59i0KBBuO222wJaQUK84ZrMyHGN6Txc40OGFO9OIJUD2jhkMh6ZqTygkQeppqQ1fnWCp0+fjo0bNwIAqqurMXbsWLz55puYPn061q1bF9AKEuINy1WzqiW8XVjWAXgfiABA37idkJnS14Sa3xkar7/+egDARx99hISEBBQUFGDjxo3485//HNAKEuINi00IRGq5FDBdEuYPqQzCl7e0ws6wrLa0zdUDJDj8CkT19fXQ6XQAgB07duDuu++GRCLBtddei4ICmh5PQs/VNVPLpU0tGr2PGyZq43G+rBZ7j5/G8aLqwFaQtMvv5PmfffYZioqKsH37dve4UGlpKc0vIqJwd80UVwUiXespatoUEQuAg9RhgaPBHNgKknb5FYiWLl2KxYsXIz09HePGjcP48eMBCK2jkSNHBrSChHhDwnGI1MihU8kAc4lwUJfg20mkcvDqaOFn85XAVpC0y6+rZv/zP/+D6667DsXFxRg+fLj7+K233opf//rXAascId4akmzAkGSDkG3xskk46GsgAsAiYgFcBldHgSiU/N7pNSEhAQkJnm/02LFjO10hQjrF1RrSRAMy32ckssYBa0l9y3xbJHj8CkR1dXVYtWoVdu3ahdLSUvC8Z9oE2gmWiMbfblkjLkJ4nLSetsQKJb8C0cMPP4w9e/bgwQcfRGJiIq1SJqL7/Ngl1FodmMoVIgrwfaC6EacTWkTShirAaRcmOpKg8ysQ/ec//8GXX36JiRMnBqwiq1atwpIlS/C73/0Oa9asCdh5Sc9QUWtDjcUOifQKwMHvFlGEzgCrVosIqUNYNKuLD2xFSav8umoWFRWF6OjogFXi0KFD+Nvf/oZhw4YF7JykZ7HYnZA766Fw1ArrPLT+BZDkKA2GZvZBekwEQONEIeNXIHr55ZexdOlS1NfXd7oCtbW1uP/++7F+/XpERUV1+nyk53E4eWG1vK0cciknZGT0Y6DaLaIxuVodjROFil9dszfffBN5eXmIj49Heno65HLPfvSRI0e8PtfChQsxbdo0TJo0CX/605/aLWu1WmG1Wt23TSaTbxUn3VJDY45pnb0cUon/rSG3iFjhex21iELFr0A0Y8aMgDz5li1bcOTIERw6dMir8itXrsTy5csD8tyk+3CtM4t0VoAD5/dANSAsFfn4ZB36Xq7AGHVU5/ZkJ17zKxAtW7as009cVFSE3/3ud9i5c6fXWxAtWbLEvZMIILSIjEZjp+tCwptrnZnBWQFA7vdANQDIpRJUMT14BjjrKiFxOgCp39PtiJf8/g1XV1fjo48+Ql5eHp5++mlER0fjyJEjiI+PR3Jyx6k5Dx8+jNLSUowaNcp9zOl04ttvv8Xbb78Nq9UKqdRz102lUgmlktLmEU+MAb0UNuhgAThFp7pmUgkHJo+AU6IQ5sdZKt2r8knw+BWIfvrpJ0yaNAkGgwEXLlzAggULEB0djU8++QSFhYXuXEXtufXWW3HixAmPY3PnzsWAAQPw7LPPtghChLQlNUaDBwerAF7fOFCt6PhB7VDIpaiXR8HJNwgD1hSIgs6vQLRo0SJkZWXhtddec6cDAYCpU6fivvvu8+ocOp0OQ4YM8TgWERGBmJiYFscJ6ZB7xb3/3TIXhUwCizwKTv4yDViHiF9jcYcOHcKjjz7a4nhycjJKSko6XSlCfOZe2uH/QLWLQiZpbBExmksUIn61iJRKZauXznNzcxEbG+t3ZXbv3u33Y0nP9fXZKzCcPoMMHUN0AFpEcqkEtbIoOBkD6ioCUEPSEb9aRHfddRdeeukl2O12AADHcSgsLMSzzz6Le+65J6AVJKQj1dVVsNWZwDN0fg4RgDidElGxicKcJEsV4HR0vpKkXX4FojfffBO1tbWIjY2FxWLBjTfeiL59+0Kn02HFihWBriMh7eIau2USbVxAFqne1D8Ov752ACJ1WiH3taWq0+ck7fOra2YwGLBz507s3bsXx48fR21tLUaNGoVJkyYFun6EdIirFQaqZYbOd8uaTsoBml6A6bIwTqT1f8iBdMznQMTzPHJycvDJJ5/gwoUL4DgOGRkZSEhIAGOMUoKQkHLyDMp6YesgRZQXW0v7IqIxENGVs6DzqWvGGMNdd92Fhx9+GJcuXcLQoUMxePBgFBQUICsri9LEkpCrt9oRYSsHB0AZ5eOuHW04VlSNv+3Jw+Gyxn+q9TRgHWw+tYhycnLw7bffYteuXbj55ps97vv6668xY8YMbNy4EQ899FBAK0lIWyy11ZDzFshkUnABGKgGhH+49TYnzJpI4QBdwg86n1pEmzdvxh/+8IcWQQgAbrnlFjz33HP44IMPAlY5QjpkLoZaLgGnjQ1YNkWFTPhY1MoaN2esrwKapUMmgeVTIPrpp58wefLkNu+fMmUKjh8/3ulKEeKtOFRhhDEK1wweFLBzKhsDUT2nBSQygHcADdUBOz9pyadAVFlZifj4tpu/8fHxqKqiS50khDqZLL81cqnwsbDyEHYDAWicKMh8CkROpxMyWdvDSlKpFA4HTf4iIcKY/7u6tsPVNbM5+KuyNdI4UTD5NFjNGENWVlabqTiuzp5ISNA1VONCSQVqrDw0A1TIDNBu5wrpVYFIEyMcpAHroPIpEM2ZM6fDMnTFjISMuQQWmxNlvAFJLHBpY1RyKeL0SqhkUmFSI0BdsyDzKRBlZ2cHqx6E+M5cDLuTR60iFhpF4AJRhFKG+8elCTdcXbK6cqErSBN2g4JS8pLwZS6BzcmjThGLCGWQ0rmqowBOImy2aKXNGoKFAhEJT4yBNxXD7mSoVfSCNliBSCIVghFA3bMgokBEwlNdOWxWC5ycHFZlDFTywP4pbz1UhL/tyUOpqQGIaBywptxEQUOBiIQn00XYHTzqlLGIUCkCvti6weFEvc0Jq4O/asCarpwFCwUiEp5qLgmJ0PTJiNIEZmnH1VyX8K0el/CpRRQstGETCU+myzCo5bh96DVAr5SAn14lF67CNdidgJ4mNQYbtYhI+LHVN7VO9IFJ/dGca72Zu0XEcYDdAtjqgvJ8PR0FIhJ+TJeF75poQKEJylO4WkRWu1NY1a9snLZN3bOgoEBEwo/pEgDgxyoNPjhQgKLK+oA/hUeLCKA1Z0FGgYiEn+pCAEAxeqHUFJz1jXq1HLE6ZdOMbRqwDioarCbhxWFzr7i/IhG2gg7GrOohyQYMSTY0HaBAFFTUIiLhxXQR4J1wKLQwQwsAiFAGbp1Zm6hrFlQUiEh4qSoAANRrUgCOg0ouhVIWgkDkmtRoNQMOSncTaBSISHhpHB8yKYVEaDpVcEYXquttyNmbj437LggH5CpAKbTAqHsWeBSISPhwWN2pYSuVwvwhvTrws6oBQCLhUFVvR3W9HYwx4aBrnIi6ZwFHgYiEj8p8YQtoTTQkKgOiIxSI1iiC8lSqxu6ek2ewO12BiNacBQtdNSPho+K88D2mT8urWgEml3KQcBx4xmB1OIU81q5V+PWVQXvenopaRCQ8MAZU5gk/x/QN+tNxHOdOLdJgb5zUSF2zoKFARMKDuVhYYyZTAAZjSJ6yaXa1Uzjg6po1VAsZG0nAUCAi4aH0jPA9ujccjMNf9+ThgwMFwk4bQdK0Ar/xORQRwtUzxqh7FmAUiEjXx1hTIIobhBqLHRabE9X1dsilwUtmHxWhQKxOCZmk8Tk4jmZYBwkNVpOur6ZImEgoUwDRfVBV0QAAiNIEPjPj1W4f3MrusZpeQM0lunIWYNQiIl1fyUnhe+wAQCpDdb0NAIKSmbFDtNQjKCgQka7N3gCUnhJ+ThgKAKisEwJRZJDmELWLumZBIWogWrlyJcaMGQOdToe4uDjMmDED586dE7NKpKsp+QlwOoSWSOPVsup64YpVVERwW0RnS0zI2ZuPr89eaTroCkSWKoAP3kB5TyNqINqzZw8WLlyI/fv3Y+fOnbDb7bjttttQV0fpOAmEAFR0UPg5+Rr3LqtVjV2zYM2qduF5uJd5uKkMgFQG8E4hGJGAEHWw+quvvvK4nZOTg7i4OBw+fBg33HCDSLUiXcblo8IgtVIHJAwDICy5SDCoUFlngyHIY0TqxqRoFruz6SDHCQPW5hKgrqxptjXplC511aympgYAEB0d3er9VqsVVmtTCgaTibYA7rasZuDCd8LPaROEVggAqYTD9BHJIamCunEekcXm9LxDGy8EotorQNyAkNSlu+syg9U8z+Opp57CxIkTMWTIkFbLrFy5EgaDwf1lNIZmhi0JMcaA3O3CantdApA4QpRquFtENmfTCnxACEQAUFsqQq26py4TiBYuXIiTJ09iy5YtbZZZsmQJampq3F9FRUUhrCEJCcaAX3YD5T8L+873nwpImv5MrY5mQSGIXC0ix9Ur8AFAK6SoRe2VVh5F/NEluma/+c1v8MUXX+Dbb79FSkrbm+UplUoolcoQ1oyElL0B+OUb4PIx4Xa/yYAu3qPIx4cvwdRgxx3DEpESFZythFzkUg4yCQcHz2CxNa7AB5oCkdUsrH8L0pZGPYmogYgxhieffBKffvopdu/ejYyMDDGrQ8TCGHDlFJD3ddMGhpm/AhKHeRRz8gwVtVY4eAZtEBLmN8dxHGJ1SjgZg/PqVphMCagjAUs1UFcKKNKDXpfuTtRAtHDhQmzatAmff/45dDodSkqE7HsGgwFqtVrMqpFQqSsXxoMaU8BCEy0EoejeLYqWmYUgpJJLYQhSZsbmZo9Nbf0ObZwQiGpLgaj0kNSlOxM1EK1btw4AcNNNN3kcz87ORlZWVugrREKH54Gi/cCF74U5OVIZkDYRSBnrvkLW3KVqYSPFpEhVUNeYeUUbD5Tl0jhRgIjeNSM9kMMGnPmXMCANCInOMn8ldHfacbHKAgBIieoCreUIGrAOpC4xWE16EKcDOPmRsC2QRAb0u02YrNhBC8fh5K8KRKEbHD5SWIWfiqoxIFGPa3tfNXnRNWBdXym06CQh2NKoG+syl+9JD8AYcPYLIQhJ5cDw2UDi8A6DEAAUVtbD5uChU8kQpwvdlVO7g0dVvR0mS7OMjCqDMGjNO2klfgBQICKhU3xMSHAmkQJD7gEzeE7VOPBLBXaduYIGu7PFQ6M0CozNiMZwY2RIx4c0CqHTYGleJ467amJjScjq011RICKh0WASLs8DQO+bUKVMxoeHilBrdQAAeJ7hxKUa/HSxBu/tK0BRZb3Hw6MiFJjYtxfGpLe+/CdY1ArhI9JimQcgzPoGAFNxCGvUPVEgIsHnXrJhA/RJqI8bgU+PXkJxTQO+yy0DIDQwbh+cgCiNHLVWBz4+chHf/1wOq8PZagspVFwtorrWApFe2OQR5sshrFH3RIGIBN+VU8KeZBIpWP+p+OpUKWosdhjUctzQLxaAMHnQGK3BfePSMCTZAMaAQxcq8f9/k4ctBwtR03yMJkQiGidO1lsdLa/y6oRtr1FbRrt6dBIFIhJc1lrg/E7h57SJOFYpQ0FFPWQSDtNHJLk/6C4KmQS/GhSPO4YlIkIpXImyOXl3ethQi1A0rTezNt8xRGUQlncwni7jdxJdvifBdX6nsIZMG4fy6JH4/tAlAMAN/WIRo2376ldmvA5947SotToQoZBBIhFnAqNMKkEvrQJSiQQ2J+/eYgiA0J/UJwvzoUzFgKHtdZKkfRSISPCUnQNKzwKcBM5+U/HV6TI4eIaMXhEYltLxdtEcx0GnEiFBfjMPjk9v+05dohCIaJyoU6hrRoLDbhEGqAEg9VrUK3uBQdi0cNKgePGXaASKvnGciK6cdQq1iEhwnN8lrKSP6AWkTYROKsN9Y1NRUWcNycr5kHENWFuqKCVIJ1CLiARe+c9AyQmA48D6TfZI8xqnU4lcOd+duFiD7L352NM41cCDXC1kDAAAE3XP/EWBiDRhDKirEFJy2PzcScVWD5z7j/BzyhjsuaLED3nl4PnwXeDMM4bqenvbUwgatzlCTWHoKtXNdKM2MukUc4kQQMxXLVfolQn0uaXpP35HGAPO/Z+7S3ZOMwpHTwmtiLSYCCRHdoFV835wTTGoa5wF3kKkESg+3pRTifiMAhEBqi4AJ7YJK+MlUmH7Hku10MWqyheCUdKojhenXvjenWu63Hgb/ntW2A11THp02AYhAO4xrdqGtgJRY/I08xUh4b+M0hn7igJRT1dfCZz4SAhC0RnAwDsBRYSwovznnUKQyt0hfO8/VRgTac3lo0IgAmBOnYRPfnbA5uBhjNZgQp/w3vtLr24MRFYHHE4eMmmzEQ2VQfhqqAFqLgIxfUSoZXijMaKejHcCpz8TlidEGoEh/yMEIUC42jV8NtB3ktBKKssFftwAVDfbOYXngfzvgHPCZpmm2FH48GIk6qxO9NIpccewRNEmIwaKWi51J843ddQqqqGdZfxBLaKe7OIhmMsvwsIUMPSbBmXzFK0cBxjHCDOGT38uXKI++r7wHz8yVeiGlJ0VWlUAnCljsbU4HWarAzFaBX49MtlzJnKY4jgOerUc5WYraix2REe0stV1pFG4UkjjRH6hQNRTWc3Ahe9RXW/HbskomA9V4NrewIiUyJYtGH0iMHqu0FW7chKoyBO+XORqoO8kSBOGYHxEDU5eqsFdw5PdGxR2B3E6JWQSDm027lwtIlOxsKRFHn7TFMREgaiHKTU3QCmTwpD/DeC0QxZlhFM+CFaLA3vOleFciRmTBsYjtnkWRJkSGHgHkDoeKD8H1JWh1g6crdcjMm0o+ib0AgAMStRjYII+7Ltjzd0+OKH9AuooQBMD1FcI42m0FbVPKBD1IA12J/517DJU9SX4NY4jQilH4qg78ZA2Aacum/Dd+TKU1DRg04FCjMmIwtj06BYDs051NPLVw3Cqqgb55XVgDNDlmZAWHw25VAKO47zJ/No9xfQWAlFlHgUiH1Eg6kG+/7kcZosdA2v2QxkjARKGAvpESAAMTTEgIzYCX58tRV5pLQ78Ugknz3B9ppAv6EJ5Hc6WmJBfXu+RqCyjVwRGGCMh62YtoLYwxtpeJxfdByg6JHRbGfMqFzcRUCDqIUpqGnDiUg2iLfkYqquFTK4G0q/3KKNVynDnsEScL63F/vxKjE5rmsj4S3ktzhSb3eX6J+gwJNnQ+sBtN2RqsOPjwxdhdfB49IberQejyFRhUwBbnTAx1LUglnSIAlEPse+XcnDMibH8T9Cr5IBxHKDStyjHcZw7F9DVH7bkSA1UMimM0RokR6q73RhQRyIUMpgsDvCMwWx1CL/D5iRSYS5WWS5QnkuByAc0j6gHKKlpwIXyeiTWnkFGhE2YK2S8tt3HNP+P3z9Bhwl9e8EYrelxQQgQFuxGRwjBp6K2nWyRsY1jQ6VnhO4Z8QoFoh7gQH4FFI5ajORPCPN6Mq4HZD2jSxVIroySFbXWdgplCtkGLFWAmXIUeYsCUTfH8wwRChn61OyDUS8Vdp5IGC52tcJSTON4WHl7LSKZQghGAFB6OgS16h4oEHVzEgmHSdGlmBRbDbVCDvSfAkjobfeHq0VU3l6LCADiBwvfr5wWltGQDtFfZHdXWwbkfgWZRAKkT2zas534LE7v6prZYGu+o8fVonsL43C2OiFvN+kQBaJu7FjuBdQeel9YWR+VDqROELtKYU2nlCElSo2BiTrYne0EIokUSBop/HzpcGgqF+YoEHVTFSWFqPlhA07+cgkWRRQwaDp1yTqJ4zjcO9qI2wYntNiPrYWkEQAnEdKCUArZDtFfZnfDGHDxR5R/+y4UznpooxOgvuY+Suoeakpd01hR/nfi1iUMUCDqTmouAkc2ou7Uf1BprkO1KgUJNy9odeIi8R/PM1yqtrTfPQOEMTlOAlT+AlQVhKZyYYoCUXdgqQJOfQoceQ/MdAn5VXbkR00EP2QmekV2vJEh8c2HPxZh66EiFFTUt19QHQUkNk6V+HmHMFZHWkWBKJzZG4T9ww6ub9xRlUOxuj92G36NCsMQXNe4YJUEVoJByDV0vrS248IZNwjd4rpyoGBvkGsWvigQhSPeCVw8DBz4K1B0ULgdlY6G4Q/hC9sI2GURuCYtCgaN+Ns1d0cDEnQAgJ+vmFFv66CVo9AAfX8l/Fy4T9hcgLRAgSicMCb8IR96V2jq2y1CbulhM4Hhs6GMTMT43r2QYFBhTIaXWwARnyXoVYjXq+DgGY4VVXf8gPhBQPIo4f07/bmQOI146BKB6J133kF6ejpUKhXGjRuHgwcPil2lroUxIcfN0feEHTfqK4T/tP1uB0bPF3JIcxw4jsPQFANmjzFC3nynCRIwHMdhdHoUAODwhSpU17ez5MOl7yRhoqPTDvy0Tdj1hBbFuon+1/rhhx9i0aJFWLZsGY4cOYLhw4fj9ttvR2lpqdhVE5/dIkyIO5wN/LQVqLkESGRA2nhg3GNA8ijUWJ346mQxzA1Nu5C2mbiLBExmnBbGaA0cPMO/jl9uf6Y1IExyHHKPsGkl7xB2PTn2gXA1jQISOMbE/S2MGzcOY8aMwdtvvw0A4HkeRqMRTz75JJ577rl2H2symWAwGFBTUwO9PowvUfNOwF4vLAmorwDqyoTdIEzFAGv8A5fIgOSRcCSNQR00uGJuwC9ldfj5ihkOniE5Uo17R6dQEAohU4MdWw8VIV6vwh3DEsFxHBxOHhzHQdpWqhTGgKIDwtwivnF8SR0l5DHSJwu76qoMgEzdLSagevsZFTUxms1mw+HDh7FkyRL3MYlEgkmTJmHfvn0tylutVlitTQsOTSaTd09UmQ/k7Wr9P4/HMdbK8daOtV/2aGE1nDzvcT9rvE+nlDUOdgq3jxVWgrdbW54KDBqFDAMz+wir5eMH4/0jZSjLv9LiJaREqXHLgDgKQiGmV8lxz6gUqBVS9+8+r6wO/zlZDJVcCplECEhXB6Wb+8fBmHotEDcQJT/tQvHPRyDhqwHktzi/MTYS0boIQCJDVb0dpbU29E/QA+Ca0tCK+Z4PnRmwOWqiBqLy8nI4nU7Ex8d7HI+Pj8fZs2dblF+5ciWWL1/u+xM5rMLizxCxW0xw8q03NJ2QAfamX7vD1gC7kwHgYJeo0CA3wCKPRK0iDvLYDAwcM9Rd1trY/JdJOERFKJAcpcbABD3i9UoKQiKJapYqt9ZqB2OAxdb6qnvXewiVAVXGSfi2qj8iGy5CZy2G1lYOpcMEhVOYn8TbrUDj8BOzWMHXWYH6LjQXiXXQHfVBWKWKXbJkCRYtWuS+bTKZYDQaO35gpFHYtRS46j/IVR9cjw9xa8d9K5vc19KirGt3C7lUAmib9rxKG2SDU6YGJ1MJA85XnVrerGl+z6hkKGVSqOQSCjxd1KjUKAxM1KPe5gTPMzh45vFPqZe2aZum1GgN7h6TASADwFWtYd4JzmmFWgVA6gSYE2qbHbFWBxChaAwATPyxJdeuwAEgaiDq1asXpFIprlzx7G5cuXIFCQkt95FSKpVQKpUtjndIESH0wUMkWed92Vgf3stIDWVV7Oo4joNGIYNG0fFHK0Ipa2fxrOcfkbrxq7sSdTRMoVDgmmuuwa5du9zHeJ7Hrl27MH78eBFrRggJJdG7ZosWLcKcOXMwevRojB07FmvWrEFdXR3mzp0rdtUIISEieiCaNWsWysrKsHTpUpSUlGDEiBH46quvWgxgE0K6L9HnEXVGt5lHREg35e1nNPxnTBFCwh4FIkKI6CgQEUJEJ/pgdWe4hre8XupBCAkp12ezo6HosA5EZrMZALybXU0IEY3ZbIbB0Hba4rC+asbzPC5fvgydTtcllzy4lqAUFRV1u6t69NrCU6hfG2MMZrMZSUlJkLSTTSCsW0QSiQQpKSliV6NDer2+2/1Bu9BrC0+hfG3ttYRcaLCaECI6CkSEENFRIAoipVKJZcuW+ZcxoIuj1xaeuuprC+vBakJI90AtIkKI6CgQEUJER4GIECI6CkSEENFRIPKRr7vSbtu2DQMGDIBKpcLQoUPxf//3fx73M8awdOlSJCYmQq1WY9KkSfj5Z3H2R/flta1fvx7XX389oqKiEBUVhUmTJrUon5WV1bhpQNPX5MmTg/0yWuXLa8vJyWlRb5VK5VEmXN+3m266qcVr4zgO06ZNc5cR5X1jxGtbtmxhCoWCbdiwgZ06dYotWLCARUZGsitXrrRafu/evUwqlbLXXnuNnT59mv3xj39kcrmcnThxwl1m1apVzGAwsM8++4wdP36c3XXXXSwjI4NZLJZQvSzGmO+v7b777mPvvPMOO3r0KDtz5gzLyspiBoOBXbx40V1mzpw5bPLkyay4uNj9VVlZGaqX5Obra8vOzmZ6vd6j3iUlJR5lwvV9q6io8HhdJ0+eZFKplGVnZ7vLiPG+USDywdixY9nChQvdt51OJ0tKSmIrV65stfzMmTPZtGnTPI6NGzeOPfroo4wxxnieZwkJCez11193319dXc2USiXbvHlzEF5B23x9bc05HA6m0+nYP//5T/exOXPmsOnTpwe6qj7z9bVlZ2czg8HQ5vm60/u2evVqptPpWG1trfuYGO8bdc285NqVdtKkSe5j7e1KCwD79u3zKA8At99+u7t8fn4+SkpKPMoYDAaMGzeuzXMGgz+vrbn6+nrY7XZER0d7HN+9ezfi4uLQv39/PP7446ioqAho3Tvi72urra1FWloajEYjpk+fjlOnTrnv607v27vvvovZs2cjIsJzX6tQv28UiLzU3q60JSUlrT6mpKSk3fKu776cMxj8eW3NPfvss0hKSvL4UEyePBkbN27Erl278Oqrr2LPnj2YMmUKnM7Wd0ENBn9eW//+/bFhwwZ8/vnneP/998HzPCZMmICLFy8C6D7v28GDB3Hy5Ek8/PDDHsfFeN/CevU96RpWrVqFLVu2YPfu3R6DurNnz3b/PHToUAwbNgx9+vTB7t27ceutt4pRVa+MHz/eY1+9CRMmYODAgfjb3/6Gl19+WcSaBda7776LoUOHYuzYsR7HxXjfqEXkJV93pQWAhISEdsu7vvtyzmDw57W5vPHGG1i1ahV27NiBYcOGtVu2d+/e6NWrF86fP9/pOnurM6/NRS6XY+TIke56d4f3ra6uDlu2bMH8+fM7fJ5QvG8UiLzkz66048eP9ygPADt37nSXz8jIQEJCgkcZk8mEAwcOhHSnW3933H3ttdfw8ssv46uvvsLo0aM7fJ6LFy+ioqICiYmJAam3NwKxm7DT6cSJEyfc9Q739w0QppVYrVY88MADHT5PSN63kA6Nh7ktW7YwpVLJcnJy2OnTp9kjjzzCIiMj3Zd2H3zwQfbcc8+5y+/du5fJZDL2xhtvsDNnzrBly5a1evk+MjKSff755+ynn35i06dPF+0ysC+vbdWqVUyhULCPPvrI4zKv2WxmjDFmNpvZ4sWL2b59+1h+fj7773//y0aNGsUyMzNZQ0NDl35ty5cvZ9u3b2d5eXns8OHDbPbs2UylUrFTp055vP5wfN9crrvuOjZr1qwWx8V63ygQ+egvf/kLS01NZQqFgo0dO5bt37/ffd+NN97I5syZ41F+69atrF+/fkyhULDBgwezL7/80uN+nufZCy+8wOLj45lSqWS33norO3fuXCheSgu+vLa0tDQGoMXXsmXLGGOM1dfXs9tuu43FxsYyuVzO0tLS2IIFC1rMxwkVX17bU0895S4bHx/Ppk6dyo4cOeJxvnB93xhj7OzZswwA27FjR4tzifW+URoQQojoaIyIECI6CkSEENFRICKEiI4CESFEdBSICCGio0BECBEdBSJCiOgoEJGwkpOTg8jISPftF198ESNGjHDfzsrKwowZM0JeL9I5FIhIq1zpQh977LEW9y1cuBAcxyErK8ujfKADQHp6OtasWeNxbNasWcjNzW3zMWvXrkVOTo779k033YSnnnoqoPUigUeBiLTJaDRiy5YtsFgs7mMNDQ3YtGkTUlNTRamTWq1GXFxcm/cbDAaPFhMJDxSISJtGjRoFo9GITz75xH3sk08+QWpqKkaOHNmpc7fWUpkxY4a7lXXTTTehoKAAv//9790J3IGWXbPmrm6ZZWVlYc+ePVi7dq37HPn5+ejbty/eeOMNj8cdO3YMHMeFNEUJaUKBiLRr3rx5yM7Odt/esGED5s6dG/Tn/eSTT5CSkoKXXnoJxcXFKC4u9vkca9euxfjx47FgwQL3OVJTU1u8JgDIzs7GDTfcgL59+wbqJRAfUCAi7XrggQfw/fffo6CgAAUFBdi7d69XOWw6Kzo6GlKpFDqdDgkJCX4lHDMYDFAoFNBoNO5zSKVSZGVl4dy5c+5td+x2OzZt2oR58+YF+mUQL1GqWNKu2NhYTJs2DTk5OWCMYdq0aejVq5fY1eqUpKQkTJs2DRs2bMDYsWPx73//G1arFffee6/YVeuxqEVEOjRv3jzk5OTgn//8Z8BaDRKJBM0z0Njt9oCc2xsPP/yweyA+Ozsbs2bNgkajCdnzE08UiEiHJk+eDJvNBrvdjttvvz0g54yNjfUY93E6nTh58qRHGYVC0emdI9o6x9SpUxEREYF169bhq6++om6ZyKhrRjoklUpx5swZ989tqampwbFjxzyOxcTEwGg0tih7yy23YNGiRfjyyy/Rp08fvPXWW6iurvYok56ejm+//RazZ8+GUqn0q0uYnp6OAwcO4MKFC9BqtYiOjoZEInGPFS1ZsgSZmZkhzTVNWqIWEfGKXq+HXq9vt8zu3bsxcuRIj6/ly5e3WnbevHmYM2cOHnroIdx4443o3bs3br75Zo8yL730Ei5cuIA+ffogNjbWr3ovXrwYUqkUgwYNQmxsLAoLC933zZ8/HzabLSRXAUn7KFUs6bG+++473HrrrSgqKmqxSSEJLQpEpMexWq0oKyvDnDlzkJCQgA8++EDsKvV41DUjPc7mzZuRlpaG6upqvPbaa2JXh4BaRISQLoBaRIQQ0VEgIoSIjgIRIUR0FIgIIaKjQEQIER0FIkKI6CgQEUJER4GIECI6CkSEENH9P2Yi2XcLg/2gAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T08:46:06.083016Z", + "iopub.status.busy": "2024-03-26T08:46:06.082012Z", + "iopub.status.idle": "2024-03-26T08:46:06.307788Z", + "shell.execute_reply": "2024-03-26T08:46:06.306819Z" + }, + "papermill": { + "duration": 0.247357, + "end_time": "2024-03-26T08:46:06.309975", + "exception": false, + "start_time": "2024-03-26T08:46:06.062618", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEnCAYAAAD4jcW1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAw5UlEQVR4nO3dfVwU5fo/8M8usAvLAoooIAErouBDgkIqnvyqHdA0DY9WVBZIxjlidDxtiofTN4i0yFQ0y+R7LDTNo5bHzKOmEmqnEp8gDBUh8AFUHkV5Wt1d2Pv3hz8m111g0YXd2b3erxcvd+65Z+YaZrmcuWfmvgWMMQZCCOEZoakDIISQB0HJixDCS5S8CCG8RMmLEMJLlLwIIbxEyYsQwkuUvAghvETJixDCS5S8CCG8ZGvqAHqaRqPB9evX4eTkBIFAYOpwCCH3YYyhsbER/fv3h1DYwfkVM7FPPvmE+fr6MrFYzEaPHs1OnDjRYf2bN2+yBQsWMA8PDyYSidigQYPYvn37DN5eeXk5A0A/9EM/Zv5TXl7e4d+ySc+8duzYAblcjoyMDIwZMwZr1qzBlClTUFRUhH79+unUV6lUiIiIQL9+/bBz5054eXnhypUr6NWrl8HbdHJyAgCUl5fD2dnZWLtidtRqNQ4dOoTJkyfDzs7O1OEQI7CWY9rQ0ABvb2/ub7U9Jk1e6enpiIuLQ2xsLAAgIyMD+/btQ2ZmJv7+97/r1M/MzERdXR2OHTvGHTyZTNalbbZdKjo7O1t88pJIJHB2drboL7o1sbZj2lmzjsmSl0qlQm5uLpKSkrgyoVCI8PBw5OTk6F1mz549CAsLw2uvvYZvv/0Wffv2xYsvvoglS5bAxsZG7zJKpRJKpZKbbmhoAHD3i6BWq424R+albd8seR+tjbUcU0P3z2TJq7a2Fq2trXB3d9cqd3d3x4ULF/Quc/HiRRw+fBhz5szB/v37UVJSggULFkCtViMlJUXvMmlpaUhNTdUpP3ToECQSycPviJnLysoydQjEyCz9mCoUCoPq8epuo0ajQb9+/fDPf/4TNjY2CAkJwbVr17BixYp2k1dSUhLkcjk33XY9PXnyZIu/bMzKykJERIRVXGJYA2s5pm1XR50xWfJyc3ODjY0NqqqqtMqrqqrg4eGhdxlPT0/Y2dlpXSIOGTIElZWVUKlUEIlEOsuIxWKIxWKdcjs7O4v+ArSxlv20JpZ+TA3dN5M9pCoSiRASEoLs7GyuTKPRIDs7G2FhYXqX+cMf/oCSkhJoNBqurLi4GJ6ennoTFyHEcpn0CXu5XI4NGzbgiy++QGFhIeLj49Hc3MzdfYyOjtZq0I+Pj0ddXR0WLlyI4uJi7Nu3D++//z5ee+01U+0CIcRETNrmFRUVhZqaGiQnJ6OyshLBwcE4cOAA14hfVlam9YStt7c3Dh48iDfeeAMjRoyAl5cXFi5ciCVLlphqFwjpFgqFQufGVdNtJY4VlKK322lIHX5vCgkMDLSKm0/3EzBmXQNwNDQ0wMXFBfX19RbfYL9//35MmzbNottHLFVeXh5CQkIMqpubm4tRo0Z1c0Q9x9C/UV7dbSTEWgQGBiI3N1errKjiFuRfFyD92UcR4NlLq641ouRFiBmSSCQ6Z1PCKzcg/vE2hgwPQrBvHxNFZj6oSxxCCC9R8iKE8BIlL0IIL1HyIoTwEiUvQggvUfIihPASJS9CCC9R8iKE8BIlL0IIL1HyIoTwEiUvQggvUfIihPASJS9CCC9R8iKE8BIlL0IIL1HyIoTwEiUvQggvUfIihPASJS9CCC9R8iKE8BIlL0IIL1HyIoTwEiUvQggvUfIihPASJS9CCC9R8iKE8BIlL0IIL1HyIoTwEiUvQggvUfIihPASJS9CCC+ZRfJat24dZDIZ7O3tMWbMGJw8ebLdups2bYJAIND6sbe378FoCSHmwOTJa8eOHZDL5UhJSUFeXh6CgoIwZcoUVFdXt7uMs7MzKioquJ8rV670YMSEEHNg8uSVnp6OuLg4xMbGYujQocjIyIBEIkFmZma7ywgEAnh4eHA/7u7uPRgxIcQc2Jpy4yqVCrm5uUhKSuLKhEIhwsPDkZOT0+5yTU1N8PX1hUajwahRo/D+++9j2LBheusqlUoolUpuuqGhAQCgVquhVquNtCemp1AoUFRUxE033VbiWEEpnHodh9RBrFU3ICAAEomkp0MkD6mlpYX715K+u/czdN9Mmrxqa2vR2tqqc+bk7u6OCxcu6F0mICAAmZmZGDFiBOrr67Fy5UqMGzcO586dwyOPPKJTPy0tDampqTrlhw4dsqg/4NLSUrz55ps65R/qqbtq1SoMHDiw+4MiRlXeBAC2OH78OK6dNXU03UehUBhUz6TJ60GEhYUhLCyMmx43bhyGDBmC//u//8PSpUt16iclJUEul3PTDQ0N8Pb2xuTJk+Hs7NwjMfcEhUKBxx9/nJsurqjH4m/OY8WfhmKwp4tWXTrz4qczZXVAwWmMHTsWQT6upg6n27RdHXXGpMnLzc0NNjY2qKqq0iqvqqqCh4eHQeuws7PDyJEjUVJSone+WCyGWCzWKbezs4OdnV3XgzZTLi4uGD16NDctunID4hwVhgePQrBvHxNGRozF1taW+9eSvrv3M3TfTNpgLxKJEBISguzsbK5Mo9EgOztb6+yqI62trSgoKICnp2d3hUkIMUMmv2yUy+WIiYlBaGgoRo8ejTVr1qC5uRmxsbEAgOjoaHh5eSEtLQ0A8O6772Ls2LHw9/fHrVu3sGLFCly5cgWvvvqqKXeDENLDTJ68oqKiUFNTg+TkZFRWViI4OBgHDhzgGvHLysogFP5+gnjz5k3ExcWhsrISvXv3RkhICI4dO4ahQ4eaahcIISZg8uQFAAkJCUhISNA77+jRo1rTq1evxurVq3sgKkKIOTOL5EUIAS7VNqNZ2dLu/NKaZu7ftsb7+zmKbTHAzbFb4jM3lLwIMQOXapsxaeVRg+q+ubOgw/lHFk20igRGyYsQM9B2xrUmKhj+/aT669xWYu/RHEyfGAZHB93Hf0qqm/C3Hfkdnr1ZEkpehJgR/35SDPdy0TtPrVajsi8wyre3RT/nZSiTv5hNCCEPgpIXIYSX6LKRxzq6O2XInSnAuu5OEctCyYunDL071dmdKcB67k4Ry0LJi6c6uzvV2Z0pwPruThHLQsmL59q7O0V3poilowZ7QggvUfIihPASJS9CCC9R8iKE8BIlL0IIL1HyIoTwEj0qwVPK1jsQ2l/DpYYiCO11n/NqaWnB9ZbrKKwrbPcJ+0sNTRDaX4Oy9Q4A/S8DE2KuKHnx1PXmK3Ac8DH+cbLjep8e+LTD+Y4DgOvNwQgBjTpO+IWSF0/1d/RF86XX8VFUMAbqecK+paUFP//0M/7w+B/aPfMqrW7Cwh356D/Jt7vDJcToKHnxlNjGHpo7XhjgHIChffQ/YX/J9hKGuA5p9wl7zZ16aO7UQGxj393hkk501gwAdN4UYG3NAJS8CDEDhjYDAB03BVhTMwAlL0LMQGfNAEDnTQHW1gxAyYsQM9BZMwDQeVOAtTUD0HNehBBeouRFCOElSl6EEF6i5EUI4SVKXoQQXqLkRQjhJUpehBBe6nLyunjxYnfEQQghXdLl5OXv749Jkybhyy+/xJ07d7ojJkII6VSXk1deXh5GjBgBuVwODw8P/OUvf8HJkwa8kNWBdevWQSaTwd7eHmPGjDF4fdu3b4dAIMDMmTMfavuEEP7pcvIKDg7GRx99hOvXryMzMxMVFRV4/PHHMXz4cKSnp6OmpqZL69uxYwfkcjlSUlKQl5eHoKAgTJkyBdXV1R0ud/nyZSxatAjjx4/v6i4QQizAAzfY29raYtasWfj666+xfPlylJSUYNGiRfD29kZ0dDQqKioMWk96ejri4uIQGxuLoUOHIiMjAxKJBJmZme0u09raijlz5iA1NRV+fn4PuguEEB574BezT58+jczMTGzfvh2Ojo5YtGgR5s2bh6tXryI1NRWRkZGdXv6pVCrk5uYiKSmJKxMKhQgPD0dOTk67y7377rvo168f5s2bhx9//LHDbSiVSiiVSm66oaEBwN2XXNVqtSG7apZaWlq4f/XtR1tZR/vY2TpIzzHkWHR2TC3leBoae5eTV3p6OjZu3IiioiJMmzYNmzdvxrRp0yAU3j2JGzBgADZt2gSZTNbpumpra9Ha2gp3d+2+h9zd3XHhwgW9y/z000/4/PPPkZ+fb1C8aWlpSE1N1Sk/dOgQJBKJQeswR+VNAGCLn376CVf096ACAMjKynrodZDu15Vj0d4xtZTjqVAoDKrX5eS1fv16vPLKK5g7dy48PT311unXrx8+//zzrq66U42NjXj55ZexYcMGuLm5GbRMUlIS5HI5N93Q0ABvb29MnjwZzs7ORo+xp5y73oCVBcfx+OOPY1h/3f1Qq9XIyspCREREuz2pdrYO0nMMORadHVNLOZ5tV0ed6XLyysrKgo+PD3em1YYxhvLycvj4+EAkEiEmJqbTdbm5ucHGxgZVVVVa5VVVVfDw8NCpX1paisuXL2PGjBlcmUajubsjtrYoKirCwIEDtZYRi8UQi8U667Kzs2v3j5oP2jqjs7W17XA/OtpPQ9dBul9XjkV7x9RSjqehsXe5wX7gwIGora3VKa+rq8OAAQO6tC6RSISQkBBkZ2dzZRqNBtnZ2QgLC9OpHxgYiIKCAuTn53M/Tz/9NCZNmoT8/Hx4e3t3dXcIITzV5TMvxpje8qamJtjbd70HR7lcjpiYGISGhmL06NFYs2YNmpubERsbCwCIjo6Gl5cX0tLSYG9vj+HDh2st36tXLwDQKSeEWDaDk1dbu5FAIEBycrJWY3draytOnDiB4ODgLgcQFRWFmpoaJCcno7KyEsHBwThw4ADXiF9WVqZziUoIIQYnr19++QXA3TOvgoICiEQibp5IJEJQUBAWLVr0QEEkJCQgISFB77yjR492uOymTZseaJuEEH4zOHkdOXIEABAbG4uPPvqI13fqCCH81+U2r40bN3ZHHIQQ0iUGJa9Zs2Zh06ZNcHZ2xqxZszqsu2vXLqMERjp2W90KADh7rV7v/ObbSpyuATyu3ISjg+6jIgBQUt3UbfER0t0MSl4uLi4QCATcZ2J6pf8/8fx9V0EHtWyxpeRUp+tyFNPwnYR/DPrW3nupSJeN5mHysLsP8Q7sJ4WDnY3O/KKKery5swCrnnkUAZ7t/4fjKLbFADfHbouTkO5C/+XylKujCM+P9ml3fttLugP7OmK4F50tE8tjUPIaOXIkd9nYmby8vIcKiBBCDGFQ8qKeSgkh5sag5JWSktLdcRBCSJfQezeEEF4y6MzL1dUVxcXFcHNzQ+/evTts/6qrqzNacIQQ0h6Dktfq1avh5OTEfTa08Z4QQrqLQcnr3o4F586d212xEEKIwbrc5mVjY6N3WLIbN27Axkb3YUlCCOkOXU5e7XVGqFQqtbrJIYSQ7mTwE/Zr164FcLczws8++wxS6e/Dk7S2tuK///0vAgMDjR8hIYToYXDyWr16NYC7Z14ZGRlal4gikQgymQwZGRnGj5AQK9BZLyFA5z2FWFsvIQYnr0uXLgEAJk2ahF27dqF3797dFhQh1sawXkIAQ3oKsZZeQrq8l209qhJCjKezXkIAw3oKsaZeQrqcvF555ZUO52dmZj5wMIRYq856CQGop5D7dTl53bx5U2tarVbj7NmzuHXrFp544gmjBUYIIR3pcvL65ptvdMo0Gg3i4+N1RqsmhJDuYpQXs4VCIeRyOXdHkhBCupvRepUoLS3lrskJIaS7dfmysW3k7DaMMVRUVGDfvn1a70ASQkh36nLyahs5u41QKETfvn2xatWqTu9EEkKIsdBzXoQQXqKeVAkhvETJixDCS5S8CCG8RMmLEMJLRkteV69exZ///GdjrY4QQjpktOR148YNfP7558ZaHSGEdMgsLhvXrVsHmUwGe3t7jBkzBidPnmy37q5duxAaGopevXrB0dERwcHB2LJlSw9GSwgxByZPXjt27IBcLkdKSgry8vIQFBSEKVOm6B3kA7g7huRbb72FnJwc/Prrr4iNjUVsbCwOHjzYw5ETQkzJ5MkrPT0dcXFxiI2NxdChQ5GRkQGJRNJuv2ATJ07En/70JwwZMgQDBw7EwoULMWLECPz00089HDkhxJQMfsJ+1qxZHc6/detWlzeuUqmQm5uLpKQkrkwoFCI8PBw5OTmdLs8Yw+HDh1FUVITly5frraNUKqFUKrnphoYGAHf7IVOr1V2OmS/aXpJvaWmx6P20JtZyTA3dN4OTl4tLxz03uri4IDo62tDVAQBqa2vR2toKd3d3rXJ3d3dcuHCh3eXq6+vh5eUFpVIJGxsbfPrpp4iIiNBbNy0tDampqTrlhw4dgkQi6VK8fFLeBAC2OH78OK6dNXU0xBis5ZgqFAqD6hmcvDZu3PjAwRibk5MT8vPz0dTUhOzsbMjlcvj5+WHixIk6dZOSkrR6wmhoaIC3tzcmT54MZ2fnHoy6Z50pqwMKTmPs2LEI8nE1dTjECKzlmLZdHXXGpMOMuLm5wcbGBlVVVVrlVVVV8PDwaHc5oVAIf39/AEBwcDAKCwuRlpamN3mJxWKIxbrDRNnZ2cHOzu7hdsCM2dracv9a8n5aE2s5pobum8HJy9DubroyAIdIJEJISAiys7Mxc+ZMAHe7lM7OzkZCQoLB69FoNFrtWoQQy2dw8tq0aRN8fX0xcuRIMMaMFoBcLkdMTAxCQ0MxevRorFmzBs3NzYiNjQUAREdHw8vLC2lpaQDutmGFhoZi4MCBUCqV2L9/P7Zs2YL169cbLSZCiPkzOHnFx8dj27ZtuHTpEmJjY/HSSy/B1fXhr7ujoqJQU1OD5ORkVFZWIjg4GAcOHOAa8cvKyiAU/v5ER3NzMxYsWICrV6/CwcEBgYGB+PLLLxEVFfXQsRBC+EPAunAapVQqsWvXLmRmZuLYsWN46qmnMG/ePEyePBkCgaA74zSahoYGuLi4oL6+3qIb7POv3MDM9cexO34sgn37mDocYgTWckwN/Rvt0kOqYrEYL7zwArKysnD+/HkMGzYMCxYsgEwmQ1NT00MHTQghhnrgJ+yFQiEEAgEYY2htbTVmTIQQ0qkuJS+lUolt27YhIiICgwcPRkFBAT755BOUlZVBKpV2V4yEEKLD4Ab7BQsWYPv27fD29sYrr7yCbdu2wc3NrTtjI4SQdhmcvDIyMuDj4wM/Pz/88MMP+OGHH/TW27Vrl9GCI4SQ9hicvKKjo3lzR5EQYvm69JAqIYSYC5P350UIIQ+CkhchhJcoeRFCeImSFyGElyh5EUJ4iZIXIYSXKHkRQniJkpcFUqlU2Pr5etRlZWDr5+uhUqlMHRIhRkfJy8IkJiZCIpFg5dK30Zi3FyuXvg2JRILExERTh0aIUVHyshAKhQIxMTFYsWKFThdFra2tWLFiBWJiYpCXl2fw0FKEmDNKXhbi119/xebNmzuss3nzZoSEhHQ4JiYhfEHJy0IcPnwYwO/DY92vrXzhwoUIDAzssbgI6S6UvCzE/v37Afw+JPz92spPnz5t0SOFE+tByctCaDQao9YjxNyZdMRsYjz3dsMdERGBGTNmoLi4GIMHD8Z//vMfZGVl6dQjhM8oeVmIU6dOcZ+///57LlkB0OpE8t56hPAZXTZaiNu3b3Of7x+K897pe+sRwmeUvCxEnz6/D0JqY2OjNe/e6XvrEcJnlLwsxPz587nPLi4umD17Np544gnMnj0bLi4ueusRwmfU5mUh/Pz8uM91dXX497//3Wk9QviMzrwshJeXl1HrEWLu6MzLQowfPx4ymQxubm64fv06rl+/zs3z8vKCp6cnbty4gfHjx5swSkKMh5KXhbCxscGqVavwzDPP4KmnnsKSJUvw22+/YdCgQcjKysK+ffuwc+dOncZ8QviKkpcFmTVrFnbu3Ik333wTe/fu5coHDBiAnTt3YtasWSaMjhDjojYvC3T/c170ShCxRJS8LMiuXbvwzDPPoKqqSqu8qqoKzzzzDHbt2mWiyAgxPrNIXuvWrYNMJoO9vT3GjBmDkydPtlt3w4YNGD9+PHr37o3evXsjPDy8w/rWorW1FfHx8WCMab0OBNx9PYgxhvj4eJ2OCgnhK5Mnrx07dkAulyMlJQV5eXkICgrClClTUF1drbf+0aNH8cILL+DIkSPIycmBt7c3Jk+ejGvXrvVw5Obl6NGj3O/s/leA2qarq6tx9OjRng6NkG5h8uSVnp6OuLg4xMbGYujQocjIyIBEIkFmZqbe+lu3bsWCBQsQHByMwMBAfPbZZ9BoNMjOzu7hyM1LW2eExqpHiLkz6d1GlUqF3NxcJCUlcWVCoRDh4eHIyckxaB0KhQJqtRqurq565yuVSiiVSm66oaEBAKBWq6FWqx8ievNSUlKiNT1y5Eg4OjqiubkZv/zyi1Y9S9pva9LWoWRLS4tFH0ND982kyau2thatra1wd3fXKnd3dze4n/UlS5agf//+CA8P1zs/LS0NqampOuWHDh2yqB5F7233EwgEWgmrrc2rrV5br6uEX8qbAMAWx48fx7Wzpo6m+xg6QAyvn/P64IMPsH37dhw9ehT29vZ66yQlJUEul3PTDQ0NXDuZs7NzT4Xa7aKjo7nPdnZ2GDduHDQaDYRCIY4dO8aN3VhXV4dp06aZKkzyEM6U1QEFpzF27FgE+ei/0rAEbVdHnTFp8nJzc4ONjY3eW/seHh4dLrty5Up88MEH+P777zFixIh264nFYojFYp1yOzs72NnZPVjgZkgo/L35UqVStdswLxQKLWq/rUnbICq2trYWfQwN3TeTNtiLRCKEhIRoNba3Nb6HhYW1u9yHH36IpUuX4sCBAwgNDe2JUM2ep6enUesRYu5MfrdRLpdjw4YN+OKLL1BYWIj4+Hg0NzcjNjYWwN3LoXsb9JcvX463334bmZmZkMlkqKysRGVlJZqamky1C2Zh3LhxRq1HiLkzeZtXVFQUampqkJycjMrKSgQHB+PAgQNcI35ZWZnWJdH69euhUqnwzDPPaK0nJSUF77zzTk+GblZqamqMWo8Qc2fy5AUACQkJSEhI0Dvv/raby5cvd39APFRRUWHUeoSYO5NfNhLjuPfREldXV9jb20MgEMDe3l7rGThDH0EhxNyZxZkXeXj3vs9YV1fHfb5z5w7u3Lmjtx4hfEZnXhZi+PDhRq1HiLmj5GUhtm3bZtR6hJg7Sl4W4vnnnzdqPULMHSUvC3H/i9kPW48Qc0fJy0LoewXqYeoRYu7obqOFiIyMxLp167jpQYMGcb1J/Pbbb1r1iPlTKBQ6j7UUVdyCsrIEhWcdoLnRiysPDAy0qB5SDEXJy0Lc3xX2vQmro3rEPF24cAEhISF65734hfZ0bm4uRo0a1QNRmRdKXhbC0IdP6SFVfggMDERubq5WWdNtJfYdycFTk8IgdRBr1bVGlLwsxL3DndnY2GgNtHHv9P3DohHzJJFIdM6m1Go1btZWI2x0qEV3iWMoarC3EL179+Y+39/tTf/+/fXWI4TP6MzLQvj7+6O8vBwAcO3aNTzxxBPw9PRERUUFjhw5olWP8I9KpcLHH3+Mw4cPo6SkBK+//jpEIpGpwzIpOvOyEEOHDuU+M8Zw+PBhbN26FYcPH9a6VLy3HuGHxMREODo6YtGiRdi/fz8WLVoER0dHJCYmmjo0kxIwK2sEaWhogIuLC+rr6y2qD/vbt29DIpFoDbZxr7ZyhUIBBwcHE0RIHkRiYiJWrFgBd3d3pKamQiwWQ6lUIiUlBVVVVVi8eDE+/PBDU4dpVIb+jVLysiAzZ87Et99+C5FIBG9vbygUCkgkEpSXl0OlUiEyMhK7d+82dZjEQCqVCo6OjujTpw+uXr0Kxhj279+PadOmQSAQ4JFHHsGNGzfQ3NxsUZeQhv6N0mWjBdm9ezciIyOhUqlQWlqKiooKlJaWUuLiqU8//RQtLS1YtmwZN/hGG1tbW7z77rtoaWnBp59+aqIITYuSl4XZvXs3FAoF5s+fj+DgYMyfPx8KhYISFw+VlpYCAKZPn653flt5Wz1rQ3cbLZCDgwPWrl3LXWLQM0H8NHDgQADA3r17ERkZicceewxVVVVwd3fHqVOnsHfvXq161obavCyUWq2m5MVzbW1eLS0t7daxtbWlNi9CiHkRiURaI2d5eHjg9ddf1xqQWSgUWlTi6gq6bCTETNXU1EClUnHTlZWV+Pjjj7XqqFQq1NTUoG/fvj0dnsnRmRchZmr06NEAgGHDhkGpVGLlypWYNm0aVq5cCaVSiSFDhmjVszZ05kWImWobIHj58uUQiUT461//Cn9/f64d87333sOsWbOsdiBhOvMixEy1XQouWbJE7/y33npLq561oeRFiJlq6zjy3LlzaGho0JrX0NCAwsJCrXrWhpIXIWaqb9++cHFxAQC4uLggKCgIOTk5CAoK0iq31jMvavMixIzdunULvXr1Qn19PQoLC7mzLeBu4rp165bpgjMxOvMixMzdunUL1dXV8PX1hb29PXx9fVFdXW3ViQugMy9CeEEqlWLq1Kk4fvw4xo4dC6lUauqQTI7OvAgxczNnzoREIkFGRgby8/ORkZEBiUSCmTNnmjo0k6LkRYgZu7ePtsTERKxfvx6JiYkQiUT49ttvrTqBmTx5rVu3DjKZDPb29hgzZkyHt33PnTuH2bNnQyaTQSAQYM2aNT0XKCE97Pbt21ziamxsxLJly+Dp6Ylly5ahsbGRS2C3b982dagmYdLktWPHDsjlcqSkpCAvLw9BQUGYMmUKqqur9dZXKBTw8/PDBx98oPVyKiGWaPHixQAAuVyu8/K1SCTC3/72N6161sakySs9PR1xcXGIjY3F0KFDuWv5zMxMvfUfe+wxrFixAs8//zzEYrHeOoRYirZRz1999VW98+fNm6dVz9qY7G6jSqVCbm4ukpKSuDKhUIjw8HDk5OQYbTtKpRJKpZKbbntSWa1WQ61WG2075qZt3yx5Hy2dn58fAOCf//wnli1bpnNMN2zYwNWzpONs6L6YLHnV1taitbUV7u7uWuXu7u5GHZI+LS0NqampOuWHDh2CRCIx2nbMVVZWlqlDIA9o4sSJyMjIQHp6OkJDQ7lLx6ysLKhUKq7Nd+LEidi/f78JIzUuhUJhUD2Lf84rKSkJcrmcm25oaIC3tzcmT55s8T2pZmVlISIignpS5bEZM2bgP//5D1566SUkJCRg4MCBKC0txSeffIKWlhbMmDEDs2bNMnWYRnX/e5ztMVnycnNzg42NDaqqqrTKq6qqjNoYLxaL9baP2dnZWcUftbXsp6Xas2cP97hEenq61jxLHRHK0O+ryRrsRSIRQkJCkJ2dzZVpNBpkZ2cjLCzMVGERYnZoRCj9THrZKJfLERMTg9DQUIwePRpr1qxBc3MzYmNjAQDR0dHw8vJCWloagLuN/OfPn+c+X7t2Dfn5+ZBKpfD39zfZfhDS3WhEKF0mTV5RUVGoqalBcnIyKisrERwcjAMHDnCN+GVlZVoDEFy/fh0jR47kpleuXImVK1diwoQJOHr0aE+HTwgxIZM32CckJCAhIUHvvPsTkkwmg5WN1EYIaYfJXw8ihJAHYfIzr57WduZm6O1YvlKr1VAoFGhoaKD2EQthLce07W+zs6ssq0tejY2NAABvb28TR0II6UhjYyPX3bU+AmZljUgajQbXr1+Hk5MTBAKBqcPpNm0P45aXl1v0w7jWxFqOKWMMjY2N6N+/v9YNu/tZ3ZmXUCjEI488Yuoweoyzs7NFf9GtkTUc047OuNpQgz0hhJcoeRFCeImSl4USi8VISUmhfs8sCB1TbVbXYE8IsQx05kUI4SVKXoQQXqLkRQjhJUpenZg7d67Rx8abOHEiN/JLe2QyGQ3tRkgHrCp5GZI0CL+88847CA4ONnUY7TK375y5xfMwrCp5EcJHKpXK1CGYJatJXnPnzsUPP/yAjz76CAKBAAKBAKWlpZg3bx4GDBgABwcHBAQE4KOPPtK7fGpqKvr27QtnZ2fMnz/f4C9Uc3MzoqOjIZVK4enpiVWrVunUqa6uxowZM+Dg4IABAwZg69atOnUEAgHWr1+PqVOnwsHBAX5+fti5cyc3//LlyxAIBPjqq68wfvx4ODg44LHHHkNxcTFOnTqF0NBQSKVSTJ06FTU1NQb+1oDMzEwMGzYMYrEYnp6eWn2vlZWVITIyElKpFM7Oznjuuee0xiRoOyvasmULZDIZXFxc8Pzzz3MvxwN33zX98MMP4e/vD7FYDB8fH7z33nvc/CVLlmDw4MGQSCTw8/PD22+/zQ2NtWnTJqSmpuLMmTPcMd20aZPB+9bdHvQ719ZU8d5776F///4ICAgAABw7dgzBwcGwt7dHaGgodu/eDYFAgPz8fG7Zs2fPYurUqZBKpXB3d8fLL7+M2traduO5fPlyT/06jI9ZiVu3brGwsDAWFxfHKioqWEVFBbtz5w5LTk5mp06dYhcvXmRffvklk0gkbMeOHdxyMTExTCqVsqioKHb27Fm2d+9e1rdvX/aPf/zDoO3Gx8czHx8f9v3337Nff/2VTZ8+nTk5ObGFCxdydaZOncqCgoJYTk4OO336NBs3bhxzcHBgq1ev5uoAYH369GEbNmxgRUVF7H//93+ZjY0NO3/+PGOMsUuXLjEALDAwkB04cICdP3+ejR07loWEhLCJEyeyn376ieXl5TF/f382f/58g2L/9NNPmb29PVuzZg0rKipiJ0+e5GJqbW1lwcHB7PHHH2enT59mx48fZyEhIWzChAnc8ikpKUwqlbJZs2axgoIC9t///pd5eHho/e4SExNZ79692aZNm1hJSQn78ccf2YYNG7j5S5cuZT///DO7dOkS27NnD3N3d2fLly9njDGmUCjYm2++yYYNG8YdU4VCYdC+9YSH/c69/PLL7OzZs+zs2bOsvr6eubq6spdeeomdO3eO7d+/nw0ePJgBYL/88gtjjLGbN2+yvn37sqSkJFZYWMjy8vJYREQEmzRpUrvxtLS0mOJXYxRWk7wYY2zChAlaSUOf1157jc2ePZubjomJYa6urqy5uZkrW79+PZNKpay1tbXDdTU2NjKRSMS++uorruzGjRvMwcGBi6OoqIgBYCdPnuTqFBYWMgA6yev+pDNmzBgWHx/PGPs9eX322Wfc/G3btjEALDs7mytLS0tjAQEBHcbdpn///uytt97SO+/QoUPMxsaGlZWVcWXnzp3T2peUlBQmkUhYQ0MDV2fx4sVszJgxjDHGGhoamFgs1kpWnVmxYgULCQnhplNSUlhQUJDBy/e0B/3Oubu7M6VSyZWtX7+e9enTh92+fZsr27Bhg1byWrp0KZs8ebLWusvLyxkAVlRUZHA8fGF1vUrcb926dcjMzERZWRlu374NlUql0wAcFBSkNUBtWFgYmpqaUF5eDl9f33bXXVpaCpVKhTFjxnBlrq6u3GUAABQWFsLW1hYhISFcWWBgIHr16qWzvvtHVQoLC9O6ZACAESNGcJ/bxgJ49NFHtcqqq6vbjblNdXU1rl+/jj/+8Y965xcWFsLb21urX7ShQ4eiV69eKCwsxGOPPQbg7l1TJycnro6npye3/cLCQiiVyna3AQA7duzA2rVrUVpaiqamJrS0tPC+RwVDvnOPPvooN8gsABQVFWHEiBGwt7fnykaPHq21zJkzZ3DkyBFIpVKdbZaWlmLw4MHG3RETs5o2L322b9+ORYsWYd68eTh06BDy8/MRGxvL6wbSe3vYbOuv7P4yjUbT6XocHByMHs/92+9sGzk5OZgzZw6mTZuGvXv34pdffsFbb73F6+Nj6HfO0dGxy+tuamrCjBkzkJ+fr/Xz22+/4X/+53+MtQtmw6qSl0gkQmtrKzf9888/Y9y4cViwYAFGjhwJf39/lJaW6ix35swZ3L59m5s+fvw4pFJpp72xDhw4EHZ2djhx4gRXdvPmTRQXF3PTgYGBaGlpQW5uLldWVFSEW7du6azv+PHjOtNDhgzpMIYH5eTkBJlMpjWu5r2GDBmC8vJylJeXc2Xnz5/HrVu3MHToUIO2MWjQIDg4OLS7jWPHjsHX1xdvvfUWQkNDMWjQIFy5ckWrzv3H1Nw86HfufgEBASgoKIBSqeTKTp06pVVn1KhROHfuHGQyGfz9/bV+2pKhuf++usKqkpdMJsOJEydw+fJl1NbWYtCgQTh9+jQOHjyI4uJivP322zpfCODurep58+bh/Pnz2L9/P1JSUpCQkNBhL48AIJVKMW/ePCxevBiHDx/G2bNnMXfuXK3lAgIC8OSTT+Ivf/kLTpw4gdzcXLz66qt6z0q+/vprZGZmori4GCkpKTh58mS7Iy8ZwzvvvINVq1Zh7dq1+O2335CXl4ePP/4YABAeHo5HH30Uc+bMQV5eHk6ePIno6GhMmDABoaGhBq3f3t4eS5YsQWJiIjZv3ozS0lIcP34cn3/+OYC7ya2srAzbt29HaWkp1q5di2+++UZrHTKZDJcuXUJ+fj5qa2u1/rjNwYN+5+734osvQqPR4M9//jMKCwtx8OBBrFy5EsDvZ9ivvfYa6urq8MILL+DUqVMoLS3FwYMHERsbyyWs++Mx5CzcbJm60a0nFRUVsbFjxzIHBwcGgF24cIHNnTuXubi4sF69erH4+Hj297//XasBOCYmhkVGRrLk5GTWp08fJpVKWVxcHLtz545B22xsbGQvvfQSk0gkzN3dnX344Yc6jaYVFRXsqaeeYmKxmPn4+LDNmzczX19fnQb7devWsYiICCYWi5lMJtO6Q9XWYN/WeMsYY0eOHGEA2M2bN7myjRs3MhcXF4N/ZxkZGSwgIIDZ2dkxT09P9vrrr3Pzrly5wp5++mnm6OjInJyc2LPPPssqKyu5+foa01evXs18fX256dbWVrZs2TLm6+vL7OzsmI+PD3v//fe5+YsXL+Z+71FRUWz16tVa8d+5c4fNnj2b9erViwFgGzduNHjfesLDfOfu9/PPP7MRI0YwkUjEQkJC2L/+9S9unW2Ki4vZn/70J9arVy/m4ODAAgMD2d/+9jem0Wj0xnPp0qVu/g10H+oShycEAgG++eYbo7+qRPhr69atiI2NRX19vdHaKPnE6u82EsIXmzdvhp+fH7y8vHDmzBksWbIEzz33nFUmLoCS10MpKyvrsHH6/Pnz8PHx6cGIukbfLfU23333HcaPH9+D0ZDOVFZWIjk5GZWVlfD09MSzzz6r9TaCtaHLxofQ0tLS4esVMpkMtrbm+/9DSUlJu/O8vLys9n90wg+UvAghvGRVj0oQQiwHJS9CCC9R8iKE8BIlL0IIL1HyIoTwEiUv0q3mzp3L9dppZ2cHd3d3REREIDMzs0vv1W3atElvN0HdrTsGYCHGQcmLdLsnn3wSFRUVuHz5Mr777jtMmjQJCxcuxPTp09HS0mLq8AhfmfLFSmL52nvJODs7mwHgelFdtWoVGz58OJNIJOyRRx5h8fHxrLGxkTH2+wvm9/6kpKQwxhjbvHkzCwkJYVKplLm7u7MXXniBVVVVcdupq6tjL774InNzc2P29vbM39+fZWZmcvPLysrYs88+y1xcXFjv3r3Z008/zb2snJKSorPdI0eOdMvviXQdnXkRk3jiiScQFBSEXbt2AQCEQiHWrl2Lc+fO4YsvvsDhw4eRmJgIABg3bhzWrFkDZ2dnVFRUoKKiAosWLQIAqNVqLF26FGfOnMHu3btx+fJlzJ07l9vO22+/jfPnz+O7775DYWEh1q9fDzc3N27ZKVOmwMnJCT/++CN+/vlnSKVSPPnkk1CpVFi0aBGee+457syxoqIC48aN69lfFGmfqbMnsWztnXkxxlhUVBQbMmSI3nlff/0169OnDzdtaFc+p06dYgC4s7YZM2aw2NhYvXW3bNnCAgICuO5iGGNMqVQyBwcHdvDgwU7jJ6ZFZ17EZBhjXEd633//Pf74xz/Cy8sLTk5OePnll3Hjxg0oFIoO15Gbm4sZM2bAx8cHTk5OmDBhAoC7L80DQHx8PLZv347g4GAkJibi2LFj3LJnzpxBSUkJnJycIJVKIZVK4erqijt37hjUuykxLUpexGQKCwsxYMAAXL58GdOnT8eIESPw73//G7m5uVi3bh2AjgdcbW5uxpQpU+Ds7IytW7fi1KlTXE+rbctNnToVV65cwRtvvMENKNJ2ydnU1ISQkBCdPt+Li4vx4osvdvPek4dlvl0eEIt2+PBhFBQU4I033kBubi40Gg1WrVrFdZH91VdfadXX1/f6hQsXcOPGDXzwwQfceAKnT5/W2Vbfvn0RExODmJgYjB8/HosXL8bKlSsxatQo7NixA/369Wt3RCJL6vPd0tCZF+l2SqUSlZWVuHbtGvLy8vD+++8jMjIS06dPR3R0NPz9/aFWq/Hxxx/j4sWL2LJlCzIyMrTWIZPJ0NTUhOzsbNTW1kKhUMDHxwcikYhbbs+ePVi6dKnWcsnJyfj2229RUlKCc+fOYe/evdygJXPmzIGbmxsiIyPx448/4tKlSzh69Cj++te/4urVq9x2f/31VxQVFaG2tpYbrZuYAVM3uhHLFhMTwz1mYGtry/r27cvCw8NZZmam1qC96enpzNPTkzk4OLApU6awzZs36/S/P3/+fNanTx+tRyX+9a9/MZlMxsRiMQsLC2N79uzRGYh1yJAhzMHBgbm6urLIyEh28eJFbp0VFRUsOjqaubm5MbFYzPz8/FhcXByrr69njDFWXV3NIiIimFQqpUclzAz150UI4SW6bCSE8BIlL0IIL1HyIoTwEiUvQggvUfIihPASJS9CCC9R8iKE8BIlL0IIL1HyIoTwEiUvQggvUfIihPDS/wO4PRRaGMsQwwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T08:46:06.348666Z", + "iopub.status.busy": "2024-03-26T08:46:06.347794Z", + "iopub.status.idle": "2024-03-26T08:46:06.605328Z", + "shell.execute_reply": "2024-03-26T08:46:06.604309Z" + }, + "papermill": { + "duration": 0.279396, + "end_time": "2024-03-26T08:46:06.607758", + "exception": false, + "start_time": "2024-03-26T08:46:06.328362", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUoAAAEmCAYAAAAEH9kkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB3c0lEQVR4nO2dd1hUx9fHv3crvStFUVCJWLALwRiNiqIxUVLUGGPFksRO7A2jeYOxBVv0ZxI1JhpLYtTYEiSWKIiK2EssKDZAQFhAYNu8fyx72bt9YWk6n+fZB3bu3Lmz5Z49c86ZcxhCCAGFQqFQDMKr7glQKBRKTYcKSgqFQjEBFZQUCoViAiooKRQKxQRUUFIoFIoJqKCkUCgUE1BBSaFQKCaggpJCoVBMIKjuCbzMKJVKPHnyBI6OjmAYprqnQ6FQNCCEID8/Hz4+PuDxjOuMVFBWIk+ePIGvr291T4NCoRjh4cOHqF+/vtE+VFBWIo6OjgBUH4STk1M1z4ZCoWgikUjg6+vL3qfGoIKyElEvt52cnKigpFBqKOaYxagzh0KhUExABSWFQqGYgApKCoVCMQG1UdZACCGQy+VQKBTVPRUKpdbC5/MhEAisEppHBWUNQyqV4unTp3jx4kV1T4VCqfXY2dnB29sbIpGoQuNQQVmDUCqVSE1NBZ/Ph4+PD0QiEQ1Up1DKASEEUqkUz549Q2pqKgICAkwGlRuDCsoahFQqhVKphK+vL+zs7Krsupn5xSiWKeHraksFM+WlwdbWFkKhEA8ePIBUKoWNjU25x6LOnBpIRX75ykOmpAS5L6QoklGbKOXlwlr3EhWUrzgKJYGytL6cTK6s5tlQKDUTKihfceSKMuEoVdCCnBSKPqigfMWRK8uEo0xRtRrliBEjEBERYdUx33rrLUyZMsVoHz8/P8TGxlr1upTqwZzP2xpQQfmKw9EoK7D0rqovLKXqWLhwIdq0aVPd06gRUEH5iiPT0CilVaxRUigAIJPJqnsKJqGCsoZDCMELqbzSHvlFchTLFCiWKZBfJENhiYw9Roh5NssRI0bgxIkTWLVqFRiGAcMwuHv3LiIjI+Hv7w9bW1s0bdoUq1at0nv+l19+iTp16sDJyQmffvoppFKpWdctLCzEsGHD4ODgAG9vb6xYsUKnT2ZmJt59913Y2trC398f27Zt0+nDMAzWr1+PPn36wNbWFo0aNcJvv/3GHr9//z4YhsGuXbvw5ptvwtbWFh07dsR///2Hc+fOoUOHDnBwcECfPn3w7Nkzs+YOAJs2bUKLFi0gFovh7e2NCRMmsMfS0tLQv39/ODg4wMnJCQMHDkRGRgZ7XK3t/fzzz/Dz84OzszM++ugj5Ofns32USiWWLl2KJk2aQCwWo0GDBvi///s/9vjMmTPx2muvwc7ODo0aNcL8+fNZobVlyxZ8+eWXuHTpEvuZbtmyxeRrunnzJjp37gwbGxs0b94cR48eBcMw2Lt3L+e93LlzJ7p27QobGxts27YN2dnZGDx4MOrVqwc7OzsEBQXh119/5YxtzuddWdA4yhpOkUyB5gv+qpZrX18UDjuR6a/IqlWr8N9//6Fly5ZYtGgRAMDV1RX169fH7t274e7ujoSEBIwdOxbe3t4YOHAge258fDxsbGxw/Phx3L9/HyNHjoS7uzvnhjbE9OnTceLECezbtw9169bFnDlzcOHCBc5yccSIEXjy5AmOHTsGoVCISZMmITMzU2es+fPnY8mSJVi1ahV+/vlnfPTRR7hy5QqaNWvG9omOjkZsbCwaNGiAUaNG4eOPP4ajoyNWrVoFOzs7DBw4EAsWLMD69etNzn39+vWIiorCkiVL0KdPH+Tl5eH06dMAVAJOLSRPnDgBuVyO8ePHY9CgQTh+/Dg7xt27d7F3714cOHAAz58/x8CBA7FkyRL2vZs9eza+//57fPvtt+jcuTOePn2Kmzdvsuc7Ojpiy5Yt8PHxwZUrVzBmzBg4OjpixowZGDRoEK5evYojR47g6NGjAABnZ2ejr0mhUCAiIgINGjRAUlIS8vPz8cUXX+jtO2vWLKxYsQJt27aFjY0NiouL0b59e8ycORNOTk44ePAghg4disaNGyM4OBiAeZ93pUEolUZeXh4BQPLy8szqX1RURK5fv06KiorYtsISGWk480C1PApLZGa/1q5du5LJkycb7TN+/HjywQcfsM+HDx9O3NzcSGFhIdu2fv164uDgQBQKhdGx8vPziUgkIrt27WLbsrOzia2tLTuPW7duEQDk7NmzbJ8bN24QAOTbb79l2wCQTz/9lDN+SEgI+eyzzwghhKSmphIA5IcffmCP//rrrwQAiY+PZ9tiYmJI06ZNjc5bjY+PD5k7d67eY3///Tfh8/kkLS2Nbbt27RrntURHRxM7OzsikUjYPtOnTychISGEEEIkEgkRi8Xk+++/N2s+hBCybNky0r59e/Z5dHQ0ad26tdnnHz58mAgEAvL06VO2LS4ujgAgf/zxByGk7L2MjY01OV7fvn3JF198QQgx7/PWh757So0l9yfVKGs4tkI+ri8Kr7Tx7zwrQLG0LNDc180OzrZC9toVYd26ddi0aRPS0tJQVFQEqVSq8+vfunVrzi6k0NBQFBQU4OHDh2jYsKHBse/evQupVIqQkBC2zc3NDU2bNmWf37hxAwKBAO3bt2fbAgMD4eLiojNeaGiozvOLFy9y2lq1asX+7+npCQAICgritOnTVrXJzMzEkydP0KNHD73Hb9y4AV9fX04ZkebNm8PFxQU3btxAx44dAai895rZub29vdnr37hxAyUlJQavAQA7d+7E6tWrcffuXRQUFEAul1cowfStW7fg6+sLLy8vtk2tDWrToUMHznOFQoGvv/4au3btwuPHjyGVSlFSUsJ+N8z5vCsTKihrOAzDmLX8LS9CHg8QAiI+D1KFEgIezyrX27FjB6ZNm4YVK1YgNDQUjo6OWLZsGZKSkqww6+pBKBSy/6u3emq3KZWmHWK2trZWn4/29U1dIzExEUOGDMGXX36J8PBwODs7Y8eOHVVm97O3t+c8X7ZsGVatWoXY2FgEBQXB3t4eU6ZMMdteXdlQZ84rDCGEjaO0E6m0x/LGUopEIk5auNOnT6NTp074/PPP0bZtWzRp0gR3797VOe/SpUsoKipin585cwYODg4mi7I1btwYQqGQI3ifP3+O//77j30eGBgIuVyO5ORktu3WrVvIzc3VGe/MmTM6zzXtk9bE0dERfn5+iI+P13u8WbNmePjwIR4+fMi2Xb9+Hbm5uWjevLlZ1wgICICtra3BayQkJKBhw4aYO3cuOnTogICAADx48IDTR/szNUXTpk3x8OFDjtPp3LlzZp17+vRp9O/fH5988glat26NRo0acT5Lcz7vyoRqlK8wCiVhPdu2IgFyi2TljqX08/NDUlIS7t+/DwcHBwQEBGDr1q3466+/4O/vj59//hnnzp2Dv78/5zypVIrIyEjMmzcP9+/fR3R0NCZMmGByj66DgwMiIyMxffp0uLu7o27dupg7dy7nvKZNm6J3794YN24c1q9fD4FAgClTpujVtnbv3o0OHTqgc+fO2LZtG86ePYsff/yxXO+FOSxcuBCffvop6tatiz59+iA/Px+nT5/GxIkTERYWhqCgIAwZMgSxsbGQy+X4/PPP0bVrV50lqyFsbGwwc+ZMzJgxAyKRCG+88QaePXuGa9euITIyEgEBAUhLS8OOHTvQsWNHHDx4EH/88QdnDD8/P6SmpuLixYuoX78+HB0dIRaLDV6zZ8+eaNy4MYYPH46lS5ciPz8f8+bNA2C6Lk1AQAB+++03JCQkwNXVFStXrkRGRgb7w2DO512ZUI3yFUatTfJ5DMQC1VehvBrltGnTwOfz0bx5c9SpUwfh4eF4//33MWjQIISEhCA7Oxuff/65znk9evRAQEAAunTpgkGDBqFfv35YuHChWddctmwZ3nzzTbz77rsICwtD586dOfZIANi8eTN8fHzQtWtXvP/++xg7dizq1q2rM9aXX36JHTt2oFWrVti6dSt+/fVXs7W38jB8+HDExsbiu+++Q4sWLfDOO+/g9u3bAFRCZd++fXB1dUWXLl0QFhaGRo0aYefOnRZdY/78+fjiiy+wYMECNGvWDIMGDWJtmP369cPUqVMxYcIEtGnTBgkJCZg/fz7n/A8++AC9e/dGt27dUKdOHZ1wHW34fD727t2LgoICdOzYEaNHj8bcuXMBwGTmnnnz5qFdu3YIDw/HW2+9BS8vL51dW+Z83pWGSXdPJbN27VrSsGFDIhaLSXBwMElKSjLaf9euXaRp06ZELBaTli1bkoMHD3KO//7776Rnz57Ezc2NACApKSk6Y3Tt2pUA4DzGjRvH6fPgwQPy9ttvE1tbW1KnTh0ybdo0IpOZ7wUmxDpe78okv0hKLj18Tm4+lZAiqZxcevicXH2cWyXXrklAwytLsS6nTp0iAMidO3eq5frW8npXq0a5c+dOREVFITo6GhcuXEDr1q0RHh5u0HOYkJCAwYMHIzIyEikpKYiIiEBERASuXr3K9iksLETnzp3xzTffGL32mDFj8PTpU/axdOlS9phCoUDfvn0hlUqRkJCAn376CVu2bMGCBQus88JrCGqNUshnIOSrvgoKJYHCDIcEhaKPP/74A3Fxcbh//z6OHj2KsWPH4o033kDjxo2re2oVozKkuLkEBweT8ePHs88VCgXx8fEhMTExevsPHDiQ9O3bl9MWEhKiow0SUhavZUijNBZ7dejQIcLj8Uh6ejrbtn79euLk5ERKSkpMvKoyarpGmSkpJpcePicPslRxjNce55JLD5+TFyXyKrm+MR48eEDs7e0NPh48eGC1a6ESNEpjcz958qRVr1VV/PLLLwZfU/PmzQkhhPz0008kICCAiMViUq9ePTJ8+HCSlZVVbXOu9XGUUqkUycnJmD17NtvG4/EQFhaGxMREveckJiYiKiqK0xYeHs5uj7KEbdu24ZdffoGXlxfeffddzJ8/n43ZSkxMRFBQEBsrp77OZ599hmvXrqFt27Z6xywpKUFJSQn7XCKRWDyvqkReqjkK+EzpXx7kSkVpe8ViKCuKj4+PThyj9nFrQczcqmkJxuZer149q1+vKujXrx8njlETdajSsGHDMGzYsKqcVpVQbYIyKysLCoWCI4wAVdCu5jYrTdLT0/X2T09Pt+jaH3/8MRo2bAgfHx9cvnwZM2fOxK1bt7Bnzx6j11EfM0RMTAy+/PJLi+ZSnchL80+qBSWfp/qrUFpfcFiKQCBAkyZNqnsa5aY2z90Qjo6OnAD3V4lXMjxo7Nix7P9BQUHw9vZGjx49cPfu3QrZUmbPns3ReCUSicl4wOpE7eEWlIZYCEoFpbwGCEoKpSZRbc4cDw8P8Pl8TnAqAGRkZHC2QGni5eVlUX9zUS8n7ty5Y/Q66mOGEIvFcHJy4jxqMprOHKBmaZQUSk2i2gSlSCRC+/btOTsHlEol4uPjdfbdqgkNDdXZaRAXF2ewv7mo7Une3t7sda5cucLxvsfFxcHJyalSY+uqGnbpXSog1Zol1SgpFC7VuvSOiorC8OHD0aFDBwQHByM2NhaFhYUYOXIkAJVhuF69eoiJiQEATJ48GV27dsWKFSvQt29f7NixA+fPn8fGjRvZMXNycpCWloYnT54AUG1ZA1SaoJeXF+7evYvt27fj7bffhru7Oy5fvoypU6eiS5cubNKDXr16oXnz5hg6dCiWLl2K9PR0zJs3D+PHjze6M6E2QUhZGJCgNDSI1ShLBahCqURaThGcbYVws69YAXkKpVZjfYe8ZaxZs4Y0aNCAiEQiEhwcTM6cOcMe69q1Kxk+fDin/65du8hrr71GRCIRadGihU7A+ebNm3WCyQGQ6OhoQgghaWlppEuXLsTNzY2IxWLSpEkTMn36dJ0Qgfv375M+ffoQW1tb4uHhQb744ouXKuBcKleQSw+fk8sPnxOlUkkIISSnoIRcevic3M3MJ4QQkpFXRC49fE4uPXxe6fOhUCoDa4UHMYRUQmwEBYDKmePs7Iy8vDyz7JXFxcVITU2Fv79/hYq1m0ORVI7bmQUQ8Hho7qOam6RIhvvZhbAV8hHg6Yj0vGJk5hcDAFrVd6m0ufj5+WHKlClszR2GYfDHH39YvfAY5dXD2D1lyf35Snq9KWV2SHVokOb/CnYPeNXPCwCePn0KV1fX6rk4haIHKihfUWRajhygzEapFqI8jYwvhBCTGWCsRUWjGKwBIQQKhQICAb1FKDR7UM2HEEBaaPWHvCQfjOwFRMpitk0gfwFG9gJEWghlcQE0lE0oTVho8vPzMWTIENjb28Pb2xvffvttuUvY6itGtWfPHnTr1g12dnZo3bq1zu6tU6dOsYW/fH19MWnSJBQWFrLHf/75Z3To0AGOjo7w8vLCxx9/zIlqOH78OBiGweHDh9G+fXuIxWKcOnXK4rlTXk7oz2VNR/YC+Np62/XU1C19aMIHEKTxnJlyH4B6OW58KR4VFYXTp09j//798PT0xIIFC6xa+Gnu3LlYvnw5AgICMHfuXAwePBh37tyBQCDA3bt30bt3b3z11VfYtGkTnj17hgkTJmDChAnYvHkzAFVJ1MWLF6Np06bIzMxEVFQURowYgUOHDnGuM2vWLCxfvhyNGjWiy38KCxWUFLMwplHm5+fjp59+wvbt29kaLeo8kNZi2rRp6Nu3LwBV7sgWLVrgzp07CAwMRExMDIYMGcJqrwEBAVi9ejW6du2K9evXw8bGBqNGjWLHatSoEVavXo2OHTuioKAADg4O7LFFixahZ8+eVps3xXpIimTIL5bB28WWYxaqCqigrOkI7YA5T6w+bGpWIQpK5PB1tYWLXVmM5J3MAhTJFHB3EMFOaAdAVabB2G6de/fuQSaTcQpJOTs7W7Xwk2ZhL/XGgMzMTAQGBuLSpUu4fPkyp2Y3IQRKpRKpqalo1qwZkpOTsXDhQly6dAnPnz9na8ukpaVxNhGYm0GcUvXcz1aZUkQCPuo4Vm08MxWUNR2GAUT2pvtZiJSnBBEqILCxB0RlRao83IRIy3mBrBLAVllW2Km6tzXqK+ylFnYFBQUYN24cJk2apHNegwYNUFhYiPDwcISHh2Pbtm2oU6cO0tLSEB4erlO8SrvoFaXmIS9nFv6KQAXlK4pca1eOGhc7EWQKgqd5RSiSlRWWMrb0btSoEYRCIc6dO4cGDRoAAPLy8vDff/+hS5culTB7Lu3atcP169cNZuy5cuUKsrOzsWTJEjZJyfnz5yt9XpRKompX3QCo1/uVREkIqyFqhgepqeMo1lnaGNMoHR0dMXz4cEyfPh3Hjh1jC1jxeLwqCSmaOXMmEhISMGHCBFy8eBG3b9/Gvn37MGHCBAAqrVIkEmHNmjW4d+8e9u/fj8WLF1f6vCgvD1RQvoKok2EwDMPGTmrj5WTDMZgrTIQHrVy5EqGhoXjnnXcQFhaGN954A82aNav0HUaAyn554sQJ/Pfff3jzzTfRtm1bLFiwgHUm1alTB1u2bMHu3bvRvHlzLFmyBMuXL6/0eVFeHugWxkqkpm5hfCGV405mAYR8Hpp5G56XTK7EjXRVlvY6jmJ4O+uWeTVEYWEh6tWrhxUrViAyMrLCc6ZQLj/KBQDUdRTDy8zvIt3CSCk32pnNDSEU8FDX0QaZ+cVQmnDmpKSk4ObNmwgODkZeXh4WLVoEAOjfv791Jk2hsFS9kZIKylcQWakjR2hG8Xg29ZoZ647ly5fj1q1bbK7Rf//9Fzdu3ECfPn0MnlNQUGDepCmUaoQKylcQczVKoGw3jimNsm3btkhOTtZpLyoqMlpoi0KpDVBB+QpSltncDI2SqVh5CFtb25ey0Bal+qjiTTkAqNe7RlLZ/jXtMrXG4LFLb+rzo9Q+rHUvUUFZg1DvPnnx4kWlXketUQoNhAZporZRmlp6Uyg1EfW9pLmzqzzQpXcNgs/nw8XFhU3/ZWdnVykB2yUlxSAKJRRyAYqLjW8Hk8kUIHIpZEoGxcXFVp8LhWIuRK7abiqTMjD1VSSE4MWLF8jMzISLiwv4fH6Frk0FZU3gRQ6QfgVo1JVNWquZK9HaPM0tgpIAvAKxzhZGbRRKgsy8YjAA+AW21WIfolAAIPO5KkFLia0A+TbmaYguLi5WSQRNBWV18/w+sPltoCgXGHcSjEcTeHt7o27dupDJZFa/3AupHKP3qBLS/jmxM+xExr8CJTIFxq3+FwCwb8IbcBBXbAlDoZSX0XuOAwBGdfbHkGYNTfYXCoUV1iTVUEFZ3Tj7Aq7+gOQU8NtIYPRRQCAGn8+32oesSUZhIR7nK2Ar5MPNycFkf7GYIPOFEjIFQQkRwKMKtiRSKPp4nK9K0lJC+FWyNVaTanfmrFu3Dn5+frCxsUFISAjOnj1rtP/u3bsRGBgIGxsbBAUF6WSo3rNnD3r16gV3d3cwDKMTw5eTk4OJEyeiadOmsLW1RYMGDTBp0iTk5eVx+jEMo/PYsWOHVV4zBx4f+OB7wM4dSL8MxC2w/jU0eJZfAgBm5/NjGAaOpcuc/GLra7gUiqVUddJeoJoF5c6dOxEVFYXo6GhcuHABrVu3Rnh4uEH7XEJCAgYPHozIyEikpKQgIiICERERuHr1KtunsLAQnTt3xjfffKN3jCdPnuDJkydYvnw5rl69ii1btuDIkSN69yNv3rwZT58+ZR+VVj7VyQeIWK/6P2kDcPNg5VwHlgtKAHC0US088ovllTInCsUSzAjWsD7WKzVuOcHBwWT8+PHsc4VCQXx8fEhMTIze/gMHDiR9+/bltIWEhJBx48bp9E1NTSUASEpKisl57Nq1i4hEIiKTydg2AOSPP/4w74UYwJIC64QQQo7MISTaiZCYBoTkPqzQtQ3xU0IqaTjzABm39bzZ57yz+l/ScOYBEn8jvVLmRKGYQqlUkoYzD5CGMw+QjSfuWmVMS+7PatMopVIpkpOTERYWxrbxeDyEhYXpVNhTk5iYyOkPAOHh4Qb7m4s6e4h2adLx48fDw8MDwcHB2LRpk8ng1ZKSEkgkEs7DInpEAz5tgeJc4LdIQGF9DY5qlJTaiObOsFdqZ05WVhYUCgU8PT057Z6enkhPT9d7Tnp6ukX9zZ3H4sWLMXbsWE77okWLsGvXLsTFxeGDDz7A559/jjVr1hgdKyYmBs7OzuxDnU3bbAQi4MNNgMgReHgGOLHE0pdjkqwClaD0cLBcUEqooKRUE5o7w6qqvrwmr7TXWyKRoG/fvmjevDkWLlzIOTZ//nz2/7Zt26KwsBDLli3TW5dFzezZsxEVFcUZ32Jh6dYI6LcK+G0UcHI54NcZaPSWZWMYoXwaJXXmUKoXzcVcdZgoq02j9PDwAJ/PR0ZGBqc9IyPDYICol5eXRf2NkZ+fj969e8PR0RF//PGHyS1OISEhePToEUpKSgz2EYvFcHJy4jzKRcsPgHbDABBgz1ig4Fn5xtEDXXpTaiOv7NJbnbMwPj6ebVMqlYiPj0doaKjec0JDQzn9ASAuLs5gf0NIJBL06tULIpEI+/fvNysm6+LFi3B1dYVYXEVlMnt/A9QJBAoygD/GAUrrVJ6riEYpKaIaJaV60Fx6V0d4ULUuvaOiojB8+HB06NABwcHBiI2NRWFhIUaOHAkAGDZsGOrVq4eYmBgAwOTJk9G1a1esWLECffv2xY4dO3D+/Hls3LiRHTMnJwdpaWl48kRVC/vWrVsAVNqol5cXKyRfvHiBX375heN0qVOnDvh8Pv78809kZGTg9ddfh42NDeLi4vD1119j2rRpVffmiOyADzcD33cD7sYDiWuANyZXaEhCCLIKVPtlLRGUTlSjpFQzRENPqA6NsloF5aBBg/Ds2TMsWLAA6enpaNOmDY4cOcI6bNLS0sDTyJnYqVMnbN++HfPmzcOcOXMQEBCAvXv3omXLlmyf/fv3s4IWAD766CMAQHR0NBYuXIgLFy4gKSkJAHTyJKampsLPzw9CoRDr1q3D1KlTQQhBkyZNsHLlSowZM6bS3gu9eDYH+nwD/DkZiF8ENOgE+HYs93CSIjmkpTWR3e1FZp9XtvSmGiWleuA4c6rh+hUqLlZQUMAWoVdTbrvcS4ilxcX0QojKsXNtD+DSABj3L2DrUq6h7mTmI2zlSTjZCHB5YbjZ5x268hSfb7uADg1d8dtnncp1bQqlIjzLL0HH/zsKAFgc0RJDXze919sUltyfFtsoU1NT0bdvX9jb28PZ2Rmurq5wdXWFi4sLXF1dyz1pigEYBng3FnD1A3LTgD8ncV2AFpBZDvskADixXm+69KZUD0rN73w1JJG2eOn9ySefgBCCTZs2wdPTs1piml45bJxV8ZU/9gKu7wOSNwMdRlk8THkcOQBdelOqH01BWR0ppC0WlJcuXUJycjKaNm1aGfOhGKJeeyBsIfD3PODIbMA3BPBsYdEQZY4cyzKv0PAgSnWjGR5UHdn2LV56d+zYEQ8fPqyMuVBM8fp4IKAXIC8Gdo8ApIUWna7WKD0czHfkAGXhQQVSOS0JQakWNF0htUKj/OGHH/Dpp5/i8ePHaNmypU6gdqtWraw2OYoWPJ4qy9CGzkDWf8DhGUD/dWafXtGlNyEqYelkZnZpCsVacJbe1SApLRaUz549w927dzkhOAzDgBAChmGgUCisOkGKFvYewPvfAz+9C6T8Avi/BbQaYNapz0r3edexYJ83ANgI+RDxeZAqlMgvpoKSUvVohgcpq0FSWrz0HjVqFNq2bYvExETcu3cPqampnL+UKsD/TaDrDNX/B6YA2XfNOq28GiWgkRiD7s6hVAPVbfKxWKN88OAB9u/fT4vaVzddZgD3TwEPTqtKSETGAQLjAlCdOai8gjK7UEodOpRqQVHNS2+LNcru3bvj0qVLlTEXiiXwBaoluK0b8PQScHSh0e4KJUF2OZfeAM0gRKleNJ051bH0tlijfPfddzF16lRcuXIFQUFBOs6cfv36WW1yFBM411M5d34dBJz5DvDvAjTto7drTqEUSqKKX3ezYPuiGidbGiJEqT5qXRzlp59+CkCV2FYb6sypBpr2Bl7/XCUo934GfHpaJUC1UNsn3e1FJmt568NRTDVKSvWhGUdZK5beSqXS4IMKyWoibCHg3QYoeg78PlpvCYln5chsrgnNck6pTrgapWFJSQjB1cd5eF4oter1LRKUMpkMAoGAU/WQUgMQiMtKSKQlACeX6nTJqoDHG9C0UVJBSal6jMVRKpUEyQ+e4/8OXkeXZcfwzppTOHD5iVWvb9HSWygUokGDBlRzrIm4N1Ylz/g9EjixVFVCwr8Le7i8MZRq6H5vSnWi0NyZQwjkCiXO3s/Bkavp+OtaOjIkZZUHbIQ8ZFtZo7TYRjl37lzMmTMHP//8M9zc3Kw6GUoFCfoQuHdMFYj++xjgs9OqAHVULIYSoPu9KdWLpo3y0qM8vB7zDxvuBgAOYgG6B9ZFn5Ze6Nq0DuxE1k21a/Foa9euxZ07d+Dj44OGDRvC3t6ec/zChQtWmxylHPRZCjw8B2TdAv74FPh4F8DjVVhQOtHwIEo1opk29+iNDBCi+vHu3cILfYK88EYTD4gF/Eq7vsWCMiIiohKmQbEaIntgwGbg++7AnTjgzDqg00SraZTUmfPq8fWhGzh+KxN/fP4G7MWVVxThv4x8rI6/jSlhAWhS15FzTF/A+cAOvpj/TvNKm48mFr/q6OjoypgHxZp4tgB6xwAHpqoC0Rt0KtuVU04bpZMt1ShfVQ5deYpHz4tw46kEHfwqz9z2e/IjHLj8FPVcbTG7TzPOMYWeLYzCcoS5lZdy/zwkJyfjxo0bAIAWLVqgbdu2VpsUxQq0HwncO65K9PvbSBTlfwlABA9qo6SUE6ncOpVADVFSOn6Bnu+YvthJEb/qkoZbLCgzMzPx0Ucf4fjx43BxcQEA5Obmolu3btixYwfq1Klj7TlSygPDAO+uBp6kALkPMEuxARMwsQJe78oJDyKEoFimhK2o8uxLFOtQUsmCUq01vpDqRtVUt0Zp8ZUmTpyI/Px8XLt2DTk5OcjJycHVq1chkUgwadKkypgjpbzYugAfbgbhCfAO/wyGCI7B2bZ8KdLUGmVBiVzvl7a8RO26hA5fxWHjybs4cvWp1calWJ8SeeWGBartkIUluj/GCj0qpVBQgwXlkSNH8N1336FZszIbQvPmzbFu3TocPnzY4gmsW7cOfn5+sLGxQUhICM6ePWu0/+7duxEYGAgbGxsEBQXh0KFDnON79uxBr1694O7uDoZhcPHiRZ0xiouLMX78eLi7u8PBwQEffPABMjIyOH3S0tLQt29f2NnZoW7dupg+fTrk8lq47KzfAU/bTwcALBBsBe/ZjXINoxaUgEpYWovEu9kolCrw9aGb+PSXC5ArKldroViOWkZVukapMKxR6kuzJqrJGqVSqdRJhAGogtG1S9eaYufOnYiKikJ0dDQuXLiA1q1bIzw8HJmZmXr7JyQkYPDgwYiMjERKSgoiIiIQERHB2SlUWFiIzp0745tvvjF43alTp+LPP//E7t27ceLECTx58gTvv/8+e1yhUKBv376QSqVISEjATz/9hC1btmDBggUWvb6awg3/4TihaAUxpKqUbNIXFo8hFvAhKv0Ft5ZDhxCC7MISThutNFFzqXRBSdSCUveHWN/3okZrlN27d8fkyZPx5EnZFqHHjx9j6tSp6NGjh0VjrVy5EmPGjMHIkSPRvHlzbNiwAXZ2dti0aZPe/qtWrULv3r0xffp0NGvWDIsXL0a7du2wdu1ats/QoUOxYMEChIWF6R0jLy8PP/74I1auXInu3bujffv22Lx5MxISEnDmzBkAwN9//43r16/jl19+QZs2bdCnTx8sXrwY69atg1Rq3Yj/quBZgQxRss+Qy3cDnt0Ejsws1zhOVnboSIrkkCm4d4CxfbyU6qVabZR6lt5V6cyxWFCuXbsWEokEfn5+aNy4MRo3bgx/f39IJBKsWbPG7HGkUimSk5M5Ao3H4yEsLAyJiYl6z0lMTNQRgOHh4Qb76yM5ORkymYwzTmBgIBo0aMCOk5iYiKCgIHh6enKuI5FIcO3aNYNjl5SUQCKRcB41gWf5JciGM3b5zgfAABe2Ald+s3gcazt0nhWU6LRVR2YYinmUyCrZRlkqKAv1aZS1LTzI19cXFy5cwNGjR3Hz5k0AQLNmzQxqcIbIysqCQqHgCCMA8PT0ZMfVJj09XW//9PR0s6+bnp4OkUjEeuz1jWPoOupjhoiJicGXX35p9lyqCrVAkni/ATSYBpxcBvw5BajXDnBrZPY41t7vna1HUFJqLlWlURbps1Hqc+bUZEEJqPJO9uzZEz179rT2fGo1s2fPRlRUFPtcIpHA19e3GmekgrMrJ2SWqoREWiLw2yhg1N+AwLxEvuptjBIrCUp1nXFNqiN7NcU8rCkol/11EwIeD1N7vsa2yUt9HIUlNS88qFyCMj4+HvHx8cjMzNRx4BiyL2rj4eEBPp+v423OyMiAl5eX3nO8vLws6m9oDKlUitzcXI5WqTmOl5eXjvddfV1j1xKLxRCLyxenWJlwauXwBcAHPwDr31DFWMZ/CYT/n1njWDvoXNuRA9Cld03GWgHnzwulWHdMVRDv066N2RhadcBDkUwBhZKAzyuzQer7ARUJarCN8ssvv0SvXr0QHx+PrKwsPH/+nPMwF5FIhPbt2yM+Pp5tUyqViI+PR2hoqN5zQkNDOf0BIC4uzmB/fbRv3x5CoZAzzq1bt5CWlsaOExoaiitXrnC873FxcXByckLz5lWzt9SaqDVKNmmvc31VCQkASFwL/PeXWeNYW1Cqc2RqQuVkzcVacZSaKdA0Q80UGkpXkZY9VF/UWI3WKDds2IAtW7Zg6NChFb54VFQUhg8fjg4dOiA4OBixsbEoLCxka4YPGzYM9erVQ0xMDABg8uTJ6Nq1K1asWIG+fftix44dOH/+PDZu3MiOmZOTg7S0NNYrf+vWLQAqTdDLywvOzs6IjIxEVFQU3Nzc4OTkhIkTJyI0NBSvv/46AKBXr15o3rw5hg4diqVLlyI9PR3z5s3D+PHja6TGaAq9CTEC3wZCPgWSNqiyDH12GnDyMTqOo5WX3s/o0rtWYa2ld46GoCwskbPfS80AiBdSORw0EnBUt43S4itJpVJ06tTJKhcfNGgQli9fjgULFqBNmza4ePEijhw5wjpO0tLS8PRp2W6NTp06Yfv27di4cSNat26N3377DXv37kXLli3ZPvv370fbtm3Rt29fAMBHH32Etm3bYsOGDWyfb7/9Fu+88w4++OADdOnSBV5eXtizZw97nM/n48CBA+Dz+QgNDcUnn3yCYcOG6a0TVNMpLJGjsNQ4rpM5qOciwKsVUJSjyl+pNK4xWH3pTb3etQJ1irMSmfUFpSGN8oWWnVL/0luP+Mq+C/w6GJBYd5eXxRrl6NGjsX37dsyfP98qE5gwYQImTJig99jx48d12gYMGIABAwYYHG/EiBEYMWKE0Wva2Nhg3bp1WLduncE+DRs21Nn1UxtR2ydthXzYa++nFoiBAVuA/3UBHpxSecPfmmVwLGuHB2Xp83pTQVljsdbS+/kLrkapRtNhox0ipM+Zo7MzR16i2lDx9BLA4wODfrHKfIFyCMri4mJs3LgRR48eRatWrXR26axcudJqk6NUHE1HDsPoMX67Nwb6rgT+GAuc+EZVQsKvs96xrB4epCddP11611ys5czhLL2l+gWldoiQ3p052oIyLlolJG3dgN6Gd+aVB4sF5eXLl9GmTRsA0CkypvdGpFQrZY4cIyFArQcBqSeAi9tUVRw/PQ3Yu+t0s/bOHOrMqV1Uho2yQGOJzdUotQSl3vAgDXlz8xCQVOqgjFivt2RzRbBYUB47dsyqE6BULmZnNu+zFHh4Fsi+raoP/vFOVao2DRytWA6iSKrQuRkAbsp/Ss3CaktvLWdOTqEUX+y6iAtpuWz7C63EK3qzB6k1yrxHwL7PVf+/Pl5V697KVJ3biFItmC0oxQ6qEhJ8MXD7L+DMdzpdnKxoo9RrnwRNilETUX8kVtMotWyUK+Nu4ditZ5w+2j+iem2UAp6qhv3vo1U17b3bAGGVU4GBCsqXnLIytTamO3sFlQWfx0UDj7mF4ti6OUUV1ygNCUqaFKPmYi2vN1ejVOB5oe73qUjLmaNvpSHk81R29bREVU37DzepHJSVABWULznP8lVfSrOLinUcDTR7F1DKVFsci8sSe6gFZaFUUeHkvdl6YigBUCNlDUZqpVyh2VrOHM0dOGXtpgPObR7+q4rUAFQ17d0bW2V++qCC8iVHrVEadeZowjBAvzWAcwPgeSpwYAob3Ki2UQL665pYAl1610yOXs/Awv3XINMjmSrDRllQIodAj6A0ZaN0Rx5s//wMAAHaDlXVtK9ELBaUJ0+e1JvpWy6X4+TJk1aZ1KtIZVU3zCpPmVpbV+DDHwGGD1z9HUj5GYDKJiQuDfKt6O4cfaFBAF16Vzejt57HloT72H3+EdvGZji3wtK7WMZ14hWWyCHQk1dSOyel5tKbgRIrhBvAFGYAHk2BPtYNBdKHxYKyW7duyMnJ0WnPy8tDt27drDKpV4mcQikm/ZqCd9ecQrGV8/0RQspfz9s3GOg+T/X/oRlApir1nbWCzp/pCQ0C6M6cmkK6pFinzRrOHM1gc0AlKPk8XTFkzJkzmn8Ib/EvAQIblQNSZF/heZnCYkFJCNEbL5mdnQ17+8qf8MuGSMDDmXvZuJ/9Aj+eSrXq2JJiOWtX8ihP9cU3pgCNugHyItWOB1mRRixlxTRKw0tvKilrKsaW3uaGdeVorSQMLr21d+aUjt+GuYMZgp2qxt4xqhr2VYDZcZTqmjIMw2DEiBGc5BAKhQKXL1+22h7wVwkHsQBz+zbD5B0Xseaf24hoWw/1XGytMrZaa3OyEcBGWI5ysDwe8P5GVUq2zOvAkVlwtFFtHzVXoywokaNEpoC7lqA25MyhcrLmUiJX6lWUrj3Jw8ffJ2FqWABGvOFvdAxtD3dhicLMpTfghEKsEa6BkFHgL7yO8PYjy/lKLMdsjdLZ2RnOzs4ghMDR0ZF97uzsDC8vL4wdOxa//GK9vZWvEv1a+yDYzw3FMiW+Pli+Kon6YHflWLrs1sShrkpYggGStyBMmQAAyC8xrVESQtBr5Qm88c0/OpqEWqN0silXSlRKZaPnF4sQQK7H2zZt92XkFcmw8M/rJodVx1CqtchCqZkapUKJr4U/wJf3DA+VdRDD/0xnQ0RlYva3dPPmzQAAPz8/TJs2jS6zrQjDMPiyfwv0Xf0vDl55io/vZOGNJh4VHrcshrKCsWWNuwGdpwKnVmJ0bix2MV+ZpVEmP3iOJ3kqW9f97EK42Zd53tXOnDqOYkg0xqJL75qHpoOtRK7U2WNtiW09p/Q7Wc/VFg+yXxi2UWplD2rzbC/e4SdBRviYKJuIEltHS15ChbHYRhkdHU2FZCXQzNsJw0L9AADRBsIzLKXcjhx9dJsL+IbAVlmINcK1KHxRZPKU35LLPKeaWoNcoWSN+nUduYHwlsjJPRceIWrXRau8VxQtDGhr+gqM6atxY4icF6qViK+rHQCVQNSXVpKjUWZcQ++HsQCAZfKBuEiaVGkuSqAcgjIjIwNDhw6Fj48PBAIB+Hw+50EpP1N7vgZ3exHuZBbgp4T7FR6PUwKiovAFwAc/oojviDa8u2h723jFzSKpAgcul+UE1Fyx5byQghCAx4CjZQKWxZtH7bqEPRce43cNgUwpPxyBZ+AXS5/nW18dbkOoYyh93VR2+EKpXG/sLGujlBYCu0dCSKQ4pmiN7xWqPLPCKixVC5QjKcaIESOQlpaG+fPnw9vbm2YMsiLOtkLM7B2IGb9fRuzR2+jX2gd1nczYemgAnRIQFcXFF0dfi8a7N6bh9fRtwO33gAD9Beb+upbOScqquaTOKt0t5GYv0jHkl2fpnfPCwC4fikWs/ue2yT7agvKfmxkc04kp1J9V/VKNkhD9ERSsoDw8E8i6BYnAHdOKPwUp1e2qWqO0WFCeOnUK//77L5tqjWJdPmxfH9vOpuHSw1wsOXwTKwe1KfdYVl16q8esF4YtV3phhOBv4I9xqpRsTt6cPoQQ/Ho2TadNTVZBmQDnaf3QlsdESc2a1uG/9HyTfbRzUo7act7s8Z/kFrFZ7eu52IJhVJ9dXpGuoC0skatqz6f8DIDBjvrzkX3TmT0u1pfdvBKx+Gq+vr40FVYlwuMxWNSvBRgG2JPyGOfu6wb3m0tlCEpHGwFi5B/jgbAx8CIb2KNbQuK35EdISs2BkM+wSyTNr4y6+qK7g0jHFEa/W9WHvlRmAPezK+82xv2XnqDTkn9w5p7q++zuIIK9SKWn5elJsuKleALy5xTVky7Tcdu+Hed4jbdRxsbGYtasWbh//34lTIcCAK19XTCog6oe+IJ918qdgMJqXm8NHG2EKIEIyxxnAkJ74P6/wL8r2OOPnr/AotIwkak9X2ON9povQb301qtRWm2mFEsx53tW3t05K/++xXnuaieCvVjl09DORiWEHGuEa8BI84EGnYCuM3WEeI0XlIMGDcLx48fRuHFjODo6ws3NjfOgWIfp4U3hZCPAjacSbE96YPH5CiVhYxfrWlGjVMc93pJ7AX1LBeTxGOBBApRKgum7LyO/RI72DV0xrktjVmPk2CjVGqW9GNoWbqpQVh/m2IfLu9/b182O89zNXgR7sf60fTMFv6IVLxVKG1fgg+8BvkAnw7mwipfeFtsoY2NjK2EaFG3cHcSYFt4UC/Zdw7K/buHtIG+d3S3GeP5CCoWSgNHjWa4InL3ebQarSkhc+hX4fTR+bbMNifeyYSvkY8WA1uDzGFZj1OfM8XAUcYpLafejVC3maZTlW3rrE5TqcrSaS+/uvAsYLTgMAMjsvhJezvVVc9OamqiKvd4Wi+Xhw4cbfZSHdevWwc/PDzY2NggJCcHZs2eN9t+9ezcCAwNhY2ODoKAgnWqJhBAsWLAA3t7esLW1RVhYGG7fLvPoHT9+HAzD6H2cO3cOAHD//n29x8+cOVOu11gePg5ugGbeTpAUy7Fca+liCrV90s1OBIEVlylOtlp7vd9eDrg3ASSP4X38CwAEc/o2g5+HKtZWLSg15R/HmaM1tfI5c6hwtQZKA8qi5rtb3gJjXlrRGzZCvo6N0gvZWC5UlZXeJO+NrHo9yuZW25beAHD37l3MmzcPgwcPRmZmJgDg8OHDuHbtmsVj7dy5E1FRUYiOjsaFCxfQunVrhIeHs+Nqk5CQgMGDByMyMhIpKSmIiIhAREQEp9DZ0qVLsXr1amzYsAFJSUmwt7dHeHg4iotVu0Q6deqEp0+fch6jR4+Gv78/OnTowLne0aNHOf3at29v8WssLwI+D4v6qzb97zj3EJcf5Zp9bmU4coAyjZJN3it2gOz9TZBBgO68ZHzldQqfhDRg++tbequdOR4OIp3wMppmrfow5MzRxJiN0tZIPgE9uxTZpbdcScCDEqtE6+DGFOAW0whL5IM5+711lt41XVCeOHECQUFBSEpKwp49e1BQUAAAuHTpEqKjLa9XsXLlSowZMwYjR45E8+bNsWHDBtjZ2WHTpk16+69atQq9e/fG9OnT0axZMyxevBjt2rXD2rVrAai0i9jYWMybNw/9+/dHq1atsHXrVjx58gR79+4FAIhEInh5ebEPd3d37Nu3DyNHjtS5cd3d3Tl9tcvzVjYd/dzwXtt6IETl2NFXjU4flScoy6w16uS9392ww2LZEADAEMkPYJ5eZPuULb3LxuA6c7jjU+Ww+qjo0ttGaFic6Ptc1c4cAJgk2IMQ3k0UEBssc5wJKYQGS9kCtUBQzpo1C1999RXi4uIgEpXZvrp3727xslQqlSI5ORlhYWFlE+LxEBYWhsTERL3nJCYmcvoDQHh4ONs/NTUV6enpnD7Ozs4ICQkxOOb+/fuRnZ2NkSN1s5H069cPdevWRefOnbF//36jr6ekpAQSiYTzsAaz+wTCXsTHxYe5+O2CebtQsirB4w2ovqDqG0JSLMOVR3lY889tbFX0wlPvHmDUJSRKVDF56qW1WqMkhGiEBxmOoySE4HFukVnLaipcrYOm1m/oLTWmUVqaoUqtUb7Ou46J/D8AAHNloyCxawiAu1NIW4aLBDXcRnnlyhW89957Ou1169ZFVlaWRWNlZWVBoVDA09OT0+7p6Yn09HS956Snpxvtr/5ryZg//vgjwsPDUb9+fbbNwcEBK1aswO7du3Hw4EF07twZERERRoVlTEwMJ6uSr6+vwb6WUNfJBlPCXgMAfHP4pt64M22skjnIAOrl97OCEkzddRFyJcHbQd7wGvoD4OwL5NwDDkwFCNGwUaq+6ZIiOWSllnl3e5Ge8CDVsV/OPMAbS/7B2n/uWH3+FP2YpVEa8XobE5SaIzuWCkgHsQBukCBWuA58hmCXvCv2KTvDrlTTLDSwswuoBRqli4sLnj59qtOekpKCevWsW3S8Knj06BH++usvREZGcto9PDwQFRWFkJAQdOzYEUuWLMEnn3yCZcuWGRxr9uzZyMvLYx8PHz602jxHvOGHJnUdkF0oxbdx/5nsXxkxlGrUy+/FB67jTmYBPBzE+CoiCIydG/DBD6oSEld2Axe3saYMtaNAHRrkKNafI1N9r87fp7J3rzDjtVKsg6ag1Pz5Mjfg3Kig1BjDtTQKw17Ix3LhBngxz3FH6YNoucoZrHbyaNootYW4qKYLyo8++ggzZ85Eeno6GIaBUqnE6dOnMW3aNAwbNsyisTw8PMDn85GRkcFpz8jIgJeXl95zvLy8jPZX/zV3zM2bN8Pd3R39+vUzOd+QkBDcuWNYwxGLxXBycuI8rIWQz8PCd1WOna2J93HjqfFlfWXZKIEyjTKltGD9Nx8ElYUgNXgd6DZH9f+h6WigUG1lVGsEWVqaru7S2/J1NF15WwdNYWToPTXm9TZmo9TE1U71/QnJ/BXd+RdRQoSYIJuEIqg843aiUo1SakSjrOlbGL/++msEBgbC19cXBQUFaN68Obp06YJOnTph3rx5Fo0lEonQvn17xMfHs21KpRLx8fEIDQ3Ve05oaCinPwDExcWx/f39/eHl5cXpI5FIkJSUpDMmIQSbN2/GsGHDzHLSXLx4Ed7e3ib7VRadAzzwdpAXlESVis2YUKlMQamZbHdQB1/0aMY1c6BzFNDoLUD2AtMkSyCGlL3xskozm7uXClYdZ47WtURm3BDURmkdzAo4NyIotX/0NNGMZnC1FwGPk9HxzmoAwGL5J7hJyiIl1IKSa6Os3qW3xQHnIpEI33//PebPn4+rV6+ioKAAbdu2RUBAQLkmEBUVheHDh6NDhw4IDg5GbGwsCgsLWcfKsGHDUK9ePcTExAAAJk+ejK5du2LFihXo27cvduzYgfPnz2Pjxo0AVElwp0yZgq+++goBAQHw9/fH/Pnz4ePjg4iICM61//nnH6SmpmL06NE68/rpp58gEonQtm1bAMCePXuwadMm/PDDD+V6ndZibt/m+OdmJs6m5mD/pSfo30a/ucOqKda0cCrVKOu72mLeO810O/B4wHsbgQ1voGHhfcwT/AJCXgegGRpUqlHydDVKzR8ADysGy1OMY05AhTFBaUzQah563UcI/DYCfCLHIUUwflFwnbN2pTZMzeS9ukvvGp5mTU2DBg3QoEED0x1NMGjQIDx79gwLFixAeno62rRpgyNHjrDOmLS0NPA0opI7deqE7du3Y968eZgzZw4CAgKwd+9etGzZku0zY8YMFBYWYuzYscjNzUXnzp1x5MgR2Nhwg15//PFHdOrUCYGBgXrntnjxYjx48AACgQCBgYHYuXMnPvywcusHm6Keiy0mdGuC5X//h68P3UCPZp7sDgc1UrkSz0sTpFotxZoGEW3r4X52IRZHtOTU+ubg6Am89z/gl/cxVHAUFx79BbQcobH0VglA3aQYKoePGjdz65FTKoxcI+J8S8J9+LraYWBHX4CT4dxYgTFzrkIwJm8V8Pw+iuzqYVbOGEBrI6t9qUapmedSOxi+RmqUUVFRWLx4Mezt7REVFWW078qVKy2exIQJEzBhwgS9x44fP67TNmDAAAwYMMDgeAzDYNGiRVi0aJHR627fvt3gsYrsNKpsRr/ZCLuTH+FB9gus+ec2ZvfhanVqrU3AY+Bia/24z57NPdGzuafpjk16YJ/DQPQv2IWg5HlASDdkFaqX3gZslAAe5b6waD76gtR/PvMATeo4ILSxu0VjvSoUlMix/+IT9Grhyf6Yagqj/GI5Zvx+Gf3a+HDOM+b1NiYn1cdiAy6Df/0PgOHjXtfVkPyhW43TVo8zp1YsvVNSUiCTydj/DUGT+FYNNkI+ot9tjlFbzmPTqVQMaO+LJnUd2OOaCXu1l7ZVzW7H4fCVXEA72R3g90g8F36lmhvrzOH2VyoJHj8vKzNhSZkBNafvZGH+XtVOrftL+pZz5i83c/Zcwf5LT/DzmQc4PPlNAPrDg7TLCpcYK7thTKUkBAHMI7z9KFb1vMd8kPodAZzS6apPo9TJHlQTk2IcO3ZM7/+U6qN7oCd6BNZF/M1MfPnnNWwdFcz+UFWmI8dSCF+ASbKJ+MdhPkSPzqGX3Q84jPdY2yMDXY3ycW6ZoCw2I1uN9v1504wEtK86R66qYoo1oyf0bWFUf5fUlFejFCiKsVa4GiJSAjTuDnSaDIcc/XWX9NkotXek1fikGJSaw4J3m0PE5+Hf21n461pZOFRlOnIshccweETqIKXNlwCA917sRhfeJYMaJSFcLcaSCn9qiiyo4UIpQ9/22AxJMRupAOjaKF3sykw7xpw5b91fiaa8R8gXuKts1zweuzNHG7M0ypq49H7//ffNHnDPnj3lngzFMhq622Nc10ZY888dLD5wHV1fqwNbEV9j6V39jhC1lvvIqxdCOkQC53/ESuF6FGIQADfdpBiEcDREcwSl9u1ZVA7h+sqhRyHTp1Gu0doZlSkpwf2sQjY7lFwj/5n69DP3suHtbIOG7qXVWq/+jlYZe6EkDP7wW4BhDnUBQMcJqcaWFZSaGiW3jzlhY9bErKtpbstzcnJCfHw8zp8vq5WRnJyM+Ph4ODs7GxmFUhl8/lYT1HOxxePcIqw/cRdAzVp6qzVGJSEo6r4YN5QN4MFIUO/YFECpMJnhvDwZtV+Uw65J0W+jvPaEu7HhVkY+3lp+HI+eqxxuUg2bJSHA07wiDP7+DN5ZcwoPc14AOalAaUmH7xT9cM+pI9vfRsjTn1WoBjpzzLra5s2b2YenpycGDhyI1NRU7NmzB3v27MG9e/fw0UcfwcPDo7LnS9HCVsTHvL4qr/eGE3eRlv2iUrcvWopmPsqsYgYTZBPxgogheHASOPWt3qW35i1hTuovALj3rAAff38GCXezyuUAetXQZ+EzNzMVAFx8mAtAVaNdDQGQXSAtrawox7Qd50B+GwWUSPDIsRW+lX/ICQdjGIYVigAw6g1//DWlC5tVyFj2oBq/hXHTpk2YNm0ap4Y3n89HVFSUwdRolMqld0svdG7iAalciUUHrmtolOUvdWstNDXK7EIp7pJ6WCkco2o89jV88i9x+mtrDoSYvoFXx99G79h/kXA3Gx9/n8TRRGhSX/Mx90cJUDnhFErCCVInhECu0dDjyf/APLkA2LjgzyaLoYDuXnBNO+XAjvXR1MuxRoYHWXw1uVyOmzdv6rTfvHkTSkMpkimVCsMwWNivOQQ8BkdvZODSwzwANWPprbZBEpTt8z7r3BsIGggQBXrdmANnFLD9Cco8smq0b+Bz93Mwass5TpvmEtBYMgWKCn2RfJa8VzwGkGmFChFSNkYP/kWMFRwEAPz3+hJIxKo8C9pRDpo5KQWlG0vUzhypXMleQ3tqwpru9R45ciQiIyOxcuVKnDp1CqdOncKKFSswevRovfkcKVVDk7qOGNXZH0CZ0KgJzhxNjVLtzXZ3EAPvrATcGsGxJANLhRuhXnATQpCaVcgZQ/MGLiiRY8CGRPxzU38GfIDrAJJTQakXbYGV90LGpr8zh8R72bqCEgQKJYEncrCitKTDFnkvDE+si9zSnWLaAlrToSMo/bLYaSzH1T96Ool7a2IcpSbLly+Hl5cXVqxYwaZb8/b2xvTp0/HFF19YfYIU85nYvQn2pjxGZo1y5pRlOM99UZbZHGJH4MPNUHwfhnD+eQxT/o2tinC9ISaaN8nq+Ns6x7XRDCuhgtI0z/JL0PH/jlp0ztbEBxj5hj+njRBALpchVvgdXCCBwjMI2wvG4Gl2MX49m6Z3HM2lN79UUIoEPAj5DGQKghdSOZxthbXPRsnj8TBjxgw8fvwYubm5yM3NxePHjzFjxgyO3ZJS9TjaCDHnbZVjx1EsMBh+UZVoJu5V207ZapI+bZDUZAoAYK5gG1ow9zlBxmrUwi5DUowfT6WavGaRRlC03NhOklcYTc0u4a5lCbfVqD3fagiAele+Qyj/OopgA/6ALRjRhZtHQXvBrKk9CjSW0+r6O2qNUtvWXCPjKA1hzXyLFOvQv40Pcl9IUc/VrkZsKWWLiylVzhyAaxK47jsYhTf/QU9+MtYIV+NsfjedMdTOnIsPc82yo2neVFSjNI24nMtYdeIVNS1lV9Hgiip12jr7zzHNowmE97nJq3WX3hpOYY0QCHuxAJJiOV6U/nDqBpzXguxBv/32G3bt2oW0tDRIpVLOsQsXLlhlYpTywTAMRmgtiaoTzaV3Vj43xRqgWqFMl43FId5sNOKlQ3blKwBDOGOohd3tDPO2JmoKR7kFdrdXFbGFtW7UqE0pAOAKCeYWrwBDlPhd8Sb+te2BaTCd/8GeY6MsE9jayXu1FwY1MuBck9WrV2PkyJHw9PRESkoKgoOD4e7ujnv37qFPnz6VMUdKLYYbHqRHUDJALhwxWToBCsKgacZBvM87yRlDbbf8L0PlHTdle9XUOrUdDpqUt0b1y0Z5NcpcVqMkWCb8H+qQbBQ6+mO+bCSbjEU7TlZbcDrosVECZQK0yMDSu8bbKL/77jts3LgRa9asgUgkwowZMxAXF4dJkyYhLy+vMuZIqcUwmgHnpXuG1bkoNY+fI4H4Vq7K9blYuBmNmCdsH7WG+F+pRhlUz/gOME2vuaGl+p3MArw27zAW7re8Fv3LgKa4Kv/SW/V5juIfQRg/BSUQIiV4JV7AhvVga++80tYv7fV4vYEyGyWrUda2OMq0tDR06tQJAGBra4v8fNWXd+jQofj111+tOztKrUd9n0gVSvbGUueiBLgax3eK/rhl2xb2TAnWCtdADFV/hYJAplDi3jOVAGxpQlBqIjcQ27vmH5X3fEvCfbPHonDJfSFDS+YeZglUeV3XCUfguZPKeaPWDk2ZyTmCkq+rUbI2ymoOD7L4al5eXsjJyQGgynKuruWdmppKd0FQdFBrFOqtbQyDskJk4C7FlOBhresMZBEnNOc9wBzBNgAqbeJBdiGkCiXsRXw0Kk3IYA4/JTyw0it5ueC87+W8bYsKnmOtcA1EjAJHFB3xG9ObNZOUCUrttTf3qYOegHOgzEb59/UMTN15kVO6FqgFAefdu3dna1uPHDkSU6dORc+ePTFo0CC99b4przZqjTEzvxgA4GYn4tiitJdmaVInfCH7DAAwXBCHcN5ZKJRK1j7ZxNPR7Gp/gCrTOcU4luzxLoPg48xv4cfLwCPigRmyMZCTMucZv1To6dgotSSlZniQZl/1HvCjNzLwR8pj3Z05vBoeHrRx40Z2q+L48ePh7u6OhIQE9OvXD+PGjbP6BCm1G7UgfKbH4606zu0vKZbjkrI1NsjfwaeCA1gq3IisvI9wK12lYbxW1wFigWVe2sUHriO0kTvCzClf8YrAqdtdjvMH8E+gS8kJyAkPU2QTIIEDhErCLpEN2Si1UTtzBDyGo32+29oHF9Kew8fFFm0buKCNrwtkCoIxW89DwGOqPHO/RYJSLpfj66+/xqhRo1C/fn0AqjrfH330UaVMjlL7YbOus9sXudsqtW8kSZHKk7pCPhCv826gDe8uhEfH466jqgpnUy9Hi0NDfjyVih9PpdKyEAYwp0ytJo2Zx1gk2AIAWCkfgBuCZoBUAbmSsE4Xdumtda623FTbIvlagq9zgAfiorpy2s7cywZQ9Y4cwMKlt0AgwNKlSyGX0wzSFPNQf/+fSfRrlNp3kqRYJShlEGCCbCIkxBZ2GcnolKYqRxzg6VhuL+3n25LxvFBquuOrgMb7bq6c5DGAGFKsE66GLSPFv4qWWK94FzalHmqFsix7EJ/Rb6PU9XqrzhWYoSGqe1S1fRIoh42yR48eOHHihFUnsW7dOvj5+cHGxgYhISE4e/as0f67d+9GYGAgbGxsEBQUhEOHDnGOE0KwYMECeHt7w9bWFmFhYbh9m7tH2M/PDwzDcB5Llizh9Ll8+TLefPNN2NjYwNfXF0uXLrXOC36FUGuM+aXGeFMapWZihkekLmbJVCnZPir5HZ15V9DU09HipbeaQ1fS8c0RVear6t+zVHMwV6MU8nmYL/gZgbyHeEacECX7HAQ8VlDKlUooSuNW+Xz9cZTa1Hexg1jAg6+bncnr89i94FW/VdpiG2WfPn0wa9YsXLlyBe3bt4e9PdcD2a9fP4vG27lzJ6KiorBhwwaEhIQgNjYW4eHhuHXrFurWravTPyEhAYMHD0ZMTAzeeecdbN++HREREbhw4QJb23vp0qVYvXo1fvrpJ/j7+2P+/PkIDw/H9evXObW9Fy1ahDFjxrDPHR0d2f8lEgl69eqFsLAwbNiwAVeuXMGoUaPg4uKCsWPHWvQaX2W0bxRTNkptDilfx3++T/Haw91YJfoObrwRyBOY7/XWJkNSXO5zXyY033ZzfTnvCJLwCeIBAFNl4/EMLgDAOtcUSgL175zAgNdbe+ntbCfE0aiucLQxLYrUp1Z1YTGgHILy888/B6C/fjfDMFAoLMsuvXLlSowZM4ZN0bZhwwYcPHgQmzZtwqxZs3T6r1q1Cr1798b06dMBAIsXL0ZcXBzWrl2LDRs2gBCC2NhYzJs3D/379wcAbN26FZ6enti7dy/Hnuro6AgvLy+989q2bRukUik2bdoEkUiEFi1a4OLFi1i5ciUVlBagfaNoZ103ZewHgG0un2LwgzMI5D0E/vgU4j4/W3WOryKan4s5YX31mUwsxP8AAN/J++GUMog9pq5xI1cSKEodveqltymvNwCztEkAcLFTrUbcqyFzv8VLb6VSafBhqZCUSqVITk5GWFhY2YR4PISFhSExMVHvOYmJiZz+ABAeHs72T01NRXp6OqePs7MzQkJCdMZcsmQJ3N3d0bZtWyxbtoxje01MTESXLl0gEpUtFdWa7vPnz/XOraSkBBKJhPN41dEWhNpLb3PydlzJKMEE2URIGTFw7xjcLq4v93xqQqKQmoYpOSmAHGuEa+GIF0hWBmBl6Q4qNTalS2FCykwnfDO93pbQpK4D/je0Pb4d1MZqY5pLtZarzcrKgkKhgKcnN2zD09MT6enpes9JT0832l/919SYkyZNwo4dO3Ds2DGMGzcOX3/9NWbMmGHyOprX0CYmJoZTiM3X19fga39VML30Nn0j3XiajzukPs42U60wHBOWoB3zX7nm8yptilj59y18seuSzmuWypXIKyrL/GPKRjlNsBtteXeQD3tMkk6AXGshqtYogbJicOpdNjrx5hWUm+EtvNCkrkPFBikHZi+9i4qKEB8fj3feeQcAMHv2bJSUlNVf5vP5WLx4MccGWJOJiopi/2/VqhVEIhHGjRuHmJgYiMXlU+1nz57NGVcikbzywlI73s2UM0cf6vKzvLZDAd41MFd/w2rRWrxd8jUksOym0Xe9G08lCPRyfOm0zdWlpWaHd2qIVvVd2HbtJLrGbJRdeZfwqeBPAMAym4l4XFxHp4+mc01d95tnpte7tmC2RvnTTz/hf//7H/t87dq1SEhIQEpKClJSUvDLL79g/XrLlkQeHh7g8/nIyMjgtGdkZBi0HXp5eRntr/5ryZgAEBISArlcjvv37xu9juY1tBGLxXBycuI8XnW0ZY+2RmmJbArwcgLe+RbE1R/1mSwsFX4PS8Ol7fQkM+6z6l/8fuGxRePUVPKLZToaZH4xN5xPvUtKjSGNsg6eY4VQdU9vlfdEgqiT3n4cjbI0abLAQPag2orZgnLbtm06Tozt27fj2LFjOHbsGJYtW4Zdu3ZZdHGRSIT27dsjPj6ebVMqlYiPj0doaKjec0JDQzn9ASAuLo7t7+/vDy8vL04fiUSCpKQkg2MCwMWLF8Hj8VhPe2hoKE6ePAmZrGyJEhcXh6ZNm8LV1dWi1/kqo6nBOYgFbChJ2XHzxnGzF6kS/to4AR9ugpTw0Zt/Dp/wLSthoN5brK3pTNt9yaJxaiI30yUIWvg3PvuFmxN2balmqUanlroeQcmDErHC7+DBSHBD2QD/Jx/COmi0sdGIa1XXayrbwmjltXc1YbagvHPnDoKCyjxdNjY24GnstwwODsb169ctnkBUVBS+//57/PTTT7hx4wY+++wzFBYWsl7wYcOGYfbs2Wz/yZMn48iRI1ixYgVu3ryJhQsX4vz585gwYQIA1Q0wZcoUfPXVV9i/fz+uXLmCYcOGwcfHBxEREQBUjprY2FhcunQJ9+7dw7Zt2zB16lR88sknrBD8+OOPIRKJEBkZiWvXrmHnzp1YtWoVZ2lNMY3mbaGv2Jm5y92Aug5sX6ZeOywnquS+8wW/oDlz3+z5aNaRftnYcvo+AODItXTcfVZW2TKxdEeLGl1BqTvWZ/z9eIN/DS+IGBNkE1ECkc7uGTWaP35qjVK9eUZnZ44Zr6MmYva3Jjc3l2OTfPbsGee4UqnkHDeXQYMG4dmzZ1iwYAHS09PRpk0bHDlyhHWcpKWlcQRyp06dsH37dsybNw9z5sxBQEAA9u7dy8ZQAsCMGTNQWFiIsWPHIjc3F507d8aRI0dY+6lYLMaOHTuwcOFClJSUwN/fH1OnTuUIQWdnZ/z9998YP3482rdvDw8PDyxYsICGBlmI5k2psysH5ntFm3o5cp7v4PVFsOIKwvgpWCNcg3el/4cXMG0ft68BdYQqC823sscKw5tCtN9ybRtlB+YmogS7AQDzZSNxl9QDwE2DpomQzwOfp6rzrbZRqjXKl8Xua/a3pn79+rh69SqaNm2q9/jly5fZ/d+WMmHCBFYj1Ob48eM6bQMGDMCAAQMMjscwDBYtWoRFixbpPd6uXTs2PZwxWrVqhX///ddkP4phNJUQbUeO9nFjBHhyBaVYKMD04nE4xJuDxrynWCTcgmmyT02Oc/6BKkXg8xcv41ZG895M7R8nTRulMwqwWrQWfIZgj6Izfle+yR4zpFEK+QwrKNVZ4w3ZKGur3DR76f32229jwYIFKC7W3dlQVFSEL7/8En370qQDFC6MlTTK17RCQsQCHp7DCZOl46EgDD7kn8R7PNM/aqfvZCNTUoxMieWrn5cFbeFVJigJlgv/Bx8mB/eUXpgvGwlN4WtoP7adSMAeU4cHsaUgtM7RF3BeGzBbUM6ZMwc5OTlo2rQpli1bhn379mHfvn1YunQpmjZtiufPn2POnDmVOVdKLURTEOrdUWHmffOalkapziB0ljTDKvkHAICvhJvgzzw1OVZWgVSvr3znuTTEHLpRq0rcPsktwv5LT8yqTqlGezmszsozgv8XevKTUUIEmCibhELYcvpp/6itH9IOAzvUx4AO9VltU730FpiZPai2YPbS29PTEwkJCfjss88wa9Ys1lPGMAx69uyJ7777TidAm0LRVCjq6F16m3fnuNpzz9WM3VuriEAo7zpC+dexVrga70kXQQqhwbE0HR2azPz9CgCVUP6gffnMSFXNm0uPQaEkyCuSmRRCd58VYMzW8zre61/PPkQLJhWzS0s6fC0fgmvET+d8bRtlz+ae6BPkDaAs9Zl66W0ww3ktxSLLtr+/P44cOYKcnBzcuaMKOWjSpAnc3NwqZXKU2o/m0kufRlneODvNVGtK8DBZNh6HebPQgvcAswXb8aV8uMFzrz+VGN2hU5vsl2pNMvFuFrsX2hDR+66xdYc0sUcR1gpXQ8zI8beiPX5S9NJ7Pl8rq7imEORrLb35hmyURmdYcynXFkY3NzcEBwcjODiYCkmKUTQVivLaKPWVftBO3psJV3xR6swZKfgLPXnnDY5nKiflVwdv4PSdLJPzqkkwjGnrn/7SvURlsuBl4DFxx3TZOBgSZ8ZyRmrbKM3NHlRbqNa93pSXH66NUl8cpfHznWwE2Db6dZ12fXWdjyvbYqNc5VBcJvwffKBf2JlTRmD4JuM5UWsaPIbRKcCljViom8fxQ/5JvMc/DTnhYbJ0PPKMbAnV9nozeo5pL71fuZ05FEp54JnQKI0R6OWIS9G90L6h7k4oQ+UglskH4aKyEVyYQqwSrQUfuhmteIz5Wb1rCzwG2HvxidE+NlrvmWZJh2/lH+I8CTR6vqGdOYCmRsl15ujU9a6lKiUVlJRKRX2jiPg8OOlLzmpCYBm6sTT3F2sigwATS0tIdOT9hymC33UvSQBi4sKW1uWpbswxYWhqlGJIsVa4BnZMCU4rWmC9wnTCbT7f8DJa20bJY5feJoetFdSubwOl1qEWdO4OIr1Cz5i4Mqb12elZRqp5SDwxRzYaADCevw+deFc5x7WTROijOgpYVQRzBJKmRjlXsA3NeGnIIk6YIvscShOioKG7HboEeBg8rq7JrZ0Uo7bGTWpTu74NlFqHeultaNltLBeiMa3PkEap5oAyFNvl3cBjiCq5A/LYY/svPWHrhBuitglKY8tiNeJSp1hv3lkME8QBAL6QfYZnMJ3k5dgXb6GuE3eLqD6vt05SDK23sbZqmLXr20CpdfA0NEp9GNMajR0zJSgBYJF8GG4p66Muk4sVwvVgYH4geXXUZakIppbeuS+k+OVMGuozz7BUqKpouUH+Lk4oW5s3Po9B1wDdXJRq1DGW6nAl9e+Mjo2ylmqYVFBSKpVgfzfUc7FF39LAZG2MLr2NHLMTmg4BLoYYE2STUERE6Mq/jLH8gybPUSOswTbK/zLyUaDl4dbW3LT55sgtCCDHauEaODEvkKJsguVyw/kS9MHjMbA1YPJw0Eo2wibFsOgKNZea+22gvBQ083bC6VndMaCD/kzvxgK/jR2zM6JRNvMuS5h8m9THl/JhAIBpgl1oy9w2dBqHmrr0Hr/9Anp9exJhWtmBTHmTrz+VIErwG9rx7kBC7DBRplvSwRwMmUOcbLg7oWgcJYViRcqrURpbeh+e/CbeaOLOPt+h6IY/Fa9DyCiwWrgWTtDdnaJNTRSUZ1NzcPCyai97ulbZ3X9vP9N3CksP4RV8LtgPAJgpG4NHRLcUdEVwsuUKXfWS+5XemUOhWI1yxjMa0ygBbdsYgzmy0XigrAtf3jMsMaOERE0MD3r0/IXBYw9zigweq4NcjMpcAgD4Rd4Dh5Uh5Z6DwMAa35BGac0qjNVJzfs2UF4pjMYzGgsPMsOZo0k+7DBRNhEywsfb/LMYwo832l9YA7eUWJIhSA0DJVYKv4OD/DluKH2xWD60QnNo28BFb7uTLVdQ8iupCmN1QQUlpVpRGnFEG196W25fu0wa4xv5RwCABYKfEcikGexbE5fepsrK6uMz/p94k38VUp4NJsgmoQTGE2eYonug/iW7m1Z2Jz6jX6OkXm8KpRwYDzgvnzPHGD8q+uAfRRuIGRnWClfDFrqJqIGaufS2NE1me+YWW9Jhf72pbEmHivB+2/qwEfLwmid3T7iXVoylgO7MoVCsh1Gvt5HzDIWpmLweeJgm+xTpxBVNeE+wULBVbz9hDYyjVGi9V71jTxrs64wCrBKtg4BRYq+iE5Ice1tlDs52Qpyf1xN7x7/Bafdy5gpKQ/koy6MV1wSooKRUK5qhPNoY3cJYTo0SAHLghCmy8VASBoMEx9GPd1pPrxooKLVUypvp+QZ6EiwVbkR9JgupSk/MlUVCVg77piEcxALYaZk+PJ30C0ptU6/+VG81HyooKdWKr5sdDkzsrPeYMUePq4kktaY4o2yONYr3AABfC39EQyadc1yhx3iamlWIvqv/xaErpstNWJsbTyXIKzK9Rx0AhvLjEM4/DynhY6JsIj4MDYRMUbmanLu9iKOF8w14vaWVPI/KokYIynXr1sHPzw82NjYICQnB2bPGcwHu3r0bgYGBsLGxQVBQEA4dOsQ5TgjBggUL4O3tDVtbW4SFheH27bJA4/v37yMyMhL+/v6wtbVF48aNER0dDalUyunDMIzOw5zqjRTLaFnPWW+7MY3S1V4ExwqWnl0tfw9JykA4MMVYK1wNEWTsMe37OUNSjG7Lj+PaEwk+33ahQte1lKR72eiz6l98e/Q/k32bM/cxT/ALACBG/jGukkZwsxeze7ArCx6PQV3HMq1SwJar5fajGmU52blzJ6KiohAdHY0LFy6gdevWCA8PR2Zmpt7+CQkJGDx4MCIjI5GSkoKIiAhERETg6tWyDDFLly7F6tWrsWHDBiQlJcHe3h7h4eFsBcmbN29CqVTif//7H65du4Zvv/0WGzZs0Fsc7ejRo3j69Cn7aN++feW8ERQdTJmzhoY2LPe5AKAAH5Ol45FDHBDEu4+Zgh3sMaXGUlVSLEPI18bDiSqTw1fTTXcCYIdirBGugZiRI07RDpsVKrvkg5xCxF3PqMwpAgA8ncoSn6jDLbW93DI5FZTlYuXKlRgzZgxGjhyJ5s2bY8OGDbCzs8OmTZv09l+1ahV69+6N6dOno1mzZli8eDHatWuHtWvXAlBpk7GxsZg3bx769++PVq1aYevWrXjy5An27t0LAOjduzc2b96MXr16oVGjRujXrx+mTZuGPXv26FzP3d0dXl5e7EMoNFy0ilK1VDSYOdDLEelwZ+uBRwoOowcvGQBw6k4WvjpwHQolwVvLjld0qlXCYuFmNOY9xVPixinpsOfC4yq5vqadUq1RUhulFZBKpUhOTkZYWBjbxuPxEBYWhsTERL3nJCYmcvoDQHh4ONs/NTUV6enpnD7Ozs4ICQkxOCYA5OXl6a3/069fP9StWxedO3fG/v37jb6ekpISSCQSzoNSeRiTk6YS8zrbCjGrjyqj9z/KdvhR3gcAsFz4P3hBVb71h1Op6L7iOHJM1NipTJRKgi0J9032e593Eh/w/4WCMJgknYBcOJo8x9poCkpqo7QiWVlZUCgUOmVuPT09kZ6uf7mRnp5utL/6ryVj3rlzB2vWrMG4cePYNgcHB6xYsQK7d+/GwYMH0blzZ0RERBgVljExMXB2dmYfvr76E0FQzMNY6BBg3C8tN3FD2on4nJv4G/lHuKz0hyujCqtRl5B4kG1422Bl8iC7EF8fuoHfLzwy2bcR8wSLhZsBALHyD3DOREmHykIzRIhvII6ytmqUFbOGvwQ8fvwYvXv3xoABAzBmzBi23cPDA1FRUezzjh074smTJ1i2bBn69dOfNn/27NmccyQSCRWWFcCk7mFEpTS13c9WxOcUy5JCiImyiTggmosQ3k1MEuzBt0bSkBFCKrX+ywfrE5BVYFqTFUOKNcI1sGdKkKBojnWKCKP9RXxepTl2vDhLb/1xlLVVUFarRunh4QE+n4+MDK6hOSMjA15eXnrP8fLyMtpf/decMZ88eYJu3bqhU6dO2Lhxo8n5hoSEsPXM9SEWi+Hk5MR5UMqPKYeMPjGl3otsKm7QTsTXkbMPiBfmyiIBABP5exHKu2bw/Kd5KsdgkVTBcfxYC3OEJADMFmxHC94DZBNHVWyoiVvanITH2kzq3sSsfvqX3tw+VFCWA5FIhPbt2yM+vsyjqFQqER8fj9DQUL3nhIaGcvoDQFxcHNvf398fXl5enD4SiQRJSUmcMR8/foy33noL7du3x+bNm8EzlfkUwMWLF+HtrT8BLcX6mLIz6jvaqjTUSG7ihrQTCvQ6g/YrO2Gn/K3SEhLr4K5RQkKTTkv+wbakB2i24AjeX59g9FqVRS/eOYwQ/A1AVdIh04ySDgF1DZejNUQdR/OqZ2ouvQ1lD5LKa6eNstqX3lFRURg+fDg6dOiA4OBgxMbGorCwECNHjgQADBs2DPXq1UNMTAwAYPLkyejatStWrFiBvn37YseOHTh//jyrETIMgylTpuCrr75CQEAA/P39MX/+fPj4+CAiIgJAmZBs2LAhli9fjmfPynL5qbXOn376CSKRCG3btgUA7NmzB5s2bcIPP/xQVW8NxQS5L8q0rt8/64TDV55iclgAANNLb5GAp1OnWs1C+TC0491GAO8xlgs3YJRsOogenWLuH6qQtIsPc/E0rwjezrblfSkW44MstqTD/+R9cVzZxqzzWtZzxvkHzy26lrkmBs3wIPXbr31qZcdzVhbVLigHDRqEZ8+eYcGCBUhPT0ebNm1w5MgR1hmTlpbG0fY6deqE7du3Y968eZgzZw4CAgKwd+9etGzZku0zY8YMFBYWYuzYscjNzUXnzp1x5MgR2NiofvHi4uJw584d3LlzB/Xr1+fMR9OBsHjxYjx48AACgQCBgYHYuXMnPvzww8p8OygamFp6Z0pK2P/bN3Tl1P+WmxCUUb1eM+gsKoINJsgmYp9oPrrxL2G08hC+V7xjdLzQmH9wf0lf4xM2k/Q8/Yk61PChwGrRWrgwhbiobIzl8kFmj20ouN8Y5ppi7UQCfNq1MfKKZKx2qa1R1tY4ymoXlAAwYcIETJgwQe+x48eP67QNGDAAAwYYNrQzDINFixZh0aJFeo+PGDECI0aMMDqn4cOHY/jw4Ub7UCoXU6a/zHzDAsXU0rtdA1dcSONqVnUcxXiWrxK+t0gDLJIPw9fCHzFDsBNnlYG4RMyz1VWUJYdvGD0+VfAbOvD+g4TYYqJsAmQW3MYN3e0sno8l8arqkCs12qfKjeXVq8FUe8A5hWIY45IyM7/E4DFz9jZrl3jd8El7jHzDj32+XdEdBxXBEDIKrBGugSOqJlRo78UnBo+9wbuCz/mqELXZsjF4SDwN9tXHczNjQqPfbc7+XxHfvvbOHGkt1SipoKTUCD5srzKBdG7iYfY5g0oLlnV9TbeMqjnZwLU1pToOYkzqHqDRwqiEkbIOGvCeIcaMEhIVpbDEcOILD+QhVvgdeAzBdnl3HFS+bvH4QfXNW3prZmeqyA4obTNwj2aWCfaaAhWUlBrB4v4tse7jdvjuk3Zsmykb5WdvNca20SFYr3GOGlM2SkC/7U0s5N4SEtizJSTe4SdhMP8fk+NWBEOzVpd0qMPk4ZayPhaVs6SDnUiAnWNNC1jN2jgVCRfVFLJTwgLwadfG5R+sGqGCklIjsBXx0beVN6dIlSlRJ+Dz8EYTD53ciIB5tjB9mpJITwmIi6QJlskHAgCiBVvxGvNQ73gyhdLkbiJjZEiKse6Y/jjdcfwD6MK/giIiwgTZJBTDvJAdbRgGCGnkbrKfQCNlWkUC6zVPHdnJv0ZmjjeH2jlryitBRYSOQo+Nsp1WYSztOE0CAoGBWjnfK/rihKIVbEpLSNhA1z4a8nU8PvkxqdxzDvk6HuuP39WdN/Mfpgl2AQAWyofjNqmv08dczBV5msKxInXWGIbBX1O6YP+EN+BsV3sTylBBSamxVMQaqG/pra0ZacthY7tWCHiIkn2GTOKC13iPEa2nhEROoRSn72SXb8IGcEIBVovWQsAosV8Rip2Ktyo0njHtcGyXRmX9NNormqWpqZcjWtV3qdAY1Q0VlJQaS0XKq5hTm0W7i2biWX1kwxlTZJ9DSRgMFhzDu7zK3pFD8I3we9RnsvBAWRdzZJGoaIkKQ2fP69sMXQLKnGKasvFlKRBWEaigpNRYKrL01ndzTwkLgINYgNGd/QGUr9BVgrIl1ir6A1CVkGjA6CbE/TnxvsXjHtGTnPcT/lH04Z+DlPAxQTYJBbA8BlIb9fsyOLgBp72jnxtc7cuWxpphPRUtu/EyUCMCzikUfVRk6a2vfrSfuz0uLujJ2iE1BeX4bmXe2B6BdRF/U3+GfQBYJf8Ar/NuIJh3C2uEa/ChdCEn6Hv+vmsYGupn9lyLZQp8+ksyp60Z8wDzS0s6fCP/CFdII32nsvwcGQxXOxHeWXPKaD/1S17YrznCW3jC180OWfklaO3rAgCY+3Yz1HPlbsV0d6CCkmqUlJqLlUMW+TyG46zRNGNODy/bUaIdIqSNqoTEBDwnDmjNu4cZGiUkysO1J9zEG7ZQ1fARMzLEK9riR8XbJscI9HIya3ui2nYrFvDxVtO6aFzHgeMFH9OlEd4O4iZ+8XAon4f9ZYIKSkqN4zVPVYabXi30p9ozB31Lb4GW+9ZQbXDtZak+nsK9tNwCMEZwCN14KZzjf156YnZKsSWHb3KeLxJsQWPeU6QTV0zTKOlgDFOZlpp5O6FVfWc42Zi3iMwrKiu0RpfedOlNqYFsG/06jt7IQL/WPuUeQ59o4WkJymbejhj5hh98tLL+vBlQB/FfdIWAx6CrkXo5R5XtsVkejpGCv7BCuB59SpYgA6pyIhN/TcGsPoFmBVhrmkojeKcwQHASCsKotFaYmdPUhPZ9sLQksLkxkdkFZeFPtTX20ZrQd4BS46jjKMbg4Aawr0A52jFddG162mEuDMMg+t0Wevs2ruOAhu72ODLlTaPXiZF/jKtKP7iVlpDgoUyLjL9huvJhwt0sNu2ZH/MUXwlVRfXWKN5DEmlm8nw1pqwUPB6j80NhjJxCmelOrxBUUFJeSiZy9myrKE/gtL2eXT+aSCHEBNlEFBAbvM67gYn8P9hjppzqcoUSH3+vClAXQYa1wjVwYIpxRtkMq+XvWzTPioRS6ePNANWee5daHCRuTaigpLyU8HkMWmkkgGjq6cjZHmnJOKa4T7wxVzYKADBJsAchjCpN2vkHz/FCajjJhWa97tmC7WjJu48c4oDJUtMlHbRxtrWuQHuraR3sHPs64qO6WnXc2goVlJSXlh+GdcCsPoE4NzcMhye/adHSU40+QamdcxEA9ik7Y7e8C/gMwSrRWrhCVap44X7DdXfUaeLCeMkYKfgLgKqkg9rOaQo/dzu8GeCBfePfYHcV/TujG3aNC8XZOT2wcWh7AMCi/i3MGk8ThmEQ0sgd7tTjDQBgSEWieilGkUgkcHZ2Rl5eHi00Vkt5kluETktUGYN6NffE4OAG6BZYF36zDur0tUUxDojmojHvKeIVbREpmwaAMZj5vOuyY5BmP8Qh8Wy4MgX4Xv42/k/+CafPxQU9sfv8IzzJK8Lm0/c5xz5oVx8rBrY2Ov9imQI2Brz7rzqW3J9Uo6RQjKC5pP1uSDt0C6xrsG8RbDBeNhklRIge/BRE8g8b7Hv5US4eZeertE+mAJeUjbBU/pFOPxc7EcZ0aYTRb+o6nMxxYFMhaR2ooKRQjGAvFuCvKV1U4UIawervGghdukkaYHGpVjhT8CuCmHs6fVLSniPpXg4mC35HMO8W8omtKuelkWi9ei62WPBOcyz9oBXbRteCVQcVlBSKCZp6OaJxHW6Z1/fb1TPY/xdFGA4pgiFiFFgrXI1uX+1na3+npD3He98l4J8jv2ECfx8AYI4sEmlmlHQY1dkfAzv6VuCVUMoLFZQUSjl4zdOR/f+riJa4ubi3xlEGs2Sj8Yh4oCEvE1NLvsPYrecAAGfu5cAdeVglXAceQ/CrvBv+VHay6Nod/VTVJgcHU6FZVdQIQblu3Tr4+fnBxsYGISEhOHv2rNH+u3fvRmBgIGxsbBAUFIRDhw5xjhNCsGDBAnh7e8PW1hZhYWG4ffs2p09OTg6GDBkCJycnuLi4IDIyEgUFBZw+ly9fxptvvgkbGxv4+vpi6dKl1nnBlFqPj3NZSrbXG7nDRsjHns87YdVHbQAAEjhgknQC5ISHfvxEuN3ehbwXMtgKgBXCDajL5OI/ZT18KR/GGXdar9dw7+u3seT9IIPB7tvHvI7E2d3Rwc887zil4lS7oNy5cyeioqIQHR2NCxcuoHXr1ggPD0dmpv7sLQkJCRg8eDAiIyORkpKCiIgIRERE4OrVq2yfpUuXYvXq1diwYQOSkpJgb2+P8PBwFBeXlTcdMmQIrl27hri4OBw4cAAnT57E2LFj2eMSiQS9evVCw4YNkZycjGXLlmHhwoXYuHFj5b0ZlFoDwzDYPjoESz9ohSZ1Vcvydg1c0b9N2ZL8AnkNy0tLSHwp+Alf/vgbnhxehrf4l1BMhDolHX6ODMb4bk3A4zH4KLgBAr30e2KFfB68tbZdUioZUs0EBweT8ePHs88VCgXx8fEhMTExevsPHDiQ9O3bl9MWEhJCxo0bRwghRKlUEi8vL7Js2TL2eG5uLhGLxeTXX38lhBBy/fp1AoCcO3eO7XP48GHCMAx5/PgxIYSQ7777jri6upKSkhK2z8yZM0nTpk3Nfm15eXkEAMnLyzP7HErtp+HMA+zDb+Z+cmLeG4REO5H785sQ6QJXQqKdyKw5U0nDmQfIJz+cYftSqhZL7s9q1SilUimSk5MRFhbGtvF4PISFhSExMVHvOYmJiZz+ABAeHs72T01NRXp6OqePs7MzQkJC2D6JiYlwcXFBhw4d2D5hYWHg8XhISkpi+3Tp0gUikYhznVu3buH58+d651ZSUgKJRMJ5UF49hoSosg/tn/AG1n7cAVGyz/GMOKMhLxNCRoEDitfxq6I7Uub3xM+RITgwsTMSZnWv5llTjFGtgjIrKwsKhQKenlyPn6enJ9LTdTM+A0B6errR/uq/pvrUrcuNhxMIBHBzc+P00TeG5jW0iYmJgbOzM/vw9aXG9leR/3svCPeX9EWr+i7o28ob8dEDMEX2OeSEh/tKTwRP+hn3l7wDV3vVj3DLes7wcaFL6ZpMtdsoXyZmz56NvLw89vHwof6yppRXC2dbIbZ9PQu5kYmoE5WAunUMB61TaibVmo/Sw8MDfD4fGRncdFQZGRnw8tKftNXLy8tof/XfjIwMeHt7c/q0adOG7aPtLJLL5cjJyeGMo+86mtfQRiwWQyyme2Mp+vFooLtHnFI7qFaNUiQSoX379oiPj2fblEol4uPjERoaqvec0NBQTn8AiIuLY/v7+/vDy8uL00cikSApKYntExoaitzcXCQnl9Up+eeff6BUKhESEsL2OXnyJGQyGec6TZs2haurawVfOYVCqVVUgXPJKDt27CBisZhs2bKFXL9+nYwdO5a4uLiQ9PR0QgghQ4cOJbNmzWL7nz59mggEArJ8+XJy48YNEh0dTYRCIbly5QrbZ8mSJcTFxYXs27ePXL58mfTv35/4+/uToqIitk/v3r1J27ZtSVJSEjl16hQJCAgggwcPZo/n5uYST09PMnToUHL16lWyY8cOYmdnR/73v/+Z/dqo15tCqblYcn9Wu6AkhJA1a9aQBg0aEJFIRIKDg8mZM2fYY127diXDhw/n9N+1axd57bXXiEgkIi1atCAHDx7kHFcqlWT+/PnE09OTiMVi0qNHD3Lr1i1On+zsbDJ48GDi4OBAnJycyMiRI0l+fj6nz6VLl0jnzp2JWCwm9erVI0uWLLHodVFBSaHUXCy5P2matUqEplmjUGouNM0ahUKhWBEqKCkUCsUEVFBSKBSKCWhd70pEbf6lWxkplJqH+r40x01DBWUlkp+fDwB0KyOFUoPJz8+Hs7Oz0T7U612JKJVKPHnyBI6OjmDMKXDyEiKRSODr64uHDx++8p5/+l6oqCnvAyEE+fn58PHxAY9n3ApJNcpKhMfjoX79+tU9jRqBk5PTKy0cNKHvhYqa8D6Y0iTVUGcOhUKhmIAKSgqFQjEBFZSUSkUsFiM6OppmVQJ9L9TUxveBOnMoFArFBFSjpFAoFBNQQUmhUCgmoIKSQqFQTEAFJYVCoZiACkqK1cnJycGQIUPg5OQEFxcXREZGoqCgwOg5GzduxFtvvQUnJycwDIPc3NyqmayVWbduHfz8/GBjY4OQkBCcPXvWaP/du3cjMDAQNjY2CAoKwqFDh6poppWLJe/DtWvX8MEHH8DPzw8MwyA2NrbqJmomVFBSrM6QIUNw7do1xMXF4cCBAzh58iTGjh1r9JwXL16gd+/emDNnThXN0vrs3LkTUVFRiI6OxoULF9C6dWuEh4frFLJTk5CQgMGDByMyMhIpKSmIiIhAREQErl69WsUzty6Wvg8vXrxAo0aNsGTJEoOF+6qdyku0TnkVuX79OgFAzp07x7YdPnyYMAxDHj9+bPL8Y8eOEQDk+fPnlTjLyiE4OJiMHz+efa5QKIiPjw+JiYnR23/gwIGkb9++nLaQkBAybty4Sp1nZWPp+6BJw4YNybfffluJsysfVKOkWJXExES4uLigQ4cObFtYWBh4PB6SkpKqcWaVi1QqRXJyMsLCwtg2Ho+HsLAwJCYm6j0nMTGR0x8AwsPDDfavDZTnfagNUEFJsSrp6emoW7cup00gEMDNzQ3p6enVNKvKJysrCwqFAp6enpx2T09Pg687PT3dov61gfK8D7UBKigpZjFr1iwwDGP0cfPmzeqeJoVSKdA0axSz+OKLLzBixAijfRo1agQvLy8do71cLkdOTk7NNdRbAQ8PD/D5fGRkZHDaMzIyDL5uLy8vi/rXBsrzPtQGqEZJMYs6deogMDDQ6EMkEiE0NBS5ublITk5mz/3nn3+gVCoREhJSja+gchGJRGjfvj3i4+PZNqVSifj4eISGhuo9JzQ0lNMfAOLi4gz2rw2U532oFVS3N4ny8tG7d2/Stm1bkpSURE6dOkUCAgLI4MGD2eOPHj0iTZs2JUlJSWzb06dPSUpKCvn+++8JAHLy5EmSkpJCsrOzq+MllIsdO3YQsVhMtmzZQq5fv07Gjh1LXFxcSHp6OiGEkKFDh5JZs2ax/U+fPk0EAgFZvnw5uXHjBomOjiZCoZBcuXKlul6CVbD0fSgpKSEpKSkkJSWFeHt7k2nTppGUlBRy+/bt6noJOlBBSbE62dnZZPDgwcTBwYE4OTmRkSNHkvz8fPZ4amoqAUCOHTvGtkVHRxMAOo/NmzdX/QuoAGvWrCENGjQgIpGIBAcHkzNnzrDHunbtSoYPH87pv2vXLvLaa68RkUhEWrRoQQ4ePFjFM64cLHkf1N8H7UfXrl2rfuIGoGnWKBQKxQTURkmhUCgmoIKSQqFQTEAFJYVCoZiACkoKhUIxARWUFAqFYgIqKCkUCsUEVFBSKBSKCaigpFAoFBNQQUl5pRgxYoTezEe9e/eu7qlRajA0exDllaN3797YvHkzp00sFuvtK5PJIBQKOW1SqRQikcji65b3PEr1QzVKyiuHWCyGl5cX5+Hq6goAYBgG69evR79+/WBvb4//+7//w8KFC9GmTRv88MMP8Pf3h42NDQAgLS0N/fv3h4ODA5ycnDBw4EBOejFD51FqH1RQUihaLFy4EO+99x6uXLmCUaNGAQDu3LmD33//HXv27MHFixehVCrRv39/5OTk4MSJE4iLi8O9e/cwaNAgzlja51FqJ3TpTXnlOHDgABwcHDhtc+bMYStAfvzxxxg5ciTnuFQqxdatW1GnTh0AqryRV65cQWpqKnx9fQEAW7duRYsWLXDu3Dl07NhR73mU2gkVlJRXjm7dumH9+vWcNjc3N/Z/zcJoaho2bMgRdjdu3ICvry8rJAGgefPmcHFxwY0bN1hBqX0epXZCBSXllcPe3h5NmjQxetycNnOvRan9UBslhVIOmjVrhocPH+Lhw4ds2/Xr15Gbm4vmzZtX48wolQHVKCmvHCUlJTqlUwUCATw8PMweIywsDEFBQRgyZAhiY2Mhl8vx+eefo2vXrnqX7pTaDdUoKa8cR44cgbe3N+fRuXNni8ZgGAb79u2Dq6srunTpgrCwMDRq1Ag7d+6spFlTqhNaCoJCoVBMQDVKCoVCMQEVlBQKhWICKigpFArFBFRQUigUigmooKRQKBQTUEFJoVAoJqCCkkKhUExABSWFQqGYgApKCoVCMQEVlBQKhWICKigpFArFBFRQUigUign+H4u90o88J0JbAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.022571, + "end_time": "2024-03-26T08:46:06.653182", + "exception": false, + "start_time": "2024-03-26T08:46:06.630611", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4267.894457, + "end_time": "2024-03-26T08:46:09.396960", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/tab_ddpm_concat/1/mlu-eval.ipynb", + "output_path": "eval/contraceptive/tab_ddpm_concat/1/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/contraceptive/tab_ddpm_concat/1", + "path_prefix": "../../../../", + "random_seed": 1, + "single_model": "tab_ddpm_concat" + }, + "start_time": "2024-03-26T07:35:01.502503", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/tab_ddpm_concat/model.pt b/contraceptive/tab_ddpm_concat/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..bcc7b900564e417090e027e510ecfcdcdfde14b5 --- /dev/null +++ b/contraceptive/tab_ddpm_concat/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:580dd6bf4bf13b726600816630964fc1bec39c2e2b26dc693ffce19581ba0d3c +size 31355458 diff --git a/contraceptive/tab_ddpm_concat/params.json b/contraceptive/tab_ddpm_concat/params.json new file mode 100644 index 0000000000000000000000000000000000000000..2acfa1f89b8473004c48ef7d48280c8fa533b5a8 --- /dev/null +++ b/contraceptive/tab_ddpm_concat/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 1, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.7494458230986923, "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "head_activation": "relu6", "loss_balancer_beta": 0.6806661100374879, "loss_balancer_r": 0.9427716710925113, "tf_activation": "relu6", "mse_mag_multiply": true, "tf_pma_low_exp_2": 2, "dataset_size_exp_2": 11, "batch_size_exp_2": 2, "epochs": 100, "lr_mul": 0.07424782199493057, "n_warmup_steps": 104, "Optim": "amsgradw", "fixed_role_model": "tab_ddpm_concat", "mse_mag_target": 0.20359405820922769, "d_model_exp_2": 7, "attn_activation": "leakyhardtanh", "tf_d_inner_exp_2": 9, "tf_n_layers_enc": 3, "tf_n_head_exp_2": 5, "tf_activation_final": "leakyhardtanh", "tf_num_inds_exp_2": 4, "ada_d_hid_exp_2": 10, "ada_n_layers": 8, "ada_activation": "softsign", "ada_activation_final": "leakyhardsigmoid", "head_d_hid_exp_2": 8, "head_n_layers": 8, "head_n_head_exp_2": 4, "head_activation_final": "leakyhardsigmoid", "mse_mag": true, "models": ["tab_ddpm_concat"], "max_seconds": 3600} \ No newline at end of file diff --git a/contraceptive/tvae/eval.csv b/contraceptive/tvae/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..1943d95fdd075482b1673b63fd9ccf44faf9ca73 --- /dev/null +++ b/contraceptive/tvae/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tvae,0.007679888527539636,0.015172706345554446,0.0017569630032327648,1.6627535820007324,0.013730231672525406,0.4312901794910431,0.022872319445014,2.7858511657541385e-06,2.7128655910491943,0.031409189105033875,0.08435197174549103,0.0419161431491375,0.09169832617044449,0.008257805369794369,4.375619173049927 diff --git a/contraceptive/tvae/history.csv b/contraceptive/tvae/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..5e4e0171a087d93d36833e8ee89a4ce746ad118e --- /dev/null +++ b/contraceptive/tvae/history.csv @@ -0,0 +1,10 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.0186110673548602,0.9859759064087645,0.0014670658553136355,0.12243372971605923,0.0,0.0,0.0,0.0,0.018859607335697446,900,225,282.9204273223877,1.257424121432834,0.3143560303582085,0.11435848254969136,0.008952803679017556,4.103192632228183,0.00013674774596836414,0.0,0.0,0.0,0.0,0.0,0.008952803679017556,450,113,94.89591884613037,0.8397868924436316,0.21087981965806749,0.022598956262412588 +1,0.005419754103058949,0.7368064368608555,6.596730340261968e-05,0.0279144444456324,0.0,0.0,0.0,0.0,0.005536770965490077,900,225,283.5930871963501,1.2604137208726671,0.3151034302181668,0.09610795313699377,0.009089643146014875,2.2618643136544057,0.00020205410188401396,0.0,0.0,0.0,0.0,0.0,0.009089643146014875,450,113,96.57411861419678,0.8546382178247502,0.21460915247599283,0.05953517459001726 +2,0.0041141705360819795,0.4941683148547074,5.9163905845158306e-05,0.01810061747660964,0.0,0.0,0.0,0.0,0.004181610228713706,900,225,290.8302204608917,1.2925787576039631,0.3231446894009908,0.09772732237146961,0.002547107368138515,1.7669729735376936,7.385972450104247e-06,0.0,0.0,0.0,0.0,0.0,0.002547107368138515,450,113,96.63251686096191,0.8551550164686895,0.21473892635769315,0.0506768064260219 +3,0.0023371195333958088,0.4912818201007985,5.350784180537504e-06,0.014647599203356853,0.0,0.0,0.0,0.0,0.002370942204983698,900,225,283.96464467048645,1.2620650874243842,0.31551627185609604,0.10408781512019535,0.0034601237480011253,2.612866154395119,1.3134922784065689e-05,0.0,0.0,0.0,0.0,0.0,0.0034601237480011253,450,113,94.63316321372986,0.8374616213604412,0.21029591825273303,0.04398040732047397 +4,0.0020195719559625205,0.2510072517482392,6.499685870238886e-06,0.013006549620121304,0.0,0.0,0.0,0.0,0.002048346129207251,900,225,285.49330401420593,1.2688591289520263,0.3172147822380066,0.10502510418287582,0.0023336571037200176,1.0392825815654005,7.885705093391342e-06,0.0,0.0,0.0,0.0,0.0,0.0023336571037200176,450,113,95.2993221282959,0.8433568329937691,0.2117762713962131,0.05800044378760246 +5,0.0019015362533779505,0.24092813788177259,5.394619782389408e-06,0.014597136508156028,0.0,0.0,0.0,0.0,0.0019251112584364213,900,225,285.9157590866089,1.2707367070515951,0.3176841767628988,0.10497349073075586,0.0023449724944657646,1.324564782714674,8.32234693904156e-06,0.0,0.0,0.0,0.0,0.0,0.0023449724944657646,450,113,95.89643502235413,0.8486410178969391,0.21310318893856472,0.059179596025476115 +6,0.001617696879960325,0.15725728821715396,4.0012195520334405e-06,0.013585555124898544,0.0,0.0,0.0,0.0,0.0016441765983391734,900,225,283.3449795246124,1.2593110201093887,0.31482775502734717,0.10613164186891583,0.002407976164379054,1.1220077265352362,7.455731951812162e-06,0.0,0.0,0.0,0.0,0.0,0.002407976164379054,450,113,94.69690179824829,0.8380256796305159,0.21043755955166288,0.048071262323651956 +7,0.0014774360275219402,0.16173793539400327,3.2265756165433642e-06,0.013837635460464905,0.0,0.0,0.0,0.0,0.001494622107436751,900,225,285.05916833877563,1.266929637061225,0.31673240926530627,0.10432403838468922,0.002109985634936796,1.285013097416944,5.779796396621073e-06,0.0,0.0,0.0,0.0,0.0,0.002109985634936796,450,113,95.49577188491821,0.8450953264152055,0.21221282641092937,0.059411970754277656 +8,0.0013566906487297577,0.1447534780750981,1.6995160905337986e-06,0.014613873372133336,0.0,0.0,0.0,0.0,0.0013734321816850246,900,225,284.22162222862244,1.2632072099049887,0.3158018024762472,0.10500639389579494,0.0019757018623446734,1.1056950748967773,7.016759447038556e-06,0.0,0.0,0.0,0.0,0.0,0.0019757018623446734,450,113,95.37925219535828,0.8440641787199847,0.21195389376746285,0.05243083165208521 diff --git a/contraceptive/tvae/mlu-eval-load.ipynb b/contraceptive/tvae/mlu-eval-load.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..43682b99c2222bb7db8237bc6ea4f42b0a3296d9 --- /dev/null +++ b/contraceptive/tvae/mlu-eval-load.ipynb @@ -0,0 +1,1897 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:09.716853Z", + "iopub.status.busy": "2024-03-26T06:14:09.716516Z", + "iopub.status.idle": "2024-03-26T06:14:09.751080Z", + "shell.execute_reply": "2024-03-26T06:14:09.750209Z" + }, + "papermill": { + "duration": 0.048366, + "end_time": "2024-03-26T06:14:09.753162", + "exception": false, + "start_time": "2024-03-26T06:14:09.704796", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:09.777238Z", + "iopub.status.busy": "2024-03-26T06:14:09.776845Z", + "iopub.status.idle": "2024-03-26T06:14:09.783683Z", + "shell.execute_reply": "2024-03-26T06:14:09.782816Z" + }, + "papermill": { + "duration": 0.021393, + "end_time": "2024-03-26T06:14:09.785713", + "exception": false, + "start_time": "2024-03-26T06:14:09.764320", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:09.807856Z", + "iopub.status.busy": "2024-03-26T06:14:09.807481Z", + "iopub.status.idle": "2024-03-26T06:14:09.811737Z", + "shell.execute_reply": "2024-03-26T06:14:09.810817Z" + }, + "papermill": { + "duration": 0.017458, + "end_time": "2024-03-26T06:14:09.813678", + "exception": false, + "start_time": "2024-03-26T06:14:09.796220", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:09.834989Z", + "iopub.status.busy": "2024-03-26T06:14:09.834707Z", + "iopub.status.idle": "2024-03-26T06:14:09.838737Z", + "shell.execute_reply": "2024-03-26T06:14:09.837852Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.016899, + "end_time": "2024-03-26T06:14:09.840633", + "exception": false, + "start_time": "2024-03-26T06:14:09.823734", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:09.862927Z", + "iopub.status.busy": "2024-03-26T06:14:09.862557Z", + "iopub.status.idle": "2024-03-26T06:14:09.868375Z", + "shell.execute_reply": "2024-03-26T06:14:09.867491Z" + }, + "papermill": { + "duration": 0.019372, + "end_time": "2024-03-26T06:14:09.870390", + "exception": false, + "start_time": "2024-03-26T06:14:09.851018", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "aa5c3c44", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:09.894235Z", + "iopub.status.busy": "2024-03-26T06:14:09.893893Z", + "iopub.status.idle": "2024-03-26T06:14:09.899183Z", + "shell.execute_reply": "2024-03-26T06:14:09.898294Z" + }, + "papermill": { + "duration": 0.019499, + "end_time": "2024-03-26T06:14:09.901153", + "exception": false, + "start_time": "2024-03-26T06:14:09.881654", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"tvae\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 3\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/tvae/3\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.01047, + "end_time": "2024-03-26T06:14:09.922153", + "exception": false, + "start_time": "2024-03-26T06:14:09.911683", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:09.945524Z", + "iopub.status.busy": "2024-03-26T06:14:09.945194Z", + "iopub.status.idle": "2024-03-26T06:14:09.955205Z", + "shell.execute_reply": "2024-03-26T06:14:09.954357Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.024262, + "end_time": "2024-03-26T06:14:09.957271", + "exception": false, + "start_time": "2024-03-26T06:14:09.933009", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/tvae/3\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:09.981306Z", + "iopub.status.busy": "2024-03-26T06:14:09.981006Z", + "iopub.status.idle": "2024-03-26T06:14:11.959845Z", + "shell.execute_reply": "2024-03-26T06:14:11.958840Z" + }, + "papermill": { + "duration": 1.993152, + "end_time": "2024-03-26T06:14:11.962022", + "exception": false, + "start_time": "2024-03-26T06:14:09.968870", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:11.986265Z", + "iopub.status.busy": "2024-03-26T06:14:11.985723Z", + "iopub.status.idle": "2024-03-26T06:14:11.998763Z", + "shell.execute_reply": "2024-03-26T06:14:11.997997Z" + }, + "papermill": { + "duration": 0.027303, + "end_time": "2024-03-26T06:14:12.000651", + "exception": false, + "start_time": "2024-03-26T06:14:11.973348", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:12.021795Z", + "iopub.status.busy": "2024-03-26T06:14:12.021532Z", + "iopub.status.idle": "2024-03-26T06:14:12.028272Z", + "shell.execute_reply": "2024-03-26T06:14:12.027403Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.01945, + "end_time": "2024-03-26T06:14:12.030145", + "exception": false, + "start_time": "2024-03-26T06:14:12.010695", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:12.052707Z", + "iopub.status.busy": "2024-03-26T06:14:12.052103Z", + "iopub.status.idle": "2024-03-26T06:14:12.148547Z", + "shell.execute_reply": "2024-03-26T06:14:12.147719Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.110291, + "end_time": "2024-03-26T06:14:12.151114", + "exception": false, + "start_time": "2024-03-26T06:14:12.040823", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:12.175195Z", + "iopub.status.busy": "2024-03-26T06:14:12.174895Z", + "iopub.status.idle": "2024-03-26T06:14:16.782406Z", + "shell.execute_reply": "2024-03-26T06:14:16.781435Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.621686, + "end_time": "2024-03-26T06:14:16.784882", + "exception": false, + "start_time": "2024-03-26T06:14:12.163196", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-26 06:14:14.369103: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-26 06:14:14.369156: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-26 06:14:14.370705: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:16.807457Z", + "iopub.status.busy": "2024-03-26T06:14:16.806914Z", + "iopub.status.idle": "2024-03-26T06:14:16.812557Z", + "shell.execute_reply": "2024-03-26T06:14:16.811835Z" + }, + "papermill": { + "duration": 0.019118, + "end_time": "2024-03-26T06:14:16.814494", + "exception": false, + "start_time": "2024-03-26T06:14:16.795376", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:16.836793Z", + "iopub.status.busy": "2024-03-26T06:14:16.836505Z", + "iopub.status.idle": "2024-03-26T06:14:25.675506Z", + "shell.execute_reply": "2024-03-26T06:14:25.674281Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.852783, + "end_time": "2024-03-26T06:14:25.677928", + "exception": false, + "start_time": "2024-03-26T06:14:16.825145", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-26T06:14:25.703937Z", + "iopub.status.busy": "2024-03-26T06:14:25.703043Z", + "iopub.status.idle": "2024-03-26T06:14:25.710091Z", + "shell.execute_reply": "2024-03-26T06:14:25.709290Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.022528, + "end_time": "2024-03-26T06:14:25.712177", + "exception": false, + "start_time": "2024-03-26T06:14:25.689649", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 46,\n", + " 'realtabformer': (24, 72, Embedding(72, 672), True),\n", + " 'lct_gan': 40,\n", + " 'tab_ddpm_concat': 10}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:25.735109Z", + "iopub.status.busy": "2024-03-26T06:14:25.734853Z", + "iopub.status.idle": "2024-03-26T06:14:25.797142Z", + "shell.execute_reply": "2024-03-26T06:14:25.796236Z" + }, + "papermill": { + "duration": 0.076158, + "end_time": "2024-03-26T06:14:25.799086", + "exception": false, + "start_time": "2024-03-26T06:14:25.722928", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/tvae/all inf False\n", + "../../../../ml-utility-loss/aug_test/contraceptive 0\n", + "Caching in ../../../../contraceptive/_cache_bs_test/tvae/all inf False\n", + "../../../../ml-utility-loss/bs_test/contraceptive 0\n", + "Caching in ../../../../contraceptive/_cache_synth_test/tvae/all inf False\n", + "../../../../ml-utility-loss/synthetics/contraceptive 600\n", + "600\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:25.824071Z", + "iopub.status.busy": "2024-03-26T06:14:25.823797Z", + "iopub.status.idle": "2024-03-26T06:14:26.145084Z", + "shell.execute_reply": "2024-03-26T06:14:26.144170Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.336301, + "end_time": "2024-03-26T06:14:26.147139", + "exception": false, + "start_time": "2024-03-26T06:14:25.810838", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'loss_balancer_beta': 0.7999999999999999,\n", + " 'loss_balancer_r': 0.95,\n", + " 'tf_pma_low': 8,\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7,\n", + " 'bias_weight_decay': 0.05,\n", + " 'head_activation': torch.nn.modules.activation.ReLU6,\n", + " 'tf_activation': torch.nn.modules.activation.Tanh,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.07,\n", + " 'n_warmup_steps': 200,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'fixed_role_model': 'tvae',\n", + " 'g_loss_mul': 0.1,\n", + " 'd_model': 256,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 512,\n", + " 'ada_n_layers': 9,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 512,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'single_model': True,\n", + " 'models': ['tvae'],\n", + " 'max_seconds': 3600,\n", + " 'Body': 'twin_encoder',\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 128,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.5, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "# params = {\n", + "# **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "# }\n", + "params = getattr(PARAMS, dataset_name).BEST_DICT[gp][gp_multiply][single_model]\n", + "if isinstance(params, (list, tuple)):\n", + " params = params[param_index]\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-26T06:14:26.172524Z", + "iopub.status.busy": "2024-03-26T06:14:26.172203Z", + "iopub.status.idle": "2024-03-26T06:14:26.579190Z", + "shell.execute_reply": "2024-03-26T06:14:26.578215Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.422351, + "end_time": "2024-03-26T06:14:26.581360", + "exception": false, + "start_time": "2024-03-26T06:14:26.159009", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n", + "[*] Embedding False True\n", + "['tvae'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:26.606253Z", + "iopub.status.busy": "2024-03-26T06:14:26.605950Z", + "iopub.status.idle": "2024-03-26T06:14:26.610070Z", + "shell.execute_reply": "2024-03-26T06:14:26.609305Z" + }, + "papermill": { + "duration": 0.018786, + "end_time": "2024-03-26T06:14:26.612019", + "exception": false, + "start_time": "2024-03-26T06:14:26.593233", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:26.635215Z", + "iopub.status.busy": "2024-03-26T06:14:26.634508Z", + "iopub.status.idle": "2024-03-26T06:14:26.641599Z", + "shell.execute_reply": "2024-03-26T06:14:26.640627Z" + }, + "papermill": { + "duration": 0.020872, + "end_time": "2024-03-26T06:14:26.643636", + "exception": false, + "start_time": "2024-03-26T06:14:26.622764", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "7608065" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:26.667149Z", + "iopub.status.busy": "2024-03-26T06:14:26.666890Z", + "iopub.status.idle": "2024-03-26T06:14:26.749092Z", + "shell.execute_reply": "2024-03-26T06:14:26.748125Z" + }, + "papermill": { + "duration": 0.096444, + "end_time": "2024-03-26T06:14:26.751321", + "exception": false, + "start_time": "2024-03-26T06:14:26.654877", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 46] --\n", + "├─Adapter: 1-1 [2, 1179, 46] --\n", + "│ └─Sequential: 2-1 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 512] 24,064\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-16 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-9 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-17 [2, 1179, 256] 131,328\n", + "│ │ │ └─LeakyHardsigmoid: 4-18 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 46] (recursive)\n", + "│ └─Sequential: 2-2 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-10 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-32 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-17 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-33 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-34 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-18 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-36 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 2048] --\n", + "│ └─Encoder: 2-3 [2, 8, 256] --\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-6 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1179, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-12 [2, 1179, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-18 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1179, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-24 [2, 1179, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-39 [2, 8, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-30 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1179, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-36 [2, 1179, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 8, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 8, 256] 2,048\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 8, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 8, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 8, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 8, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 8, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-42 [2, 8, 256] --\n", + "│ └─Encoder: 2-4 [2, 8, 256] (recursive)\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-40 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-8 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-21 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-22 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-48 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-23 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 294, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-54 [2, 294, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-25 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-60 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 294, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-66 [2, 294, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-31 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-42 [2, 8, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-72 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 294, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-78 [2, 294, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-14 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-40 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 8, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 8, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-84 [2, 8, 256] --\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-21 [2, 512] --\n", + "│ │ │ └─Linear: 4-43 [2, 512] 1,049,088\n", + "│ │ │ └─ReLU6: 4-44 [2, 512] --\n", + "│ │ └─FeedForward: 3-22 [2, 512] --\n", + "│ │ │ └─Linear: 4-45 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-46 [2, 512] --\n", + "│ │ └─FeedForward: 3-23 [2, 512] --\n", + "│ │ │ └─Linear: 4-47 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-48 [2, 512] --\n", + "│ │ └─FeedForward: 3-24 [2, 512] --\n", + "│ │ │ └─Linear: 4-49 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-50 [2, 512] --\n", + "│ │ └─FeedForward: 3-25 [2, 512] --\n", + "│ │ │ └─Linear: 4-51 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-52 [2, 512] --\n", + "│ │ └─FeedForward: 3-26 [2, 512] --\n", + "│ │ │ └─Linear: 4-53 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-54 [2, 512] --\n", + "│ │ └─FeedForward: 3-27 [2, 512] --\n", + "│ │ │ └─Linear: 4-55 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-56 [2, 512] --\n", + "│ │ └─FeedForward: 3-28 [2, 512] --\n", + "│ │ │ └─Linear: 4-57 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-58 [2, 512] --\n", + "│ │ └─FeedForward: 3-29 [2, 1] --\n", + "│ │ │ └─Linear: 4-59 [2, 1] 513\n", + "│ │ │ └─LeakyHardsigmoid: 4-60 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 7,608,065\n", + "Trainable params: 7,608,065\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 24.25\n", + "========================================================================================================================\n", + "Input size (MB): 0.54\n", + "Forward/backward pass size (MB): 257.33\n", + "Params size (MB): 30.43\n", + "Estimated Total Size (MB): 288.30\n", + "========================================================================================================================" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = test_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "357e54f8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:26.778603Z", + "iopub.status.busy": "2024-03-26T06:14:26.778295Z", + "iopub.status.idle": "2024-03-26T06:14:26.831158Z", + "shell.execute_reply": "2024-03-26T06:14:26.830255Z" + }, + "papermill": { + "duration": 0.068301, + "end_time": "2024-03-26T06:14:26.833171", + "exception": false, + "start_time": "2024-03-26T06:14:26.764870", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torch\n", + "model.load_state_dict(torch.load(\"model.pt\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:14:26.858952Z", + "iopub.status.busy": "2024-03-26T06:14:26.858654Z", + "iopub.status.idle": "2024-03-26T06:16:44.623466Z", + "shell.execute_reply": "2024-03-26T06:16:44.622670Z" + }, + "papermill": { + "duration": 137.78106, + "end_time": "2024-03-26T06:16:44.626154", + "exception": false, + "start_time": "2024-03-26T06:14:26.845094", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:16:44.655948Z", + "iopub.status.busy": "2024-03-26T06:16:44.655227Z", + "iopub.status.idle": "2024-03-26T06:16:44.677244Z", + "shell.execute_reply": "2024-03-26T06:16:44.676393Z" + }, + "papermill": { + "duration": 0.039357, + "end_time": "2024-03-26T06:16:44.679186", + "exception": false, + "start_time": "2024-03-26T06:16:44.639829", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tvae0.007680.0151730.0017571.6627540.013730.431290.0228720.0000032.7128660.0314090.0843520.0419160.0916980.0082584.375619
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "tvae 0.00768 0.015173 0.001757 1.662754 0.01373 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "tvae 0.43129 0.022872 0.000003 2.712866 0.031409 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "tvae 0.084352 0.041916 0.091698 0.008258 4.375619 " + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:16:44.705227Z", + "iopub.status.busy": "2024-03-26T06:16:44.704948Z", + "iopub.status.idle": "2024-03-26T06:16:45.161394Z", + "shell.execute_reply": "2024-03-26T06:16:45.160413Z" + }, + "papermill": { + "duration": 0.472067, + "end_time": "2024-03-26T06:16:45.163651", + "exception": false, + "start_time": "2024-03-26T06:16:44.691584", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:16:45.192401Z", + "iopub.status.busy": "2024-03-26T06:16:45.191629Z", + "iopub.status.idle": "2024-03-26T06:19:09.687602Z", + "shell.execute_reply": "2024-03-26T06:19:09.686823Z" + }, + "papermill": { + "duration": 144.513058, + "end_time": "2024-03-26T06:19:09.690106", + "exception": false, + "start_time": "2024-03-26T06:16:45.177048", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/tvae/all inf False\n", + "Caching in ../../../../contraceptive/_cache_bs_test/tvae/all inf False\n", + "Caching in ../../../../contraceptive/_cache_synth_test/tvae/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:19:09.719210Z", + "iopub.status.busy": "2024-03-26T06:19:09.718326Z", + "iopub.status.idle": "2024-03-26T06:19:09.737849Z", + "shell.execute_reply": "2024-03-26T06:19:09.737164Z" + }, + "papermill": { + "duration": 0.035973, + "end_time": "2024-03-26T06:19:09.739735", + "exception": false, + "start_time": "2024-03-26T06:19:09.703762", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:19:09.766150Z", + "iopub.status.busy": "2024-03-26T06:19:09.765514Z", + "iopub.status.idle": "2024-03-26T06:19:09.770530Z", + "shell.execute_reply": "2024-03-26T06:19:09.769641Z" + }, + "papermill": { + "duration": 0.020945, + "end_time": "2024-03-26T06:19:09.772633", + "exception": false, + "start_time": "2024-03-26T06:19:09.751688", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tvae': 0.4056767049431801}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:19:09.798521Z", + "iopub.status.busy": "2024-03-26T06:19:09.798201Z", + "iopub.status.idle": "2024-03-26T06:19:10.247597Z", + "shell.execute_reply": "2024-03-26T06:19:10.246597Z" + }, + "papermill": { + "duration": 0.46518, + "end_time": "2024-03-26T06:19:10.249767", + "exception": false, + "start_time": "2024-03-26T06:19:09.784587", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABA60lEQVR4nO3deXxU9b34/9eZfbLMZCE7CQm7yq6CqAgoVZYq1Fax2gq1qLfFay2X+7D08a2I1oKta631tl4B/bngUtRerVtVwI1FEFlU1hACJCRA9mT28/vjzAwMSSDLZE4meT8fj3kkOedk5n0yyTuf/aOoqqoihBA6MugdgBBCSCISQuhOEpEQQneSiIQQupNEJITQnSQiIYTuJBEJIXQniUgIoTtJREII3UkiEkLoThKRiKrPP/+ce++9l+rqar1DEXFEEpGIqs8//5wlS5ZIIhLtIolICKE7SUQiau69917++7//G4CioiIURUFRFJKSkpg8eXKz6wOBAHl5efzoRz8KH3vooYe4+OKLSU9Px263c/755/Paa6+1+HrPP/88559/Pna7nbS0NG644QZKS0u75uZEl1JkGRARLdu2bWPZsmW89NJLPProo/Tp0weAffv2cd9993H48GGys7PD169bt46JEyfy6quvhpNRfn4+11xzDeeeey4ej4dVq1axceNG3nrrLWbMmBH+3gceeIDf/e53XH/99UycOJHKykqeeOIJkpKS+Oqrr0hJSYnpvYtOUoWIoj/96U8qoBYXF4eP7dq1SwXUJ554IuLaX/7yl2pSUpLa2NgYPnbq56qqqh6PRx02bJh6+eWXh48dOHBANRqN6gMPPBBx7fbt21WTydTsuOj+pGomutzgwYMZNWoUL7/8cviY3+/ntdde4+qrr8Zut4ePn/p5VVUVNTU1TJgwgS1btoSPr169mkAgwPXXX8+xY8fCj+zsbAYNGsTHH38cmxsTUWPSOwDRO8yePZvf/va3HD58mLy8PNasWUNFRQWzZ8+OuO6tt97i97//PVu3bsXtdoePK4oS/nzPnj2oqsqgQYNafC2z2dw1NyG6jCQiEROzZ89m0aJFvPrqq9x111288sorOJ1Opk6dGr7mk08+4ZprruGyyy7jr3/9Kzk5OZjNZlasWMGLL74Yvi4QCKAoCu+88w5Go7HZayUlJcXknkT0SCISUXVqyeVURUVFjB07lpdffpk77riD1atXM2vWLKxWa/iaf/zjH9hsNt57772I4ytWrIh4rgEDBqCqKkVFRQwePLhrbkTElLQRiahKTEwEaHFA4+zZs1m/fj3Lly/n2LFjzaplRqMRRVHw+/3hYwcOHOCNN96IuO7aa6/FaDSyZMkS1NM6fVVV5fjx49G5GREz0n0vomrTpk2MHTuW6dOnc8MNN2A2m7n66qtJTEzk0KFDFBQUkJSUhNlspry8PKI956OPPuKKK65gwoQJ3HjjjVRUVPDkk0+SnZ3Ntm3bIpLOsmXLWLRoERdffDGzZs0iOTmZ4uJiXn/9dW677TYWLlyox+2LjtKxx070UPfff7+al5enGgyGZl35l1xyiQqo8+bNa/F7n3nmGXXQoEGq1WpVhw4dqq5YsUJdvHix2tKv6j/+8Q/10ksvVRMTE9XExER16NCh6vz589Vdu3Z11a2JLiIlIiGE7qSNSAihO0lEQgjdSSISQuhOEpEQQneSiIQQupNEJITQXVxP8QgEAhw5coTk5ORWpxYIIfSjqip1dXXk5uZiMLRe7onrRHTkyBHy8/P1DkMIcRalpaX07du31fNxnYiSk5MB7SYdDofO0QghTldbW0t+fn74b7U1cZ2IQtUxh8MhiUiIbuxsTSfSWC2E0J0kIiGE7iQRCSF0F9dtRG2hqio+ny9isS3RNkajEZPJJEMjRJfr0YnI4/FQVlZGY2Oj3qHErYSEBHJycrBYLHqHInqwHpuIAoEAxcXFGI1GcnNzsVgs8p+9HVRVxePxUFlZSXFxMYMGDTrjgDQhOkP3RHT48GHuvvtu3nnnHRobGxk4cCArVqzgggsu6NTzejweAoEA+fn5JCQkRCna3sVut2M2mykpKcHj8WCz2fQOKSoOHGtg04ETZCRbuXRgH0xGSbB60zURVVVVcckllzB58mTeeecdMjIy2LNnD6mpqVF7Dfkv3jk97edX1eDhrW1H8PpVDlU1YTIYuHRQH73D6vV0TUQPPvgg+fn5EdvFFBUV6RiR6Ok2l1Th9auYDAq+gMrXh6q5oDAVm7n5/mgidnT9d/fPf/6TCy64gOuuu47MzExGjx7N008/3er1breb2traiIcQbeUPqJSc0DouZo3OIy3RgscXYH9lg86RCV0T0f79+3nqqacYNGgQ7733Hr/4xS+48847efbZZ1u8funSpTidzvBDJrx2XmFhIY899pjeYcSE0aBwyyWF3DSugL6pdgZmJpGeZKGH1T7jkq5Vs0AgwAUXXMAf/vAHAEaPHs2OHTv4n//5H+bMmdPs+kWLFrFgwYLw16EJdUK0laIoZDq0Rvfx/dO5ZKC0D3UHuv4vyMnJ4dxzz404ds4553Dw4MEWr7dareEJrjLR9SSPx6N3CHHJYJDhHN2FronokksuYdeuXRHHdu/eTb9+/br0dT2+QKsPnz/Q5mu9bbi2IyZNmsQdd9zBHXfcgdPppE+fPvzud78L73RaWFjI/fffz80334zD4eC2224D4NNPP2XChAnY7Xby8/O58847aWg42f5RUVHB1Vdfjd1up6ioiBdeeKFD8cWjOpeX5Z8W8+6OsmbbVPsDKi6vjLzXk65Vs1//+tdcfPHF/OEPf+D6669n48aN/P3vf+fvf/97l77ukx/vbfVcUZ9EZo3OC3/993X78Ppb3oOyb6qd6y44WTVc/lkxTZ7IX+hff29wh2J89tln+fnPf87GjRv58ssvue222ygoKODWW28F4KGHHuKee+5h8eLFAOzbt4+pU6fy+9//nuXLl1NZWRlOZqFeyblz53LkyBE+/vhjzGYzd955JxUVFR2KL96U17ioafJiMRkiBrZ+eeAEX+w7zqiCFCYMytAxwt5N10R04YUX8vrrr7No0SLuu+8+ioqKeOyxx7jpppv0DKtbyM/P59FHH0VRFIYMGcL27dt59NFHw4no8ssv57/+67/C18+bN4+bbrqJu+66C4BBgwbx5z//mYkTJ/LUU09x8OBB3nnnHTZu3MiFF14IwDPPPMM555wT83vTQ1mNC4AcZ+SgTIvJgC+gcqJBqrd60n1k9fe//32+//3vx/Q1508e2Oq505sNbrtsQKvXnj5j5JZLojcG6qKLLor4zz1+/Hgefvjh8OTd00eef/3112zbti2iuqWqaniqy+7duzGZTJx//vnh80OHDiUlJSVqMXdn5eFEZI84npqgzaE7Xi+JSE+6JyI9WExtbxrrqms7KzExMeLr+vp6br/9du68885m1xYUFLB79+5YhdbtBAIqFXVaIso+rUSUnqQlolqXF68/gFmme+iiVyaieLBhw4aIr9evX8+gQYMwGlseATxmzBi++eYbBg5subQ3dOhQfD4fmzdvDlfNdu3aRXV1dVTj7o6qm7x4/Spmo0JqgjninN1sxG4x0uTxU9XgCXfti9iS9N9NHTx4kAULFrBr1y5eeuklnnjiCX71q1+1ev3dd9/N559/zh133MHWrVvZs2cPb775JnfccQcAQ4YMYerUqdx+++1s2LCBzZs3M2/ePOx2e6vP2VMcr3cDkJ5kbbYCg6IopAWrZ1WN3pjHJjSSiLqpm2++maamJsaOHcv8+fP51a9+Fe6mb8mIESNYu3Ytu3fvZsKECYwePZp77rmH3Nzc8DUrVqwgNzeXiRMncu2113LbbbeRmZkZi9vRlaJApsNKlsPa4nmHXasY1LokEelFUU8fVBFHamtrcTqd1NTUNBvc6HK5KC4upqioKO6Wr5g0aRKjRo3qFlMv4vnn2FbbD9Vw8EQjQ7KTGJh55m1vRPuc6W/0VNJGJHq94X2dDO/r1DuMXk2qZqJHCwRU/IG4LfT3GlIi6obWrFmjdwg9RmW9m5c3lZKXYueH57e+5bGqqtS7fSRZZbMAPUiJSPRoJxo8+AMqgTM0hQYCKn9ds4///aSYercvhtGJEElEokeratRGTIdGULfEYFDCKzTWuiQR6UESkejRaoJjg1JOG8h4umSr1krRKCUiXUgiEj1aVTgRnXlftsRgIpKqmT4kEYkeS1XVU6pmZy4RJVq1qpkkIn1IIhI9VpPXj8cXQFHAaT9zIkoKlogaJBHpQrrvRY/lC6gMzkrG6w+cdRPFk1UzWalRD5KIRI/lsJmZMSKnTdemJ1oYnJVMZivz0UTXkkQkBJDpsLU5aYno611tRKoKPk/sH+2YV/zcc8+Rnp6O2+2OOD5r1ix++tOfRvsn0qO5vP5mC+WL7ql3lYj8Xvjk4di/7oT/AtOZu49DrrvuOu68807++c9/ct111wHa7htvv/0277//fldG2eO8uvkQNY0eZo3Oo29qgnbQ74WjO4Nrg5wHxpN/AoGASpPXj9VkOGubkogu+Wl3M3a7nRtvvDG88wbA888/T0FBAZMmTdIvsDijqio1jR68fpVESzDZ+H3w9SrY9Q589y/Y/goETm759PyGEv6+bn94oX0RO72rRGQ0a6UTPV63HW699VYuvPBCDh8+TF5eHitXrmTu3LkyGbMd6t0+vH4Vg6LgCHXdH9oINYe0UpAKVJVA2VbIGwNoy8YCNHikCz/WelciUpQ2V5H0NHr0aEaOHMlzzz3HlVdeyc6dO3n77bf1DiuuVAdHVDvtJowGRSsNlQbXAR88DXwu2PMBHPwCckeDopAQLDmdvjed6Hq9KxHFkXnz5vHYY49x+PBhpkyZQn5+/tm/SYSFRlSHp3ZUfgdeF9gckHkuqAEoXgeuWqgqhrT+2C1aS4UkotiTNqJu6sYbb+TQoUM8/fTT3HLLLXqHE3eqTp/senSn9jFnJBgMWvUs81ztWKW21ZLdHJz4Koko5nRNRPfeey+KokQ8hg4dqmdI3YbT6eSHP/whSUlJzJo1S+9w4k71qct/+L1QfVA7kXHK71d6cOulE/tBVUmwaG1EjV5JRLGme9XsvPPO49///nf4a5NJ95C6jcOHD3PTTTdhtcpo3/bqm5qAoihkJFu1RumAD2xOSEg/eVFqPzCYwFUDjcdJsGg/Z5eUiGJO9796k8lEdna23mF0K1VVVaxZs4Y1a9bw17/+Ve9w4tL5/VI5v1+q9sXu/drH9AGR+4QbzeDsC1UHoLqEFMcwhmYnk54kiT/WdE9Ee/bsITc3F5vNxvjx41m6dCkFBQUtXut2uyNGHNfW1sYqzJgaPXo0VVVVPPjggwwZMkTvcOJfdYn2MbWo+TlnnpaIag6TkXc+04bLNA896JqIxo0bx8qVKxkyZAhlZWUsWbKECRMmsGPHDpKTm+8vtXTpUpYsWaJDpLF14MABvUOIay6vH19AJdFiRPG5oeGYdsKZ1/xiR/BY7ZHYBSia0bWxetq0aVx33XWMGDGCq666in/9619UV1fzyiuvtHj9okWLqKmpCT9KS0tjHLGIB9+W1fL0uv38a3s51B7WDtpTwZLY/GJHcCfcpirwNBAIqDS4fbIFUYzpXjU7VUpKCoMHD2bv3r0tnrdare1uuJVJj50Tjz+/6iat695hN51MRC2VhgDMdkhIg8YTUH+UZ3Zoo7JvuqiAzOSeubNtd9StxhHV19ezb98+cnI6X083m7XxI42NjZ1+rt4s9PML/TzjQajrPsVugZpgInK0kogAkjK1j/WV2IJd+DKoMbZ0LREtXLiQq6++mn79+nHkyBEWL16M0Wjkxz/+caef22g0kpKSQkVFBQAJCQkyV6sdVFWlsbGRiooKUlJSMBqNeofUZlUNwcGMdhPUl2sHk8/wzy0xE/gOGipIMGsJSwY1xpauiejQoUP8+Mc/5vjx42RkZHDppZeyfv16MjIyovL8oWEBoWQk2i8lJSWuhlf4Ayq1Li0RpZrd2rQOxQCJZ/idCpeIKrDbgiUiGdQYU7omolWrVnXp8yuKQk5ODpmZmXi93i59rZ7IbDbHVUkIoKbJi6qCxWQg0XNcO5iQFrHuUDOhJNV4HHuS9qlUzWKrWzVWdxWj0Rh3f1CiY0KTXZ12M0pDsH0oVOJpjc2prcrg8+AI1AAGqZrFWK9IRKL3cNjMjOmXSqLFCPVHtYNJWWf+JkWBhD5Qe4REfy2QgkuqZjEliUj0KBnJViYmB6taGyq1j2dqHwpJSIfaI6QZ6hianU+2U7ruY0kSkeiZ/F5oOqF9frYSEWjtSECmsZFp58g0j1jrVuOIhOis8hoXjR4fan2FtnuKJQGsSWf/RruWiGg80bUBihZJiUj0GF5/gJc2ausO/cfQRuzQtmoZhEtENB7H7w/g8gVIsBhl7FmMSIlI9BihdaqtZgM2b4120J7atm+2p4Ki4PM08dcPtvP3dfvxyXyzmJFEJHqMmqaTqzIqTVXawVCV62yMZrA6MBoUEn3a8jIyqDF2JBGJHiO8TrXdrM2mh7aXiAAS0lBQcCpaIpIu/NiRRCR6jKqG0GTXDiaiYOnJodYB4PYGznS1iCJJRKLHCC3/kWb1gc+tDVRsVyJKASAxUA9IiSiWJBGJHqM2mIhSFC2RYE0+8xyz09mcACQEE5G0EcWOdN+LHuPCwjSqm7ykEFz2tT2lIQBbCgAJfq1q5pKqWcxIIhI9xsj8FO2T/d9oH9vaYxYSLBGlmH2ck2klLbH7b0/eU0giEj1PRxqqAcw2MNvITIapA+yQ1IYR2SIqpI1I9AhVDR7Kapq0BuaOJiIIV89oqo5WaKINJBGJHmHb4RpWbSxl4/7jJye7JrSzagbh6pm/qVp6zWJIEpHoEWqaQsvD+sDn0bruQ6Wb9rCnUNPk5Z/rd/LKl7JdVaxIIhI9Qk1nu+5DbE5MRgWrr15KRDEkiUjEPVVVw2OInMFR0e3uMQuxpWAyKFh9tbi8gbjc1y0eSSISca/J68fjC6AokBgIJaIONFRDMBEZsPrq8fsDePwyligWJBGJuBeqliVZTRjd1drBDiciBwYDmPFiDLhx+yQRxYIkIhH3QonIaTd3rscMwGhGsSRiNChY/fUy8TVGuk0iWrZsGYqicNddd+kdiogzWck2Jg3JYHieo3NjiEKsjnD1TBqsY6NbjKzetGkTf/vb3xgxYoTeoYg4lJpoITXRAp4G+K4TXfchNgcpCWaGpKrYzLIfXizoXiKqr6/npptu4umnnyY1tRP/xYQILXxvdXSs6z7E6qAwPZHxeWYykq3RiU2cke6JaP78+cyYMYMpU6ac9Vq3201tbW3EQ4i9FXWU17jwhxJRZ6ploCUyAJf8fsWKrlWzVatWsWXLFjZt2tSm65cuXcqSJUu6OCoRT3z+AG9tK0NV4Rf9jmOEzicim5aIAq5a/P4AZqPu/697PN1+wqWlpfzqV7/ihRdewGZr266aixYtoqamJvwoLZUh+L1drcunbV9mMmD1BHfu6GiPWYjVQWlVI+u27+fTvcc6H6Q4K91KRJs3b6aiooIxY8aEj/n9ftatW8df/vIX3G43RmNkQ6HVasVqlTq7OCnUde+wm1FcUegxA7A5MCoKFn8DtR5fJyMUbaFbIrriiivYvn17xLGf/exnDB06lLvvvrtZEhKiJeExRDYT1EUpEVmSMJpMKKj4XbVAbueeT5yVbokoOTmZYcOGRRxLTEwkPT292XEhWnNy1r23c7PuT6UoGKzJQC1qkzRYx4K0wom4FkpEaYYG7UBnu+6DDHZtXSLVVdPp5xJn1y0GNIasWbNG7xBEnAmXiAgu/9HZalmQKSGUiOqi8nzizLpVIhKivSYM7MOJRg+pTcEe1M72mAUZE1IAUDxSIooFSUQirhX2SaSQRNgZbMuJUonIkugkNcFMQoIPVVVRFCUqzytaJolI9AzRmOx6CktCCkOzHZBk0BrARZfqUGP1/v37ox2HEO12rN7NrvI6jte5Ts4zi1IiCo2uxi1tRLHQoUQ0cOBAJk+ezPPPP4/L5Yp2TEK0yb6Kev61vYyv9h8Gvzc6XfchwflmAU8jAa87Os8pWtWhRLRlyxZGjBjBggULyM7O5vbbb2fjxo3Rjk2IMwr1mKUr0e26B8BsY1t5ExuKT3Co/Gh0nlO0qkOJaNSoUTz++OMcOXKE5cuXU1ZWxqWXXsqwYcN45JFHqKysjHacQjQT3rnDEN2u+xC/Wdvp1dskPWddrVMDGk0mE9deey2vvvoqDz74IHv37mXhwoXk5+dz8803U1ZWFq04hWgmPM9MDSaiKHXdh6jB6pm/QRJRV+tUIvryyy/55S9/SU5ODo888ggLFy5k3759fPDBBxw5coSZM2dGK04hIvj8Aerd2oTUpM7u3NEKxZYMEJxvJrpShyrUjzzyCCtWrGDXrl1Mnz6d5557junTp2MwaHmtqKiIlStXUlhYGM1YhQirO2X5D4unWjsY5URkCPac+aVq1uU6lIieeuopbrnlFubOnUtOTk6L12RmZvLMM890KjghWhOultlMKOExRNGtmhlswWke0oXf5TqUiD744AMKCgrCJaAQVVUpLS2loKAAi8XCnDlzohKkEKfLcti4ZlQuiqcBdoe67p1RfQ1jcL4ZMvG1y3WojWjAgAEcO9Z85boTJ05QVFTU6aCEOBu7xciAjCT6JwbH+ESz6z4o2ZlKaoKZFKOMI+pqHXrnWtsPvL6+vs3LvgoRFaFqWZR7zADys7MhOzjC2usCs/xud5V2JaIFCxYAoCgK99xzDwkJCeFzfr+fDRs2MGrUqKgGKERLdhyuwWIyUFh3HAtEvaEaAJNFSz5elzbVQxJRl2lXIvrqq68ArUS0fft2LBZL+JzFYmHkyJEsXLgwuhEKcRpVVVm7uxKPL8Ctace6LhEBWJMJeJowuGshKaNrXkO0LxF9/PHHgLa29OOPP47D4eiSoIQ4kyavH49P25Pe7g8t/xH9qlmjx8en+5pwNJxg3JBaZA5+1+lQG9GKFSuiHYcQbRbquk+2GjG6uq6NyGI00KgkkAx4G2uwnPU7REe1ORFde+21rFy5EofDwbXXXnvGa1evXt3pwIRoTXidarMHXD5QDFHvugcwGQ0n55tJIupSbU5ETqczvEqd0xn9N12Itqpp1BJRH2NwjpnNCYYu2n7KKqOrY6HNiejU6phUzYSemi2Y3wXVspDwfDNJRF2qQwMam5qaaGxsDH9dUlLCY489xvvvvx+1wIRoTXhTxfDOHV2ZiLTSf6CpFloZPyc6r0OJaObMmTz33HMAVFdXM3bsWB5++GFmzpzJU089FdUAhTjd5UMzuXpkLhmm4IJoCV3Udc8pE199HvA2ddnr9HYdXqFxwoQJALz22mtkZ2dTUlLCc889x5///OeoBijE6dKTrAzMTMLu7bqu+5Cs1CSSkx2YjQZZv7oLdSgRNTY2kpys1Z3ff/99rr32WgwGAxdddBElJSVtfp6nnnqKESNG4HA4cDgcjB8/nnfeeacjIYneJhDo0ukdIRf1T2dY/76kJlgkEXWhDi+e/8Ybb1BaWsp7773HlVdeCUBFRUW7Bjn27duXZcuWsXnzZr788ksuv/xyZs6cyc6dOzsSlugFjtW7+fLACQ6VlYEa0Ca6Wrt4YG3o+d3SYN1VOpSI7rnnHhYuXEhhYSHjxo1j/PjxgFY6Gj16dJuf5+qrr2b69OkMGjSIwYMH88ADD5CUlMT69es7EpboBQ5XNfHJnmN8V3xAO2BP7fp9x6wOVFQpEXWhDo2s/tGPfsSll15KWVkZI0eODB+/4oor+MEPftChQPx+P6+++ioNDQ3hxHY6t9uN231ySYbaWlnCs7cJD2ZUur7HDGD30Tp2fFPDOU11nJMlv29dpcMLuGRnZ5OdnR1xbOzYse1+nu3btzN+/HhcLhdJSUm8/vrrnHvuuS1eu3TpUpYsWdKheEXP0KzrvgvbhwCMBoV6EvH5pUTUlTqUiBoaGli2bBkffvghFRUVBAKBiPPt2Ql2yJAhbN26lZqaGl577TXmzJnD2rVrW0xGixYtCi9FAlqJKD8/vyO3IOJUeJ5ZILp73bfGajLgMSbiDwTALSWirtKhRDRv3jzWrl3LT3/6U3JycsJTPzrCYrEwcOBAAM4//3w2bdrE448/zt/+9rdm11qtVqxWa4dfS8Q3VVXDiSjRV60dTOjTpa9pMxvxmJLwBYIlIlXt+japXqhDieidd97h7bff5pJLLol2PAQCgYh2ICFCXN4AHl8Ao9+FzdikJYTErk1EWokoAV8A1IAPxdsIlsQufc3eqEOJKDU1lbS0ztfNFy1axLRp0ygoKKCuro4XX3yRNWvW8N5773X6uUXPEyoN9THUYVQUsDnA1LUlZJvZiKoYcRvs+AMqJletJKIu0KHu+/vvv5977rknYr5ZR1RUVHDzzTczZMgQrrjiCjZt2sR7773H9773vU49r+iZ+iRZuOmiAi4vCP7adnG1DMBkUDAalGA7kTRYd5UOlYgefvhh9u3bR1ZWFoWFhZjN5ojzW7ZsadPzyL5noj1MRgOZyTYILf+RmN7lr6koCv3SE3A2pQPl0mDdRTqUiGbNmhXlMIRoh4bgVlaJsVlDeuaoPEgshEOVkoi6SIcS0eLFi6MdhxBntbmkClVVGVZzFBvEpGoWFpyFj0sSUVfoUBsRaMt//O///i+LFi3ixIkTgFYlO3z4cNSCE+JUW0ur+fy7Q3gbg8mgi3vMIliTZZpHF+pQiWjbtm1MmTIFp9PJgQMHuPXWW0lLS2P16tUcPHgwvFaRENHiD6jUubwk+aqx2owx6TEL+XhXBSX7jnGlx0WuTUpEXaFDJaIFCxYwd+5c9uzZE7Gz6/Tp01m3bl3UghMipM7lRVUh2V+N2ajEtlqmQoMSmuZRry1BIqKqQ4lo06ZN3H777c2O5+XlUV5e3umghDhddXDB/AylGgUlppsdWs3BQY0q2tIj3oaYvXZv0aFEZLVaW5z5vnv3bjIyZDdMEX3VoVn3anAxtKTsM1wdXVaTERQDLkNwi3VpsI66DiWia665hvvuuw+vV/vlUBSFgwcPcvfdd/PDH/4wqgEKAVDd6AE1QIo/mIiSY5eIbGbtzySciKTBOuo6lIgefvhh6uvrycjIoKmpiYkTJzJw4ECSk5N54IEHoh2jENQ0ebF7q7EbVTCau3zW/alsZm3PtEYlOLVDxhJFXYd6zZxOJx988AGfffYZX3/9NfX19YwZM4YpU6ZEOz4hAJg6LJuG5HKSii2QlBXTGfBWk/b/ulGxawckEUVduxNRIBBg5cqVrF69mgMHDqAoCkVFRWRnZ6OqaqeWBBGiNVaTEataBUZDTKtlAIkWE7kpNpIs6RA4KG1EXaBdVTNVVbnmmmuYN28ehw8fZvjw4Zx33nmUlJQwd+7cDi8TK0Sb1B3VPiZlxfRlUxMtzL6wgIvPLdIOSBtR1LWrRLRy5UrWrVvHhx9+yOTJkyPOffTRR8yaNYvnnnuOm2++OapBit6tos7FjkNVnHfkAFkJsS8RhVm1LbSkahZ97SoRvfTSS/z2t79tloQALr/8cn7zm9/wwgsvRC04IQCO1rjZs7+Yqtp6raE6loMZTxXaVsjTAAG/PjH0UO1KRNu2bWPq1Kmtnp82bRpff/11p4MS4lQ1TV6S3Uexmo3gyANDh6dIdtgLG0r4y6dHqPeq2nKxnvqYx9CTtesdPXHiBFlZrdfPs7KyqKqq6nRQQpyquslDsucoNpMBnH11icHnV/EGwGsKduFLg3VUtSsR+f1+TKbWm5WMRiM+n6/TQQlxqupGL8nucm08jzNPlxhCgxo9xtBYImmwjqZ2NVarqsrcuXNb3UlDFr0X0aaqKg21VVh99djMqVrVTAdWkzao0W1MgoAskBZt7UpEc+bMOes10mMmoqm2yUdC4yEMClhTc2K29MfpwtM8jAngRUpEUdauRLRixYquikOIFlU3eXC6DmMzGzGkFekWR6hE1KQkaQdcNbrF0hN1eMtpIWKhX1oCuXlN+JuSQc9EFCwRNRlC0zykRBRNse8HFaI96isw+5qw2azg0KfHDCDFbiE3xYY9KTjZVtqIokpKRKJ7O7Zb+5haBEb9fl3PzXVwbq4DPI1wFO2j36drTD2JriWipUuXcuGFF5KcnExmZiazZs1i165deoYkuhFVVfn6qw0UH2/AnTpI73A0ZjsYgsnHI9WzaNE1Ea1du5b58+ezfv16PvjgA7xeL1deeSUNDbIUp4CG6qM0VpVTVuvBmNFNElFoq2uQQY1RpGu58t133434euXKlWRmZrJ582Yuu+wynaIS3UV98WYAfM5CTNYEXWOpbvTw2uZDKIrCz5Od0HgCXNVAP13j6im6VQW3pkbrEk1LS2vxvNvtjhg02dK62aKH8PvwHNqqfZ47Ss9IADAaFOpcPgyKgprhRAHpwo+ibtNrFggEuOuuu7jkkksYNmxYi9csXboUp9MZfuTn58c4ShEzR3fQ1NiA25hEYs4QvaMJLxcbUFW8lmDVrKlav4B6mG6TiObPn8+OHTtYtWpVq9csWrSImpqa8KO0tDSGEYqY8fug5DMa3D7Kk88j02HXOyLMRoO2nxrgNobaiKr1C6iH6RZVszvuuIO33nqLdevW0bdv62NFrFZrq/PcRA9y+Eu8DdXUqTaOJp1HpqN7vOc2sxGv30eTMZlkkKpZFOlaIlJVlTvuuIPXX3+djz76iKIi/UbOim6ivhKKP8HjD3AsYzzOJHt4eoXe7Jbgbh6G0Az8evB7dYyo59C1RDR//nxefPFF3nzzTZKTk8O7xDqdTux2/YvjIsbcdbD9VQj4SMwZwrXDrsLlU/WOKswe2lYoYAGTBXwerVSUqNOKkT2IriWip556ipqaGiZNmkROTk748fLLL+sZltBDXTlsflb7w7anwtAZKAZDuBTSHfRJspKbYsNiNoItRTso1bOo0LVEpKrd57+d0FHFt/DdW1ojdUI66vDrwJxAd9uY6rLBp2ynftQJ9RXScxYl3aKxWvRSqgoln0PxOu3rtP5w7kwO1Ph5f9N+BmclM3lopr4xtsaeon2UnrOokEQk9KGqsPffcOhL7ev8C6H/5WAwsLfiKI0ePyrduMQcrppV6xlFjyGJSOij5HMtCSkKDJwCfS8AIBBQ2V+p7ZAxMCNZzwibOXCsgX9/e5S0RAvXFqRoB6VqFhWSiETsHd93sjo26HuQd3741IHjDTR6/NjMRvJSu1fPaWiah8VkOKVqJo3V0dBtRlaLXsLTCN+9rX2ed35EEgL46mA1AOflOjAauldzdWiaR5PHDzandtDnBm+TjlH1DJKIRGwVr9V2Sk3sAwMidww+cKyBgycaMSgKI/um6BPfGYSGEri8AVSDCSyhPc6kVNRZkohE7NRXQllwJ+DBU7Xto4NUVWVD8XEARuY7cSaYW3oGXdlM2p9LQFVx+wInS0XSTtRpkohE7Oz7CFQVV+ogvm1y8umeY2w5qO0MrCgKM0flMSo/hUsGds+RyiajQWsfIlg9SwguV9N0QseoegZprBaxUVtG09E9lFQ18bGrH65KbTpPlsPGmAJtQXqb2dh9xw0F2cxGPL4ATV4/qfbgQvqNkog6SxKR6HKqqrJvy0ccP1RNRcJAXCYnWQ4b2U4rGUk2vcNrlyyHlWSrCYOigD1UIqrSN6geQBKR6HIbvj1AYN9WFMDU7yJuHt6P9KTusbRHe31/RO7JLwxSNYsWSUSiyw1nL98ZFTILhnDRRSNRlO7VLd9hoRKRp1Hrwjd3r3FP8UQaq0WX8PkDwU88JB7fzqiCFPJHTOw5SQi0pUCswS2opXrWKZKIRNTVNHl59osS9lbUQ/k28LkxJqZD+kC9Q+u0nUdq+N9P9vP+Tq2xPVwqkgbrTpFEJKKqwe1j9ZZD1DZ5Wb+vErV0o3ai74XavLI4p6BN86h3+7QD0oUfFZKIRNR4/QFe/+ow1Y1eHHYzP8hvRHHVgNkG2cP1Di8qEq3a6OoGj187ICWiqJBEJKJm7a5KKuvcJFiM/HBMHokV2gaJ5I6JGEUdzxIsWv9Oo5SIokoSkYiKXeV1bD9cg6LAtGE5pHgroeYwGIyQN0bv8KImVCJq8voJBNTIEpGsONphkohEp9U0evn3t0cBGFuYRkF6ApRu0E5mngvW7rWuUGfYTEZtt1cVGr1+bTkQRdF28/DU6x1e3JJEJDotyWZiRF8neal2LuqfrpUOju3WTuaP0ze4KDMYFBJC2wq5fVqJL7RaY+Nx/QKLczKgUXSa0aAwYVAG/oCKwaBA6UatmpI+EJIyzv4EcSbLacPp9Z88kNhHG0fUcBxSC3WLK55JIhKR/D7wubS1ds7S3V5Z5yYt0RJewMxoULS1hsq3axfkj+3qaHVxzcjcyAOJfeDYHmio1CegHkASkdAEAlDyqVaa8XvB5oCiiZA9rMXLa5q8vLb5EKkJZq4ZlRvuTaLkCwj4wJEDKQUxvAEdJQZLfZKIOkzaiIRWjdr1Nhz47OQWyq5a+Pb/YM+/m/UG+fwB3t5WhsvrR4WTW0I3VcORLdrnRZf1iAGMbXJqIpKesw7RNRGtW7eOq6++mtzcXBRF4Y033tAznN6rbCuU7wDFAOdcDZcthMJLtXOHNmkLmgWpqsrHuyo5WuvCZjYyfXjOybWli9dBwA+p/SC1KPb3ESPfldfy9Lr9vLujTDuQkK797Hxubdts0W66JqKGhgZGjhzJk08+qWcYvZunAfav0T7vP0mrihnNUDQBhs7QjpduhIPrAfhi/3F2BMcLTR+ejdMeHKh4fB8c3amVgvpP7tGlIQWFerePWldwUKPBeHJgo1TPOkTXNqJp06Yxbdo0PUMQpRvB64KkTG0+2KlyRmjLW+z7CHXvR2w+1MCGhnwAJg/JpF96cPF4TyPsflf7PO8CrX2oBwtP8wiNrgatwbrhmPZIH6BTZPErrtqI3G43tbW1EQ/RCX7fycXsCy8FQwu/DgXjoOAiVMCw930y67/jssEZjMxP0c773LDjH1qbkj1Vaxvq4ZJtWimwzuVDDbUJSYN1p8RVr9nSpUtZsmSJ3mH0HMd2aSUeaxKkD2p22uX1YzUZUPpPwuD30t/1BYWBLaQ1GqB6pPa9xWu1UoDJCsN/pK3R08MlWU3aYOqASqPHT6LVJImok+KqRLRo0SJqamrCj9LSUr1Dim9HvtI+5oxqVhraW1HPs58f4OtDNVp7z6DvkXLOZNISrdo4oa+e10pCDcfAkgAjb9CqJ72A0aCQGByuUBdqJwonomPaUAjRLnFVIrJarVit8bnWcbdTXwnVpVpvT87I8GFVVfmypIpP9xwD4LuyWkb2dWorKxZdBmkDoHQ91JZpjdrpA6Bg/MnNBnuJZJuJerePercXsGnVUpMFfB5oPKa1uYk2i6tEJKIoVBrqM1AbvBj0VWl1OAmNLkjh0oF9Ipd3deaB84exjLRbynLaMBgUTKGSpKJAcg5UlUBdmSSidtI1EdXX17N3797w18XFxWzdupW0tDQKCnrJqFw9+DxwNDgNI3d0+PCBYw2s2621cVwysA9ji9L0iC4uTB7SQqJJzg4movKIUqY4O10T0ZdffsnkySf3P1+wYAEAc+bMYeXKlTpF1QtU7NSSkT01PPDQ5fXzwTdHUVUYlufkwsJUnYOMQ0nZ2se6Mn3jiEO6JqJJkyad7P4UsaGqJ6tluaPDAw/3HK2n3u0jLdHCpCEZPWu3jS4UCK04AFqJCLT2t4BfG+go2kTaiHqbujKoOwoGU8Q60sP7OkmwGkmwGDEb46ozVRfH692s3nIYRYF5E/prB+2p2jAGn1vrxg8lJnFW8hvX2xwOTkrNHKp1u59iQEYSOU7ZJLAtEq1ar1mdy4fHF+yuDzVYA9Qe0S+4OCSJqDfxNkHFt9rnwUbqiloXjR7fGb5JtMRmNmIza1Wv6ibPyRPOPO1jzSEdoopfkoh6k/Lt2lpBSRngyCMQUHlnRzkrPjtA6YlGvaOLO6kJ2lSPmkbvyYPOvtpHSUTtIomotwgE4PAp2/soCt+U1XKiwYNBUchIloGi7ZUSTETVTackIkdfbZCoq0Z7iDaRRNRbnNinLVxmtkHWMHz+AOv3a4u9jy1KDVczRNulJGjz6qoaTqmamSyQnKV9Xi1TkNpKElFvcehL7WPOSDBZ2H64hjqXjySriRF9U3QNLV61WCKCU6pnkojaShJRb1BzGKoOaL06uWPw+AJsLNZ2Jh3XP0266zsoLdFCXoqdbIct8kRKofbxRLEsHdtGMo6oNzjwifYxaxjYU/hy3zEaPX5SEsycl+vUN7Y4lpls4/oL85ufSCnQBjO6arQ93hLTYx9cnJF/hT1d1QHtP7NigMJLtGMqGBSFSwb2ObnetIgekwWcwQR1Yp++scQJKRH1ZH4f7H5P+zx3tDbyF7h4YB/Oy3XisMvbHw1efwCvP3BySyXQlkepOgAn9vfY/d2iSUpEPVnxWq1qYEmEossi5vU5E8wynywKthys4smP94aXTglLC077qD6orQkuzkgSUU9V8Z22MD7A4KuocMErX5ZyrN6tb1w9TLLVhKrCsXpP5ImEdG3FyoBfW5JXnJEkop7oxH5tc0SA/LHUJPXnn1uPcKTaxWd7j535e0W7ZCZrPWbH6t14/acsEasokHmu9nloWo1olSSinqbiW9j+D20qR59BVGSO59UvS6lz+UhPsnDVeTIjPJocdhOJViP+gMrR2tOqYJnnaB+rSsBdH/vg4ogkop5CVWH/Wtj5BgR8+NIG8rl1Aqs2HQ4noR+MzpMR1FGmKAq5KdqKBUeqT0tECWngyAU1AOXbdIgufkgi6gm8Lm1HjZLPAWjKvoD/r24MG0pq8AdU+mckcv0F+eH9uER05YUTUVMLJ8doHw9v0dqLRIuk/zbeNRzXklDjcW2xsyFTsWUNI3nLYfyqh0lDMhiQkSQ9ZF0oL1VLRIeqGvH6A5Ej1TPOgX0fgbsOKndB1rk6Rdm9SSKKZ5W74bv/Q/W5qfRacV5wPda0fBTgqvOysJgMWE1SFetqGUlWzslxhEtGEYwmbbWDA59qj4yhLe+o28vJTyQeqSoUr4Md/8DrdvF1nYOXA1P49OjJXVaTbWZJQjGiKApTh2UzvK+z5Xl7+WPBbNdKrWVbYx5fPJBEFG9C7UEHPuN4g5v3a/NZY7uCgCmBRKtJNiPojkxWKLxU+7x4Lbhq9Y2nG5KqWTwJtgd56yrZf8LNJvOFVCYPoU+ylavOywqPaRH6cHn97DxSQ5LVzJDs5MiTuaO1FTLryuG7t2HEbKminUISUTxQVa37d88H1DY0suO4yo6Uq2iyZTKuKJVxRekyebUb+LaslnW7j5FsM1HUJxGL6ZREYzDCOdfA5uXaHLTd78KQaeHtnHq7bpGSn3zySQoLC7HZbIwbN46NGzfqHVL30XgCtr8G3/0L/F6sGYV8k/tD7Ol53DA2n4sHyAz67mJYnhOH3Uydy8cXwdUvIySmwzkzteRT9jV88yb4vc2v64V0T0Qvv/wyCxYsYPHixWzZsoWRI0dy1VVXUVFRoXdo+qo7ivrd29R98hQlu79GVQwwYDLWMT9h5oWD+PHYArJOX5BL6MpsNDBpSAYAW0qq2FxS1bzNLmMwDJ2hLctS8S1sfBqOftPrxxgpqs6tm+PGjePCCy/kL3/5CwCBQID8/Hz+8z//k9/85jdn/N7a2lqcTic1NTU4HI5YhBt9AT94Gwm46nHXHsVddYSm8j24aiupbvDg8gWotuUzfOIP6Nu3QO9oRRt8vu8YG/ZrK2DmpdoZlZ/C4KzT2oyqD2rzAUMN19YkSBsAKfnahFlbCphscd+O1Na/UV3biDweD5s3b2bRokXhYwaDgSlTpvDFF180u97tduN2n5w9Xlvbxt6HE8Ww78OWl+1UVY7WuSg51gCEzqvhSxVgUFYSqQkWUFUq69zsr6wLXqWdD34HiqoyMDOJ9CQrqCrHGjzsPVob8Vqn6p+RQGawUFPV4Gb30ZPzkVQUTiQUUdlnODn9BmFPkb3o48X4/unYzEY+23OMw1VNWIyGcCLy+gM890UJRgUMge+R2fA1fWp2Yg5Uw75DpCZYKEjTNr4MqCpby10EFBMoBlQUVMWAigEUSE2wMDgzKfy6Gw+coKVSxXk5DpKsUf5TVxQYfj3YolMA0DURHTt2DL/fT1ZWVsTxrKwsvvvuu2bXL126lCVLlrT/hXxubT/yVqgNLvwNDa1/f5MPFG2MjupyEXCdvPbUN14F8ABurd6veNyonjPsF+Y1gM0GioLRmkStNRl/Qh8Mqf2wZw1gaJ8Urkw7rdFTdHuKojCmIJWBmUlsP1RD5ilbNXn9AWpPWWz/uHUkSsZ5OFxlpLhKGWRtAIsfPNrvmLup9d8fNWCBxJPz2wJ1x1tMRDg84OuCP3U1cPZr2iiues0WLVrEggULwl/X1taSn9/CmsGnS8mHkTdon4d7KU428CZ5/BQ0erVTp51XFLAlWCE4WTTZ56eoyRcxZUI5paktMcEMwYGEib4AA91eUBQUTj5v6FsTrWawJ4LJjgOF7wEGaXjuMRw2M5cM7BNxzGoycsPYfPwBLWWcLCQXAuOxW4yQZIWAH8XbRP7gKu1zNQAEtD/+gJYArBYFEk8mufx+LS/AZk62wKn/zKLVGmNJjM7zoHMi6tOnD0ajkaNHj0YcP3r0KNnZzZersFqtWK0d2AjQkghpRa2eTgo+2sIefLSFLfhoCynz9A5Gg0KOsw2/QQYjijWJvn3b+psJeXFce9f1999isXD++efz4Ycfho8FAgE+/PBDxo8fr2NkQohY0r1qtmDBAubMmcMFF1zA2LFjeeyxx2hoaOBnP/uZ3qEJIWJE90Q0e/ZsKisrueeeeygvL2fUqFG8++67zRqwhRA9l+7jiDqjR4wjEqIHa+vfqLSRCiF0J4lICKE7SURCCN3p3ljdGaHmrTZP9RBCxFTob/NsTdFxnYjq6rQ5X20aXS2E0E1dXR1Op7PV83HdaxYIBDhy5AjJycndcpeK0BSU0tLSHterJ/cWn2J9b6qqUldXR25uLoYzrCQQ1yUig8FA37599Q7jrBwOR4/7hQ6Re4tPsby3M5WEQqSxWgihO0lEQgjdSSLqQlarlcWLF3dsxYBuTu4tPnXXe4vrxmohRM8gJSIhhO4kEQkhdCeJSAihO0lEQgjdSSJqp/buSvvqq68ydOhQbDYbw4cP51//+lfEeVVVueeee8jJycFutzNlyhT27NnTlbfQqvbc29NPP82ECRNITU0lNTWVKVOmNLt+7ty5KIoS8Zg6dWpX30aL2nNvK1eubBa3zRa5+ni8vm+TJk1qdm+KojBjxozwNbq8b6pos1WrVqkWi0Vdvny5unPnTvXWW29VU1JS1KNHj7Z4/WeffaYajUb1j3/8o/rNN9+o/+///T/VbDar27dvD1+zbNky1el0qm+88Yb69ddfq9dcc41aVFSkNjU1xeq2VFVt/73deOON6pNPPql+9dVX6rfffqvOnTtXdTqd6qFDh8LXzJkzR506dapaVlYWfpw4cSJWtxTW3ntbsWKF6nA4IuIuLy+PuCZe37fjx49H3NeOHTtUo9GorlixInyNHu+bJKJ2GDt2rDp//vzw136/X83NzVWXLl3a4vXXX3+9OmPGjIhj48aNU2+//XZVVVU1EAio2dnZ6p/+9Kfw+erqatVqtaovvfRSF9xB69p7b6fz+XxqcnKy+uyzz4aPzZkzR505c2a0Q2239t7bihUrVKfT2erz9aT37dFHH1WTk5PV+vr68DE93jepmrVRaFfaKVOmhI+daVdagC+++CLieoCrrroqfH1xcTHl5eUR1zidTsaNG9fqc3aFjtzb6RobG/F6vaSlpUUcX7NmDZmZmQwZMoRf/OIXHD9+PKqxn01H762+vp5+/fqRn5/PzJkz2blzZ/hcT3rfnnnmGW644QYSEyP3KIv1+yaJqI3OtCtteXl5i99TXl5+xutDH9vznF2hI/d2urvvvpvc3NyIP4qpU6fy3HPP8eGHH/Lggw+ydu1apk2bht/vj2r8Z9KRexsyZAjLly/nzTff5PnnnycQCHDxxRdz6NAhoOe8bxs3bmTHjh3Mmzcv4rge71tcz74X3cOyZctYtWoVa9asiWjUveGGG8KfDx8+nBEjRjBgwADWrFnDFVdcoUeobTJ+/PiIffUuvvhizjnnHP72t79x//336xhZdD3zzDMMHz6csWPHRhzX432TElEbtXdXWoDs7OwzXh/62J7n7AodubeQhx56iGXLlvH+++8zYsSIM17bv39/+vTpw969ezsdc1t15t5CzGYzo0ePDsfdE963hoYGVq1axc9//vOzvk4s3jdJRG3UkV1px48fH3E9wAcffBC+vqioiOzs7Ihramtr2bBhQ0x3uu3ojrt//OMfuf/++3n33Xe54IILzvo6hw4d4vjx4+Tk5EQl7raIxm7Cfr+f7du3h+OO9/cNtGElbrebn/zkJ2d9nZi8bzFtGo9zq1atUq1Wq7py5Ur1m2++UW+77TY1JSUl3LX705/+VP3Nb34Tvv6zzz5TTSaT+tBDD6nffvutunjx4ha771NSUtQ333xT3bZtmzpz5kzduoHbc2/Lli1TLRaL+tprr0V089bV1amqqqp1dXXqwoUL1S+++EItLi5W//3vf6tjxoxRBw0apLpcrm59b0uWLFHfe+89dd++fermzZvVG264QbXZbOrOnTsj7j8e37eQSy+9VJ09e3az43q9b5KI2umJJ55QCwoKVIvFoo4dO1Zdv359+NzEiRPVOXPmRFz/yiuvqIMHD1YtFot63nnnqW+//XbE+UAgoP7ud79Ts7KyVKvVql5xxRXqrl27YnErzbTn3vr166cCzR6LFy9WVVVVGxsb1SuvvFLNyMhQzWaz2q9fP/XWW29tNh4nVtpzb3fddVf42qysLHX69Onqli1bIp4vXt83VVXV7777TgXU999/v9lz6fW+yTIgQgjdSRuREEJ3koiEELqTRCSE0J0kIiGE7iQRCSF0J4lICKE7SURCCN1JIhJC6E4SkYgrK1euJCUlJfz1vffey6hRo8Jfz507l1mzZsU8LtE5kohEi0LrFv/Hf/xHs3Pz589HURTmzp0bcX20E0BhYSGPPfZYxLHZs2eze/fuVr/n8ccfZ+XKleGvJ02axF133RXVuET0SSISrcrPz2fVqlU0NTWFj7lcLl588UUKCgp0iclut5OZmdnqeafTGVFiEvFBEpFo1ZgxY8jPz2f16tXhY6tXr6agoIDRo0d36rlbKqnMmjUrXMqaNGkSJSUl/PrXvw7vJAHNq2anO7VkNnfuXNauXcvjjz8efo7i4mIGDhzIQw89FPF9W7duRVGUmK6VJE6SRCTO6JZbbmHFihXhr5cvX87PfvazLn/d1atX07dvX+677z7KysooKytr93M8/vjjjB8/nltvvTX8HAUFBc3uCWDFihVcdtllDBw4MFq3INpBEpE4o5/85Cd8+umnlJSUUFJSwmeffdamxbQ6Ky0tDaPRSHJyMtnZ2R1a+dDpdGKxWEhISAg/h9FoZO7cuezatSu8/5fX6+XFF1/klltuifZtiDaSNavFGWVkZDBjxgxWrlyJqqrMmDGDPn366B1Wp+Tm5jJjxgyWL1/O2LFj+b//+z/cbjfXXXed3qH1WlIiEmd1yy23sHLlSp599tmolRoMBgOnL4Xl9Xqj8txtMW/evHBD/IoVK5g9ezYJCQkxe30RSRKROKupU6fi8Xjwer1cddVVUXnOjIyMiHYfv9/Pjh07Iq6xWCyd3sKmteeYPn06iYmJPPXUU7z77rtSLdOZVM3EWRmNRr799tvw562pqalh69atEcfS09PJz89vdu3ll1/OggULePvttxkwYACPPPII1dXVEdcUFhaybt06brjhBqxWa4eqhIWFhWzYsIEDBw6QlJREWloaBoMh3Fa0aNEiBg0aFNNF70VzUiISbeJwOHA4HGe8Zs2aNYwePTrisWTJkhavveWWW5gzZw4333wzEydOpH///kyePDnimvvuu48DBw4wYMAAMjIyOhT3woULMRqNnHvuuWRkZHDw4MHwuZ///Od4PJ6Y9AKKM5M1q0Wv9cknn3DFFVdQWlrabLdUEVuSiESv43a7qaysZM6cOWRnZ/PCCy/oHVKvJ1Uz0eu89NJL9OvXj+rqav74xz/qHY5ASkRCiG5ASkRCCN1JIhJC6E4SkRBCd5KIhBC6k0QkhNCdJCIhhO4kEQkhdCeJSAihu/8fcsWlO9+vuX8AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:19:10.278496Z", + "iopub.status.busy": "2024-03-26T06:19:10.278173Z", + "iopub.status.idle": "2024-03-26T06:19:10.594828Z", + "shell.execute_reply": "2024-03-26T06:19:10.594002Z" + }, + "papermill": { + "duration": 0.333253, + "end_time": "2024-03-26T06:19:10.596989", + "exception": false, + "start_time": "2024-03-26T06:19:10.263736", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/h0lEQVR4nO3deXhU9b348feZPZPMTBayQkICAQTZi1BwA6UKWEXtdWlpC271Vr29lB9PlXtbLdoWbdVKW6+ttoDeilQt1dtaxR0qILIIArIGEraEhJA9mf38/jgzA0MSyDKZk+Xzep55kpxzMvM5meST7/5VVFVVEUIIHRn0DkAIISQRCSF0J4lICKE7SURCCN1JIhJC6E4SkRBCd5KIhBC6k0QkhNCdSe8AOiMYDHLixAkcDgeKougdjhDiHKqqUldXR05ODgZD6+WeHp2ITpw4QW5urt5hCCEu4OjRowwYMKDV8z06ETkcDkC7SafTqXM0Qohz1dbWkpubG/lbbU2PTkTh6pjT6ZREJEQ3dqGmE2msFkLoThKREEJ3koiEELrr0W1EbaGqKn6/n0AgoHcovZrRaMRkMskwCtEhvToReb1eSktLaWxs1DuUPsFut5OdnY3FYtE7FNHD9NpEFAwGOXz4MEajkZycHCwWi/y37iKqquL1eqmoqODw4cMMGTLkvIPXhDiX7ono+PHjPPjgg7z99ts0NjZSWFjI8uXLmTBhQqee1+v1EgwGyc3NxW63xyha0ZqEhATMZjMlJSV4vV5sNpveIbWq+FQDm4tPk+6wcllhP0xGSZp60zURVVVVcemllzJt2jTefvtt0tPTOXDgACkpKTF7DfnPHD894Wdd1eDlH1+cwBdQOVbVhMlg4LIh/fQOq8/TNRE98cQT5Obmsnz58sixgoICHSMSvd3Wkip8ARWTQcEfVNlxrJoJ+SnYzEa9Q+vTdP0X9n//939MmDCBW265hYyMDMaNG8cLL7zQ6vUej4fa2tqohxBtFQiqlJzWOi5uHNef1EQLXn+QQxUNOkcmdE1Ehw4d4rnnnmPIkCGsWbOG73//+/zgBz/gxRdfbPH6JUuW4HK5Ig+Z8Craw2hQuPPSfOZMymNASgKFGUmkJVnoATXKXk/Rc18zi8XChAkT2LBhQ+TYD37wAzZv3szGjRubXe/xePB4PJGvwxPqampqms01c7vdHD58mIKCgm7dcNqSqVOnMnbsWJ555hm9Q2mXnvYzDwZVDAbpSe1KtbW1uFyuFv9Gz6br/4Ls7GxGjBgRdWz48OEcOXKkxeutVmtkgqtMdBWdJUmo+9A1EV166aXs27cv6tj+/fsZOHBgl76u1x9s9eEPBNt8ra8N17bXvHnzWLt2LUuXLkVRFBRFYcCAATz33HNR133++ecYDAZKSkoAePrppxk1ahSJiYnk5uZy3333UV9fH/U9n3zyCZdffjkJCQnk5ubygx/8gIaGvtE+Uuf2seyTw7yzq5RzKwGBoIrbJyPv9aRrr9kPf/hDpkyZwi9+8QtuvfVWPvvsM55//nmef/75Ln3dZz862Oq5gn6J3Diuf+Tr59cV4Qu0XHsdkJLALRPOtFMtW3+YJm/0L/QPvza0XbEtXbqU/fv3M3LkSB599FEAnnrqKVauXMn3v//9yHUvv/wyl156aSRpGwwGfvOb31BQUMChQ4e47777+NGPfsT//M//AFBUVMSMGTP42c9+xrJly6ioqOCBBx7ggQceiOq17K3KatzUNPmwmAxRA1u3FJ9mY1ElY/OSuXxIuo4R9m26loguueQS/va3v/HKK68wcuRIHnvsMZ555hnmzJmjZ1i6crlcWCwW7HY7WVlZZGVlMWfOHNavXx+psgaDQVatWhX1c5o/fz7Tpk0jPz+fq666ip/97Ge8+uqrkfNLlixhzpw5zJ8/nyFDhjBlyhR+85vf8NJLL+F2u+N+n/FWWqPdY7Yruu3KYjLgD6qcbvDqEZYI0X1k9de//nW+/vWvx/U1759W2Oq5c5sNvnfF4FavPXfGyJ2Xds0YqLFjxzJ8+HBWrlzJQw89xNq1aykvL+eWW26JXPP++++zZMkS9u7dS21tLX6/H7fbTWNjI3a7nR07dvDFF1/w8ssvR75HVdXIVJjhw4d3SezdRVkkESVEHU+xa/PiKuslEempT3ZcWkyGVh/nDvc/37XmNlwbK3PmzGHlypUArFy5khkzZpCWlgZAcXExX//61xk9ejR//etf2bp1K88++yygTXUBqK+v595772X79u2Rx44dOzhw4ACDB7eebHuDYFClvE5LRFnnlIjSkrREVOv2NWvzE/Gje4lINGexWJotW/Ktb32LH//4x2zdupXXX3+d3//+95FzW7duJRgM8tRTT0WmWZxdLQMYP348X375JYWFrZcGe6vqJh++gIrZqJBiN0edSzAbSbAYafIGqGrwkuHs/sMOeqM+WSLq7vLz89m0aRPFxcWcOnWKYDBIfn4+U6ZM4a677iIQCHDDDTdEri8sLMTn8/Hb3/6WQ4cO8b//+79RiQrgwQcfZMOGDTzwwANs376dAwcO8Oabb/LAAw/E+/birrJeG3uWlmRttgKDoiikhqpnVY2+uMcmNJKIuqGFCxdiNBoZMWIE6enpkUbqOXPmsGPHDm666SYSEs60dYwZM4ann36aJ554gpEjR/Lyyy+zZMmSqOccPXo0a9euZf/+/Vx++eWMGzeOhx9+mJycnLjemx4UBTKcVjKd1hbPOxO0ikGtWxKRXnQdWd1Z5xu12dNG+fYGPfVnvvNYDUdONzIsK4nCjPNveyPap60jq6WNSPR5owa4GDXApXcYfZpUzUSvFgyqBII9ttDfZ0giEr1aRb2HZz86yF+3HjvvdaqqUuf2NZv+IeJDEpHo1U43eAkEVYLnSTDBoMr/fFzEH/91mHqPP47RiTBJRKJXq2rUBnSGR1C3xGBQIis01rolEelBEpHo1WpCY4OSzxnIeC6HVeu3aZQSkS4kEYlerSqSiM6/11piKBFJ1UwfkohEr6Wq6llVs/OXiBKtWtVMEpE+JBGJXqvJF8DrD6Io4Eo4fyJKCpWIGiQR6UISUTc0depU5s+fr3cYEd0tnrbyB1WGZjrIT0u84CaKZ6pmslKjHmRkdS/l9Xr7/B70TpuZ60Znt+natEQLQzMdZLQyH010LSkRdTMtrVldVFTEXXfdRUFBAQkJCQwbNoylS5c2+74bb7yRn//85+Tk5DBs2DAANmzYwNixY7HZbEyYMIE33ngDRVHYvn175Ht37drFzJkzSUpKIjMzk+985zucOnWq1XiKi4vj9eOImwynjetGZ3NJfqreofRJfatEpKoQ0GGGtdHcfDnHVrS0ZnVKSgoDBgzgtddeIy0tjQ0bNvC9732P7Oxsbr311sj3fvDBBzidTt577z1Am3B4/fXXM2vWLFauXElJSUmzKlZ1dTVXXXUVd999N7/+9a9pamriwQcf5NZbb+XDDz9sMZ709J6xtrPbF8B6zhrVonvqW4ko4IN/PRX/1738/4GpbdWkc9esDlu8eHHk84KCAjZu3Mirr74alYgSExP54x//GKmS/f73v0dRFF544QVsNhsjRozg+PHj3HPPPZHv+d3vfse4ceP4xS9+ETm2bNkycnNz2b9/P0OHDm0xnp7gta3HqGn0cuO4/gxIsWsHAz44uTu0NsjFYDzzJxAMqjSFkteF2pREbPWtRNSDPfvssyxbtowjR47Q1NSE1+tl7NixUdeMGjUqql1o3759jB49OmpJjokTJ0Z9z44dO/joo49ISkpq9ppFRUUMHdq+XUi6C1VVqWn04guoJFpCv+YBP+xYBTWheWcnd8Po2wlv9frnTSVU1nv5t68MIDfVrlPkfVPfSkRGs1Y60eN1O2HVqlUsXLiQp556ismTJ+NwOPjVr37Fpk2boq5LTExs93PX19dz/fXX88QTTzQ7l53dtobe7qje48cXUDEoCs5w1/2xz7QkZDSBClSVQOl26D8e0JaNBWjwShd+vPWtRKQoba4i6encNavXr1/PlClTuO+++yLHioqKLvg8w4YN489//jMejwerVesN2rx5c9Q148eP569//Sv5+fmYTC3/OrS0hnZ3Vx0aUe1KMGE0KFpp6GgocQ+dCX43HHgPjmyEnHGgKNhDJadz96YTXU8qwt3QuWtWDxkyhC1btrBmzRr279/PT37yk2YJpSXf+ta3CAaDfO9732PPnj2sWbOGJ598EiDSgHv//fdz+vRpvvnNb7J582aKiopYs2YNd9xxRyT5tLSGdncXHlEdmdpRsRd8brA5IWMEZI8FkxXctVB1GIAEi/bnIIko/nRNRD/96U8jXcLhx0UXXaRnSN3CuWtWX3vttdx8883cdtttTJo0icrKyqjSUWucTid///vf2b59O2PHjuW///u/efjhhwEi7UY5OTmsX7+eQCDANddcw6hRo5g/fz7JycmRHUFaW0O7O6s6d7Lryd3ax+wxWpuQ0aQlJICK/QAkmEMTXyURxZ3uVbOLL76Y999/P/J1a9WDvmTo0KFs3Lgx6tjy5cubbQ199gL5K1asaPG5pkyZwo4dOyJfv/zyy5jNZvLy8iLHhgwZwurVq9sVT3dXffbyHwEfVIeSZ/pZ/+jSCuHE53D6EKgqdovWRtTok0QUb7r/1ZtMph7XLdyTvPTSSwwaNIj+/fuzY8eOyBihs3cB6Y0GpNhRFIV0h1VrlA76weYCe9qZi1IGgsEE7hporMRu0drR3FIiijvd24gOHDhATk4OgwYNYs6cOT2i2N+TlJWV8e1vf5vhw4fzwx/+kFtuuYXnn39e77C63FcGpnDDmBxykhO0Eg9A2uDogaVGM7gGaJ9Xl5Bst3BRloP8fu3vfRSdo2uJaNKkSaxYsYJhw4ZRWlrK4sWLufzyy9m1axcOR/NtXTweDx6PJ/J1bW1tPMPtkX70ox/xox/9SO8w9FVdon1MKWh+ztUfqoqh5jjp/b/CzFE9d8hCT6ZrIpo5c2bk89GjRzNp0iQGDhzIq6++yl133dXs+iVLlkSNMBaiJW5fAH9QJdFiRPF7oEGbN4erf/OLnaFjtSfiF6BoRveq2dmSk5MZOnQoBw8ebPH8okWLqKmpiTyOHj0a5whFT7CntJYX1h3inzvLoPa4djAhBSwtVLmcoZ1um6rA20AwqNLg8csWRHHWrRJRfX09RUVFrY7otVqtOJ3OqMeFyPYw8dNdftbVTVrXvTPBdCYRtVQaAjAngD00477+JH/65DDPrztEZYOn5etFl9A1ES1cuJC1a9dSXFzMhg0buOmmmzAajXzzm9/s9HObzdr4kcbGxk4/l2ib8M86/LPXS7jrPjnBAjWhRORsJREBJGVoH+srsIW68GVQY3zp2kZ07NgxvvnNb1JZWUl6ejqXXXYZn376aUyWmTAajSQnJ1NeXg6A3W6X5SC6iKqqNDY2Ul5eTnJyMkajUdd4qhpCgxkTTFBfph10nKcROjED2AsN5djNWsKSQY3xpWsiWrVqVZc+f3h8UjgZia6VnJys+5iwQFCl1q0lohSzR5vWoRgg8Tz/3CIlonISbKESkQxqjCvdBzR2JUVRyM7OJiMjA59PhwXR+hCz2ax7SQigpsmHqoLFZCDRW6kdtKdGrTvUTDhJNVaSEFoNRapm8dWrE1GY0WjsFn8kouuFJ7u6EswoDaH2oXCJpzU2l7Yqg9+LM1gDGKRqFmd9IhGJvsNpMzN+YAqJFiPUn9QOJmWe/5sUBez9oPYEiYFaIBm3VM3iShKR6FXSHVaudISqWpsqtI/nax8Ks6dB7QlSDXVclJVLlst24e8RMSOJSPROAR80ndY+v1CJCCJjiTKMjcwcLtM84q1bDWgUorPKatw0ev2o9eXari0WO1ibr8fdTEJoUGPj6a4NULRISkSi1/AFgrzymbZ6w79f1EgCtK1aBmdGVzdWEggEcfuD2C1GGXsWJ1IiEr1GeJ1qq9mAzVejHUxIads3J6SAouD3NvE/7+3k+XWH8Mt8s7iRRCR6jZqmM6syKk1V2sGENu7cajSD1YnRoJDo15aXkUGN8SOJSPQakXWqE8zabHpoe4kIwJ6KgoJL0RKRdOHHjyQi0WtUNYQnu3YwEYVKT061DgCPr/vvVtJbSCISvUZ4+Y9Uqx/8Hm2gYrsSUTIAicF6QEpE8SSJSPQataFElKxoiQSr4/xzzM5lcwFgDyUiaSOKH+m+F73GJfmpVDf5SCa07Gt7SkMAtmQA7AGtauaWqlncSCISvcaY3GTtk0Nfah/b2mMWFioRJZv9DM+wkprY/bcn7y0kEYnepyMN1QBmG5htZDhgxuAESGrDiGwRE9JGJHqFqgYvpTVNWgNzRxMRRKpnNFXHKjTRBpKIRK/wxfEaVn12lM8OVZ6Z7GpvZ9UMItWzQFO19JrFkSQi0SvUNIWXh/WD36t13YdLN+2RkExNk4//+3Q3r26R7ariRRKR6BVqOtt1H2ZzYTIqWP31UiKKI0lEosdTVTUyhsgVGhXd7h6zMFsyJoOC1V+L2xfsNnu19XaSiESP1+QL4PUHURRIDIYTUQcaqiGUiAxY/fUEAkG8ARlLFA+SiESPF66WJVlNGD3V2sEOJyInBgOY8WEMevD4JRHFgyQi0eOFE5Erwdy5HjMAoxnFkojRoGAN1MvE1zjpNono8ccfR1EU5s+fr3cooofJdNiYOiydUf2dnRtDFGZ1Rqpn0mAdH91iZPXmzZv5wx/+wOjRo/UORfRAKYkWUhIt4G2AvZ3oug+zOUm2mxmWomIzy3548aB7iai+vp45c+bwwgsvkJLSif9iQoQXvrc6O9Z1H2Z1kp+WyOT+ZtId1tjEJs5L90R0//33c9111zF9+vQLXuvxeKitrY16CHGwvI6yGjeBcCLqTLUMtEQG4Jbfr3jRtWq2atUqtm3bxubNm9t0/ZIlS1i8eHEXRyV6En8gyD++KEVV4fsDKzFC5xORTUtEQXctgUAQs1H3/9e9nm4/4aNHj/Kf//mfvPzyy9hsbdtVc9GiRdTU1EQeR4/KEPy+rtbt17YvMxmwekM7d3S0xyzM6uRoVSPrdh7ik4OnOh+kuCDdSkRbt26lvLyc8ePHR44FAgHWrVvH7373OzweD0ZjdEOh1WrFapU6uzgj3HXvTDCjuGPQYwZgc2JUFCyBBmq9/k5GKNpCt0R09dVXs3Pnzqhjd9xxBxdddBEPPvhgsyQkREsiY4hsJqiLUSKyJGE0mVBQCbhrgZzOPZ+4IN0SkcPhYOTIkVHHEhMTSUtLa3ZciNacmXXv69ys+7MpCgarA6hFbZIG63iQVjjRo4UTUaqhQTvQ2a77EEOCti6R6q7p9HOJC+sWAxrDPv74Y71DED1MpEREaPmPzlbLQkz2cCKqi8nzifPrVolIiPa6vLAfpxu9pDSFelA722MWYrQnA6B4pUQUD5KIRI+W3y+RfBJhd6gtJ0YlIkuiixS7Gbvdj6qqKIoSk+cVLZNEJHqHWEx2PYvFnsxFWU5IMmgN4KJLdaix+tChQ7GOQ4h2O1XvYV9ZHZV17jPzzGKUiMKjq/FIG1E8dCgRFRYWMm3aNP785z/jdrtjHZMQbVJUXs8/d5by+aHjEPDFpus+LDTfLOhtJOjzxOY5Ras6lIi2bdvG6NGjWbBgAVlZWdx777189tlnsY5NiPMK95ilKbHtugfAbOOLsiY2HT7NsbKTsXlO0aoOJaKxY8eydOlSTpw4wbJlyygtLeWyyy5j5MiRPP3001RUVMQ6TiGaiezcYYht131YwKzt9Oprkp6zrtapAY0mk4mbb76Z1157jSeeeIKDBw+ycOFCcnNz+e53v0tpaWms4hSimcg8MzWUiGLUdR+mhqpngQZJRF2tU4loy5Yt3HfffWRnZ/P000+zcOFCioqKeO+99zhx4gSzZ8+OVZxCRPEHgtR7tAmpSZ3duaMVis0BEJpvJrpShyrUTz/9NMuXL2ffvn3MmjWLl156iVmzZmEwaHmtoKCAFStWkJ+fH8tYhYioO2v5D4u3WjsY40RkCPWcBaRq1uU6lIiee+457rzzTubNm0d2dnaL12RkZPCnP/2pU8EJ0ZpItcxmQomMIYpt1cxgC03zkC78LtehRPTee++Rl5cXKQGFqarK0aNHycvLw2KxMHfu3JgEKcS5Mp02bhibg+JtgP3hrntXTF/DGJpvhkx87XIdaiMaPHgwp041X7nu9OnTFBQUdDooIS4kwWJkcHoSgxJDY3xi2XUf4nClkGI3k2yUcURdrUPvXGv7gdfX17d52VchYiJcLYtxjxlAblYWZIVGWPvcYJbf7a7SrkS0YMECABRF4eGHH8Zut0fOBQIBNm3axNixY2MaoBAt2XW8BovJQH5dJRaIeUM1ACaLlnx8bm2qhySiLtOuRPT5558DWolo586dWCyWyDmLxcKYMWNYuHBhbCMU4hyqqrJ2fwVef5B7Uk91XSICsDoIepsweGohKb1rXkO0LxF99NFHgLa29NKlS3E6nV0SlBDn0+QL4PVre9InBMLLf8S+atbo9fNJURPOhtNMGlaLzMHvOh1qI1q+fHms4xCizcJd9w6rEaO769qILEYDjYodB+BrrMFywe8QHdXmRHTzzTezYsUKnE4nN99883mvXb16dacDE6I1kXWqzV5w+0ExxLzrHsBkNJyZbyaJqEu1ORG5XK7IKnUuV+zfdCHaqqZRS0T9jKE5ZjYXGLpo+ymrjK6OhzYnorOrY1I1E3pqtmB+F1TLwiLzzSQRdakODWhsamqisbEx8nVJSQnPPPMM7777bswCE6I1kU0VIzt3dGUi0kr/waZaaGX8nOi8DiWi2bNn89JLLwFQXV3NxIkTeeqpp5g9ezbPPfdcTAMU4lxXXZTB9WNySDeFFkSzd1HXPWdNfPV7wdfUZa/T13V4hcbLL78cgNdff52srCxKSkp46aWX+M1vftPm53nuuecYPXo0TqcTp9PJ5MmTefvttzsSkuhD0pKsFGYkkeDruq77sMyUJBwOJ2ajQdav7kIdSkSNjY04HFrd+d133+Xmm2/GYDDw1a9+lZKSkjY/z4ABA3j88cfZunUrW7Zs4aqrrmL27Nns3r27I2GJviQY7NLpHWFfHZTGyEEDSLFbJBF1oQ4vnv/GG29w9OhR1qxZwzXXXANAeXl5uwY5Xn/99cyaNYshQ4YwdOhQfv7zn5OUlMSnn37akbBEH3Cq3sOW4tMcKy0FNahNdLV28cDa8PN7pMG6q3QoET388MMsXLiQ/Px8Jk2axOTJkwGtdDRu3LgOBRIIBFi1ahUNDQ2R5xPiXMermvjXgVPsPVysHUhI6fp9x6xOVFQpEXWhDo2s/rd/+zcuu+wySktLGTNmTOT41VdfzU033dSu59q5cyeTJ0/G7XaTlJTE3/72N0aMGNHitR6PB4/nzJIMtbWyhGdfExnMqHR9jxnA/pN17PqyhuFNdQzPlN+3rtLhBVyysrLIysqKOjZx4sR2P8+wYcPYvn07NTU1vP7668ydO5e1a9e2mIyWLFnC4sWLOxqy6AWadd13YfsQgNGgUE8i/oCUiLpShxJRQ0MDjz/+OB988AHl5eUEg8Go8+3ZCdZisVBYWAjAV77yFTZv3szSpUv5wx/+0OzaRYsWRZYiAa1ElJub25FbED1UZJ5ZMLZ73bfGajLgNSYSCAbBIyWirtKhRHT33Xezdu1avvOd75CdnR2Z+hELwWAwqvp1NqvVitVqjdlriZ5FVdVIIkr0V2sH7f269DVtZiNeUxL+YKhEpKpd3ybVB3UoEb399tu89dZbXHrppZ168UWLFjFz5kzy8vKoq6tj5cqVfPzxx6xZs6ZTzyt6J7cviNcfxBhwYzM2aQkhsWsTkVYisuMPghr0o/gawZLYpa/ZF3UoEaWkpJCa2vm6eXl5eWQjRpfLxejRo1mzZg1f+9rXOv3covcJl4b6GeowKgrYnGDq2hKyzWxEVYx4DAkEgiomd60koi7QoUT02GOP8fDDD/Piiy9GLRfbXrLdkGiPfkkW5nw1j+DxSiily6tlACaDgtGghNqJfJg8dUDLW2iJjutQInrqqacoKioiMzOT/Px8zGZz1Plt27bFJDghzmYyGshw2CC8/EdiWpe/pqIoDEyz42pKA8qkwbqLdCgR3XjjjTEOQ4h2aAhtZZUYnzWkZ4/tD4n5cKxCElEX6VAieuSRR2IdhxAXtLWkClVVGVlzEhvEpWoWEZqFj1sSUVfo0BQP0Jb/+OMf/8iiRYs4ffo0oFXJjh8/HrPghDjb9qPVbNh7DF9jKBl0cY9ZFKtDpnl0oQ6ViL744gumT5+Oy+WiuLiYe+65h9TUVFavXs2RI0ciaxUJESuBoEqd20eSvxqrzRiXHrOwj/aVU1J0imu8bnJsUiLqCh0qES1YsIB58+Zx4MCBqJ1dZ82axbp162IWnBBhdW4fqgqOQDVmoxLfapkKDUp4mke9tgSJiKkOJaLNmzdz7733Njvev39/ysrKOh2UEOeqDi2Yn65Uo6DEdbNDqzk0qFFFW3rE1xC31+4rOpSIrFZrizPf9+/fT3q67IYpYq86POteDS2GlpR1nqtjy2oygmLAbQiNmZMG65jrUCK64YYbePTRR/H5tF8ORVE4cuQIDz74IN/4xjdiGqAQANWNXlCDJAdCicgRv0RkM2t/JpFEJA3WMdehRPTUU09RX19Peno6TU1NXHnllRQWFuJwOPj5z38e6xiFoKbJR4KvmgSjCkZzl8+6P5vNrO2Z1qiEpnbIWKKY61Cvmcvl4r333mP9+vXs2LGD+vp6xo8fz/Tp02MdnxAAzBiZRYOjjKTDFkjKjOsMeKtJ+3/dqCRoByQRxVy7E1EwGGTFihWsXr2a4uJiFEWhoKCArKwsVFWN6ZIgQoRZTUasahUYDXGtlgEkWkzkJNtIsqRB8Ii0EXWBdlXNVFXlhhtu4O677+b48eOMGjWKiy++mJKSEubNm9fuZWKFaJe6k9rHpMy4vmxKooXbLsljyogC7YC0EcVcu0pEK1asYN26dXzwwQdMmzYt6tyHH37IjTfeyEsvvcR3v/vdmAYp+rbyOje7jlVx8YliMu3xLxFFWLUttKRqFnvtKhG98sor/Nd//VezJARw1VVX8dBDD/Hyyy/HLDghAE7WeDhw6DBVtfVaQ3U8BzOeLbytkLcBggF9Yuil2pWIvvjiC2bMmNHq+ZkzZ7Jjx45OByXE2WqafDg8J7GajeDsD4YOT5HssJc3lfC7T05Q71O15WK99XGPoTdr1zt6+vRpMjNbr59nZmZSVVXV6aCEOFt1kxeH9yQ2kwFcA3SJwR9Q8QXBZwp14UuDdUy1KxEFAgFMptablYxGI36/v9NBCXG26kYfDk+ZNp7H1V+XGMKDGr3G8FgiabCOpXY1Vquqyrx581rdSaO13TeE6ChVVWmorcLqr8dmTtGqZjqwmrRBjR5jEgRlgbRYa1cimjt37gWvkR4zEUu1TX7sjccwKGBNyY7b0h/nikzzMNrBh5SIYqxdiWj58uVdFYcQLapu8uJyH8dmNmJILdAtjnCJqElJ0g64a3SLpTfq8JbTQsTDwFQ7Of2bCDQ5QM9EFCoRNRnC0zykRBRL8e8HFaI96ssx+5uw2azg1KfHDCA5wUJOso2EpNBkW2kjiikpEYnu7dR+7WNKARj1+3UdkeNkRI4TvI1wEu1jwK9rTL2JriWiJUuWcMkll+BwOMjIyODGG29k3759eoYkuhFVVdnx+SYOVzbgSRmidzgacwIYQsnHK9WzWNE1Ea1du5b777+fTz/9lPfeew+fz8c111xDQ4MsxSmgofokjVVllNZ6MaZ3k0QU3uoaZFBjDOlarnznnXeivl6xYgUZGRls3bqVK664QqeoRHdRf3grAH5XPiZrx7c2j4XqRi+vbz2Goijc5XBB42lwVwMDdY2rt+hWFdyaGq1LNDU1tcXzHo8natBkS+tmi14i4Md7bLv2ec5YPSMBwGhQqHP7MSgKaroLBaQLP4a6Ta9ZMBhk/vz5XHrppYwcObLFa5YsWYLL5Yo8cnNz4xyliJuTu2hqbMBjTCIxe5je0USWiw2qKj5LqGrWVK1fQL1Mt0lE999/P7t27WLVqlWtXrNo0SJqamoij6NHj8YxQhE3AT+UrKfB46fMcTEZzgS9I8JsNGj7qQEeY7iNqFq/gHqZblE1e+CBB/jHP/7BunXrGDCg9bEiVqu11Xluohc5vgVfQzV1qo2TSReT4ewe77nNbMQX8NNkdOAAqZrFkK4lIlVVeeCBB/jb3/7Ghx9+SEGBfiNnRTdRXwGH/4U3EORU+mRcSQmR6RV6S7CEdvMwhGfg10PAp2NEvYeuJaL777+flStX8uabb+JwOCK7xLpcLhIS9C+Oizjz1MHO1yDoJzF7GDePvBa3X9U7qoiE8LZCQQuYLOD3aqWiRJ1WjOxFdC0RPffcc9TU1DB16lSys7Mjj7/85S96hiX0UFcGW1/U/rATUuCi61AMhkgppDvol2QlJ9mGxWwEW7J2UKpnMaFriUhVu89/O6Gj8j2w9x9aI7U9DXXULWC20902prpi6FnbqZ90QX259JzFSLdorBZ9lKpCyQY4vE77OnUQjJhNcU2AdzcfYmimg2kXZegbY2sSkrWP0nMWE5KIhD5UFQ6+D8e2aF/nXgKDrgKDgYPlJ2n0BlDpxiXmSNWsWs8oeg1JREIfJRu0JKQoUDgdBkwAIBhUOVSh7ZBRmO7QM8Jmik818P6ek6QmWrg5L1k7KFWzmJBEJOKvsuhMdWzI16D/VyKniisbaPQGsJmN9E/pXj2n4WkeFpPhrKqZNFbHQrcZWS36CG8j7H1L+7z/V6KSEMDnR6oBuDjHidHQvZqrw9M8mrwBsLm0g34P+Jp0jKp3kEQk4uvwWm2n1MR+MDh6x+DiUw0cOd2IQVEYMyBZn/jOIzyUwO0LohpMYAnvcSalos6SRCTip74CSkM7AQ+doW0fHaKqKpsOVwIwJteFy25u6Rl0ZTNpfy5BVcXjD54pFUk7UadJIhLxU/QhqCrulCHsaXLxyYFTbDui7QysKAqzx/ZnbG4ylxZ2z5HKJqNBax8iVD2zh5araTqtY1S9gzRWi/ioLaXp5AFKqpr4yD0Qd4U2nSfTaWN8nrYgvc1s7L7jhkJsZiNef5AmX4CUhNBC+o2SiDpLEpHocqqqUrTtQyqPVVNuL8RtcpHptJHlspKeZNM7vHbJdFpxWE0YFAUSwiWiKn2D6gUkEYkut2lPMcGi7SiAaeBX+e6ogaQldY+lPdrr66NzznxhkKpZrEgiEl1uFAfZa1TIyBvGV786BkXpXt3yHRYuEXkbtS58c/ca99STSGO16BL+QDD0iZfEyp2MzUsmd/SVvScJgbYUiDW0BbVUzzpFEpGIuZomHy9uLOFgeT2UfQF+D8bENEgr1Du0Ttt9ooY//usQ7+7WGtsjpSJpsO4USUQipho8flZvO0Ztk49PiypQj36mnRhwiTavrIdT0KZ51Hv82gHpwo8JSUQiZnyBIH/7/DjVjT6cCWZuym1EcdeA2QZZo/QOLyYSrdro6gZvQDsgJaKYkEQkYmbtvgoq6jzYLUa+Mb4/ieXaBonkjI8aRd2T2S1a/06jlIhiShKRiIl9ZXXsPF6DosDMkdkk+yqg5jgYjNB/vN7hxUy4RNTkCxAMqtElIllxtMMkEYlOq2n08f6ekwBMzE8lL80ORzdpJzNGgLV7rSvUGTaTUdvtVYVGX0BbDkRRtN08vPV6h9djSSISnZZkMzF6gIv+KQl8dVCaVjo4tV87mTtJ3+BizGBQsIe3FfL4tRJfeLXGxkr9AuvhZECj6DSjQeHyIekEgioGgwJHP9OqKWmFkJR+4SfoYTJdNly+wJkDif20cUQNlZCSr1tcPZkkIhEt4Ae/W1tr5wLd7RV1HlITLZEFzIwGRVtrqGyndkHuxK6OVhc3jMmJPpDYD04dgIYKfQLqBSQRCU0wCCWfaKWZgA9sTii4ErJGtnh5TZOP17ceI8Vu5oaxOZHeJEo2QtAPzmxIzovjDegoMVTqk0TUYbq2Ea1bt47rr7+enJwcFEXhjTfe0DOcvktVYd9bULz+zBbK7lrY83c48H6z3iB/IMhbX5Ti9gVQ4cyW0E3VcGKb9nnBFb1iAGObnJ2IpOesQ3RNRA0NDYwZM4Znn31WzzBE6XYo2wWKAYZfD1cshPzLtHPHNmsLmoWoqspH+yo4WevGZjYya1T2mbWlD6+DYABSBkJKQfzvI072ltXywrpDvLOrVDtgT9N+dn6Ptm22aDddq2YzZ85k5syZeoYgvA1w6GPt80FTz1TFCi7XlkLd+5ZWXbMkQt5X2Xiokl2h8UKzRmXhSggNVKwsgpO7tVLQoGm9ujSkoFDv8VPrDg1qNBi1gY0Np7RSkc2pb4A9kHTf93VHPwOfG5IytPlgZ8seDYOvAkA9+CFbNnzApkPaCOJpwzIYmBZaPN7bCPvf0T7vP0FrH+rFItM8wqOrQWuwBi0ZiXbrUYnI4/FQW1sb9RCdEPCfWcw+/zIwtPDrkDcJ8r6KChgOvktG/V6uGJrOmNxk7bzfA7v+qrUpJaRobUO9nMOmlQLr3H7UcJuQNFh3So/qNVuyZAmLFy/WO4ze49Q+bUEvaxKkDWl22u0LYDUZUAZNxRDwMci9kfzgNlIbDVA9Rvvew2u1UoDJCqP+TVujp5dLspq0wdRBlUZvgESrSRJRJ/WoEtGiRYuoqamJPI4ePap3SD3bic+1j9ljm5WGDpbX8+KGYnYcq9Hae4Z8jeTh00hNtGrjhD7/s1YSajgFFjuMuf1M9aSXMxoUEkPDFerC7USRRHRKGwoh2qVHlYisVitWa89c67jbqa+A6qNab0/2mMhhVVXZUlLFJwe0to69pbWMGeDSVlYsuAJSB8PRT6G2VJtRnzYY8iaf2Wywj3DYTNR7/NR7fIBNq5aaLOD3QuMprc1NtJmuiai+vp6DBw9Gvj58+DDbt28nNTWVvLw+MhhOL+HSUL/CqF6ez49WR5LQuLxkLivsF728q6s/uL4Rz0i7pUyXDYNBwRQuSSoKOLKhqgTqSiURtZOuiWjLli1Mm3Zm2+EFCxYAMHfuXFasWKFTVH2A3wsnQ9MwcsZFDhefamDdfq2N49LCfkwsSNUjuh5h2rAWEo0jK5SIyqJKmeLCdE1EU6dOPdPrIOKnfLeWjBJSIgMP3b4A7315ElWFkf1dXJKfonOQPVBSlvaxrlTfOHqgHtVYLWJAVc9Uy3LGRQYeHjhZT73HT2qihanD0nvXbhtdKBg86x+pI5SI6iu0EeaizXpUY7WIgbpSqDsJBlPUOtKjBriwW43YLUbMRvn/dCGV9R5WbzuOosDdlw/SDiakaMMY/B6tGz+cmMQFyW9cX3M8NCk14yKt2/0sg9OTyHbJJoFtkWjVes3q3H68/lB3fbjBGqD2hH7B9UCSiPoSXxOU79E+DzVSl9e6afT6z/NNoiU2sxGbWZvqUd3kPXPC1V/7WHNMh6h6LklEfUnZTm2toKR0cPYnGFR5e1cZy9cXc/R0o97R9Tgpdm2qR02j78xB1wDtoySidpFE1FcEg3D8rO19FIUvS2s53eDFoCikO2SgaHslhxJRddNZicg5QBsk6q7RHqJNJBH1FaeLtIXLzDbIHIk/EOTTQ9pi7xMLUiLVDNF2yXZtXl1Vw1lVM5MFHJna59UyBamtJBH1Fce2aB+zx4DJws7jNdS5/SRZTYwekKxraD1ViyUiOKt6JomorSQR9QU1x6GqWOvVyRmP1x/ks8PaukKTBqVKd30HpSZa6J+cQJbTFn0iOV/7ePqwLB3bRjKOqC8o/pf2MXMkJCSzpegUjd4AyXYzF+e49I2tB8tw2Lj1ktzmJ5LztFUb3TXaHm+JafEProeRf4W9XVWx9p9ZMUD+pdoxFQyKwqWF/c6sNy1ix2QBVyhBnS7SN5YeQkpEvVnAD/vXaJ/njNNG/gJTCvtxcY4LZ4K8/bHgCwTxBYJntlQCbXmUqmI4fajX7u8WS1Ii6s0Or9WqBpZEKLgiaoKxy26W+WQxsO1IFc9+dDCydEpEamjaR/URbU1wcV6SiHqr8r3awvgAQ6+l3A2vbjnKqXqPvnH1Mg6rCVWFU/Xe6BP2NG3FymBAW5JXnJckot7o9CFtc0SA3InUJA3i/7af4ES1m/UHZZeJWMpwaD1mp+o9+AJnLRGrKJAxQvs8PK1GtEoSUW9Tvgd2/lWbytFvCOUZk3lty1Hq3H7Skixce7HMCI8lZ4KJRKuRQFDlZO05VbCM4drHqhLw1Mc/uB5EElFvoapwaC3sfgOCfvyphWywXs6qzccjSeimcf1lBHWMKYpCTrK2YsGJ6nMSkT0VnDmgBqHsCx2i6zkkEfUGPre2o0bJBgCasibwv3Xj2VRSQyCoMig9kVsn5Eb24xKx1T+SiJpaODle+3h8myyWdh7Sf9vTNVRqSaixUlvsbNgMbJkjcWw7TkD1MnVYOoPTk6SHrAv1T9ES0bGqRnyBYPRI9fThUPQheOqgYh9kjtApyu5NElFPVrEf9v4d1e+hwmfFNeFWrKm5KMC1F2diMRmwmqQq1tXSk6wMz3ZGSkZRjCZttYPiT7RH+kUt76jbx8lPpCdSVTi8Dnb9FZ/HzY46J38JTueTk2d2WXXYzJKE4kRRFGaMzGLUAFfL8/ZyJ4I5QSu1lm6Pe3w9gSSinibcHlS8nsoGD+/W5vKx7WqCJjuJVpPsitIdmayQf5n2+eG14K7VN55uSKpmPUmoPchXV8Gh0x42my+hwjGMfg4r116cGRnTIvTh9gXYfaKGJKuZYVmO6JM547QVMuvKYO9bMPo2qaKdRRJRT6CqWvfvgfeobWhkV6XKruRrabJlMKkghUkFaTJ5tRvYU1rLuv2ncNhMFPRLxGI6K9EYjDD8Bti6TJuDtv8dGDYzsp1TX9ctUvKzzz5Lfn4+NpuNSZMm8dlnn+kdUvfReBp2vg57/wkBH9b0fL7M+QYJaf25fWIuUwbLDPruYmR/F84EM3VuPxtDq19GSUyD4bO15FO6A758EwK+5tf1Qbonor/85S8sWLCARx55hG3btjFmzBiuvfZaysvL9Q5NX3UnUfe+Rd2/nqNk/w5UxQCDp2Ed/21mXzKEb07MI/PcBbmErsxGA1OHpQOwraSKrSVVzdvs0ofCRddpy7KU74HPXoCTX/b5MUaKqnPr5qRJk7jkkkv43e9+B0AwGCQ3N5f/+I//4KGHHjrv99bW1uJyuaipqcHpdMYj3NgLBsDXSNBdj6f2JJ6qEzSVHcBdW0F1gxe3P0i1LZdRV97EgAF5ekcr2mBD0Sk2HdJWwOyfksDY3GSGZp7TZlR9RJsPGG64tiZB6mBIztUmzNqSwWTr8e1Ibf0b1bWNyOv1snXrVhYtWhQ5ZjAYmD59Ohs3bmx2vcfjweM5M3u8traNvQ+nD0PRBy0v26mqnKxzU3KqAQifVyOXKsCQzCRS7BZQVSrqPByqqAtdpZ0PfQeKqlKYkURakhVUlVMNXg6erI16rbMNSreTESrUVDV42H/yzHwkFYXT9gIq+o0ie+AQEpJlL/qeYvKgNGxmI+sPnOJ4VRMWoyGSiHyBIC9tLMGogCH4NTIadtCvZjfmYDUUHSPFbiEvVdv4MqiqbC9zE1RMoBhQUVAVAyoGUCDFbmFoRlLkdT8rPk1LpYqLs50kWWP8p64oMOpWsMWmAKBrIjp16hSBQIDMzMyo45mZmezdu7fZ9UuWLGHx4sXtfyG/R9uPvBVqg5tAQ0Pr39/kB0Ubo6O63QTdZ649+41XAbyAR6v3K14Pqvc8+4X5DGCzgaJgtCZRa3UQsPfDkDKQhMzBXNQvmWtSz2n0FN2eoiiMz0uhMCOJncdqyDhrqyZfIEjtWYvtV1rHoKRfjNNdSrL7KEOsDWAJgFf7HfM0tf77owYtkHhmfluwrrLFRITTC/4u+FNXgxe+po16VK/ZokWLWLBgQeTr2tpacnNbWDP4XMm5MOZ27fNIL8WZBt4kb4C8Rp926pzzigI2uxVCk0Ud/gAFTf6oKRPKWU1tiXYzhAYSJvqDFHp8oCgonHne8LcmWs2QkAimBJwofA0wSMNzr+G0mbm0sF/UMavJyO0TcwkEtZRxppCcD0wmwWKEJCsEAyi+JnKHVmmfq0EgqP3xB7UEYLUokHgmyeUObHkBNrPDAmf/M4tVa4wlMTbPg86JqF+/fhiNRk6ePBl1/OTJk2RlNV+uwmq1YrV2YCNASyKkFrR6Oin0aIuE0KMtbKFHW0iZp28wGhSyXW34DTIYUaxJDBjQ1t9M6N+Da++6/v5bLBa+8pWv8MEHH0SOBYNBPvjgAyZPnqxjZEKIeNK9arZgwQLmzp3LhAkTmDhxIs888wwNDQ3ccccdeocmhIgT3RPRbbfdRkVFBQ8//DBlZWWMHTuWd955p1kDthCi99J9HFFn9IpxREL0Ym39G5U2UiGE7iQRCSF0J4lICKE73RurOyPcvNXmqR5CiLgK/21eqCm6RyeiujptzlebRlcLIXRTV1eHy+Vq9XyP7jULBoOcOHECh8PRLXepCE9BOXr0aK/r1ZN765nifW+qqlJXV0dOTg6G86wk0KNLRAaDgQEDBugdxgU5nc5e9wsdJvfWM8Xz3s5XEgqTxmohhO4kEQkhdCeJqAtZrVYeeeSRjq0Y0M3JvfVM3fXeenRjtRCid5ASkRBCd5KIhBC6k0QkhNCdJCIhhO4kEbVTe3elfe2117jooouw2WyMGjWKf/7zn1HnVVXl4YcfJjs7m4SEBKZPn86BAwe68hZa1Z57e+GFF7j88stJSUkhJSWF6dOnN7t+3rx5KIoS9ZgxY0ZX30aL2nNvK1asaBa3zRa9+nhPfd+mTp3a7N4UReG6666LXKPL+6aKNlu1apVqsVjUZcuWqbt371bvueceNTk5WT158mSL169fv141Go3qL3/5S/XLL79Uf/zjH6tms1nduXNn5JrHH39cdblc6htvvKHu2LFDveGGG9SCggK1qakpXrelqmr77+1b3/qW+uyzz6qff/65umfPHnXevHmqy+VSjx07Frlm7ty56owZM9TS0tLI4/Tp0/G6pYj23tvy5ctVp9MZFXdZWVnUNT31fausrIy6r127dqlGo1Fdvnx55Bo93jdJRO0wceJE9f777498HQgE1JycHHXJkiUtXn/rrbeq1113XdSxSZMmqffee6+qqqoaDAbVrKws9Ve/+lXkfHV1tWq1WtVXXnmlC+6gde29t3P5/X7V4XCoL774YuTY3Llz1dmzZ8c61HZr770tX75cdblcrT5fb3rffv3rX6sOh0Otr6+PHNPjfZOqWRuFd6WdPn165Nj5dqUF2LhxY9T1ANdee23k+sOHD1NWVhZ1jcvlYtKkSa0+Z1foyL2dq7GxEZ/PR2pqatTxjz/+mIyMDIYNG8b3v/99KisrYxr7hXT03urr6xk4cCC5ubnMnj2b3bt3R871pvftT3/6E7fffjuJidF7lMX7fZNE1Ebn25W2rKysxe8pKys77/Xhj+15zq7QkXs714MPPkhOTk7UH8WMGTN46aWX+OCDD3jiiSdYu3YtM2fOJBAIxDT+8+nIvQ0bNoxly5bx5ptv8uc//5lgMMiUKVM4duwY0Hvet88++4xdu3Zx9913Rx3X433r0bPvRffw+OOPs2rVKj7++OOoRt3bb7898vmoUaMYPXo0gwcP5uOPP+bqq6/WI9Q2mTx5ctS+elOmTGH48OH84Q9/4LHHHtMxstj605/+xKhRo5g4cWLUcT3eNykRtVF7d6UFyMrKOu/14Y/tec6u0JF7C3vyySd5/PHHeffddxk9evR5rx00aBD9+vXj4MGDnY65rTpzb2Fms5lx48ZF4u4N71tDQwOrVq3irrvuuuDrxON9k0TURh3ZlXby5MlR1wO89957kesLCgrIysqKuqa2tpZNmzbFdafbju64+8tf/pLHHnuMd955hwkTJlzwdY4dO0ZlZSXZ2dkxibstYrGbcCAQYOfOnZG4e/r7BtqwEo/Hw7e//e0Lvk5c3re4No33cKtWrVKtVqu6YsUK9csvv1S/973vqcnJyZGu3e985zvqQw89FLl+/fr1qslkUp988kl1z5496iOPPNJi931ycrL65ptvql988YU6e/Zs3bqB23Nvjz/+uGqxWNTXX389qpu3rq5OVVVVraurUxcuXKhu3LhRPXz4sPr++++r48ePV4cMGaK63e5ufW+LFy9W16xZoxYVFalbt25Vb7/9dtVms6m7d++Ouv+e+L6FXXbZZeptt93W7Lhe75skonb67W9/q+bl5akWi0WdOHGi+umnn0bOXXnllercuXOjrn/11VfVoUOHqhaLRb344ovVt956K+p8MBhUf/KTn6iZmZmq1WpVr776anXfvn3xuJVm2nNvAwcOVIFmj0ceeURVVVVtbGxUr7nmGjU9PV01m83qwIED1XvuuafZeJx4ac+9zZ8/P3JtZmamOmvWLHXbtm1Rz9dT3zdVVdW9e/eqgPruu+82ey693jdZBkQIoTtpIxJC6E4SkRBCd5KIhBC6k0QkhNCdJCIhhO4kEQkhdCeJSAihO0lEokdZsWIFycnJka9/+tOfMnbs2MjX8+bN48Ybb4x7XKJzJBGJFoWXC/33f//3Zufuv/9+FEVh3rx5UdfHOgHk5+fzzDPPRB277bbb2L9/f6vfs3TpUlasWBH5eurUqcyfPz+mcYnYk0QkWpWbm8uqVatoamqKHHO73axcuZK8vDxdYkpISCAjI6PV8y6XK6rEJHoGSUSiVePHjyc3N5fVq1dHjq1evZq8vDzGjRvXqeduqaRy4403RkpZU6dOpaSkhB/+8IeRBdyhedXsXGeXzObNm8fatWtZunRp5DkOHz5MYWEhTz75ZNT3bd++HUVR4rpEiThDEpE4rzvvvJPly5dHvl62bBl33HFHl7/u6tWrGTBgAI8++iilpaWUlpa2+zmWLl3K5MmTueeeeyLPkZeX1+yeAJYvX84VV1xBYWFhrG5BtIMkInFe3/72t/nkk08oKSmhpKSE9evXt2kNm85KTU3FaDTicDjIysrq0IJjLpcLi8WC3W6PPIfRaGTevHns27cvsu2Oz+dj5cqV3HnnnbG+DdFGslSsOK/09HSuu+46VqxYgaqqXHfddfTr10/vsDolJyeH6667jmXLljFx4kT+/ve/4/F4uOWWW/QOrc+SEpG4oDvvvJMVK1bw4osvxqzUYDAYOHcFGp/PF5Pnbou777470hC/fPlybrvtNux2e9xeX0STRCQuaMaMGXi9Xnw+H9dee21MnjM9PT2q3ScQCLBr166oaywWS6d3jmjtOWbNmkViYiLPPfcc77zzjlTLdCZVM3FBRqORPXv2RD5vTU1NDdu3b486lpaWRm5ubrNrr7rqKhYsWMBbb73F4MGDefrpp6muro66Jj8/n3Xr1nH77bdjtVo7VCXMz89n06ZNFBcXk5SURGpqKgaDIdJWtGjRIoYMGRLXtaZFc1IiEm3idDpxOp3nvebjjz9m3LhxUY/Fixe3eO2dd97J3Llz+e53v8uVV17JoEGDmDZtWtQ1jz76KMXFxQwePJj09PQOxb1w4UKMRiMjRowgPT2dI0eORM7dddddeL3euPQCivOTpWJFn/Wvf/2Lq6++mqNHjzbbpFDElyQi0ed4PB4qKiqYO3cuWVlZvPzyy3qH1OdJ1Uz0Oa+88goDBw6kurqaX/7yl3qHI5ASkRCiG5ASkRBCd5KIhBC6k0QkhNCdJCIhhO4kEQkhdCeJSAihO0lEQgjdSSISQuhOEpEQQnf/H02N4t4zJIM5AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:19:10.626736Z", + "iopub.status.busy": "2024-03-26T06:19:10.626406Z", + "iopub.status.idle": "2024-03-26T06:19:10.849835Z", + "shell.execute_reply": "2024-03-26T06:19:10.848888Z" + }, + "papermill": { + "duration": 0.240531, + "end_time": "2024-03-26T06:19:10.851761", + "exception": false, + "start_time": "2024-03-26T06:19:10.611230", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEnCAYAAAD4jcW1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAtZklEQVR4nO3dfVwU9d4//tcusAsLgjfISoiu4g1YR1C8VDx1xBQw0zS7Ia3AzWMnjY61BzV+5wou0xOVhGYpXFlolilXRemlphBKlymmQhoaN+EdJveirIDuLruf7x/+mNO2CywGzM7s+/l48MCd+czue5zdFzOfnfmMhDHGQAghAiPluwBCCLkbFF6EEEGi8CKECBKFFyFEkCi8CCGCROFFCBEkCi9CiCBReBFCBInCixAiSM58F9DbTCYTKisr0adPH0gkEr7LIYT8DmMMN2/exD333AOptIP9K8az999/nw0dOpTJ5XI2ceJE9sMPP3TY/vr162zZsmVs0KBBTCaTsZEjR7J9+/bZ/HpXrlxhAOiHfujHzn+uXLnS4WeZ1z2vzMxMaDQapKenY9KkSdiwYQOioqJQWloKHx8fi/Z6vR4RERHw8fHBF198AT8/P1y+fBl9+/a1+TX79OkDALhy5Qo8PT27a1XsjsFgQHZ2NiIjI+Hi4sJ3OaQbOMo21Wq18Pf35z6r7eE1vFJTU7FkyRKo1WoAQHp6Ovbt24eMjAy8+uqrFu0zMjLQ0NCAY8eOcRtPpVJ16TXbDhU9PT1FH14KhQKenp6ifqM7Ekfbpp116/AWXnq9HgUFBUhISOCmSaVSzJgxA/n5+VaX2bNnD8LCwvDiiy9i9+7dGDhwIBYuXIhVq1bBycnJ6jI6nQ46nY57rNVqAdx5IxgMhm5cI/vStm5iXkdH4yjb1Nb14y286uvrYTQaoVQqzaYrlUqUlJRYXebChQs4dOgQnn76aezfvx/l5eVYtmwZDAYDkpKSrC6TnJyM1atXW0zPzs6GQqH44yti53JycvgugXQzsW/TlpYWm9oJ6ttGk8kEHx8ffPDBB3ByckJoaCiuXr2KdevWtRteCQkJ0Gg03OO24+nIyEjRHzbm5OQgIiLCIQ4xHIGjbNO2o6PO8BZe3t7ecHJyQk1Njdn0mpoaDBo0yOoyvr6+cHFxMTtEDAoKQnV1NfR6PWQymcUycrkccrncYrqLi4uo3wBtHGU9HYnYt6mt68bbSaoymQyhoaHIzc3lpplMJuTm5iIsLMzqMn/+859RXl4Ok8nETSsrK4Ovr6/V4CKEiBevZ9hrNBps2bIFH3/8MYqLi7F06VI0Nzdz3z7GxMSYdegvXboUDQ0NWL58OcrKyrBv3z688cYbePHFF/laBUIIT3jt84qOjkZdXR0SExNRXV2NkJAQHDhwgOvEr6ioMDvD1t/fHwcPHsQrr7yCsWPHws/PD8uXL8eqVav4WgVCekRLS4vFF1dNt3Q4VnQe/bxPwcPt310hgYGBDvHl0+9JGHOsG3BotVp4eXmhsbFR9B32+/fvx6xZs0TdPyJWhYWFCA0NtaltQUEBxo8f38MV9R5bP6OC+raREEcRGBiIgoICs2mlVTeg+bwIqU/8CaN9+5q1dUQUXoTYIYVCYbE3Jb18DfIjtxB0XzBChg7gqTL7QUPiEEIEicKLECJIFF6EEEGi8CKECBKFFyFEkCi8CCGCROFFCBEkCi9CiCBReBFCBInCixAiSBRehBBBovAihAgShRchRJAovAghgkThRQgRJAovQoggUXgRQgSJwosQIkgUXoQQQaLwIoQIEoUXIUSQKLwIIYJE4UUIESQKL0KIINFNZ0WipaUFJSUl3OOmWzocKzqPft6n4OEmN2sbGBgIhULR2yUS0q0ovESipKQEoaGhFtPfttK2oKDA4m7MhAgNhZdIBAYGoqCggHtcWnUDms+LkPrEnzDat69FW0KEjsJLJBQKhdnelPTyNciP3ELQfcEIGTqAx8oI6RnUYU8IESQKL0KIIFF4EUIEyS7Ca9OmTVCpVHB1dcWkSZNw4sSJdttu27YNEonE7MfV1bUXqyWE2APewyszMxMajQZJSUkoLCxEcHAwoqKiUFtb2+4ynp6eqKqq4n4uX77cixUTQuwB7+GVmpqKJUuWQK1WY8yYMUhPT4dCoUBGRka7y0gkEgwaNIj7USqVvVgxIcQe8HqqhF6vR0FBARISErhpUqkUM2bMQH5+frvLNTU1YejQoTCZTBg/fjzeeOMN3HvvvVbb6nQ66HQ67rFWqwUAGAwGGAyGbloT+9Pa2sr9FvN6OhJH2aa2rhuv4VVfXw+j0Wix56RUKs0udfmt0aNHIyMjA2PHjkVjYyNSUlIwZcoUnDt3DoMHD7Zon5ycjNWrV1tMz87OFvUlMleaAMAZx48fx9WzfFdDuoOjbNOWlhab2gnuJNWwsDCEhYVxj6dMmYKgoCD893//N9asWWPRPiEhARqNhnus1Wrh7++PyMhIeHp69krNfDhT0QAUncLkyZMRPKQ/3+WQbuAo27Tt6KgzvIaXt7c3nJycUFNTYza9pqYGgwYNsuk5XFxcMG7cOJSXl1udL5fLIZfLLaa7uLjAxcWl60ULhLOzM/dbzOvpSBxlm9q6brx22MtkMoSGhiI3N5ebZjKZkJuba7Z31RGj0YiioiL4+vr2VJmEEDvE+2GjRqNBbGwsJkyYgIkTJ2LDhg1obm6GWq0GAMTExMDPzw/JyckAgNdffx2TJ0/GiBEjcOPGDaxbtw6XL1/GX//6Vz5XgxDSy3gPr+joaNTV1SExMRHV1dUICQnBgQMHuE78iooKSKX/3kG8fv06lixZgurqavTr1w+hoaE4duwYxowZw9cqEEJ4wHt4AUBcXBzi4uKszsvLyzN7vH79eqxfv74XqiKE2DO7CC9CCHCxvhnNutZ255+va+Z+t3Xe/5673BnDvN17pD57Q+FFiB24WN+MaSl5NrX9xxdFHc4/HB/uEAFG4UWIHWjb49oQHYIRPh7W29zSYW9ePmaHh8HdzfL0n/LaJrycebrDvTcxofAixI6M8PHAfX5eVucZDAZUDwTGD+0n6vO8bEXhJWAd9ZHY0j8COFYfCREXCi+BsrWPpLP+EcBx+kiIuFB4CVRnfSSd9Y8AjtdHQsSFwkvg2usjof4RIna8D0ZICCF3g8KLECJIFF6EEEGi8CKECBKFFyFEkCi8CCGCROFFCBEkCi9CiCBReBFCBInCixAiSBRehBBBomsbCbEDOuNtSF2v4qK2FFJX64MRtra2orK1EsUNxVaHObqobYLU9Sp0xtsArI8JJiYUXgLV2Zu9szc64HhvdntW2XwZ7sPew/93ovO2mw9sbnee+zCgsjkEoVB2Y3X2icJLoGx9s3f0Rgcc681uz+5xH4rmiy/h3egQBLQzDHRrayuOfn8Uf77/z1b/IJ2vbcLyzNO4Z9rQni7XLlB4CVRnb/bO3uiA473Z7ZncyRWm234Y5jkaYwa0Pwz0ReeLCOofZHWYI9PtRphu10Hu5NrT5doFCi+B6uzN3tkbHXC8NzsRF/q2kRAiSBRehBBBovAihAgShRchRJAovAghgkThRQgRpC6H14ULF3qiDkII6ZIuh9eIESMwbdo0fPrpp7h9+3ZP1EQIIZ3qcngVFhZi7Nix0Gg0GDRoEP72t7/hxAkbLsjqwKZNm6BSqeDq6opJkybZ/Hy7du2CRCLBvHnz/tDrE0KEp8vhFRISgnfffReVlZXIyMhAVVUV7r//ftx3331ITU1FXV1dl54vMzMTGo0GSUlJKCwsRHBwMKKiolBbW9vhcpcuXUJ8fDweeOCBrq4CIUQE7rrD3tnZGfPnz8fnn3+Ot956C+Xl5YiPj4e/vz9iYmJQVVVl0/OkpqZiyZIlUKvVGDNmDNLT06FQKJCRkdHuMkajEU8//TRWr16N4cOH3+0qEEIE7K6vbTx16hQyMjKwa9cuuLu7Iz4+HosXL8avv/6K1atXY+7cuZ0e/un1ehQUFCAhIYGbJpVKMWPGDOTn57e73Ouvvw4fHx8sXrwYR44c6fA1dDoddDod91ir1QK4c+2fwWCwZVXtUmtrK/fb2nq0TetoHTt7DtJ7bNkWnW1TsWxPW2vvcnilpqZi69atKC0txaxZs7B9+3bMmjULUumdnbhhw4Zh27ZtUKlUnT5XfX09jEYjlErz4ViUSiVKSkqsLvP999/jo48+wunTp22qNzk5GatXr7aYnp2dDYVCYdNz2KMrTQDgjO+//x6XrY+gAgDIycn5w89Bel5XtkV721Qs27OlpcWmdl0Or7S0NDz33HNYtGgRfH19rbbx8fHBRx991NWn7tTNmzfx7LPPYsuWLfD29rZpmYSEBGg0Gu6xVquFv78/IiMj4enp2e019pZzlVqkFB3H/fffj3vvsVwPg8GAnJwcREREtDuqRGfPQXqPLduis20qlu3ZdnTUmS6HV05ODoYMGcLtabVhjOHKlSsYMmQIZDIZYmNjO30ub29vODk5oaamxmx6TU0NBg0aZNH+/PnzuHTpEubMmcNNM5lMd1bE2RmlpaUICAgwW0Yul0Mul1s8l4uLS7sfaiFoG6PL2dm5w/XoaD1tfQ7S87qyLdrbpmLZnrbW3uUO+4CAANTX11tMb2howLBhw7r0XDKZDKGhocjNzeWmmUwm5ObmIiwszKJ9YGAgioqKcPr0ae7nkUcewbRp03D69Gn4+/t3dXUIIQLV5T0vxpjV6U1NTXB17fqgdhqNBrGxsZgwYQImTpyIDRs2oLm5GWq1GgAQExMDPz8/JCcnw9XVFffdd5/Z8n379gUAi+mEEHGzObza+o0kEgkSExPNOruNRiN++OEHhISEdLmA6Oho1NXVITExEdXV1QgJCcGBAwe4TvyKigqLQ1RCCLE5vH788UcAd/a8ioqKIJPJuHkymQzBwcGIj4+/qyLi4uIQFxdndV5eXl6Hy27btu2uXpMQImw2h9fhw4cBAGq1Gu+++66gv6kjhAhfl/u8tm7d2hN1EEJIl9gUXvPnz8e2bdvg6emJ+fPnd9g2KyurWwojhJCO2BReXl5ekEgk3L8JIYRvNoXXbw8V6bCREGIP6BwEQogg2bTnNW7cOO6wsTOFhYV/qCBim1sGIwDg7NVGq/Obb+lwqg4YdPk63N0sL48CgPLaph6rj5CeZlN40Uil9uf8/x88r2YVddDKGZ+Un+z0udzldz0yEiG8seldm5SU1NN1kC6KvPfOhesBPh5wc3GymF9a1Yh/fFGEdx7/E0b7tv8li7vcGcO83XusTkJ6Cv3JFaj+7jI8NXFIu/PbBqYLGOiO+/zoG2IiPjaFV//+/VFWVgZvb2/069evw/6vhoaGbiuOEELaY1N4rV+/Hn369OH+bWvnPSGE9BSbwuu3AwsuWrSop2ohhBCbdfk8LycnJ6u3Jbt27RqcnCw7jgkhpCd0ObzaG4xQp9OZDZNDCCE9yeZvGzdu3AjgzmCEH374ITw8/n17EqPRiP/7v/9DYGBg91dICCFW2Bxe69evB3Bnzys9Pd3sEFEmk0GlUiE9Pb37KyTEAXR2xQTQ+VUTjnbFhM3hdfHiRQDAtGnTkJWVhX79+vVYUYQ4GtuumABsuWrCUa6Y6PJato2oSgjpPp1dMQHYdtWEI10x0eXweu655zqcn5GRcdfFEOKoOrtiAqCrJn6vy+F1/fp1s8cGgwFnz57FjRs38OCDD3ZbYYQQ0pEuh9dXX31lMc1kMmHp0qUWd6smhJCe0i2DEUqlUmg0Gu4bSUII6WndNpLq+fPnuWNyQgjpaV0+bGy7c3Ybxhiqqqqwb98+s2sgCSGkJ3U5vNrunN1GKpVi4MCBeOeddzr9JpIQQroLnedFCBEkunsQIUSQKLwIIYJE4UUIESQKL0KIIHVbeP366694/vnnu+vpCCGkQ90WXteuXcNHH33UXU9HCCEdsovDxk2bNkGlUsHV1RWTJk3CiRMn2m2blZWFCRMmoG/fvnB3d0dISAg++eSTXqyWEGIPeA+vzMxMaDQaJCUlobCwEMHBwYiKirJ6kw/gzj0k//nPfyI/Px8//fQT1Go11Go1Dh482MuVE0L4xHt4paamYsmSJVCr1RgzZgzS09OhUCjaHRcsPDwcjz76KIKCghAQEIDly5dj7Nix+P7773u5ckIIn2w+w37+/Pkdzr9x40aXX1yv16OgoAAJCQncNKlUihkzZiA/P7/T5RljOHToEEpLS/HWW29ZbaPT6aDT6bjHWq0WwJ1xyAwGQ5drFoq2i+RbW1tFvZ6OxFG2qa3rZnN4eXl1PHKjl5cXYmJibH06AEB9fT2MRiOUSqXZdKVSiZKSknaXa2xshJ+fH3Q6HZycnLB582ZERERYbZucnIzVq1dbTM/OzoZCoehSvUJypQkAnHH8+HFcPct3NaQ7OMo2bWlpsamdzeG1devWuy6mu/Xp0wenT59GU1MTcnNzodFoMHz4cISHh1u0TUhIMBsJQ6vVwt/fH5GRkfD09OzFqnvXmYoGoOgUJk+ejOAh/fkuh3QDR9mmbUdHneH1NiPe3t5wcnJCTU2N2fSamhoMGjSo3eWkUilGjBgBAAgJCUFxcTGSk5OthpdcLodcbnmbKBcXF7i4uPyxFbBjzs7O3G8xr6cjcZRtauu62Rxetg5305UbcMhkMoSGhiI3Nxfz5s0DcGdI6dzcXMTFxdn8PCaTyaxfixAifjaH17Zt2zB06FCMGzcOjLFuK0Cj0SA2NhYTJkzAxIkTsWHDBjQ3N0OtVgMAYmJi4Ofnh+TkZAB3+rAmTJiAgIAA6HQ67N+/H5988gnS0tK6rSZCiP2zObyWLl2KnTt34uLFi1Cr1XjmmWfQv/8fP+6Ojo5GXV0dEhMTUV1djZCQEBw4cIDrxK+oqIBU+u8zOpqbm7Fs2TL8+uuvcHNzQ2BgID799FNER0f/4VoIIcIhYV3YjdLpdMjKykJGRgaOHTuGhx9+GIsXL0ZkZCQkEklP1tlttFotvLy80NjYKOoO+9OXr2Fe2nF8vXQyQoYO4Lsc0g0cZZva+hnt0kmqcrkcCxYsQE5ODn7++Wfce++9WLZsGVQqFZqamv5w0YQQYqu7PsNeKpVCIpGAMQaj0didNRFCSKe6FF46nQ47d+5EREQERo0ahaKiIrz//vuoqKiAh4dHT9VICCEWbO6wX7ZsGXbt2gV/f38899xz2LlzJ7y9vXuyNkIIaZfN4ZWeno4hQ4Zg+PDh+O677/Ddd99ZbZeVldVtxRFCSHtsDq+YmBjBfKNICBG/Lp2kSggh9oL38bwIIeRuUHgRQgSJwosQIkgUXoQQQaLwIoQIEoUXIUSQKLwIIYJE4UUIESQKL0KIIFF4EUIEicKLECJIFF6EEEGi8CKECBKFFyFEkCi8CCGCROFFCBEkCi9CiCBReBFCBInCixAiSBRehBBBovAihAgShZcI6fV67PgoDQ056djxURr0ej3fJRHS7Si8RGblypVQKBRIWfMabhbuRcqa16BQKLBy5Uq+SyOkW1F4iURLSwtiY2Oxbt06GI1Gs3lGoxHr1q1DbGwsCgsL0dLSwlOVhHQfCi+R+Omnn7B9+/YO22zfvh2hoaEoKSnppaoI6TkUXiLx7bff2tTu73//OwIDA3u4GkJ6nl2E16ZNm6BSqeDq6opJkybhxIkT7bbdsmULHnjgAfTr1w/9+vXDjBkzOmzvKDIzM21qd+jQISgUih6uhpCex3t4ZWZmQqPRICkpCYWFhQgODkZUVBRqa2utts/Ly8OCBQtw+PBh5Ofnw9/fH5GRkbh69WovV25fGhoaurUdIfaO9/BKTU3FkiVLoFarMWbMGKSnp0OhUCAjI8Nq+x07dmDZsmUICQlBYGAgPvzwQ5hMJuTm5vZy5faFMdat7Qixd858vrher0dBQQESEhK4aVKpFDNmzEB+fr5Nz9HS0gKDwYD+/ftbna/T6aDT6bjHWq0WAGAwGGAwGP5A9falqanJ5nZiWm9H0trayv0W8za0dd14Da/6+noYjUYolUqz6Uql0uZvxFatWoV77rkHM2bMsDo/OTkZq1evtpienZ0tqr4fW09/aGlpwf79+3u4GtITrjQBgDOOHz+Oq2f5rqbn2Ppe5jW8/qg333wTu3btQl5eHlxdXa22SUhIgEaj4R5rtVqun8zT07O3Su1xbm5uNu19ubm5YdasWb1QEeluZyoagKJTmDx5MoKHWD/SEIO2o6PO8Bpe3t7ecHJyQk1Njdn0mpoaDBo0qMNlU1JS8Oabb+Lbb7/F2LFj220nl8shl8stpru4uMDFxeXuCrdDAQEBOHPmjE3txLTejsTZ2Zn7LeZtaOu68dphL5PJEBoaatbZ3tb5HhYW1u5yb7/9NtasWYMDBw5gwoQJvVGq3Zs5c2a3tiPE3vH+baNGo8GWLVvw8ccfo7i4GEuXLkVzczPUajUAICYmxqxD/6233sJrr72GjIwMqFQqVFdXo7q62uYOa7GKiIjo1naE2Dve+7yio6NRV1eHxMREVFdXIyQkBAcOHOA68SsqKiCV/jtj09LujJLw+OOPmz1PUlIS/uu//qs3S7cr4eHh8PLyQmNjY7ttvLy8EB4e3ntFEdKDeA8vAIiLi0NcXJzVeXl5eWaPL1261PMFCVTbOVwSicTsfK62x3SOFxET3g8bSffIy8uDVquFn58fnJyczOY5OTnBz88PWq3W4o8BIUJlF3te5I9rC6XKykrMmjULw4cPR2lpKUaPHo0LFy5w53bl5eVh+vTpPFZKSPeg8BIJk8kEABg5ciTOnj2Lffv2AbhzMu7QoUMxcuRIlJWVce0IEToKL5FouzyqrKwMDz/8MDQaDX755ReMHDkS2dnZXJi1dxkVIUJD4SUSPj4+3L9zcnK4sALunE9nrR0hQkYd9iJx7do17t+/v+HGbx//th0hQkbhJRIDBgzg/u3m5mY277ePf9uOECGjw0aRqKur4/794IMPIjIy0mqf12/bEfvV0tJiMbJKadUN6KrLUXzWDaZrfbnpgYGBohohxVYUXiLRdjg4atQonDt3zqzPa9iwYRg1ahTKysrosFEgSkpKEBoaanXewo/NHxcUFGD8+PG9UJV9ofASibZLqMrKyjB79my88sor3J5XTk4O9u7da9aO2LfAwEAUFBSYTWu6pcO+w/l4eFoYPNzkZm0dEYWXSISHh2Pt2rUIDAzE2bNnubAC7ux5BQYGoqSkhK5tFAiFQmGxN2UwGHC9vhZhEyeIekgcW1F4iUR4eDh8fHxQUlKChx9+2GzPq63Py8fHh8KLiAaFl0g4OTkhLS0Njz/+OA4dOmTW56VQKCCRSJCWlmZx3SMRBr1ej/feew+HDh1CeXk5XnrpJbPz9xwSczCNjY0MAGtsbOS7lB7x5ZdfMpVKxQBwP8OGDWNffvkl36WRu7RixQrm7Oxstk2dnZ3ZihUr+C6tR9j6GZUw5ljjpGi1Wm7cKzGNYf9bt27dgkajwfHjxzF58mSkpqZanPtFhGHlypVYt24dlEolVq9eDblcDp1Oh6SkJNTU1GDFihV4++23+S6zW9n8Ge2VKLUjYt/zWrFiBXNycjL7K+3k5CTav9JiptPpmLOzM1MqlcxgMDC9Xs++/vprptfrmcFgYEqlkjk7OzOdTsd3qd3K1s8ofW8uIm1/pY1Go9l0o9GIdevWYeXKlTxVRu7G5s2b0drairVr13I332jj7OyM119/Ha2trdi8eTNPFfKLwksk9Ho9UlJSOmyTkpJicd0jsV/nz58HAMyePdvq/Lbpbe0cDYWXSGzcuJEb5tnHxwfp6enYunUr0tPTuZEkGGPYuHEjn2WSLggICAAA7N27F3V1dRg5ciSeeuopjBw5EnV1ddy5fG3tHA112IvElClTkJ+fDw8PD1y/fh2MMezfvx+zZs2CRCJB37590dzcjLCwMBw7dozvcokN9Ho93N3d0dra2m4bZ2dnNDc3i+q0CVs/o7TnJRJVVVUAgIceeshq/0hUVJRZO2L/ZDKZ2eVcgwYNwksvvWR2Q2apVCqq4OoKCi+R8PX1BQB88803Fn+pW1tbkZ2dbdaO2L+6ujqzPsrq6mq89957qK6u5qbp9XqHHSmEwkskHn30UQBAU1MTBg8ejA8//BANDQ348MMPMXjwYO6mvG3tiP2bOHEiAODee++FTqdDSkoKZs2ahZSUFOh0OgQFBZm1czTU5yUSer0erq6uHd6bUSKR4Pbt2w57mCE0Hh4eaG5uxt69e/Hwww/DYDBw/ZguLi746quvMH/+fLi7u4vqjvHU5+VgZDIZ4uPjO2wTHx9PwSUgAwcOBACsWrXK6vx//vOfZu0cDYWXiLz99ttYsWKFxcXXzs7OoryMROxOnDgBADh37hy0Wq3ZPK1Wi+LiYrN2joYOG0XotyMQPPjggzQCgYD17dsXjY2NAICgoCA88sgj2LNnDxdcXl5euHHjBo8Vdj9bP6M0JI4IOTk5ITg4GJWVlQgODqZhcATsxo0bXIAVFxdzoQWIM7i6gg4bRSYrKwsBAQGIiIhAamoqIiIiEBAQgKysLL5LI3fpxo0bqK2txdChQ+Hq6oqhQ4eitrbWoYMLoPASlaysLDz22GOora01m15bW4vHHnuMAkzAPDw88NBDDyEwMBAPPfQQPDw8+C6Jd9TnJRJGoxG+vr6oq6uDm5sbbt26xc1re+zj44PKyko6jBSYefPmYffu3RbT586di6+//rr3C+phdKqEg8nLy+POtJ4+fTqOHDmCnTt34siRI5g+fTqAO3tgeXl5PFZJuqotuGQyGVauXIm0tDSsXLkSMpkMu3fvxrx58/gukTe8h9emTZugUqng6uqKSZMmdfi177lz5/DYY49BpVJBIpFgw4YNvVeonTt06BAAYPLkydi9ezcmTZoENzc3TJo0Cbt378bkyZPN2hH7d+vWLS64bt68ibVr18LX1xdr167FzZs3uQD77V62I+E1vDIzM6HRaJCUlITCwkIEBwcjKirKos+mTUtLC4YPH44333zT7OJUAly5cgUA8PTTT1vcm1EqlWLBggVm7Yj9W7FiBQBAo9FYnOoik8nw8ssvm7VzNLyGV2pqKpYsWQK1Wo0xY8YgPT0dCoUCGRkZVtv/x3/8B9atW4ennnoKcrncahtH5e/vDwDYsWMHTCaT2TyTyYSdO3eatSP275dffgEA/PWvf7U6f/HixWbtHA1v53np9XoUFBQgISGBmyaVSjFjxgzk5+d32+vodDrodDrucduZygaDAQaDodteh29/+ctf8MYbb+D48eOYM2cO4uPjcevWLXz//fdISUnB8ePHuXZiWm8xGz58OADggw8+wNq1a7nt1vZ7y5YtXDsxbVNb14W38Kqvr4fRaIRSqTSbrlQqUVJS0m2vk5ycjNWrV1tMz87OhkKh6LbX4ZvRaISnpye0Wi2+/fZb7N+/n5vXdsjh5eWF5uZms3nEfoWHhyM9PR2pqamYMGECtx1zcnKg1+u5Pt/w8HBRbdOWlhab2on+DPuEhARoNBrusVarhb+/PyIjI0V1qgRw5y9xdHS0xakQbY8/+OADzJkzh4/SyF2aM2cO/vd//xfPPPMM4uLiEBAQgPPnz+P9999Ha2sr5syZg/nz5/NdZrf6/XWc7eEtvLy9veHk5ISamhqz6TU1Nd3aGS+Xy632j7m4uMDFxaXbXscePPnkk3B2doZGo8Hly5e56UqlEu+8847o3uSOYM+ePdzpEqmpqWbzxHqel62fS9467GUyGUJDQ5Gbm8tNM5lMyM3NRVhYGF9lCd78+fNx/vx55OTkQKPRICcnB+Xl5RRcAvb111+jpaUFL7zwAkJCQvDCCy+gpaVFlMHVFbweNmo0GsTGxmLChAmYOHEiNmzYgObmZqjVagBATEwM/Pz8kJycDOBOJ//PP//M/fvq1as4ffo0PDw8MGLECN7Ww944OTlh6tSpaG5uxtSpU+mMehFwc3PDxo0bzQYjdHS8hld0dDTq6uqQmJiI6upqhISE4MCBA1wnfkVFhdk5S5WVlRg3bhz3OCUlBSkpKZg6dSqdOU6Ig+G9wz4uLg5xcXFW5/0+kFQqVYfDHBNCHAfvlwcRQsjd4H3Pq7e17bnZ+nWsUBkMBrS0tECr1VL/iEg4yjZt+2x2dpTlcOF18+ZNAHSZDCH27ubNm/Dy8mp3vsON52UymVBZWYk+ffpAIpHwXU6PaTsZ98qVK6I7GddROco2ZYzh5s2buOeeeywGGfgth9vzkkqlGDx4MN9l9BpPT09Rv9EdkSNs0472uNpQhz0hRJAovAghgkThJVJyuRxJSUk07pmI0DY153Ad9oQQcaA9L0KIIFF4EUIEicKLECJIFF6EEEGi8BKA8PBw7jZXRFjsbdvZWz1/BIUXIXZOr9fzXYJ9YsSuxcbGMgBmP35+fmzz5s1m7QoLC5lEImGXLl1ijDH2zjvvsPvuu48pFAo2ePBgtnTpUnbz5k2zZY4cOcLuv/9+5urqygYPHsxeeukl1tTU1GvrJnbWtl15eTl77rnnmEqlYq6urmzUqFFsw4YNFsvNnTuXrV27lvn6+jKVSsUYY+zo0aMsODiYyeVyFhoayr766isGgP3444/cskVFRWzmzJnM3d2d+fj4sGeeeYbV1dW1W8/Fixd767+j21F42bkbN26wsLAwtmTJElZVVcWqqqpYfHw8u//++83a/eMf/zCbtn79enbo0CF28eJFlpuby0aPHs2WLl3KzS8vL2fu7u5s/fr1rKysjB09epSNGzeOLVq0qNfWTeysbbvbt2+zxMREdvLkSXbhwgX26aefMoVCwTIzM7nlYmNjmYeHB3v22WfZ2bNn2dmzZ1ljYyPr378/e+aZZ9i5c+fY/v372ahRo8zC6/r162zgwIEsISGBFRcXs8LCQhYREcGmTZvWbj2tra18/Nd0CwovAZg6dSpbvnw59/jHH39kEomEXb58mTHGmNFoZH5+fiwtLa3d5/j888/ZgAEDuMeLFy9mzz//vFmbI0eOMKlUym7dutW9K+DAfr/trHnxxRfZY489xj2OjY1lSqWS6XQ6blpaWhobMGCA2bbZsmWLWXitWbOGRUZGmj33lStXGABWWlpqcz1CQX1eAhQSEoKgoCB89tlnAIDvvvsOtbW1eOKJJ7g23377LaZPnw4/Pz/06dMHzz77LK5du8bd0PPMmTPYtm0bPDw8uJ+oqCiYTCZcvHiRl/VyFJs2bUJoaCgGDhwIDw8PfPDBB6ioqDBr86c//Ym7ySwAlJaWYuzYsXB1deWmTZw40WyZM2fO4PDhw2bbNDAwEABw/vz5HlwjflB4CdTTTz/Nhddnn32GmTNnYsCAAQCAS5cuYfbs2Rg7diy+/PJLFBQUYNOmTQD+3fnb1NSEv/3tbzh9+jT3c+bMGfzyyy8ICAjgZ6UcwK5duxAfH4/FixcjOzsbp0+fhlqttuiUd3d37/JzNzU1Yc6cOWbb9PTp0/jll1/wl7/8pbtWwW443HheQiSTyWA0Gs2mLVy4EP/5n/+JgoICfPHFF0hPT+fmFRQUwGQy4Z133uEGc/uf//kfs+XHjx+Pn3/+mW4Z18N+v+2OHj2KKVOmYNmyZdw0W/aKRo8ejU8//RQ6nY67MPvkyZNmbcaPH48vv/wSKpUKzs7WP9rW3ktCRXteAqBSqfDDDz/g0qVLqK+vh8lkgkqlwpQpU7B48WIYjUY88sgjXPsRI0bAYDDgvffew4ULF/DJJ5+YhRsArFq1CseOHUNcXBz313n37t3t3smJ3J3fb7uRI0fi1KlTOHjwIMrKyvDaa69ZhJA1CxcuhMlkwvPPP4/i4mIcPHgQKSkpAMCNCPziiy+ioaEBCxYswMmTJ3H+/HkcPHgQarWaCyxr7yXB4rvTjXSutLSUTZ48mbm5uZl9vb1582YGgMXExFgsk5qaynx9fZmbmxuLiopi27dvZwDY9evXuTYnTpxgERERzMPDg7m7u7OxY8eyf/3rX720Vo7h99uupKSELVq0iHl5ebG+ffuypUuXsldffZUFBwdzy7SdKvF7R48eZWPHjmUymYyFhoayzz77jHvONmVlZezRRx9lffv2ZW5ubiwwMJC9/PLLzGQyWa1HyKdK0JA4hAjUjh07oFar0djYCDc3N77L6XXU50WIQGzfvh3Dhw+Hn58fzpw5g1WrVuHJJ590yOACKLwIEYzq6mokJiaiuroavr6+eOKJJ/Cvf/2L77J4Q4eNhBBBom8bCSGCROFFCBEkCi9CiCBReBFCBInCixAiSBRepEctWrQIEokEEokELi4uUCqViIiIQEZGRpcuTdm2bRv69u3bc4W2Y9GiRZg3b16vvy7pHIUX6XEzZ85EVVUVLl26hG+++QbTpk3D8uXLMXv2bLS2tvJdHhEqfq9OImLX3nV6ubm5DADbsmULY6zjYasPHz5sMXxxUlISY4yx7du3s9DQUObh4cGUSiVbsGABq6mp4V6noaGBLVy4kHl7ezNXV1c2YsQIlpGRwc2vqKhgTzzxBPPy8mL9+vVjjzzyCHe9X1JSksXrHj58uEf+n0jX0Z4X4cWDDz6I4OBgZGVlAQCkUik2btyIc+fO4eOPP8ahQ4ewcuVKAMCUKVOwYcMGeHp6oqqqClVVVYiPjwcAGAwGrFmzBmfOnMHXX3+NS5cuYdGiRdzrvPbaa/j555/xzTffoLi4GGlpafD29uaWjYqKQp8+fXDkyBEcPXoUHh4emDlzJvR6PeLj4/Hkk09ye45VVVWYMmVK7/5HkfbxnZ5E3Nrb82KMsejoaBYUFGR13u+Hrd66dSvz8vLq9PVOnjzJAHB7bXPmzGFqtdpq208++YSNHj2aG3GBMcZ0Oh1zc3NjBw8e7LR+wi/a8yK8YYxxY1F1Nmx1ewoKCjBnzhwMGTIEffr0wdSpUwGAG1Z56dKl2LVrF0JCQrBy5UocO3aMW/bMmTMoLy9Hnz59uGGT+/fvj9u3b4ty2GSxofAivCkuLsawYcNsGrbamubmZkRFRcHT0xM7duzAyZMn8dVXX5kt99BDD+Hy5ct45ZVXUFlZienTp3OHnE1NTQgNDbUYNrmsrAwLFy7s4bUnfxSNKkF4cejQIRQVFeGVV16xadhqa8MXl5SU4Nq1a3jzzTfh7+8PADh16pTFaw0cOBCxsbGIjY3FAw88gBUrViAlJQXjx49HZmYmfHx84OnpabVOMQ2bLDa050V6nE6nQ3V1Na5evYrCwkK88cYbmDt3LmbPno2YmBibhq1WqVRoampCbm4u6uvr0dLSgiFDhkAmk3HL7dmzB2vWrDFbLjExEbt370Z5eTnOnTuHvXv3IigoCMCdm5h4e3tj7ty5OHLkCC5evIi8vDz8/e9/x6+//sq97k8//YTS0lLU19fDYDD0zn8a6RzfnW5E3H57l2ZnZ2c2cOBANmPGDJaRkcGMRiPXzpZhq1944QU2YMAAs1MlPvvsM6ZSqZhcLmdhYWFsz549FvcyDAoKYm5ubqx///5s7ty57MKFC9xzVlVVsZiYGObt7c3kcjkbPnw4W7JkCWtsbGSMMVZbW8sNlQ06VcKu0HhehBBBosNGQoggUXgRQgSJwosQIkgUXoQQQaLwIoQIEoUXIUSQKLwIIYJE4UUIESQKL0KIIFF4EUIEicKLECJIFF6EEEH6f2mVz1c6Do8JAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T06:19:10.881754Z", + "iopub.status.busy": "2024-03-26T06:19:10.881471Z", + "iopub.status.idle": "2024-03-26T06:19:11.146144Z", + "shell.execute_reply": "2024-03-26T06:19:11.145004Z" + }, + "papermill": { + "duration": 0.282331, + "end_time": "2024-03-26T06:19:11.148374", + "exception": false, + "start_time": "2024-03-26T06:19:10.866043", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABivElEQVR4nO2dd3RU1dqHnzMtjYRQkwDB0KRIlRLBrvGC4kWw0QRBPlG42BARUEFUBBUQUYQrVwQVBOwdhQgIUg1BunSpKZT0Mu18f0zmZHpJm2TYz1pZyZzZ55ydyeQ3795vk2RZlhEIBIIgRBXoCQgEAkFlIQROIBAELULgBAJB0CIETiAQBC1C4AQCQdAiBE4gEAQtQuAEAkHQIgROIBAELZpAT6A6YjabOXfuHJGRkUiSFOjpCAQCB2RZJjc3l0aNGqFSubfThMC54Ny5c8THxwd6GgKBwAunT5+mSZMmbp8XAueCyMhIwPLiRUVFBXg2AoHAkZycHOLj45X/VXcIgXOBdVkaFRUlBE4gqMZ420ISTgaBQBC0CIETCARBixA4gUAQtIg9OEGNx2QyYTAYAj0NQQWiVqvRaDTlDtMSAieo0eTl5XHmzBlE3dbgIzw8nLi4OHQ6XZmvIQROUGMxmUycOXOG8PBwGjRoIIKygwRZltHr9WRmZnLixAlatWrlMZjXE0LgrgA2H7nA4k3HmTGgPU3qhAd6OhWGwWBAlmUaNGhAWFhYoKcjqEDCwsLQarX8888/6PV6QkNDy3Qd4WS4Anjow+1sPJzJhM//CvRUKgVhuQUnZbXa7K5RAfMQ1BAycooDPQWBoEoRAncFIbbhBVcaQuAEAkGVM2LECPr371/p9xECdwUhQimqB7fccgtPP/10oKdxRSAETiAQlImaEFwtBE4QNMiyTIHeGJAvX63jESNGsHHjRt555x0kSUKSJJo0acLChQvtxqWmpqJSqfjnn38AmDt3Lh06dCAiIoL4+HjGjh1LXl6e3TmbN2/mxhtvJCwsjPj4eJ588kny8/N9mtf58+fp27cvYWFhNGvWjBUrVpCQkMC8efOUMZIksXDhQvr160dERAQzZszAZDIxatQomjVrRlhYGK1bt+add96xu7bJZGL8+PFER0dTr149Jk6cWGWrCREHJwgaCg0m2k39JSD3PvBKb8J13v+d3nnnHQ4fPkz79u155ZVXAJgzZw4rVqxgzJgxyrjly5dz/fXXc9VVVwGWkIn58+fTrFkzjh8/ztixY5k4cSLvv/8+AMeOHaNPnz689tprLFmyhMzMTMaNG8e4ceP46KOPvM5r+PDhXLhwgQ0bNqDVahk/fjwZGRlO415++WVmzZrFvHnz0Gg0mM1mmjRpwueff069evXYsmULo0ePJi4ujgcffFD5/ZYuXcqSJUto27Ytc+bM4euvv+a2227z/sKWEyFwAkEVUrt2bXQ6HeHh4cTGxgIwdOhQ5syZw6lTp2jatClms5mVK1fy4osvKufZ7tklJCTw2muv8fjjjysCN3PmTIYOHaqMa9WqFfPnz+fmm29m4cKFHgNlDx06xLp169i5cyfdunUD4H//+x+tWrVyGjtkyBBGjhxpd2z69OnKz82aNWPr1q2sXr1aEbh58+YxefJk7r33XgAWLVrEL79UzQeREDhB0BCmVXPgld4Bu3dZ6dy5M23btmXFihVMmjSJjRs3kpGRwQMPPKCMWbduHTNnzuTQoUPk5ORgNBopKiqioKCA8PBw/vrrL/bs2cPy5cuVc2RZxmw2c+LECdq2bev2/n///TcajYZrr71WOdayZUvq1KnjNNYqgLYsWLCAJUuWcOrUKQoLC9Hr9XTu3BmA7Oxszp8/T2JiojJeo9HQrVu3KlmmCoG7ggh2H6okST4tE6sjQ4cOVQRuxYoV9OnTh3r16gFw8uRJ7r77bsaMGcOMGTOoW7cumzdvZtSoUej1esLDw8nLy+Oxxx7jySefdLp206ZNK2yeERERdo9XrlzJhAkTmDNnDj179iQyMpK33nqL7du3V9g9y0PNfDcIBDUYnU6HyWSyOzZkyBBefPFFUlJS+OKLL1i0aJHyXEpKCmazmTlz5ijpS6tXr7Y7/9prr+XAgQO0bNnS7/m0bt0ao9FIamoqXbt2BeDo0aNcvnzZ67l//PEHvXr1YuzYscqxY8eOKT/Xrl2buLg4tm/fzk033QSA0WgkJSXFzmKsLIQX9QrCcUWQX2zk7nc3MXft4cBM6AolISGB7du3c/LkSS5cuIDZbCYhIYFevXoxatQoTCYT/fr1U8a3bNkSg8HAu+++y/Hjx/nkk0/sBBDg+eefZ8uWLYwbN47du3dz5MgRvv32W8aNG+d1Pm3atCEpKYnRo0ezY8cOUlNTGT16NGFhYV7zfFu1asWff/7JL7/8wuHDh3nppZfYuXOn3ZinnnqKWbNm8c0333Do0CHGjh1LVlaW7y9YORACdwVz8HwO+87m8HXqmUBP5YpiwoQJqNVq2rVrR4MGDTh16hRgWab+9ddfDBgwwK46SqdOnZg7dy5vvPEG7du3Z/ny5cycOdPumh07dmTjxo0cPnyYG2+8kS5dujB16lQaNWrk05w+/vhjYmJiuOmmmxgwYACPPvookZGRXqt4PPbYY9x7770MHDiQxMRELl68aGfNATz77LMMGzaMhx9+WFnGDhgwwKd5lRdJFuHtTuTk5FC7dm2ys7ODoqtWwqQfAWhaN5zfJ96qHN958hIPLNpKTFQI26ckBWp6ZaaoqIgTJ07QrFmzMpfTEbjmzJkzxMfHs27dOm6//faAzMHT39fX/1GxB3cFY/1o0xvNgZ2IIOD89ttv5OXl0aFDB86fP8/EiRNJSEhQ9s1qKkLgrmCsxrsQuOBm06ZN3HnnnW6fz8vLw2AwMGXKFI4fP05kZCS9evVi+fLlaLXaKpxpxSME7grGujdRLAQuqOnWrRu7d+/2OKZ379707h2YGMLKRAhcEPHRHyf47VAGHwzrRpjOOfBUdoiEM5dYcEazjNkso1KJyrjBSFhYWJnCR4IB4UUNIj764ySbjlxgz5ks306w0Tu9SVhxguBDCFwQkV1oKV/jzi3u6C+3fSiWqYJgRAhckGA2y+QU+Vefy1bwhKNBEIwIgQsScouNimD5GtlouycnlqiCYEQIXJCQU+jdenNaogoLThDkCIELErJ9EDhHbPVOCFzNwVWl3W+++SZg86nOiDCRIMFW4BzDQaw45k3bZukJgau5nD9/3mXtNoEQuKChTEtUm5+LjfblewQ1B2tl4EAiyzImkwmNpnpJiliiBgllWaISbHtwsgz6/MB8+VmzIjc3l6FDhxIREUFcXBxvv/12mdsJ2i5RT548iSRJfPXVV9x6662Eh4fTqVMntm7daneOtwY1n3zyCd26dSMyMpLY2FiGDBli16Nhw4YNSJLEzz//TNeuXQkJCWHz5s1+z72yqRZyu2DBAt566y3S0tLo1KkT7777Lj169HA5dvHixXz88cfs27cPgK5du/L666/bjZdlmWnTprF48WKysrK4/vrrWbhwocsa88GCncCVwYtaHAxeVEMBvO5beaAKZ8o50EV4H1fC+PHj+eOPP/juu++IiYlh6tSp7Nq1Syn1XV5eeOEFZs+eTatWrXjhhRcYPHgwR48eRaPR+NSgxmAw8Oqrr9K6dWsyMjIYP348I0aM4KeffrK7z6RJk5g9ezbNmzevlsvkgFtwq1atYvz48UybNo1du3bRqVMnevfu7bKjD1g+OQYPHsz69evZunUr8fHx/Otf/+Ls2bPKmDfffJP58+ezaNEitm/fTkREBL1796aoqKiqfq0qp0xOhmCz4GoIubm5LFu2jNmzZ3P77bfTvn17PvroI6cqv+VhwoQJ9O3bl6uvvprp06fzzz//cPToUcC+QU2rVq3o1asX8+fP5+OPP1b+Rx555BHuvPNOmjdvznXXXcf8+fP5+eefnVoVvvLKK9xxxx20aNGCunXrVtj8K4qAW3Bz587l0UcfVTr1LFq0iB9//JElS5YwadIkp/G2TTXA0v3nyy+/JDk5meHDhyPLMvPmzePFF1/knnvuAUqL+X3zzTcMGjSo8n+pAGDvZPCNoBM4bbjFkgrUvX3k+PHjGAwGu1VH7dq1ad26dYVNp2PHjsrPcXFxAGRkZNCmTRufGtSkpKTw8ssv89dff3H58mXMZsv749SpU7Rr1045z1UTmupEQAVOr9eTkpLC5MmTlWMqlYqkpCSnPQN3FBQUYDAYlE+PEydOkJaWRlJSaQHH2rVrk5iYyNatW10KXHFxMcXFxcrjnJycsv5KAcMXC86xtmnQhYlIkl/LxGDGtsyRtey4VaS8NajJz89XqossX75cqTrcu3dv9Hq93XjHJjTVjYAK3IULFzCZTMTExNgdj4mJ4dChQz5d4/nnn6dRo0aKoKWlpSnXcLym9TlHZs6cadfbsSaSU2T0+xy7MJFg2IOrITRv3hytVsvOnTuVjlfZ2dkcPny4SgpMemtQs3fvXi5evMisWbOIj48H4M8//6z0eVUGAd+DKw+zZs1i5cqVfP311+UqWT158mSys7OVr9OnT1fgLKsGuyWqz06GUoLCgqshREZG8vDDD/Pcc8+xfv169u/fz6hRo1CpVF6bvFQE3hrUNG3aFJ1OpzS5+e6773j11VcrfV6VQUAFrn79+qjVatLT0+2Op6ene43tmT17NrNmzeLXX3+122+wnufPNUNCQoiKirL7qmn4EgfniK0Qiji4qmXu3Ln07NmTu+++m6SkJK6//nratm1bJb0lvDWoadCgAUuXLuXzzz+nXbt2zJo1i9mzZ1f6vCqDgC5RdTodXbt2JTk5mf79+wOWfYLk5GSP7c7efPNNZsyYwS+//OK0ydmsWTNiY2NJTk5WXO45OTls376dMWPGVNavEnDKFAeH90wGo8lMsdFMREjA/VFBRWRkpN0mf35+PtOnT2f06NFezz158qTdY9uthoSEBKe91ujoaKdj3bt359dff3V7j8GDBzN48GC397nllluqpDN9eQn4u3b8+PE8/PDDdOvWjR49ejBv3jzy8/MVr+rw4cNp3Lix0ibtjTfeYOrUqaxYsYKEhARlX61WrVrUqlULSZJ4+umnee2112jVqhXNmjXjpZdeolGjRoqIBhuyLPuUquV41Bcv6p3vbOJIRh6pL91BnQhdeacqKCE1NZVDhw7Ro0cPsrOzeeWVVwAUz7+gYgi4wA0cOJDMzEymTp1KWloanTt3Zs2aNYqT4NSpU0o3b4CFCxei1+u5//777a4zbdo0Xn75ZQAmTpxIfn4+o0ePJisrixtuuIE1a9YEbWu5fL0Jk9n/T1O7VC03ToYjGZa4py3HLtK3Y1xZpidww+zZs/n777+VlcymTZs4ePCg1wYxAt8JuMABShS1KzZs2GD32NE8d4UkSbzyyivKp2KwU7blqX9xcGrRr6FC6dKlCykpKU7HCwsLvTaIEfhOtRA4QflwdDC42xpxTrb3vZqIVi0Eriq4khvEVAY1OkxEYKEqLLjq3HGrJmx2C/ynIv6uQuCCgDILnM3P3gJ9NdVQ4NRqS2tEx+h6QXBQUFAAUK7m02KJGgQ4Cpzvuaiel6hmG8eFRlX9Pgs1Gg3h4eFkZmai1WrtnFGCmossyxQUFJCRkUF0dLTyQVYWhMAFAWUJ8nXEVdtAo43AVUcngyRJxMXFceLECf75559AT0dQwURHR5e7mKcQuCDA1yWqY3yctz04o7n0WHUUOLAEi7dq1UosU4MAvdGMTmOxwrVabbksNytC4IIAZy+qb4tUb15UWwuuOntRVSpV0MY4XikYTWa6vv4r8XXDWf5/idQLLb+4gXAyBAUV4UV1FehrNFXvJaogeDiWmU++3sSZy4XUCa+4jBkhcEGAz0tUP/ui2i5RJYTACSqPfWezAWjXKKpCQ5KEwAUBZfai2vysd1FNpCzpXwJBWdh3ziJw1zSq2Eo+QuCCAN+dDA6PvRS8tF2iCgSVyf5zlira7RvVrtDrCoELArILLdV8/d0n81bw0tbJ4K5CiUBQXsxmmQNWgWssBE7gQE6RxYKrHVYS8e1Gi5zkz67gpbPAmcyiyq+g8jl1qYC8YiMhGhUtGlRsjwchcDWcIoNJsb4UgXOD0xLVS5iIQSxRBVWAdf+tTVwUGnXFSpIQuBqOdf9NrZKICPEvdsibF9XWySDy2QWVxb6zluVpRTsYQAhcjccqcFGhGlQlDUt83S+zHWU0y3a5p9ZjAkFls7/EgqtoBwMIgavxWAXO2/IUPMfBgbMn1ShaCQoqGVmWSz2ojSvegitXqlZeXp7STNZKTexIVZPJLiix4HwQOEccLb1io5lQbekyV1hwgsomLaeIS/l61CqJq2MiK/z6fltwJ06coG/fvkRERFC7dm3q1KlDnTp1iI6Opk6dOhU+QYFnbC04q5fU/X6Z+2R7cN6HE4G+gsrGuv/WqmEtuw/XisJvC+6hhx5ClmWWLFlCTExMlTSqFbjHGiISFab1u2ySo3w5LlENYokqqGSsKVrXVML+G5RB4P766y9SUlJo3bp1ZcxH4Ce2FtwZr6MdPowcTDhhwQmqmsrcf4MyLFG7d+/O6dOnK2MugjJg52SwelF9XaI6POvY3d4owkQElYziQa3gDAYrfltw//vf/3j88cc5e/Ys7du3d6qX3rFjxwqbnMA7/nhRHfG2BydyUQWVycW8Ys5nFyFJ0Dauciw4vwUuMzOTY8eOKZ3nwVI6WpZlJEnCZHKuSiGoPHKUOLiyhIl4XqLalksa9MFWUqf+S6m4KhCUF+vytFm9CGqFVE7tXb+v+sgjj9ClSxc+++wz4WSoBrj0ovp4rpOTwcMeXL7exJr9afTr1KhsExUIHLCmaLWrhAwGK34L3D///MN3330nmtNWE3JKKom4W6KezSp0e66jD8Gxqq/jEtXgpXeqQOAP+yupgogtfq83brvtNv7666/KmIugDHjbg7t+1m/Kz57qwYGrJapnp4RAUB72n628FC0rfltw//73v3nmmWfYu3cvHTp0cHIy9OvXr8ImJ/CO3RK1ZI1qFS5/O4M7L1HtH4sO8oKKIqfIwMmLlsbOlZFkb8VvgXv88ccBeOWVV5yeE06GqkVvNFNosLzeriw4b2Fs3ryoolySoLI4WLI8bRwdRp2Iimsy44jfAueYeyoIHLalymuFOv8pzV4sLle5qLY4BvoKuRNUFPtKBK4yHQzg5x6cwWBAo9Gwb9++ypqPwA+sAhcZqkGtkpy8qN5WlM4WnPtAX4GgIqnMEkm2+CVwWq2Wpk2bimVoNcGpVLkDjhac4x6at1xUp3JJQu8EFcT+s5WbomXFby/qCy+8wJQpU7h06VJlzEfgB96CfP234Lx5UYXCCcpPkcHE0cw8oPKS7K34LXDvvfcev//+O40aNaJ169Zce+21dl/+smDBAhISEggNDSUxMZEdO3a4Hbt//37uu+8+EhISkCSJefPmOY15+eWXkSTJ7qtNmzZ+z6smUFTiYAjXWcrMSA65qN4EyfF5b8n2wokqqAgOpeViMsvUr6UjJiqkUu/lt5Ohf//+FXbzVatWMX78eBYtWkRiYiLz5s2jd+/e/P333zRs2NBpfEFBAc2bN+eBBx7gmWeecXvda665hnXr1imPNZrKSQMJNPoSL6fWTaMOxy005zg4+8eOgb4G4VASVAKlXexrV3omlN//+dOmTauwm8+dO5dHH31UyWtdtGgRP/74I0uWLGHSpElO47t370737t0BXD5vRaPREBsbW2HzrK5YLS53+aHevKjurmfFZBJeVEHFU+pgqPzq32U2bVJSUjh48CBgsZi6dOni1/l6vZ6UlBQmT56sHFOpVCQlJbF169ayTguAI0eO0KhRI0JDQ+nZsyczZ86kadOmbscXFxdTXFysPM7JySnX/asKa0FKqwVX+llYEujrxQDzO5NBKJygAqiKFC0rfgtcRkYGgwYNYsOGDURHRwOQlZXFrbfeysqVK2nQoIFP17lw4QImk4mYmBi74zExMRw6dMjfaSkkJiaydOlSWrduzfnz55k+fTo33ngj+/btIzLSdc33mTNnMn369DLfM1CUWnCuzXyve3COS1QP1UR8uZ5A4A2Dycyh87lA5WYwWPHbyfDEE0+Qm5vL/v37uXTpEpcuXWLfvn3k5OTw5JNPVsYc/eLOO+/kgQceoGPHjvTu3ZuffvqJrKwsVq9e7facyZMnk52drXzVlIKeVgtO5+MenCP+VBMRCCqCoxl56E1mIkM1NK0bXun389uCW7NmDevWraNt27bKsXbt2rFgwQL+9a9/+Xyd+vXro1arSU9Ptzuenp5eoftn0dHRXH311Rw9etTtmJCQEEJCKtebUxnoHZeoSi6q5btzHBweH3tL1RJLVEF5URwMcVFVUmrNbwvObDY7JdiDJQjYnzQunU5H165dSU5Otrt2cnIyPXv29HdabsnLy+PYsWPExcVV2DWrC+V1MliXnFYL0DHQV1hwgoqmKvffoIzlkp566inOnTunHDt79izPPPMMt99+u1/XGj9+PIsXL2bZsmUcPHiQMWPGkJ+fr3hVhw8fbueE0Ov17N69m927d6PX6zl79iy7d++2s84mTJjAxo0bOXnyJFu2bGHAgAGo1WoGDx7s769a7XF0MjjhY6BvSIlAinJJgsqmtAdD1fRP9nuJ+t5779GvXz8SEhKIj48H4PTp07Rv355PP/3Ur2sNHDiQzMxMpk6dSlpaGp07d2bNmjWK4+HUqVOoVKX/vOfOnbPz1s6ePZvZs2dz8803s2HDBgDOnDnD4MGDuXjxIg0aNOCGG25g27ZtPjs/ahJWQbIKlDUb1SpETnFwblK1QrRqcouNLnoyOKZqCYkTlB2zWeZAiQVX2RkMVvwWuPj4eHbt2sW6desUb2fbtm1JSkoq0wTGjRvHuHHjXD5nFS0rCQkJXmuSrVy5skzzqIkYvAb6+mbCWQXSqaKvsOAEFcjJi/nk602EalU0rx9RJfcsUxycJEnccccd3HHHHRU9H4EfFJd7D85CiNb1ElXswQkqEmuJpDaxUWjcbatUMGUSuOTkZJKTk8nIyHByLCxZsqRCJibwjtMenIMX1ddk+xCNJZfVsVySY2d7sUKtWRxOz+XLlDOMuaUF0eGVV1TSV5QS5VW0/wZlELjp06fzyiuv0K1bN+Li4kRXrQBSKnBuAn29xsE5LFGFBRdUfPD7cb5IOUODyBD+78bmgZ5OqQe1ivbfoAwCt2jRIpYuXcqwYcMqYz4CP3B0MjjiFAfn8LzfXlRhwtUoCvUWi/zUpYIAz8Ty3rG2CawqBwOUIUxEr9fTq1evypiLwE/c5aJaLTPf9+BKlqheCl6aZRj24XZGfLRDiF0NwGqBn/PQOrKqOJddRFaBAY1K4urYWlV2X78F7v/+7/9YsWJFZcxF4CfenAzeJMibBee4RE3PLWLTkQts+DtT6ccqqL6YSv7AZy4HXuCsGQxXx0Qqe75Vgd9L1KKiIj744APWrVtHx44dnbIa5s6dW2GTE3jGW6Cvk5Xl9NByINRqwXlZotqeLxLvqz/WDyhPzb+rCquDoSoS7G3xW+D27NlD586dAZyazwiHQ9XimKrlnIvq5QIOFpzRLGM2y6hUlgs5drYX1CysApdbZCSnyOC2tH1VUNUpWlb8Frj169dXxjwEZcAa6Ou+mogXJ0PJd1snhd5kJlRlsegcyyXZnetB+zb8ncGC9Ud5476ONG9QdfstAntstxjOXi4kKi5wAlfqYKhaC65qou0ElYL3Jarn82Ulk6F0T8Q2VMRTmIinS4/4aCc7T17mmdV/eZ6AoFJxFLhAkZlbTHpOMZIEbeOEwAl8xGmJ6pSL6lvBS61Nwcxim2Df8na2zy7Ql+t8Qfkw2fz9A7kPZ02wb14/goiQqu2PIgSuBqMvd6CvBQlJEUm9BwvO9pEvYSJVlY4jcI3t3y+QoSL7qzjB3hbxDqzBlLseXMnTkuQ6VMRToK8vtp1GJZxOgcRW4M5UAwvOa4pW2j64dKJC7+23wP3+++8Yjc4xUEajkd9//71CJiXwDceS5aVe1JKmMz6maknYCJzJVuDKl4vqtk6doEqoLntw+876YMGl74dl/4alfStU5Px+B956660uu9pnZ2dz6623VsikBL7hrwXnVA/OxoJTqvraLlEd9uBsDTpfxE4tLLiAYidwAbLgsgsNSqqYWw9qxiFY1g8KL0GtGAivW2H391vgZFl2Ge928eJFIiKqpsaTwIL3enC+XcfdHpxzPTjZ5c/ucLc3KKgabD/gMnOLKTKYPIyuHKwFLpvUCXNd0STzsMVyK7gAcZ1g2FcQWnF7dT67NO69917AEsw7YsQIuyYtJpOJPXv2iBzVKkSWZWU56RjoazvG7rGLa1jPcy1w5VuialRiiRpIHD+gzmcX0ayKCk1a2e8p/u3CUYu45WdAbAcY9g2E1anQ+/sscLVrW1RVlmUiIyMJCwtTntPpdFx33XU8+uijFTq5YCe3yMCGvzO5rU1Dv93ntiEcbuPgvFyj1ItaKnDFJvcWnN2S1wex07pZOguqBrPD3+9cVmEABM5NiaSLx2DZ3ZCXBg2vgWHfVujS1IrP/1UfffQRYCkbPmHCBLEcrQCWbD7J2+sOM+WuNoy+qYVf59oWo3TMZFBStbysURW9kiSnPTizWfbaZtAbWrEHF1CsH1CRoRpyi4wBcTRYk+yvsfWgXjphsdxyz0ODtvDwdxBRr1Lu7/dH7LRp04S4VRBpOZY33IU8/wNibZeSjoG+Vrw3fi71ouocil4aXKRp+ZtgL5wMgcXqZLA2WK7qUJFCvYljmXmAjQV3+R+LuOWchfpXl4hb/Uqbg98Cl56ezrBhw2jUqBEajQa1Wm33JfCd/GLLpq9jaXBfsJ6jktwLibdgXPs4OPuKIq7StMz+rVBFmEiAcRS4qrbgDqblYJahQWQIDaNCIeu0RdyyT0O9lvDw91CrYaXOwe+8iREjRnDq1CleeuklUbK8nBToyy5wnmrBWS0tdyJkNsucvlzgMZPBqVQSDoG+PiicRnhRA4o1VatpvRKBy6rayr52JZKyz1rELesfqNvcIm6RsZU+B78FbvPmzWzatEkpmSQoO4UGS8B0WcoSuUq0d/yscY6Ds3x/4Zu9fLbjtN15pQJncjsnD8VFXCK8qIHF7GjBVfES1epgSKyvt4jb5RNQJwEe/gGiGlXJHPx+B8bHx4ty1RWE1YJzLBXuC9Zz3PVjAPd7cLbiBiV7cFYng8lqwTnPyWyXqiXi4Ko7RgeBS8suqtJGQvvOZdOALIb9PQ4uHYPaTS2WW+3GVTYHvwVu3rx5TJo0iZMnT1bCdGoYZhNsmgtFOWU63doUpEwWnNF9kG9pwUv3gbq2SJJzsUxXc7JPtvc+R7VK4qM/TrBg/VHvgwUVjtWCi6sdhkYlYTDJZOYWV8m99UYzmWlnWK6bQa28ExDVBEZ8D9FNq+T+Vvxeog4cOJCCggJatGhBeHi4U8lyV2lcQcuaSbDjAzi8Bh76EkIi/To9X1+yRPV37YdtJREPn1G+ZjJIklOpJVef9P4a7ipJYvr3BwAY0KUxjaLDvJwhqEisFlyIRkVs7VDOXC7kbFYBsbVDK/3ex0/9wzL1DK5WnUWObIQ04nvL8rSK8Vvg5s2bVwnTqKF0Hgp7VsHp7bD8QXjoC9D5HkJjteD0Rv8tOG95qOC9mogtThacNycDsPXYRaLCNHZJ1LZjbL27hQFIE7rSsToZVCqJRtFhnLlcyJnLhXS9qpJvXHCJmG8GUkd1msuqutQZ8YPFsRAA/Ba4hx9+uDLmUSMxx3Yi695VRH/xAKpTW2DFQBiyGnThPp1v3YMriwXnqZqvzz0ZSpAk55aDji0Dwb6AYlp2IYMXbwPg5Ky+pWNsbmorvmLbtuqxLlHVkkST6DB2UAWOhsLL8PE91Mn5m0y5Nl+0e58x9fwLYq9IyuTmOnbsGC+++CKDBw8mIyMDgJ9//pn9+/dX6OSqO/OSj3Dtkkt80PQt0EXCyU3w2SAweH8Tmc2yYtWUJUzElQXnGLLjqwUnIaEqOdeTBWd7yF0rOtvzRKBvYLH+LdQqicZ1LNsDlRoLV5gFnwyAtD1kqaIZrH+BuBYdK+9+PuC3wG3cuJEOHTqwfft2vvrqK/LyLJHKf/31F9OmTavwCVZnGkdb9jK26ltY9uB0teDERlg5BAxFHs8tMpoUMSlLafDSWnDuRcRXq8neyWA5yXWgr2xzjnfxsh8hTLiqxOzwQdO4ZP+z0iy4omz49F44l4ocXo/hhhc4KjfxXuSykvFb4CZNmsRrr73G2rVr0elKy5/cdtttbNu2rUInV91pHG0TX9Q0EYZ+DtpwOPYbrHoIjO49VtblKZTRgjN5CvQt+e4mDs4RCec9OFdzsr2eMM6qN46WtNWCq5TS5cW58On9cDYFwupw+u6V7DE0Jkyrpln9wHZV81vg9u7dy4ABA5yON2zYkAsXLlTIpGoKtma/LMtwVS/LHpwmDI6uhdXDweg6z7TQRuDKEiZiXaLaBfo6jPFnDw4fvKi2W4UqGwvOW1K/oOoxy24sOOt7taIozoPlD8CZHRAaDcO/JVVviXNrGxcZ8G0KvwUuOjqa8+fPOx1PTU2lceOqC+CrDjQqWaIWGkxcLjBYDja7EYasBE2oJXzk8xFgMjidW14LzltPVHCOe3P3tpaQfPKimt1YcL7s9QknQ9Vi+wGlKfGiAuTrTWQXOr8fy4Q+H1Y8CKe2QkhtGP4NxHVSKohUdZNnV/gtcIMGDeL5558nLS0NSZIwm8388ccfTJgwgeHDh1fGHKstIRo1DSMthT/PXLbJ82t+CwxaAeoQ+PtH+OIRJ5Er0Jf2tSibk8EikK5qrlk/ocvnRfV9D87kRr2EpgUO2w8olSQRqlVTv5ZlS8mdg8gv9AWWqIF//oCQKBj2NTTqAth20Qrs/huUQeBef/112rRpQ3x8PHl5ebRr146bbrqJXr168eKLL/o9gQULFpCQkEBoaCiJiYns2LHD7dj9+/dz3333kZCQgCRJbmPy/LlmeXHrnWp5OwxaDmodHPwOvnoUTKWiZmvBubKWvOHKgvNW0dcTzhacq1St0p/VdktU19cUVlvgcHQyABXnaDAUwsrBlqgBXSQ89BU06QpY3nNKDbgAtAl0xG+B0+l0LF68mGPHjvHDDz/w6aefcujQIT755BO/yyWtWrWK8ePHM23aNHbt2kWnTp3o3bu3EnriSEFBAc2bN2fWrFnExrquRODvNcuLxzdNqzvgwU9ApYX9X8PXj1nSu3BYohrL4WTwsER1Wjq6czL4mMlgb8GVHndnwQkCh70FZ/leIaEihiKLA+34BtBGWILb47srT5+5XEhOkRGtWuLqGP8yeyqDMpd7aNq0KXfddRcPPvggrVq1KtM15s6dy6OPPsrIkSNp164dixYtIjw8nCVLlrgc3717d9566y0GDRpk1xOiPNcsL03qlBQTdPemad0HHlwGKg3s+wK+GQtmk/0StQwWnOJk0Dhv4pZ6UX27lq0X1XqSq9AVk8Oyx9VxdwgJrFqsH0ZqlaRsJ1g/jMvsSTUWWxxnR9dZogWGfg5Nr7MbYu3BcHVMpMcsm6rCp0yG8ePH8+qrrxIREcH48eM9jp07d65PN9br9aSkpDB58mTlmEqlIikpia1bt/p0jYq6ZnFxMcXFpSEdOTm+J89bPxU97mu06Qv3f2RxOOxZCSo1hXETlafLFSZiYzWXx4tqPdd6jrdcVFsLTlSXqX6YbLIYrJRriWrUW96/R36xRAkMWQUJ1zsNs/ZAderBECB8ErjU1FQMBoPyszv8KX554cIFTCYTMTExdsdjYmI4dOiQz9epiGvOnDmT6dOnl+me9SIsG7c53jxT7frB/R/CF6Ng93K6ZBQgcR8yqjJWE3FvwVnxPZOh9G+nOBm8lEuSfLDg7NoMCg2sUqwrhDBd6Qdgo7IKnMkAX4yEv3+yRAcM/gya3eRyqNJFK8ABvlZ8Erj169e7/DlYmDx5sp1lmpOTQ3x8vE/nOnofPXLNAMse3FeP0vrc17yqyeFF4yPlKlke4moPrmQqvlpWkuQiTMTLEtVWVt3uwQlRCxjWUJDaYaXVfsq0B2cywpf/B4d+sDjMBi2HFu4bvO9TPKg1yIKrDOrXr49arSY9Pd3ueHp6ulsHQmVdMyQkxO2enjckhxxOr3S4H8xG5K8f5yFNMkbUvGLyv4CBq3JJjha0s4/B9SQtS1T/yiXZPluGWgGCSsaVwDUpyby5mK+nUG+ys+5cYjLC16PhwDcWR9nA5dAyye3wjJwiMnOLUUmWIN/qgE8CZ2367AtfffWVT+N0Oh1du3YlOTmZ/v37A2A2m0lOTmbcuHE+36+yr+kNq6b4U5qIToP4dtcp+p18nRGaXzGjwmS6C7UfTVqsJZY89R51tXJ0JVyuUrW8Bfq6+1lQPXAlcFFhGmqFaMgrNnI2q5CWDT2kUZlN8O1Y2Pdlibh9Alf/y+M9rfFvzRvUIlwXMNvJDp/+o2rXrq18RUVFkZyczJ9//qk8n5KSQnJystIc2lfGjx/P4sWLWbZsGQcPHmTMmDHk5+czcuRIAIYPH27nMNDr9ezevZvdu3ej1+s5e/Ysu3fv5ujRoz5fs6IpXaL6x9bIPkw2/h8Aj2jWIP/6ol8bVZ7CRKyWmivhcbkcliTnQF8ve3C2uN+Dc56ToGrIKnAWOEnyMenebIZvx1lqHao08MBH0PpOr/dUMhiqQYCvFZ9k1tr0GeD555/nwQcfZNGiRUrcm8lkYuzYsURF+feLDRw4kMzMTKZOnUpaWhqdO3dmzZo1ipPg1KlTqGwal5w7d44uXbooj2fPns3s2bO5+eab2bBhg0/XrGgcywwBFBlMhGo9m/8FBhPfm25FjZnXtR+i2b4ANFpIetk5YtcFpU4G97morpLtXVlm9mEilm+uMxlsL2Z7XIhXdcNqwUWF2VfcblwnjL/Tc92HipjN8P2T8NcKkNRw34fQ9t8+3XN/Ndt/gzLswS1ZsoTNmzfbBfWq1WrGjx9Pr169eOutt/y63rhx49wuH62iZSUhIcGnjXNP16xoHMsMvZt8hLfXHWb+4C7c3dF956CCYouXa4XpdtSYeFW7FP6YZ/nEvO1FryKn9+RkKMHVK+UqqNhSLsmXPTgbr6jN1d1acEL4AoarJSqU5k+7dDSYzfDjM5D6CUgquG8xXNPf53vuq2YeVChDoK/RaHQZcnHo0CHMV+Bus8pGGH7cc545aw9jluGb1LMez7PNZPjE9C9yb33V8mDTbNj4htf7KhV9XQX6luhKscH57+GqY31pHkOpKLkaZ+stle0sOK/TFVQx7gTOrsSXLbIMPz8HKUst4jbgA2h/n8/3yyrQK7GgNdqCGzlyJKNGjeLYsWP06NEDgO3btzNr1qxK2+eq1pQow7msQiZ8/pdyePuJS5jMsttyMQUOPQpyOz1KpFYFv74AG2Zalgc3P+f2tkrjZ9tAX4dbXS5wLtXkaukpSaW/hxLo60dfVFFNpPqR407gXIWKyLKlgdLO/wES3PM+dHzAr/sdKFmextcNc7pnIPFb4GbPnk1sbCxz5sxRyibFxcXx3HPP8eyzz1b4BKs7Vk25kGcRkxta1uevM1nkFhk5cC6HDk3sP83+uZjP8Qv5FNqkakGJRdZrHJiNsG4arH8NVGq40XXmSGlPBvdLWaWEk4vzHH8HJUzEKnAuFMlvJ4MQtYDh3oJzcDLIMvzyAmxfZHl8z3vQebDf97MuT6tLBoMVvwVOpVIxceJEJk6cqKQ0+etcCCZsczKvqhfOe0O6MOHzv1h3MIOtxy/YCVyRwcSgD7ZxPtu5nLmS+3nD0xaR++1VSJ5u2ZO7/kkX471X9M1ysOBkXOeYSlJpQnapB9b5d7WLg7P52RcvqqBqcSdwTUosuLScIoxGE5rfXoZtCyxP/vsd6PJQme5XnUok2VKubNioqKgrWtwAIkMtnxEROjWLh3cjOlzHdc3rAbDtuH2P2M//PO1S3MDBsrppAtwyxfLz2pdg6/tO45WmM3ZOBntrztUS1aUF5yKTwZWDQMTB1RzcCVyDWiFo1RIms5nCX16GLfMtT/SdC11HlPl+SomkalDk0pYyReN98cUXrF69mlOnTqHX2/8T7dq1q0ImVlPoHB/NjAHt6dQkWikPYxW4HScuYTSZ0ahV6I1mFm445vY6TntjtzxvseR+fxN+mWyx5BJHK09bLTFPgb5ZLpaoLvfgwMbNYMGVaLlLyfLFySA0sGpxJ3AqlURMVCgP5H5C5M6SoPw734Luo8p8r/xiI8cv5APVb4nqtwU3f/58Ro4cSUxMDKmpqfTo0YN69epx/Phx7rzTezBgsCFJEkMTr7Irz9w2LoqoUEvEuDU378tdZzjnwnqLKEmX0bsKwL11CtxQsgf383Ow80PlKdcWnAWrmDhacLIsu/ai2llwPi5RbY77Ui5J4BsVEVpTbDRRVOJBd7Xh/3/mz3lKUyJuvWfafXCWhUNpOcgyNIwMoUFk2VIeKwu/Be7999/ngw8+4N1330Wn0zFx4kTWrl3Lk08+SXZ2dmXMscahVkkkKsvUixhMZhast2RbdE+oYzfW+gZ01WgZSYLbp0Kvkj24H8dDyjLAXS5q6amyLLt2MriKg8M5I8OlBWcjZPZhIt6dDCKTwTt7z2TT7bV1rN55ulzXsVpvklS6haKwaQ4jilcAcKLLJOg5tlz3ApsSSdVseQplELhTp07Rq1cvAMLCwsjNzQVg2LBhfPbZZxU7uxqMdZm69dhFvk49y5nLhdSvFcLEPm2UMZY3YInAubOCJAnueAWuK3kjfv8UpH7qsvGzLYUGkzLGFpeZDDapDKV7cM7XtBcyHwJ9haj5xbjPdnExX8/EL/eU6zrWEJGoUC0q2zClP96B5FcAeMMwiH/alH1ZaotSIqmaORigDAIXGxvLpUuWzfOmTZsqvVBPnDghItdt6FkicDtPXlKst8duam5XxlmWSwN1XS5RrUgS9H4deowGZPh2HH1MGwD3uaiX8l23K3TtZHAu++SqFaA7L6ovTobRH6fwv03HvY67krFtJVkeXO6/bV0Aa6cC8Gn4MBaa+pWpDqErrBZcdQrwteK3wN1222189913gCXo95lnnuGOO+5g4MCBLvulXqm0iY0kOlxLgd7EPxcLqBuhY+h1TZ32RDQlubZe32ySBHe+Cd1GATKvS+/TT/WHnQVn6yZw5WAAN2EiNm0DrbrmyiizFTLbp31JYDmbVchrPx70PvAKprgMvTlc4SRw2/8Lv5R45W+exDeRQ4CyNTtypNho4nC6ZRUX6C72rvDbi/rBBx8oKVn/+c9/qFevHlu2bKFfv3489thjFT7BmopKJZHYrC6/7LfUpnv0xuYuS8hYLTCfil5KEtw1G9lsRL1rGW9r36fgSCfoNtBpqNXBEKZVU1iSNSHjeq/Prh6cInC+78GJpjMVQ7GxEiy4HYvh55Ly+DdOgFsmoT5sWXW5qhjjL0fS8zCaZWqHaZUg4uqEXxac0WjktddeIy0tTTk2aNAg5s+fzxNPPIFOp6vwCdZkrPtw0eFahvW8SjleJ7zUitOUZCL4XNVXpUJ/5xxWG29GLcnU+nEMHPjWbogsl2Yx1I+0/5u4a3BTulUjl1zDczUR2/01d53the75h6s907KQXfK3v0u/Bn6aYDl4/dNKEQerY6oilqilTZ6j/GpZUFX4JXAajYY333wTo9HofbCA+7s2YUCXxsy+vxO1Qkqtt3q1Sl3pWsWC8/3NpjfBJOOjfGm6EUk2WRpLH/zBzotqzWKoX8vebe+6mohzoK/rMBH/vKiH0nxv3iOouKIF2YVGHlBvYEhGSQOonuPsynBZ86MrYolaHUsk2eL3Htztt9/Oxo0bK2MuQUdkqJa3B3YmqZ19LTqrAwJKc0ldhom4wWCSMaPiOcNjyO0fsAQEfz6CjvlblDGX80ssOBuBs9SDcxMmInlfovqbi+qYySGoGq468w1vaBZbHiSOgX+9ZhdDVJb3nDv2VWMPKpRhD+7OO+9k0qRJ7N27l65duxIREWH3fL9+/SpscsHK83daQkX6dW7Eks0nAP96o1qXMiqVGmnAIpBNsP8rRqe9wk7V08i0V/bgnCw4d9VESvCUi+puiiJVqxrx1yr6nZyBSpLZ2+hBOvSZ6VRmpqIsOJNZ5uD56hsDB2UQuLFjLfFYrvqfSpKEyVQxG6XBTK0QDa/2bw/Ax1v/Afzrbm+wDfJVa+DexSCb0B74lv9q57EpszkpBW0BqF9L5/JcW2y9qGXJRa0AQ0BQEez9Ar55HBUynxpvR9XxRTq42BfTKHtw5fvDHc/Mo8hgJlynplm9CO8nBAC/l6hms9ntlxA3/1GWC354tPSOlUTUGrjvQ1IjbiBEMnBL6tPEXdoOOFtwrj60XXXVcrlEdfOJL7yo1YD9X8NXo0E2sy6sDy8ZR1I73LXTT1NBFpx1edouLso+oLgaUa5qIoLyo1WVwclgtLHgrKi1LIl7ibWma9GYi3k6YyrXqQ445Qa6ssxsezJYhc3Vez/fTSCqCPAuPzN+PFD2kw98Z2koLpug81De0j6OjMpt4Ukl9rK8AqcE+FbP/TfwY4laWFhIcnIyd999N2BpllxcXKw8r1arefXVVwkNDa34WQYxfoeJ2IwNcUjT0mhD+Y/hKdY1XEzTi5tZon2LUwUd6H1NjBKP524PziFKxOu+mi/14AT2yLJMgd5ERIjzv93iTSeUnx3/rh459KOl67xsgo6DoN+7ZM1aD7hOtIdSC668f7fSLvbVc/8N/LDgli1bxn//+1/l8XvvvceWLVtITU0lNTWVTz/9lIULF1bKJIMZrT+BviWUWnD2y4IQjQo9Wn5s+wab6US4VMzVa0cyv1fpB5HrfRebPbiSI96MMl+azgjseeyTFK6Z9gunLhZ4HOcuv9iJv9fA6octXvQOD0D/90GldlsqyUpZPlQdkWVZCRGpbiWSbPFZ4JYvX87o0fZlVVasWMH69etZv349b731FqtXr67wCQY7pS57f+LgXCxRKf3kzzFqGFX0DJtM7VEZC9CtGkgX6QjgKRfVGiZiXaL6bsEJL6pv/HrAYkWv2HHK47gQjZeO8wBH1sHqYWA2wDUDoP8iUKm9lkqCirHgTl8qJLfIiE6tolWMhwbSAcZngTt69CgdOnRQHoeGhtr1LO3RowcHDpRjH+EKxSpSu05dVlzu3nBXSSSkpBdrek4RxegYbXwWc8KNSPpclulm0VE6ht5dwUsHC87be1901So7jilZjs4br0vUY7/ByiFg0lt6lt672OJowkuppBI0ZQgud8TqYGgdG+n0QVud8HlmWVlZdntumZmZJCQkKI/NZrPd8wLfsL7Zdp68zL3vbyEz1/trqFTzdXhjWfNa00oKa+pCa6Easgqa9iJKKuQT3Uzq5zh/CFkyGbwH+rpDLFH9wzGpPqfIvjBCiNbDv+XxjfDZYDAVQ+u+mO/9kByb062lkiJDNG49m6UWXNmXqPPWHQawq45THfFZ4Jo0acK+ffvcPr9nzx6aNGlSIZO6orARkkKDiWVbTno9xa0FV/I4PccicHXCtaCLgKGrSZFbU1sqYOjhJ2kn2d/DVcFLb55Ru2oiYonqF9Z+tUcz8igymJSObFZclcAC4ORm+GwQGIvg6j7wwFImfnOIrq+u5WRJyXBl/y3cfeu+0j24sv3djCYzh9PzAGhXjT2o4IfA3XXXXUydOpWiIuey24WFhUyfPp2+fftW6OSuBDo0iQbg6pJ9jI+3niSv2HOur9JRy3EPTmsVOIsVGG2NgwqJ5D/SZHaZWxJmyuVT3eu0kUr3gSyd7S0/K3twXj7cZVk4GcpKsdHEnjNZJM3dyITP/+Jinr3V7nKJ+s9WWP4gGAqg5R3w4Meg0bHvbDYGk8xfZ7IA970YbFGXbC2V9e+26egF5echPZqW6RpVhc8CN2XKFC5dukTr1q156623+Pbbb/n222958803ad26NZcvX2bKlCmVOdeg5I52Mfz5YhJrnrqJ5vUjyCkystLLJvS7v1kcBo5B6tbNaatA2lYt0atr8bB+EqfC2lBXymO5bgZXS6eV6/hSstwW22dt/1GE2Hmn2GhW9ltPXMh3Kk7q5GQ4vQOW3w+GfGh+Kwz8FDSW+EarNW/dlvBF4LRKoG/ZlqhfpJwBYESvBMJ0PjhEAojPAhcTE8OWLVto27YtkyZNYsCAAQwYMIDJkyfTrl07Nm/eTExMjPcLCZyoXysElUpi9E3NAfhw8wm3LnyTWeZYpmU5cqJkWWLF8ZO/jk0ku1YtkUs4C+Nns8fcjHpSLst1M2ghnS1J1XL0ovo+f1stXFviKbySOXkhn+3HL7p9vthoVvZa84uNXHAQOLuthzN/wif3gj4Pmt0Egz8DbajdtaDUareWSvJowZXBc28lu8Cg/I3v71r9t6T8cn80a9aMNWvWkJmZybZt29i2bRuZmZmsWbOG5s2bV9Ycrxj6d2lMg8gQzmcX8d3ucy7H/HOxVNTUDpvIjpvT0TYCZ41ezyWCYfrJ7DdfRQMph890MwjLPe5TLqodbgpeWi0Id6TnFHH6kuc4sJrOLbM3MPCDbRzNyHX5fLHBVCpwepP7JerZXRR9dA/oczE1vQEGrwKtfVFJReByrRacxXr3bMGVPZPh+z3n0BvNtImNrNYZDFbK5N+tW7cuPXr0oEePHtStW7ei53TFEqpVM/L6BAD++/sxlyJzKK30n0bjIHA6tf1ywXaJqrUJ7symFkP1UzhobkpDKYsuvw0jKt+S9O+rF9VdoK+3qrSJrydz45vrnTyHwYjt38qWYqOZzBJRKyg2ctHRyaBRwbnd8El/Qk15bDe3Ydt1C0AXTrHRZLektb7eGTn2S9Qoj3twZc9F/XKXZXl637VNqmWBS0eqbwDLFcrQxKuoFaLhcHoeG/7OdHreNlbOXaCvlegIGwvOoYprFpEM1U/hb3MTQgozuOPPR4mX0j2WS7Jl7YEM5ee3fvlb8dz6WpXWWzR/MKB2IwB2S1S9ySk0KN5wDD7pD0XZ7KY1j+if40y+5e/3f8v+pOfMZOX1dlqi+rIHV8Z6cMcy80g9lYVaJXFPl0Z+nRsohMBVM2qHaRmSaPFMLdx4zOl5W4HTOKZqaR334GwtOMtztt27LhHFUP0L5Ee1IKIonc90Mzh17BDFRpNiwT15eyueTmrlNI91B+332qZ/v99yfR8F7u53N3PgnH1gsyUKv2ZWpDmbVchHf5wg38YD7i4OzXaJCnAmq1TsW0unePL0s1B4GRp34/9Mk8gnjLNZFkE7lpFHsdHMX6ezkGW51MmQU4Qsy355Uf214L4scS7cfHUDGkbWjJxzIXDVkEeub4ZWLbHjxCV2nbps99zB87ZLVEcLznGJau9kAOeN5QvUZm/Sp2RHJNBEusAH5peZszpZWaq2aBDBYB9CAaz/gMV+WAVzfv1b+dlklun+2jq6z1hXIZVmq5q7529i+vcHmPlzaecwlQ8WHJRasy2lMyzXvU4tcw40uhb5oS+5YLB4S89eLgRK9zv/uVhg92GlN5rJLjSQXWhZvkaHue+PoimDBWcyy3y16yxgWZ7WFKqFwC1YsICEhARCQ0NJTExkx44dHsd//vnntGnThtDQUDp06MBPP/1k9/yIESOU6HzrV58+fSrzV6hQYmuH0r9zYwA+2FjaSzS70MDZrELlsatke1uibZvbqNwnWBvDGrI+8UOOm2OJV2Uy9NB/iDaWLkFjokJp6FB2yRHr9a1BrL5gu4dzuUBPTpGR3CKjV0dFdcTa5GfzkdIYMXfxupcL9Halp3KKjLSQzvKZbgb1pRxOhbSCYV9RrCnNEjhX8ne3/vn+uZTvZC2n5xT7ZMGVpR7clmMXSMsponaYltvbNvT5vEATcIFbtWoV48ePZ9q0aezatYtOnTrRu3dvMjIyXI7fsmULgwcPZtSoUaSmptK/f3/69+/vlGXRp08fzp8/r3x99tlnVfHrVBjWkJFfDqRxLNMSNX7IIVfVmxfV1oJT8g/ddLYvCm3IEP0L/GNuyFWqDF7InEhDLitWyLP/utrjfK1z8djA2sV9rdimL2mqcW6jN2z/Ju424Qsc6uo1k86zQjeDBlI2+81X8W6T2RBWh98Olf4PnMu2CJxsY8E5pnyl5xT5JnBl6KpljX37d6c4QrXVO/bNloC/k+bOncujjz7KyJEjadeuHYsWLSI8PJwlS5a4HP/OO+/Qp08fnnvuOdq2bcurr77Ktddey3vvvWc3LiQkhNjYWOWrTp06VfHrVBitYiJJatsQWUbpCO+YjO/sZHC/RNV5KFNtTbZPox6D9S9y2tyAONM5PtO9RliRxdHhuBx2xGoV+NP6zvbf3/a8C3nF9J2/iY+3nvT5WtUFW4GzXaK6C7u5SkrjM91rxEhZHDTH85B+MgXqKH7ae56xy3cp485nFWE2y3ZL1DILnJ/VRHKLDPyy39Iq9P6u8T6dU10IqMDp9XpSUlJISkpSjqlUKpKSkti6davLc7Zu3Wo3HqB3795O4zds2EDDhg1p3bo1Y8aM4eJF94GX1ZXHbm4BwJcpZ8nILXIKO3AKE7FZooZoVHZR5h5rgNmUSzpHfQYbXiRT1ZAWqvNct3kk5GU4OTQcUZdB4GwFwPa8t9ceZv+5HKZ+u9/na1UXbH8nWy+qKy1pImWwQjeDWOkyh82NeUg/hctYYsusyexW9CYzF/KLlcojZ7MKKXBI6Tt9udBrqSSw2a7wMZPhp73nKTKYadEggk5Nqm/tN1cEVOAuXLiAyWRyyoCIiYmxay5tS1pamtfxffr04eOPPyY5OZk33niDjRs3cuedd7rtGVFcXExOTo7dV3Wge0Jdul5VB73JzNI/TjpZcGonJ0Pp4zoO9fg1HkqjS0h25tQZuQHT6szinFyXWrnHYFk/wg1ZHudaJgvO5p625zku4WoStgJn++dxtJYak8lK3Ws0li5y1NyIIfoXuYhFPGyT2W05e7lQEUqTWea4QybLkXTLB6CnUklQ+mHnqwVnXZ7e3zW+RsS+2RLwJWplMGjQIPr160eHDh3o378/P/zwAzt37mTDhg0ux8+cOZPatWsrX/Hx1ccMf6xkL+6Tbf/wd7q9BefJyRDtUE1C68GCk+z1DYBzqliG6F+gKLQhZB7kus0jqIN74bf+r5R1D67IJkC4Jvd4OGDzIaSys+BKf6c4LvKZ7jWaSBc4Zo5jsP4FLlBqGW055nq1cS6ryO46RxzeD9b3h6dSSeD5w86Rkxfy2XnyMioJBnRp7HV8dSOgAle/fn3UajXp6fYxVenp6cTGxro8JzY21q/xAM2bN6d+/focPXrU5fOTJ08mOztb+Tp9+rSfv0nlkdQ2hhYNIsgtMlJkMBOmtV12ut+Dc7TgtB42lm0bP1sxmWVOynHsuuVjqBVDZPZhPtXNpDbOlgVAYYnV5U+y/UabQGbb+Ldgyde33Y+zvi4xXOIz3Ws0VWVy0hzDEP0LZGK/P5xb5LqazLmsQrvX19HK+6ck3MRTqSTwrx7cVyWZCze0akBs7ZoR+2ZLQAVOp9PRtWtXkpOTlWNms5nk5GR69uzp8pyePXvajQdYu3at2/EAZ86c4eLFi8TFxbl8PiQkhKioKLuv6oJKJfHYTS2Ux61jS0MHtA6f0lp1aW+FuhEOS9QSC+5ivnNBTUmSnCw4awhBUXRzePh79KH1uEb1D5/qXifKhcgVGDyXeHKFbahEkU14SZDoG7Z/HpMs04DLfKZ7jQRVOqfMDXjI+BLp+J7qeDar0M6CO+xgwVnFz9P+G/juRTWbZb5UYt9qnvUG1WCJOn78eBYvXsyyZcs4ePAgY8aMIT8/n5EjRwIwfPhwJk+erIx/6qmnWLNmDXPmzOHQoUO8/PLL/Pnnn4wbNw6AvLw8nnvuObZt28bJkydJTk7mnnvuoWXLlvTu3Tsgv2N5uadLIyUOrW1cqfg6bvxLkqQsU52WqJ724CTn0kvWT3dJkqBBa/YnfcpFOZIOqpN8rJtFJPapVtZ9s7KKU6Gh5i5R3VVhtrWK5Zw0PtPNoLkqjTNyfQbrX0SK9k80LAJX+vhIhuWDxjFG0ZvA+ZqLuu3ERc5mFRIZoqH3Ne5XSNWZgAvcwIEDmT17NlOnTqVz587s3r2bNWvWKI6EU6dOcf78eWV8r169WLFiBR988AGdOnXiiy++4JtvvqF9e0uneLVazZ49e+jXrx9XX301o0aNomvXrmzatImQEM/BqtWVEI2ayXe1ISpUw787lVqhsVHOSwZrOIiTk8GDF1TCOereag1YjxvqtWGo/gUuybXorDrOMt0satmIXKHehCzLdtVO/MF+iVr9Be5/m45z6+wNpGUXOVlSVhQval4mEavuo6XqHGflegzSv8hZGhBfJ9yvezouUa0/N64TZmctehM4X3NRv0yxWG9317DYN1t87otamYwbN06xwBxx5Rh44IEHeOCBB1yODwsL45dffqnI6VULBnRpwoAulhSZ94dey097z/P4LS2cxoVo1VBkdOFkcP9Z5sqCszoLrP+kGrXEIbkpD+mnsEI3g2tVR/lI9yYP6ydRQCgFehOfbPtH2Qfyl2I7C65Ml6hSXvvRkpI159e/6e9m812tkiD/InzcD83Fvzkv12Ww/kXOyJZMgCZ1SksfhWnV3NO5ESt3Ou//huvUFOhNdlksjs/XrxVCRokl6asFl1tkZNbPh+jSNNrJQssvNvLzPothURPqvrkj4BacwH/u6hDHe0OuJVzn/PlkXaI6Oxksb+o4lxvFztbdhVxLTmOdCMs/i3WJe0BOYKh+CtlyON1Vh/lI9yZhFFGoN/H6TwedruMrtntwNcGCs/J5yhku5LleoqqLLsPH90DGATKpw2D9C5ySS0Oc6tcqXVFoVJLd9kM9mz3URtEWIcwqcJ3CFqJRE2NjzXsqlQSlVn5usZFFG4/xn+W7nDyyP+9Lo0Bvoln9CK5tWrOC5G0RAhdkKAIXYf8m73pVXdQqiWfucE65slhw9iJn3RNrULK/Y7vE3S83Y5h+MjlyGImqQ3yonY3KVOi1j4MnijxYcHqj2W3xyKrmsU/+ZNTSnXbHnlq522lcFHkk/DQU0vdCREMGFr/ASdneyWXb4V7tsIVgFTWAqFANkSH2H2a2TiSdWkVMVKlYerPgmjeoRe9rYugcH02b2EiMZplp3+232/v8IsViSd53beMaF/tmixC4IKN/58a0iY2ka1N771yf9rEcfKUPD3ZzjvGz7apld1yCuiWWoGPWxB65BQ/rJ5FHGL3UB1isnYNkcm5I5A1rgK99HJz9mIc+3E7S3N8Z+dGOgJZTyi828sv+dJIPuc6TthJFPp/oZhF2cR9ENICHv+e47Fw/zTbkx7F2nK2lLUkSjevYV/JtVj9C+TlEq7Kz4HxZov53WDe++c/1LB7ejRCNii3HLvLjXsuS9PSlArYdv4QkwYAaVDnEFULggownbm/FmqdvchkL5dhm0Iql4orz8XoROiWkwFUCfKrciifVL5Anh3Kjeh8faOcSgt5pnCfGfJqCwWT2uETdceISAOv/zmRmOZbB5cWxuIErIingY90sOqmOYwytC8O/g4ZtXI4Nt0mlcwzMtbXgXD3unlD6AWax4HwXOFvi64YzpmQv97UfDpJfbFTKIvVqUY/GDvetaQiBE5RYcM7/vI57RK7YI7VlLJMpkEO4Wb2Hhdp56PC93FHyoQzumLvRZy/qL/sD19TG29ZgBIUs1b1BZ9UxLsu1OHHXZxDTzu1421xhx1ATOwsOaBRtv3dq267vxIV8v5aojjx+cwua1g0nLaeI+b8dsStLXtMRAidAbzK7tOAa2MRXefLCHtS15xHDcxTKOm5T72aB9h20+B74e/Jigc+BvtXVARFOER/p3qSr6ghZcgQP6adQVLet53NcOImsxNlYTpLkbMHFRYcSVZJv2rReOA3LaMGBpRfItH9bhPi/G49z6lIBETo1fdrXzNg3W4TAXYEsGdHNLoykyGByuQfXwMaCc7c8kyTLUmubuR2jDBMokrXcod7Fe9r5aPwQOds9uOqaqiW7kd6wEnHrofqbHDmch/ST2S8nKM+72zcM1br/92tkY8FpVCqnpaJKkkh+9hbG3NKC53q3touJ9FfgAG5vG8PtbUoLWd7VIc6jANcUhMBdgdzWJobUl+5QHhcZXFtw9e0sODcCR+lm+RZzex41PEuxrKW3+k/ma9/zWeR+3FMazO1pLVjdtC+UYv6nnUOi6hA5chjD9JPYJ1sKJFhrt2XkuA4jyXPIObUN4o2LDmPq3e2oF6Hj5X7XuBA4i4X9fJ82xNUOK/MenC3T/n2Nsk9bk2PfbKn5Ei0oE7au/2KjyeUS1NaCc1dl12rBWdlk7shjhqf5r/Zt7lLvwMT7PG34DyZ8j4T3JGKVkcZlNsseq2+U3tv+cQh6PtDO5Xr1fvLkUEbon+cvuaXy/JcpZ+gcH01ajmvvcnxd+0wG23i6mMgQHrmhGSOvT0CSJKVkObgO66kboWNwj6bIsmzXD9cfmtYLZ+mI7pzNKiSxeb0yXaO6ISw4AVfVjXC5RK0fads42rUAmGXnvaQN5i6MNTyFXlbzb/U25mgXosL3IDlP+2wVrW+/7k+j0/RfST7on/NCh4H/at/mJvVe8uUQRugnsku2jzH860wWgNLiz5H2je2LR57PLh1n/UCxClnDyBBlm8BdO8KZ93Zg1n0d/fo9HOnVsj4PuAglqqkIgbuC+eGJG1g49Fo6NKntct+rQS3bfSDX/1SZucV23kAryeaujDM8iUFW01+9hbe0i3wWOU8BwxVtv43+JIXcYiOjlv3pdaz13joMLNTO4xb1XxTIIYzUT+RP2TkUxCpO7gTOEU/jNGqVss/mrluXwBkhcFcw7RvX5s4Oluh6V4Uqbb2onmLAwl0IHMCv5u48YXgCo6ziPvVmZmkWI/kgcp4tuKrdhVu44Rh3v7tJ6XWgxch72vncrk6lUNYxyjCBHbJrb6n1JfNV4Aq9BDFb9+G8tMcQ2CBeKgHgutR4/VqlS1RP6TruBA5gjbkHTxnGYZIlHtRsZIbmQ68id+ay66RyqHwng8ksczQjTxHSN9YcYt/ZHJb+cRLZqGe+9l3+pU6hSNbyf4Zn2Wq+xu21VIoF59rJ4Ii3cu/WWDhhwfmOEDgB4FzKvG6Ezilh30pCPfvN8TCtZ1/Vj+breMYwFpMsMUSznlc0S/EkVXk2zVQcLbbKNOBW/3maxNeTSZq7kaVbTto99+mWY/w+qz93qndSLGsYbRjPH+YOHq9nlSFPFpxtRRFv5ZOssXDu9uAEzggvqgCwtx52vpCEWiW59Sy2jYvipE1ZJE8WnJXvzNejNpiZo13EMM06jKiZbhyO6yzYUhxb41XmEnXiF3uUn9/97Sgjr28GgBoTLxnepa96O3pZzWOGZ/jd3Mnr9ZrVj2DvmWy2l6SauaJD49qKxfpK/2sID1EzNPEql2OtAif0zXeEBScA7AWuQWSIU8lzW2zL+gAunQyu+Np8IxONozHLEiM1v/CS5lO8LTod9warcgvufHYhKsy8pf0v/dRb0ctqxhieZoO5i0/n7z2bzb/f26w83jDhFsY61PCzXW42jAxl7oOd6XqV6/JE1oR7X3JiBRaEwAkA37ph9WphiY0a1N0+jMAXC87KF6abmWz8PwBGaX5msmYFnkTOnzaE/vDh5hN88Psxj2MWJB/mTe0H3KvejEFW84ThSZLNXX2+h2Mf25ioUKd4Q0ugbihT7nKdkG9Lywa1kCSo4+HDR2CPWKIKAN+EZNkjPSgymIgM1dKiQQTHMi3lyf0ROIBVpltRY+Z17Yc8pvkRE2reNA7E1XLVcV7+GnBms8zag+l0ahKtdIXKLzby6g8HPJ53Ob+I9rumcr/md4yyiicN4/jF3N3Pu9sTplM7VXRpWi+cLZNu86nmWnzdcJaN7FEju1sFCmHBCQDflj1atYrIUEsaUP/OpWW6wxwCfe9oZ9+Y2xUrTLfzkmEEAGM13/GM5guX4xx7wfqbbP/lrjM89kkKN725XjnmTcwlzMzQLGGQZgMmWeJpw3/42Zzo133d0aSOc/khfwpK3nR1A66OifQ+UAAIC05QwsO9Evhlfxp3d3QuzOgK2//JcIeGJIuHd+OZVbv5OvWsx2t8YvoXGkxM037CU5qvMclq5pvutRsz8iP76rn+7sH9fuQCYL8E93wJmemaZQzR/IZJlnjGMJYfzO5bUvrLvzs24sD5nBpdBrwmIQROAFgStH988kafx9taHa6WqHMe6MTu01mcuOC5y9ZHpjtRYeYl7XLGa7/AhIoFpv5ux7ur6OErZrPMlK/2ur36NM3HDNesxSxLPGd4jO/M15frflZe7GsJBlapJCbf6bmMkqDiEEtUQbmx9aImtbUsT1UqichQ3z4/PzT1ZZZhEADPaVfzuPo7t2PL60VduPEYa/anuboyL2iWM1Jj6cj2vPFRvjLfVL6b2fB/NzavsGsJfEcInKBM6Gy8gbYC9+b9pcneB8/n+Hy9RaZ+vGV4EIBJ2pX8n/pHl+Mc4+K84bi79dYvf7sYJTNJs5JHNT8BMNkwis9Nt/h1H0H1RAicoEwMTmxK65hIxt7Swq55iq2gGEz+mVsLTP1523AfAC9qlzNS/XNFTNULMhM0q3lc873lvoaRfGa6vQruK6gKxB6coEzUCtHwyzOWJVyGTSqSraQN6NLYq6PBkXdM96KWTDyp+YZp2k8wouYT07/cjt90JJOzlwsZZNOjwBZvDsqnNV8yTvMtANMMD/Op6Q7PJ7hh3fib2Xc2m/ScIhKb16P/gj+U5wb3CJ7yQzUNIXCCchNqs0S1DcF4fUAHvwUOJOYaH0CDmbGa73hVuxQTala4saqGfbgDgE7x0U4ZFt54Qv0VT2u+AuBVw0MsM/X2c64Wrm0aTcuGtWjZsJZybHjPq9hx4hK3tG7Ik7e39HC2oDIRAicoN5EhGu5oF0ORwWTX3cnXFC5nJN40DkSNicc0P/K69kOMqFhtutVulG1e6uV87+0K95QUoAQYq/6WZ7WW2LsZhiF8aLqrjHOFpnWdk+Rfuad9ma8nqDiEwAnKjSRJLB7erVzXuL1NQ4eGyhIzjUPQYGaU5mdmaf6HGRVfmG5m/Krd1Kul4+FeCcro8BDvb+V+71mWjaPV3zNRuwqANwyDWGy6u1xzf6Gv+9aAgsAiBE5QqeyYcjtzfj3Mqj9PA9C8fgTHHWLjVj/Wkx7N6nLiQj63zt5g84zEq8aHUGNihOZX3tR8gFFW81XqDQAs3nRCGRnmEGx88HyOy4yFUeqfmKL9DIDZhgdYaOrn8++iVkmYzDLXNIpi9WM9MZjMZe5/IKgahMAJKpWGUaG8cX9HReDaN65tJ3CP3tiM7gmWqP5m9SNcXEHiZePDqDEzTLOOOdqFmA0qvjP3cntPk1nmznc2ARZniJWH1b/wkvZTAOYZ7+U90wCff4+ezevx8agebPw7k+4JdYnwwWIUBB4RJiKoEqwOgNkPdGLRQ9dyXfO6bJt8Oy/0bWeXFfH4zS1cnC0x1TiCFcZbUUsyb2sX0Fe1zW6EbYaDreVmLZ75kHot07XLAHjX2J95xvt8mveIkmXwC33bolWrSGoXQ+3wsrXlE1Q9klzVRe5rADk5OdSuXZvs7GyiovzzzAnKT8Ik10G+Embe0CzmQc1GjLKKcYYnWWPuYXlOgjE3t2BinzbkFBno+PKvynmD1cnM1H4IwELjv3nDOAhvhTb3T++tWGl6o9mpCoggsPj6Pyr+aoJqx5djXC8/ZVRMMj7Kl6Yb0Uhm3tW+yx0qSzcsWYb3N1jqu53ILF0CP6DeoIjbB8a+PonbQ9c1tVuCCnGruYiNBEG1o+tVdTj++l2oVBJbjl7gkWU7KTJYlp1mVDxneAw1Jvqrt7BA+w6PG57hN/O1yvnvJB8B4D7V77yhWQzAEmMfXjcOwVHcEpvVVUqKn5h5F0cz8mjeoBaC4KBafDQtWLCAhIQEQkNDSUxMZMeOHR7Hf/7557Rp04bQ0FA6dOjATz/9ZPe8LMtMnTqVuLg4wsLCSEpK4siRI5X5KwgqGGs/iF4t63Po1TuZMaA0rsyMimcNY/jedB06yWTpUararTx/Ia+Y/qrNvKX9LypJZpnxDl4xDsNR3E7MvItVj/Vkyl1teH1AByRJolVMpCgJHkQEXOBWrVrF+PHjmTZtGrt27aJTp0707t2bjIwMl+O3bNnC4MGDGTVqFKmpqfTv35/+/fuzb98+Zcybb77J/PnzWbRoEdu3byciIoLevXtTVORbf0pB9WNo4lWcnNWXdeMt6WEm1Dxt+A8/mnoQIhn5r/ZtblTt4WhGHled+5k52oWoJJlPjbczzTgCW3H7V7sYTs7qqzg3Rt/UgiGJrlO9BDWbgDsZEhMT6d69O++99x4AZrOZ+Ph4nnjiCSZNmuQ0fuDAgeTn5/PDDz8ox6677jo6d+7MokWLkGWZRo0a8eyzzzJhwgQAsrOziYmJYenSpQwaNMjrnISToXqTW2SgQ4kTQYORBdr59Fb/SZGs5UPTnTym/gGNZOYz461MMY5CLvkcPzLjTqeeCIKaSY1wMuj1elJSUkhKSlKOqVQqkpKS2Lp1q8tztm7dajceoHfv3sr4EydOkJaWZjemdu3aJCYmur1mcXExOTk5dl+C6ktkqJZ903szqHs8RjSMMzzJWtO1hEoG/qP5Do1k5nPjTYq4jbqhGd+Pu0GI2xVIQJ0MFy5cwGQyERNjX8M/JiaGQ4cOuTwnLS3N5fi0tDTleesxd2McmTlzJtOnTy/T7yAIDLVCNMy6ryOz7utIod7EQ/+NQp3xMrepd/OV6Qa2XDONowO7iv20KxzhRQUmT57M+PHjlcc5OTnEx4sSNzWFMJ2aL5+4FbPpJs6f2Mu9LTpxr+iOLCDAAle/fn3UajXp6el2x9PT04mNjXV5TmxsrMfx1u/p6enExcXZjencubPLa4aEhBASEuLyOUHNQaVWE9eyc6CnIahGBHRTQqfT0bVrV5KTk5VjZrOZ5ORkevZ03cmoZ8+eduMB1q5dq4xv1qwZsbGxdmNycnLYvn2722sKBIIgRQ4wK1eulENCQuSlS5fKBw4ckEePHi1HR0fLaWlpsizL8rBhw+RJkyYp4//44w9Zo9HIs2fPlg8ePChPmzZN1mq18t69e5Uxs2bNkqOjo+Vvv/1W3rNnj3zPPffIzZo1kwsLC32aU3Z2tgzI2dnZFfvLCgSCCsHX/9GA78ENHDiQzMxMpk6dSlpaGp07d2bNmjWKk+DUqVOoVKWGZq9evVixYgUvvvgiU6ZMoVWrVnzzzTe0b18aCDpx4kTy8/MZPXo0WVlZ3HDDDaxZs4bQUNERXCC4kgh4HFx1RMTBCQTVmxoRBycQCASViRA4gUAQtAiBEwgEQUvAnQzVEeu2pEjZEgiqJ9b/TW8uBCFwLsjNzQUQ2QwCQTUnNzeX2rVru31eeFFdYDabOXfuHJGRkXb9AqoT1nSy06dPC09vCeI1cU0wvi6yLJObm0ujRo3swsgcERacC1QqFU2aNAn0NHwiKioqaN60FYV4TVwTbK+LJ8vNinAyCASCoEUInEAgCFqEwNVQQkJCmDZtmqiCYoN4TVxzJb8uwskgEAiCFmHBCQSCoEUInEAgCFqEwAkEgqBFCJxAIAhahMDVEC5dusTQoUOJiooiOjqaUaNGkZeX5/GcDz74gFtuuYWoqCgkSSIrK6tqJluJLFiwgISEBEJDQ0lMTGTHjh0ex3/++ee0adOG0NBQOnTowE8//VRFM61a/Hld9u/fz3333UdCQgKSJDFv3ryqm2gVIwSuhjB06FD279/P2rVr+eGHH/j9998ZPXq0x3MKCgro06cPU6ZMqaJZVi6rVq1i/PjxTJs2jV27dtGpUyd69+5NRkaGy/Fbtmxh8ODBjBo1itTUVPr370///v3Zt29fFc+8cvH3dSkoKKB58+bMmjXLbXOnoKFyK6cLKoIDBw7IgLxz507l2M8//yxLkiSfPXvW6/nr16+XAfny5cuVOMvKp0ePHvJ//vMf5bHJZJIbNWokz5w50+X4Bx98UO7bt6/dscTERPmxxx6r1HlWNf6+LrZcddVV8ttvv12JswsswoKrAWzdupXo6Gi6deumHEtKSkKlUrF9+/YAzqzq0Ov1pKSkkJSUpBxTqVQkJSWxdetWl+ds3brVbjxA79693Y6viZTldbmSEAJXA0hLS6Nhw4Z2xzQaDXXr1iUtLS1As6paLly4gMlkUpoRWYmJiXH7GqSlpfk1viZSltflSkIIXACZNGkSkiR5/Dp06FCgpykQ1FhEuaQA8uyzzzJixAiPY5o3b05sbKzThrHRaOTSpUvBv0lcQv369VGr1aSnp9sdT09Pd/saxMbG+jW+JlKW1+VKQlhwAaRBgwa0adPG45dOp6Nnz55kZWWRkpKinPvbb79hNptJTEwM4G9Qdeh0Orp27UpycrJyzGw2k5ycTM+ePV2e07NnT7vxAGvXrnU7viZSltfliiLQXg6Bb/Tp00fu0qWLvH37dnnz5s1yq1at5MGDByvPnzlzRm7durW8fft25dj58+fl1NRUefHixTIg//7773Jqaqp88eLFQPwK5WblypVySEiIvHTpUvnAgQPy6NGj5ejoaDktLU2WZVkeNmyYPGnSJGX8H3/8IWs0Gnn27NnywYMH5WnTpslarVbeu3dvoH6FSsHf16W4uFhOTU2VU1NT5bi4OHnChAlyamqqfOTIkUD9CpWGELgawsWLF+XBgwfLtWrVkqOiouSRI0fKubm5yvMnTpyQAXn9+vXKsWnTpsmA09dHH31U9b9ABfHuu+/KTZs2lXU6ndyjRw9527ZtynM333yz/PDDD9uNX716tXz11VfLOp1Ovuaaa+Qff/yximdcNfjzuljfK45fN998c9VPvJIR5ZIEAkHQIvbgBAJB0CIETiAQBC1C4AQCQdAiBE4gEAQtQuAEAkHQIgROIBAELULgBAJB0CIETiAQBC1C4AQ1hhEjRrisuNKnT59AT01QTRHVRAQ1ij59+vDRRx/ZHXPXsd1gMKDVau2O6fV6dDqd3/ct63mCwCIsOEGNIiQkhNjYWLuvOnXqACBJEgsXLqRfv35EREQwY8YMXn75ZTp37sz//vc/mjVrRmhoKACnTp3innvuoVatWkRFRfHggw/alRxyd56gZiEEThBUvPzyywwYMIC9e/fyyCOPAHD06FG+/PJLvvrqK3bv3o3ZbOaee+7h0qVLbNy4kbVr13L8+HEGDhxody3H8wQ1D7FEFdQofvjhB2rVqmV3bMqUKUrnsCFDhjBy5Ei75/V6PR9//DENGjQALDXh9u7dy4kTJ4iPjwfg448/5pprrmHnzp10797d5XmCmocQOEGN4tZbb2XhwoV2x+rWrav8bNuYx8pVV11lJ1IHDx4kPj5eETeAdu3aER0dzcGDBxWBczxPUPMQAieoUURERNCyZUuPz/tyzNd7CWo2Yg9OcMXRtm1bTp8+zenTp5VjBw4cICsri3bt2gVwZoKKRlhwghpFcXGxUzs8jUZD/fr1fb5GUlISHTp0YOjQocybNw+j0cjYsWO5+eabXS5xBTUXYcEJahRr1qwhLi7O7uuGG27w6xqSJPHtt99Sp04dbrrpJpKSkmjevDmrVq2qpFkLAoUoWS4QCIIWYcEJBIKgRQicQCAIWoTACQSCoEUInEAgCFqEwAkEgqBFCJxAIAhahMAJBIKgRQicQCAIWoTACQSCoEUInEAgCFqEwAkEgqBFCJxAIAha/h8DwSb/TWgIcQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.015349, + "end_time": "2024-03-26T06:19:11.179031", + "exception": false, + "start_time": "2024-03-26T06:19:11.163682", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 305.644608, + "end_time": "2024-03-26T06:19:13.915777", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/tvae/3/mlu-eval-load.ipynb", + "output_path": "eval/contraceptive/tvae/3/mlu-eval-load.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/contraceptive/tvae/3", + "path_prefix": "../../../../", + "random_seed": 3, + "single_model": "tvae" + }, + "start_time": "2024-03-26T06:14:08.271169", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/tvae/mlu-eval.ipynb b/contraceptive/tvae/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..665508421a3f46ec7324d72e10b08bf054fe5a38 --- /dev/null +++ b/contraceptive/tvae/mlu-eval.ipynb @@ -0,0 +1,2262 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:22.450987Z", + "iopub.status.busy": "2024-03-24T03:52:22.450233Z", + "iopub.status.idle": "2024-03-24T03:52:22.483258Z", + "shell.execute_reply": "2024-03-24T03:52:22.482499Z" + }, + "papermill": { + "duration": 0.047848, + "end_time": "2024-03-24T03:52:22.485228", + "exception": false, + "start_time": "2024-03-24T03:52:22.437380", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:22.510014Z", + "iopub.status.busy": "2024-03-24T03:52:22.509647Z", + "iopub.status.idle": "2024-03-24T03:52:22.516729Z", + "shell.execute_reply": "2024-03-24T03:52:22.515964Z" + }, + "papermill": { + "duration": 0.021478, + "end_time": "2024-03-24T03:52:22.518584", + "exception": false, + "start_time": "2024-03-24T03:52:22.497106", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:22.541720Z", + "iopub.status.busy": "2024-03-24T03:52:22.541450Z", + "iopub.status.idle": "2024-03-24T03:52:22.545507Z", + "shell.execute_reply": "2024-03-24T03:52:22.544660Z" + }, + "papermill": { + "duration": 0.017743, + "end_time": "2024-03-24T03:52:22.547346", + "exception": false, + "start_time": "2024-03-24T03:52:22.529603", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:22.570612Z", + "iopub.status.busy": "2024-03-24T03:52:22.570349Z", + "iopub.status.idle": "2024-03-24T03:52:22.574203Z", + "shell.execute_reply": "2024-03-24T03:52:22.573402Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017688, + "end_time": "2024-03-24T03:52:22.575966", + "exception": false, + "start_time": "2024-03-24T03:52:22.558278", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:22.599001Z", + "iopub.status.busy": "2024-03-24T03:52:22.598704Z", + "iopub.status.idle": "2024-03-24T03:52:22.604355Z", + "shell.execute_reply": "2024-03-24T03:52:22.603465Z" + }, + "papermill": { + "duration": 0.019515, + "end_time": "2024-03-24T03:52:22.606320", + "exception": false, + "start_time": "2024-03-24T03:52:22.586805", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "d3fb48eb", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:22.634188Z", + "iopub.status.busy": "2024-03-24T03:52:22.633494Z", + "iopub.status.idle": "2024-03-24T03:52:22.639700Z", + "shell.execute_reply": "2024-03-24T03:52:22.638855Z" + }, + "papermill": { + "duration": 0.021842, + "end_time": "2024-03-24T03:52:22.641603", + "exception": false, + "start_time": "2024-03-24T03:52:22.619761", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"tvae\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 3\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/tvae/3\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011643, + "end_time": "2024-03-24T03:52:22.664864", + "exception": false, + "start_time": "2024-03-24T03:52:22.653221", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:22.689861Z", + "iopub.status.busy": "2024-03-24T03:52:22.689154Z", + "iopub.status.idle": "2024-03-24T03:52:22.698942Z", + "shell.execute_reply": "2024-03-24T03:52:22.698104Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.024484, + "end_time": "2024-03-24T03:52:22.700881", + "exception": false, + "start_time": "2024-03-24T03:52:22.676397", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/tvae/3\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:22.726018Z", + "iopub.status.busy": "2024-03-24T03:52:22.725701Z", + "iopub.status.idle": "2024-03-24T03:52:24.735502Z", + "shell.execute_reply": "2024-03-24T03:52:24.734467Z" + }, + "papermill": { + "duration": 2.025307, + "end_time": "2024-03-24T03:52:24.738565", + "exception": false, + "start_time": "2024-03-24T03:52:22.713258", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:24.774162Z", + "iopub.status.busy": "2024-03-24T03:52:24.773653Z", + "iopub.status.idle": "2024-03-24T03:52:24.787027Z", + "shell.execute_reply": "2024-03-24T03:52:24.786075Z" + }, + "papermill": { + "duration": 0.029079, + "end_time": "2024-03-24T03:52:24.789233", + "exception": false, + "start_time": "2024-03-24T03:52:24.760154", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:24.814317Z", + "iopub.status.busy": "2024-03-24T03:52:24.813979Z", + "iopub.status.idle": "2024-03-24T03:52:24.821764Z", + "shell.execute_reply": "2024-03-24T03:52:24.820799Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.022496, + "end_time": "2024-03-24T03:52:24.823753", + "exception": false, + "start_time": "2024-03-24T03:52:24.801257", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:24.848221Z", + "iopub.status.busy": "2024-03-24T03:52:24.847474Z", + "iopub.status.idle": "2024-03-24T03:52:24.946969Z", + "shell.execute_reply": "2024-03-24T03:52:24.945975Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.114511, + "end_time": "2024-03-24T03:52:24.949504", + "exception": false, + "start_time": "2024-03-24T03:52:24.834993", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:24.976909Z", + "iopub.status.busy": "2024-03-24T03:52:24.976092Z", + "iopub.status.idle": "2024-03-24T03:52:29.751296Z", + "shell.execute_reply": "2024-03-24T03:52:29.750111Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.791354, + "end_time": "2024-03-24T03:52:29.753739", + "exception": false, + "start_time": "2024-03-24T03:52:24.962385", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-24 03:52:27.361338: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-24 03:52:27.361398: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-24 03:52:27.363100: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:29.778913Z", + "iopub.status.busy": "2024-03-24T03:52:29.777968Z", + "iopub.status.idle": "2024-03-24T03:52:29.783768Z", + "shell.execute_reply": "2024-03-24T03:52:29.782925Z" + }, + "papermill": { + "duration": 0.020125, + "end_time": "2024-03-24T03:52:29.785661", + "exception": false, + "start_time": "2024-03-24T03:52:29.765536", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:29.811634Z", + "iopub.status.busy": "2024-03-24T03:52:29.811363Z", + "iopub.status.idle": "2024-03-24T03:52:38.196253Z", + "shell.execute_reply": "2024-03-24T03:52:38.195215Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.40106, + "end_time": "2024-03-24T03:52:38.198807", + "exception": false, + "start_time": "2024-03-24T03:52:29.797747", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-24T03:52:38.227529Z", + "iopub.status.busy": "2024-03-24T03:52:38.226599Z", + "iopub.status.idle": "2024-03-24T03:52:38.234285Z", + "shell.execute_reply": "2024-03-24T03:52:38.233457Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.024299, + "end_time": "2024-03-24T03:52:38.236284", + "exception": false, + "start_time": "2024-03-24T03:52:38.211985", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 46,\n", + " 'realtabformer': (24, 72, Embedding(72, 672), True),\n", + " 'lct_gan': 40,\n", + " 'tab_ddpm_concat': 10}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:38.261575Z", + "iopub.status.busy": "2024-03-24T03:52:38.261306Z", + "iopub.status.idle": "2024-03-24T03:52:38.265800Z", + "shell.execute_reply": "2024-03-24T03:52:38.264944Z" + }, + "papermill": { + "duration": 0.01926, + "end_time": "2024-03-24T03:52:38.267590", + "exception": false, + "start_time": "2024-03-24T03:52:38.248330", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:38.292916Z", + "iopub.status.busy": "2024-03-24T03:52:38.292074Z", + "iopub.status.idle": "2024-03-24T03:52:38.814031Z", + "shell.execute_reply": "2024-03-24T03:52:38.812992Z" + }, + "papermill": { + "duration": 0.536746, + "end_time": "2024-03-24T03:52:38.816213", + "exception": false, + "start_time": "2024-03-24T03:52:38.279467", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/tvae/all inf False\n", + "../../../../ml-utility-loss/aug_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_bs_test/tvae/all inf False\n", + "../../../../ml-utility-loss/bs_test/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../contraceptive/_cache_synth_test/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/contraceptive [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:38.843542Z", + "iopub.status.busy": "2024-03-24T03:52:38.843234Z", + "iopub.status.idle": "2024-03-24T03:52:39.166434Z", + "shell.execute_reply": "2024-03-24T03:52:39.165569Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.339125, + "end_time": "2024-03-24T03:52:39.168588", + "exception": false, + "start_time": "2024-03-24T03:52:38.829463", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'loss_balancer_beta': 0.7999999999999999,\n", + " 'loss_balancer_r': 0.95,\n", + " 'tf_pma_low': 8,\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7,\n", + " 'bias_weight_decay': 0.05,\n", + " 'head_activation': torch.nn.modules.activation.ReLU6,\n", + " 'tf_activation': torch.nn.modules.activation.Tanh,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.07,\n", + " 'n_warmup_steps': 200,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'fixed_role_model': 'tvae',\n", + " 'g_loss_mul': 0.1,\n", + " 'd_model': 256,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 512,\n", + " 'ada_n_layers': 9,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 512,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'single_model': True,\n", + " 'models': ['tvae'],\n", + " 'max_seconds': 3600,\n", + " 'Body': 'twin_encoder',\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 128,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.5, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "# params = {\n", + "# **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "# }\n", + "params = getattr(PARAMS, dataset_name).BEST_DICT[gp][gp_multiply][single_model]\n", + "if isinstance(params, (list, tuple)):\n", + " params = params[param_index]\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:39.196741Z", + "iopub.status.busy": "2024-03-24T03:52:39.195984Z", + "iopub.status.idle": "2024-03-24T03:52:39.303252Z", + "shell.execute_reply": "2024-03-24T03:52:39.302115Z" + }, + "papermill": { + "duration": 0.12382, + "end_time": "2024-03-24T03:52:39.305523", + "exception": false, + "start_time": "2024-03-24T03:52:39.181703", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_train/tvae/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/contraceptive [400, 0]\n", + "Caching in ../../../../contraceptive/_cache_aug_val/tvae/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/contraceptive [0, 200]\n", + "Caching in ../../../../contraceptive/_cache_bs_train/tvae/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/contraceptive [100, 0]\n", + "Caching in ../../../../contraceptive/_cache_bs_val/tvae/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/contraceptive [0, 50]\n", + "Caching in ../../../../contraceptive/_cache_synth/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/contraceptive [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-24T03:52:39.335359Z", + "iopub.status.busy": "2024-03-24T03:52:39.334329Z", + "iopub.status.idle": "2024-03-24T03:52:39.765650Z", + "shell.execute_reply": "2024-03-24T03:52:39.764664Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.44856, + "end_time": "2024-03-24T03:52:39.767806", + "exception": false, + "start_time": "2024-03-24T03:52:39.319246", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['tvae'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:39.796226Z", + "iopub.status.busy": "2024-03-24T03:52:39.795851Z", + "iopub.status.idle": "2024-03-24T03:52:39.800934Z", + "shell.execute_reply": "2024-03-24T03:52:39.800035Z" + }, + "papermill": { + "duration": 0.021479, + "end_time": "2024-03-24T03:52:39.802768", + "exception": false, + "start_time": "2024-03-24T03:52:39.781289", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:39.829368Z", + "iopub.status.busy": "2024-03-24T03:52:39.828616Z", + "iopub.status.idle": "2024-03-24T03:52:39.835800Z", + "shell.execute_reply": "2024-03-24T03:52:39.834940Z" + }, + "papermill": { + "duration": 0.022397, + "end_time": "2024-03-24T03:52:39.837683", + "exception": false, + "start_time": "2024-03-24T03:52:39.815286", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "7608065" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:39.864099Z", + "iopub.status.busy": "2024-03-24T03:52:39.863512Z", + "iopub.status.idle": "2024-03-24T03:52:39.943403Z", + "shell.execute_reply": "2024-03-24T03:52:39.942525Z" + }, + "papermill": { + "duration": 0.095531, + "end_time": "2024-03-24T03:52:39.945552", + "exception": false, + "start_time": "2024-03-24T03:52:39.850021", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 46] --\n", + "├─Adapter: 1-1 [2, 1179, 46] --\n", + "│ └─Sequential: 2-1 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 512] 24,064\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 512] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 512] 262,656\n", + "│ │ │ └─Softsign: 4-16 [2, 1179, 512] --\n", + "│ │ └─FeedForward: 3-9 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-17 [2, 1179, 256] 131,328\n", + "│ │ │ └─LeakyHardsigmoid: 4-18 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 46] (recursive)\n", + "│ └─Sequential: 2-2 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-10 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-32 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-17 [2, 294, 512] (recursive)\n", + "│ │ │ └─Linear: 4-33 [2, 294, 512] (recursive)\n", + "│ │ │ └─Softsign: 4-34 [2, 294, 512] --\n", + "│ │ └─FeedForward: 3-18 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-36 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 2048] --\n", + "│ └─Encoder: 2-3 [2, 8, 256] --\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-6 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1179, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-12 [2, 1179, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-18 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1179, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-24 [2, 1179, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─Tanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-39 [2, 8, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 128, 256] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 128, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 128, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 128, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-30 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 128, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1179, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1179, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-36 [2, 1179, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 8, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 8, 256] 2,048\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 8, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 8, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 8, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 8, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 8, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-42 [2, 8, 256] --\n", + "│ └─Encoder: 2-4 [2, 8, 256] (recursive)\n", + "│ │ └─ModuleList: 3-20 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-40 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-8 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-21 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-22 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-48 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-23 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 294, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-54 [2, 294, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-25 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-60 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 294, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-66 [2, 294, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─Tanh: 6-31 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-42 [2, 8, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 128, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 128, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-72 [2, 128, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 128, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 294, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 294, 128] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-78 [2, 294, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-14 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-40 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 8, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 8, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-84 [2, 8, 256] --\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-21 [2, 512] --\n", + "│ │ │ └─Linear: 4-43 [2, 512] 1,049,088\n", + "│ │ │ └─ReLU6: 4-44 [2, 512] --\n", + "│ │ └─FeedForward: 3-22 [2, 512] --\n", + "│ │ │ └─Linear: 4-45 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-46 [2, 512] --\n", + "│ │ └─FeedForward: 3-23 [2, 512] --\n", + "│ │ │ └─Linear: 4-47 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-48 [2, 512] --\n", + "│ │ └─FeedForward: 3-24 [2, 512] --\n", + "│ │ │ └─Linear: 4-49 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-50 [2, 512] --\n", + "│ │ └─FeedForward: 3-25 [2, 512] --\n", + "│ │ │ └─Linear: 4-51 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-52 [2, 512] --\n", + "│ │ └─FeedForward: 3-26 [2, 512] --\n", + "│ │ │ └─Linear: 4-53 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-54 [2, 512] --\n", + "│ │ └─FeedForward: 3-27 [2, 512] --\n", + "│ │ │ └─Linear: 4-55 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-56 [2, 512] --\n", + "│ │ └─FeedForward: 3-28 [2, 512] --\n", + "│ │ │ └─Linear: 4-57 [2, 512] 262,656\n", + "│ │ │ └─ReLU6: 4-58 [2, 512] --\n", + "│ │ └─FeedForward: 3-29 [2, 1] --\n", + "│ │ │ └─Linear: 4-59 [2, 1] 513\n", + "│ │ │ └─LeakyHardsigmoid: 4-60 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 7,608,065\n", + "Trainable params: 7,608,065\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 24.25\n", + "========================================================================================================================\n", + "Input size (MB): 0.54\n", + "Forward/backward pass size (MB): 257.33\n", + "Params size (MB): 30.43\n", + "Estimated Total Size (MB): 288.30\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:52:39.975551Z", + "iopub.status.busy": "2024-03-24T03:52:39.974781Z", + "iopub.status.idle": "2024-03-24T04:58:47.719344Z", + "shell.execute_reply": "2024-03-24T04:58:47.717981Z" + }, + "papermill": { + "duration": 3967.777203, + "end_time": "2024-03-24T04:58:47.736984", + "exception": false, + "start_time": "2024-03-24T03:52:39.959781", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0186110673548602, 'avg_role_model_std_loss': 0.9859759064087645, 'avg_role_model_mean_pred_loss': 0.0014670658553136355, 'avg_role_model_g_mag_loss': 0.12243372971605923, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.018859607335697446, 'n_size': 900, 'n_batch': 225, 'duration': 282.9204273223877, 'duration_batch': 1.257424121432834, 'duration_size': 0.3143560303582085, 'avg_pred_std': 0.11435848254969136}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008952803679017556, 'avg_role_model_std_loss': 4.103192632228183, 'avg_role_model_mean_pred_loss': 0.00013674774596836414, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008952803679017556, 'n_size': 450, 'n_batch': 113, 'duration': 94.89591884613037, 'duration_batch': 0.8397868924436316, 'duration_size': 0.21087981965806749, 'avg_pred_std': 0.022598956262412588}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005419754103058949, 'avg_role_model_std_loss': 0.7368064368608555, 'avg_role_model_mean_pred_loss': 6.596730340261968e-05, 'avg_role_model_g_mag_loss': 0.0279144444456324, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005536770965490077, 'n_size': 900, 'n_batch': 225, 'duration': 283.5930871963501, 'duration_batch': 1.2604137208726671, 'duration_size': 0.3151034302181668, 'avg_pred_std': 0.09610795313699377}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.009089643146014875, 'avg_role_model_std_loss': 2.2618643136544057, 'avg_role_model_mean_pred_loss': 0.00020205410188401396, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009089643146014875, 'n_size': 450, 'n_batch': 113, 'duration': 96.57411861419678, 'duration_batch': 0.8546382178247502, 'duration_size': 0.21460915247599283, 'avg_pred_std': 0.05953517459001726}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0041141705360819795, 'avg_role_model_std_loss': 0.4941683148547074, 'avg_role_model_mean_pred_loss': 5.9163905845158306e-05, 'avg_role_model_g_mag_loss': 0.01810061747660964, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004181610228713706, 'n_size': 900, 'n_batch': 225, 'duration': 290.8302204608917, 'duration_batch': 1.2925787576039631, 'duration_size': 0.3231446894009908, 'avg_pred_std': 0.09772732237146961}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002547107368138515, 'avg_role_model_std_loss': 1.7669729735376936, 'avg_role_model_mean_pred_loss': 7.385972450104247e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002547107368138515, 'n_size': 450, 'n_batch': 113, 'duration': 96.63251686096191, 'duration_batch': 0.8551550164686895, 'duration_size': 0.21473892635769315, 'avg_pred_std': 0.0506768064260219}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0023371195333958088, 'avg_role_model_std_loss': 0.4912818201007985, 'avg_role_model_mean_pred_loss': 5.350784180537504e-06, 'avg_role_model_g_mag_loss': 0.014647599203356853, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002370942204983698, 'n_size': 900, 'n_batch': 225, 'duration': 283.96464467048645, 'duration_batch': 1.2620650874243842, 'duration_size': 0.31551627185609604, 'avg_pred_std': 0.10408781512019535}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0034601237480011253, 'avg_role_model_std_loss': 2.612866154395119, 'avg_role_model_mean_pred_loss': 1.3134922784065689e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0034601237480011253, 'n_size': 450, 'n_batch': 113, 'duration': 94.63316321372986, 'duration_batch': 0.8374616213604412, 'duration_size': 0.21029591825273303, 'avg_pred_std': 0.04398040732047397}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0020195719559625205, 'avg_role_model_std_loss': 0.2510072517482392, 'avg_role_model_mean_pred_loss': 6.499685870238886e-06, 'avg_role_model_g_mag_loss': 0.013006549620121304, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002048346129207251, 'n_size': 900, 'n_batch': 225, 'duration': 285.49330401420593, 'duration_batch': 1.2688591289520263, 'duration_size': 0.3172147822380066, 'avg_pred_std': 0.10502510418287582}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023336571037200176, 'avg_role_model_std_loss': 1.0392825815654005, 'avg_role_model_mean_pred_loss': 7.885705093391342e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023336571037200176, 'n_size': 450, 'n_batch': 113, 'duration': 95.2993221282959, 'duration_batch': 0.8433568329937691, 'duration_size': 0.2117762713962131, 'avg_pred_std': 0.05800044378760246}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0019015362533779505, 'avg_role_model_std_loss': 0.24092813788177259, 'avg_role_model_mean_pred_loss': 5.394619782389408e-06, 'avg_role_model_g_mag_loss': 0.014597136508156028, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019251112584364213, 'n_size': 900, 'n_batch': 225, 'duration': 285.9157590866089, 'duration_batch': 1.2707367070515951, 'duration_size': 0.3176841767628988, 'avg_pred_std': 0.10497349073075586}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023449724944657646, 'avg_role_model_std_loss': 1.324564782714674, 'avg_role_model_mean_pred_loss': 8.32234693904156e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023449724944657646, 'n_size': 450, 'n_batch': 113, 'duration': 95.89643502235413, 'duration_batch': 0.8486410178969391, 'duration_size': 0.21310318893856472, 'avg_pred_std': 0.059179596025476115}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001617696879960325, 'avg_role_model_std_loss': 0.15725728821715396, 'avg_role_model_mean_pred_loss': 4.0012195520334405e-06, 'avg_role_model_g_mag_loss': 0.013585555124898544, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016441765983391734, 'n_size': 900, 'n_batch': 225, 'duration': 283.3449795246124, 'duration_batch': 1.2593110201093887, 'duration_size': 0.31482775502734717, 'avg_pred_std': 0.10613164186891583}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002407976164379054, 'avg_role_model_std_loss': 1.1220077265352362, 'avg_role_model_mean_pred_loss': 7.455731951812162e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002407976164379054, 'n_size': 450, 'n_batch': 113, 'duration': 94.69690179824829, 'duration_batch': 0.8380256796305159, 'duration_size': 0.21043755955166288, 'avg_pred_std': 0.048071262323651956}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0014774360275219402, 'avg_role_model_std_loss': 0.16173793539400327, 'avg_role_model_mean_pred_loss': 3.2265756165433642e-06, 'avg_role_model_g_mag_loss': 0.013837635460464905, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001494622107436751, 'n_size': 900, 'n_batch': 225, 'duration': 285.05916833877563, 'duration_batch': 1.266929637061225, 'duration_size': 0.31673240926530627, 'avg_pred_std': 0.10432403838468922}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002109985634936796, 'avg_role_model_std_loss': 1.285013097416944, 'avg_role_model_mean_pred_loss': 5.779796396621073e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002109985634936796, 'n_size': 450, 'n_batch': 113, 'duration': 95.49577188491821, 'duration_batch': 0.8450953264152055, 'duration_size': 0.21221282641092937, 'avg_pred_std': 0.059411970754277656}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013566906487297577, 'avg_role_model_std_loss': 0.1447534780750981, 'avg_role_model_mean_pred_loss': 1.6995160905337986e-06, 'avg_role_model_g_mag_loss': 0.014613873372133336, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013734321816850246, 'n_size': 900, 'n_batch': 225, 'duration': 284.22162222862244, 'duration_batch': 1.2632072099049887, 'duration_size': 0.3158018024762472, 'avg_pred_std': 0.10500639389579494}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0019757018623446734, 'avg_role_model_std_loss': 1.1056950748967773, 'avg_role_model_mean_pred_loss': 7.016759447038556e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019757018623446734, 'n_size': 450, 'n_batch': 113, 'duration': 95.37925219535828, 'duration_batch': 0.8440641787199847, 'duration_size': 0.21195389376746285, 'avg_pred_std': 0.05243083165208521}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0011172247426859232, 'avg_role_model_std_loss': 0.1346889167763815, 'avg_role_model_mean_pred_loss': 1.8536731677009785e-06, 'avg_role_model_g_mag_loss': 0.013761755622625868, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011291193025641001, 'n_size': 900, 'n_batch': 225, 'duration': 284.7550492286682, 'duration_batch': 1.2655779965718588, 'duration_size': 0.3163944991429647, 'avg_pred_std': 0.10835976392237677}\n", + "Time out: 3719.8355207443237/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tvae', 'n_size': 1050, 'n_batch': 263, 'role_model_metrics': {'avg_loss': 0.0014795975680352144, 'avg_g_mag_loss': 0.030702529888277683, 'avg_g_cos_loss': 0.008277697899691357, 'pred_duration': 4.806691408157349, 'grad_duration': 2.9375720024108887, 'total_duration': 7.744263410568237, 'pred_std': 0.10091648250818253, 'std_loss': 0.008207244798541069, 'mean_pred_loss': 2.38593247559038e-06, 'pred_rmse': 0.03846553713083267, 'pred_mae': 0.029553567990660667, 'pred_mape': 0.09155531227588654, 'grad_rmse': 0.02409490942955017, 'grad_mae': 0.013917120173573494, 'grad_mape': 0.45717620849609375}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0014795975680352144, 'avg_g_mag_loss': 0.030702529888277683, 'avg_g_cos_loss': 0.008277697899691357, 'avg_pred_duration': 4.806691408157349, 'avg_grad_duration': 2.9375720024108887, 'avg_total_duration': 7.744263410568237, 'avg_pred_std': 0.10091648250818253, 'avg_std_loss': 0.008207244798541069, 'avg_mean_pred_loss': 2.38593247559038e-06}, 'min_metrics': {'avg_loss': 0.0014795975680352144, 'avg_g_mag_loss': 0.030702529888277683, 'avg_g_cos_loss': 0.008277697899691357, 'pred_duration': 4.806691408157349, 'grad_duration': 2.9375720024108887, 'total_duration': 7.744263410568237, 'pred_std': 0.10091648250818253, 'std_loss': 0.008207244798541069, 'mean_pred_loss': 2.38593247559038e-06, 'pred_rmse': 0.03846553713083267, 'pred_mae': 0.029553567990660667, 'pred_mape': 0.09155531227588654, 'grad_rmse': 0.02409490942955017, 'grad_mae': 0.013917120173573494, 'grad_mape': 0.45717620849609375}, 'model_metrics': {'tvae': {'avg_loss': 0.0014795975680352144, 'avg_g_mag_loss': 0.030702529888277683, 'avg_g_cos_loss': 0.008277697899691357, 'pred_duration': 4.806691408157349, 'grad_duration': 2.9375720024108887, 'total_duration': 7.744263410568237, 'pred_std': 0.10091648250818253, 'std_loss': 0.008207244798541069, 'mean_pred_loss': 2.38593247559038e-06, 'pred_rmse': 0.03846553713083267, 'pred_mae': 0.029553567990660667, 'pred_mape': 0.09155531227588654, 'grad_rmse': 0.02409490942955017, 'grad_mae': 0.013917120173573494, 'grad_mape': 0.45717620849609375}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T04:58:47.781597Z", + "iopub.status.busy": "2024-03-24T04:58:47.781115Z", + "iopub.status.idle": "2024-03-24T04:58:47.786340Z", + "shell.execute_reply": "2024-03-24T04:58:47.785305Z" + }, + "papermill": { + "duration": 0.026332, + "end_time": "2024-03-24T04:58:47.788412", + "exception": false, + "start_time": "2024-03-24T04:58:47.762080", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T04:58:47.819371Z", + "iopub.status.busy": "2024-03-24T04:58:47.819109Z", + "iopub.status.idle": "2024-03-24T04:58:47.895456Z", + "shell.execute_reply": "2024-03-24T04:58:47.894437Z" + }, + "papermill": { + "duration": 0.094592, + "end_time": "2024-03-24T04:58:47.897905", + "exception": false, + "start_time": "2024-03-24T04:58:47.803313", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T04:58:47.935107Z", + "iopub.status.busy": "2024-03-24T04:58:47.934691Z", + "iopub.status.idle": "2024-03-24T04:58:48.244700Z", + "shell.execute_reply": "2024-03-24T04:58:48.243579Z" + }, + "papermill": { + "duration": 0.332463, + "end_time": "2024-03-24T04:58:48.247096", + "exception": false, + "start_time": "2024-03-24T04:58:47.914633", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATYAAAESCAYAAAB3gfmJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABFGElEQVR4nO3deVxU9f4/8NfMMBvbgGwDioAJYopiGBNIekuuZJRRll2vV81MrbA0ssxSKb2J6+9a6s028fYtQu2qmZqFpHUT3BDccENBVBZFhWGdgZnP749hjowMywwDw/J+Ph4nznzO55zzOcS8Ped8Nh5jjIEQQroRvrULQAghlkaBjRDS7VBgI4R0OxTYCCHdDgU2Qki3Q4GNENLtUGAjhHQ7NtYuQGei1WpRUFAABwcH8Hg8axeHENIAYwzl5eXw8vICn9/8PRkFtgYKCgrg7e1t7WIQQppx7do19OnTp9k8FNgacHBwAKD7xTk6Olq5NISQhpRKJby9vbnvaXMosDWgf/x0dHSkwEZIJ9Wa10RUeUAI6XYosBFCuh0KbISQbofesRGL0Gq1UKvV1i4G6eKEQiEEAkGbj0OBjbSZWq1Gbm4utFqttYtCugEnJyfI5fI2tSWlwEbahDGGwsJCCAQCeHt7t9hwkpCmMMZQVVWFmzdvAgA8PT3NPhYFNjPcKlfhXKEScpkEAR4tt6npzurq6lBVVQUvLy/Y2tpauziki5NKpQCAmzdvwt3d3ezHUvrn1Qz/PpiDKZuOYtvxa9YuitVpNBoAgEgksnJJSHeh/weytrbW7GNQYDODv7vuLu1icYWVS9J5UN9aYimW+FuiwGaGAA97AEDOTQpshHRGFNjM0N9dF9hulFajQlVn5dIQQu5Hgc0MTrYiuDmIAdBdGzEfj8fDzp07rV0Mi/rwww8RHBxs7WJQYDOXf/1d26XiciuXhBDzbd68GU5OThY73rx585Cammqx45mLApuZ9M086I6N9ASt7VVib28PFxeXdi5NyyiwmUn/nu0SBTYDjDFUqeussjDGTCrrvn37EBERAScnJ7i4uOCpp57C5cuXAQDh4eGYP3++Qf5bt25BKBTijz/+AAAUFhYiOjoaUqkUfn5+SEpKgq+vL9auXWvW7+706dN4/PHHIZVK4eLigpkzZ6Ki4t7f18GDBxEaGgo7Ozs4OTlhxIgRuHr1KgDg5MmTeOyxx+Dg4ABHR0eEhITg+PHjzZ7v4MGDmDZtGsrKysDj8cDj8fDhhx8CAHx9fbF06VJMmTIFjo6OmDlzJgBg/vz5CAgIgK2tLfr164dFixYZNMu4/1H0pZdeQkxMDFavXg1PT0+4uLggNja2TU05WoMa6JpJ/yh6kR5FDVTXavDg4l+scu7sJVGwFbX+T7qyshJxcXEYMmQIKioqsHjxYjz77LPIysrCpEmTsHLlSixfvpxrfrBlyxZ4eXnh0UcfBQBMmTIFJSUlOHjwIIRCIeLi4rhW86aqrKxEVFQUwsLCcOzYMdy8eROvvPIKZs+ejc2bN6Ourg4xMTGYMWMGvv/+e6jVahw9epQr26RJkzBs2DB89tlnEAgEyMrKglAobPac4eHhWLt2LRYvXowLFy4A0N1x6a1evRqLFy9GfHw8l+bg4IDNmzfDy8sLp0+fxowZM+Dg4IB33323yfMcOHAAnp6eOHDgAHJycvDiiy8iODgYM2bMMOt31RoU2MzkX/8oev1uNarUdSZ9oUjnMH78eIPPmzZtgpubG7KzszFhwgTMnTsXf/75JxfIkpKSMHHiRPB4PJw/fx779+/HsWPHMHz4cADAV199BX9/f7PKkpSUhJqaGnzzzTews7MDAKxfvx5PP/00VqxYAaFQiLKyMjz11FN44IEHAAADBw7k9s/Pz8c777yDwMBAAGhVOUQiEWQyGXg8HuRyeaPtjz/+ON5++22DtIULF3Lrvr6+mDdvHpKTk5sNbM7Ozli/fj0EAgECAwMRHR2N1NRUCmydUS87EVztRSipUOPyzUoE9ZFZu0idglQoQPaSKKud2xSXLl3C4sWLceTIEZSUlHCd+PPz8zF48GCMGTMG3333HR599FHk5uYiPT0dn3/+OQDgwoULsLGxwUMPPcQdr3///nB2djar7OfOncPQoUO5oAYAI0aMgFarxYULFzBy5Ei89NJLiIqKwl//+ldERkZiwoQJXH/KuLg4vPLKK/i///s/REZG4oUXXuACoLn0AbuhLVu24NNPP8Xly5dRUVGBurq6FkebHjRokEHXKE9PT5w+fbpNZWsJvWNrg3vv2ehxVI/H48FWZGOVxdQW608//TTu3LmDL7/8EkeOHMGRI0cA3HtRPmnSJPzwww+ora1FUlISgoKCEBQUZPHfWWslJiYiPT0d4eHh2LJlCwICAnD48GEAundbZ8+eRXR0NH777Tc8+OCD2LFjR5vO1zDIAkB6ejomTZqEJ598Ert370ZmZiY++OCDFisW7n8k5vF47T4SDAW2NqCuVV3X7du3ceHCBSxcuBCjR4/GwIEDcffuXYM8zzzzDGpqarBv3z4kJSVh0qRJ3LYBAwagrq4OmZmZXFpOTk6jY7TWwIEDcfLkSVRWVnJphw4dAp/Px4ABA7i0YcOGYcGCBUhLS8PgwYORlJTEbQsICMBbb72FX3/9Fc899xwSExNbPK9IJOL6+7YkLS0NPj4++OCDDzB8+HD4+/tzlRedjVmBbcOGDfD19YVEIoFCocDRo0ebzb9t2zYEBgZCIpEgKCgIe/fuNdi+fft2jBkzBi4uLuDxeMjKyjLYnpeXx9Xa3L9s27aNy2dse3JysjmX2Cr+XNcqumPrapydneHi4oIvvvgCOTk5+O233xAXF2eQx87ODjExMVi0aBHOnTuHiRMnctsCAwMRGRmJmTNn4ujRo8jMzMTMmTMhlUrN6us4adIkSCQSTJ06FWfOnMGBAwfwxhtvYPLkyfDw8EBubi4WLFiA9PR0XL16Fb/++isuXbqEgQMHorq6GrNnz8bBgwdx9epVHDp0CMeOHTN4B9cUX19fVFRUIDU1FSUlJaiqqmoyr7+/P/Lz85GcnIzLly/j008/bfNdYXsxObBt2bIFcXFxiI+Px4kTJzB06FBERUU1WRuUlpaGiRMnYvr06cjMzERMTAxiYmJw5swZLk9lZSUiIiKwYsUKo8fw9vZGYWGhwfLRRx/B3t4eY8eONcibmJhokC8mJsbUS2w1/R0bNfnoevh8PpKTk5GRkYHBgwfjrbfewqpVqxrlmzRpEk6ePIlHH30Uffv2Ndj2zTffwMPDAyNHjsSzzz7L1RBKJBKTy2Nra4tffvkFd+7cwcMPP4znn38eo0ePxvr167nt58+fx/jx4xEQEICZM2ciNjYWs2bNgkAgwO3btzFlyhQEBARgwoQJGDt2LD766KMWzxseHo5XX30VL774Itzc3LBy5com844bNw5vvfUWZs+ejeDgYKSlpWHRokUmX2uHYCYKDQ1lsbGx3GeNRsO8vLxYQkKC0fwTJkxg0dHRBmkKhYLNmjWrUd7c3FwGgGVmZrZYjuDgYPbyyy8bpAFgO3bsaPkimlBWVsYAsLKyslblv1Vew3zm72a+7+1mVao6s8/blVVXV7Ps7GxWXV1t7aJY3bVr1xgAtn//fmsXpUtr6m/KlO+nSXdsarUaGRkZiIyM5NL4fD4iIyORnp5udJ/09HSD/AAQFRXVZP7WyMjIQFZWFqZPn95oW2xsLFxdXREaGopNmzY122hTpVJBqVQaLKZwsRPB2VYIxoDLt+iuraf57bffsGvXLuTm5iItLQ1/+9vf4Ovri5EjR1q7aD2eSYGtpKQEGo0GHh4eBukeHh4oKioyuk9RUZFJ+Vvj66+/xsCBAxEeHm6QvmTJEmzduhUpKSkYP348Xn/9daxbt67J4yQkJEAmk3GLt7e3SeXg8XhcezbqWtXz1NbW4v3338egQYPw7LPPws3NjWus+91338He3t7oMmjQoA4r49ixY5ssx7JlyzqsHB2ty7Vjq66uRlJSktFn+4Zpw4YNQ2VlJVatWoU333zT6LEWLFhg8MJYqVSaHNz83e1xNPcONfnogaKiohAVZbzN3rhx46BQKIxua6lHgCV99dVXqK6uNrqtV69eHVaOjmZSYHN1dYVAIEBxcbFBenFxsdGWywAgl8tNyt+SH374AVVVVZgyZUqLeRUKBZYuXQqVSgWxWNxou1gsNppuintdq+iOjdzj4OAABwfrz4fRu3dvaxfBKkx6FBWJRAgJCTEYlkSr1SI1NRVhYWFG9wkLC2s0jElKSkqT+Vvy9ddfY9y4cXBzc2sxb1ZWFpydndscvJpDj6KEdD4mP4rGxcVh6tSpGD58OEJDQ7F27VpUVlZi2rRpAHQdg3v37o2EhAQAwJw5czBq1CisWbMG0dHRSE5OxvHjx/HFF19wx7xz5w7y8/NRUFAAAFyHXLlcbnBnl5OTgz/++KNROzgA+Omnn1BcXIxHHnkEEokEKSkpWLZsGebNm2fqJZpE35bt6u1K1NRqIDGxWw8hpB2YUx27bt061rdvXyYSiVhoaCg7fPgwt23UqFFs6tSpBvm3bt3KAgICmEgkYoMGDWJ79uwx2J6YmMgANFri4+MN8i1YsIB5e3szjUbTqEw///wzCw4OZvb29szOzo4NHTqUbdy40Wjeppja3IMxxrRaLRvy4S/MZ/5ull3Q+v26C2ruQSzNEs09eIyZOIhVN6ZUKiGTyVBWVtZix96Gnv8sDcev3sUnfwvGM8E9651GTU0NcnNz4efnZ1bDVELu19TflCnfT+oragH+NGsVIZ0KBTYL4LpWUc0oMUF3nMyls6DAZgH6OzZqy0a6GktP5gLohhzn8XgoLS216HFNQYHNAvR3bHm3q6Cqa90QMISQ9kOBzQI8HMVwkNhAo2XIK2l62JcegTFAXWmdhSZzsdhkLiqVCvPmzUPv3r1hZ2cHhUKBgwcPcvtevXoVTz/9NJydnWFnZ4dBgwZh7969yMvLw2OPPQZANzQUj8fDSy+9ZNbvoy26XJeqzojH48Hf3R4n8ktx6WY5Bsit3+LcamqrgGVe1jn3+wWAyK7lfPVoMpemJ3OZPXs2srOzkZycDC8vL+zYsQNPPPEETp8+DX9/f8TGxkKtVuOPP/6AnZ0dsrOzYW9vD29vb/z3v//F+PHjceHCBTg6OkIqlZr1O2kLCmwW4u/ugBP5pdS1qguhyVyMT+aSn5+PxMRE5Ofnw8tL94/UvHnzsG/fPiQmJmLZsmXIz8/H+PHjuaHS+/Xrx+2v74Pq7u5u8fd3rUWBzUJoNN16QlvdnZO1zm0CmszFuNOnT0Oj0SAgIMAgXaVScZMhv/nmm3jttdfw66+/IjIyEuPHj8eQIUPMOl97oHdsFqLvM9rjm3zweLrHQWssNJmLRSZzqaiogEAg4MY91C/nzp3DJ598AgB45ZVXcOXKFUyePBmnT5/G8OHDmx0irKNRYLMQ/SgfuSWVqNW07ww8pO1oMhcdY5O5DBs2DBqNBjdv3kT//v0NloaPrN7e3nj11Vexfft2vP322/jyyy+5YwJo9SQx7YECm4V4yiSwEwlQp2XIK6lseQdiVTSZi46xyVwCAgIwadIkTJkyBdu3b0dubi6OHj2KhIQE7NmzBwAwd+5c/PLLL8jNzcWJEydw4MAB7nw+Pj7g8XjYvXs3bt26ZVCz22Hapxtr12ROJ/iGxq3/k/nM3832nCqwcMk6r67cCT4lJYUNHDiQicViNmTIEHbw4MFG82bs3buXAWAjR45stH9BQQEbO3YsE4vFzMfHhyUlJTF3d3e2cePGVp3//nOdOnWKPfbYY0wikbBevXqxGTNmsPLycsYYY0VFRSwmJoZ5enoykUjEfHx82OLFi5lGo2EqlYr97W9/Y97e3kwkEjEvLy82e/bsVv8/efXVV5mLi4vBwBNqtZotXryY+fr6MqFQyDw9Pdmzzz7LTp06xRhjbPbs2eyBBx5gYrGYubm5scmTJ7OSkhLumEuWLGFyuZzxeLxGg2K0hDrBW5i5neD13tl2EtsyruOtyADMiTSvdqyroU7w91y/fh3e3t7Yv38/Ro8ebe3idFmW6ARPtaIWRF2repbffvsNFRUVCAoKQmFhId59912azKWToHdsFkSd4XsWmsyl86I7NgvS37FdKalAnUYLGwH9u9Gd0WQunRcFNgvykklhKxKgSq3B1TtVeMDN3tpFIlZCk7lYF91SWBCfz0P/+vZsl4p71ns2qoMilmKJvyUKbBZ2L7D1jPdsAoFu8hp9a31C2qqqSjdCTlse2c16FN2wYQNWrVqFoqIiDB06FOvWrUNoaGiT+bdt24ZFixYhLy8P/v7+WLFiBZ588klu+/bt27Fx40ZkZGTgzp07yMzMRHBwsMEx/vKXv+D33383SJs1axY2btzIfc7Pz8drr72GAwcOwN7eHlOnTkVCQgJsbDruiTtA37WqhwwTbmNjA1tbW25IHz6f/q0k5mGMoaqqCjdv3oSTkxP3j6Y5TP7Gb9myBXFxcdi4cSMUCgXWrl2LqKgoXLhwAe7u7o3yp6WlYeLEiUhISMBTTz2FpKQkxMTE4MSJExg8eDAA3ZAtERERmDBhAmbMmNHkuWfMmIElS5Zwn21t73V61mg0iI6OhlwuR1paGgoLCzFlyhQIhcIOrf3Rd63qKYGNx+PB09MTubm53NhghLSFk5OT2ROq65ncQFehUODhhx/G+vXrAegmTPb29sYbb7yB9957r1H+F198EZWVldi9ezeX9sgjjyA4ONjgbgsA8vLy4Ofn1+QdW3BwcJOD+P3888946qmnUFBQAA8PDwDAxo0bMX/+fNy6dYvrv9actjbQBYD821UYueoARDZ8nFvyBAR807vXdEVarZYeR0mbCYXCJu/U2q2BrlqtRkZGBhYsWMCl8fl8REZGIj093eg+6enpjfrgRUVFmTWJxXfffYdvv/0WcrkcTz/9NBYtWsTdtaWnpyMoKIgLavrzvPbaazh79iyGDRvW6HgqlQoqlYr7rFQqTS7T/fo4SyER8lFTq0X+nSr4ubZ+4MOujM/n9/ieB6TzMCmwlZSUQKPRGAQPAPDw8MD58+eN7lNUVGQ0f1FRkUkF/fvf/w4fHx94eXnh1KlTmD9/Pi5cuIDt27c3ex79NmMSEhLw0UcfmVSOluhrRs/cUOJScXmPCWyEdCZdph3bzJkzufWgoCB4enpi9OjRuHz5stkD6i1YsMDgblKpVMLb27vNZfV3d9AFtpsVGNNxjcwJIfVMqsJydXWFQCBAcXGxQXpxcXGTL/vkcrlJ+VtL36o7Jyen2fPotxkjFovh6OhosFhCT23LRkhnYVJgE4lECAkJQWpqKpem1WqRmpqKsLAwo/uEhYUZ5AeAlJSUJvO3VlZWFgBwQyOHhYXh9OnTBpNppKSkwNHREQ8++GCbzmWqntbkg5DOxuRH0bi4OEydOhXDhw9HaGgo1q5di8rKSkybNg2Abuae3r17IyEhAQAwZ84cjBo1CmvWrEF0dDSSk5Nx/PhxfPHFF9wx79y5g/z8fBQU6MbK18+YI5fLIZfLcfnyZSQlJeHJJ5+Ei4sLTp06hbfeegsjR47kxlkfM2YMHnzwQUyePBkrV65EUVERFi5ciNjYWIjF4rb9lkykb/KRc7MCGi3rMTWjhHQaJo0AV2/dunWsb9++TCQSsdDQUHb48GFu26hRoxoNLLd161YWEBDARCIRGzRoENuzZ4/B9sTERAag0aIf9C4/P5+NHDmS9erVi4nFYta/f3/2zjvvNBpwLi8vj40dO5ZJpVLm6urK3n77bVZbW9vq62rrQJN6dRot8/9gL/OZv5vllVS06ViEEB0aaNJMlmjHpjf2k//hXKESX00ZjsgHPVregRDSLFO+n9T/pZ0EePSsHgiEdCYU2NrJva5VVDNKSEejwNZO+tNouoRYDQW2dnJvZvgKaLX0GpOQjkSBrZ349LKFSMBHda0GN0qND81MCGkfFNjaiY2Aj35uun6iOVSBQEiHosDWjvRdqy5S1ypCOhQFtnZEXasIsQ4KbO2op42mS0hnQYGtHXE1o8XlNIsTIR2IAls78nGxg1DAQ6Vag4KyGmsXh5AegwJbOxIK+NwIujQ2GyEdhwJbO/Ov74FATT4I6TgU2NoZNfkgpONRYGtn/jTKByEdjgJbO9O3ZcsprqCaUUI6CAW2dubrYgcBn4dyVR2KlaqWdyCEtBkFtnYmsuHD10U3qTO9ZyOkY1Bg6wDUtYqQjmVWYNuwYQN8fX0hkUigUChw9OjRZvNv27YNgYGBkEgkCAoKwt69ew22b9++HWPGjIGLiwt4PB43tZ7enTt38MYbb2DAgAGQSqXo27cv3nzzTZSVlRnk4/F4jZbk5GRzLtGi7s1aRXdshHQEkwPbli1bEBcXh/j4eJw4cQJDhw5FVFSUwXyeDaWlpWHixImYPn06MjMzERMTg5iYGJw5c4bLU1lZiYiICKxYscLoMQoKClBQUIDVq1fjzJkz2Lx5M/bt24fp06c3ypuYmIjCwkJuiYmJMfUSLa5//R3bRRpNl5COYeoUWKGhoSw2Npb7rNFomJeXF0tISDCaf8KECSw6OtogTaFQsFmzZjXKm5ubywCwzMzMFsuxdetWJhKJDKbXA8B27NjRugsxwlLT790vu6CM+czfzYLi9zGtVmvRYxPSU5jy/TTpjk2tViMjIwORkZFcGp/PR2RkJNLT043uk56ebpAfAKKioprM31r6KbhsbAznfI6NjYWrqytCQ0OxadOmZptYqFQqKJVKg6U99HOzA58HKGvqcKucakYJaW8mBbaSkhJoNBp4eBjOk+nh4YGioiKj+xQVFZmUv7XlWLp0KWbOnGmQvmTJEmzduhUpKSkYP348Xn/9daxbt67J4yQkJEAmk3GLt7e32WVqjthGAF+X+j6jVIFASLuzaTlL56JUKhEdHY0HH3wQH374ocG2RYsWcevDhg1DZWUlVq1ahTfffNPosRYsWIC4uDiDY7dXcOvvbo8rJZW4WFyOEf1d2+UchBAdk+7YXF1dIRAIUFxcbJBeXFwMuVxudB+5XG5S/uaUl5fjiSeegIODA3bs2AGhUNhsfoVCgevXr0OlMv74JxaL4ejoaLC0F+paRUjHMSmwiUQihISEIDU1lUvTarVITU1FWFiY0X3CwsIM8gNASkpKk/mbolQqMWbMGIhEIuzatQsSiaTFfbKysuDs7AyxWGzSudpDw65VhJD2ZfKjaFxcHKZOnYrhw4cjNDQUa9euRWVlJaZNmwYAmDJlCnr37o2EhAQAwJw5czBq1CisWbMG0dHRSE5OxvHjx/HFF19wx7xz5w7y8/NRUFAAALhw4QIA3d2eXC7nglpVVRW+/fZbgxf9bm5uEAgE+Omnn1BcXIxHHnkEEokEKSkpWLZsGebNm9e235CFcKN83NSNpsvj8axcIkK6MXOqXdetW8f69u3LRCIRCw0NZYcPH+a2jRo1ik2dOtUg/9atW1lAQAATiURs0KBBbM+ePQbbExMTGYBGS3x8PGOMsQMHDhjdDoDl5uYyxhj7+eefWXBwMLO3t2d2dnZs6NChbOPGjUyj0bT6utqruQdjjFWr65jve7uZz/zd7KayxuLHJ6S7M+X7yWOMhpzQUyqVkMlkXFMSSxu16gCu3q5C0gwFwh+gCgRCTGHK95P6inYgGk2XkI5Bga0DcTWjVIFASLuiwNaB/GmYcEI6BAW2DsQ1+aBHUULaFQW2DvSAmz14POB2pRq3K6jPKCHthQJbB5KKBOjjLAVAPRAIaU8U2DqYvmaUAhsh7YcCWwfT14zmUAUCIe2GAlsHozs2QtofBbYOdq/JBwU2QtoLBbYOpu8MX1Khwt1KtZVLQ0j3RIGtg9mJbdDbSVczmnOL7toIaQ8U2KyAulYR0r4osFkBda0ipH1RYLMCf+paRUi7osBmBfo7tks0Mzwh7YICmxXoa0aLlSqUVddauTSEdD8U2KzAQSKEp0w3GU0O3bURYnEU2KxE/56NakYJsTyzAtuGDRvg6+sLiUQChUKBo0ePNpt/27ZtCAwMhEQiQVBQEPbu3Wuwffv27RgzZgxcXFzA4/GQlZXV6Bg1NTWIjY2Fi4sL7O3tMX78+Ebzlebn5yM6Ohq2trZwd3fHO++8g7q6OnMusd3de89GgY0QSzM5sG3ZsgVxcXGIj4/HiRMnMHToUERFReHmzZtG86elpWHixImYPn06MjMzERMTg5iYGJw5c4bLU1lZiYiICKxYsaLJ87711lv46aefsG3bNvz+++8oKCjAc889x23XaDSIjo6GWq1GWloa/vOf/2Dz5s1YvHixqZfYIajJByHtyNQpsEJDQ1lsbCz3WaPRMC8vL5aQkGA0/4QJE1h0dLRBmkKhYLNmzWqUNzc3lwFgmZmZBumlpaVMKBSybdu2cWnnzp1jAFh6ejpjjLG9e/cyPp/PioqKuDyfffYZc3R0ZCqVqlXX1p7T793veN5t5jN/N3tk2f52Pxch3YEp30+T7tjUajUyMjIQGRnJpfH5fERGRiI9Pd3oPunp6Qb5ASAqKqrJ/MZkZGSgtrbW4DiBgYHo27cvd5z09HQEBQXBw8PD4DxKpRJnz541elyVSsVNvtxwEuaO0L9+lI/CshqU11DNKCGWZFJgKykpgUajMQgeAODh4YGioiKj+xQVFZmUv6ljiEQiODk5NXmcps6j32ZMQkICZDIZt3h7e7e6TG0lkwrh4SgGQA11CbG0Hl0rumDBApSVlXHLtWvXOvT83NhsVDNKiEWZFNhcXV0hEAga1UYWFxdDLpcb3Ucul5uUv6ljqNVqlJaWNnmcps6j32aMWCyGo6OjwdKRuM7w1JaNEIsyKbCJRCKEhIQgNTWVS9NqtUhNTUVYWJjRfcLCwgzyA0BKSkqT+Y0JCQmBUCg0OM6FCxeQn5/PHScsLAynT582qJ1NSUmBo6MjHnzwwVafqyPRaLqEtBNTayaSk5OZWCxmmzdvZtnZ2WzmzJnMycmJq42cPHkye++997j8hw4dYjY2Nmz16tXs3LlzLD4+ngmFQnb69Gkuz+3bt1lmZibbs2cPA8CSk5NZZmYmKyws5PK8+uqrrG/fvuy3335jx48fZ2FhYSwsLIzbXldXxwYPHszGjBnDsrKy2L59+5ibmxtbsGBBq6+tI2tFGWPsaK6uZjQ8IbVDzkdIV2bK99PkwMYYY+vWrWN9+/ZlIpGIhYaGssOHD3PbRo0axaZOnWqQf+vWrSwgIICJRCI2aNAgtmfPHoPtiYmJDECjJT4+nstTXV3NXn/9debs7MxsbW3Zs88+axD4GGMsLy+PjR07lkmlUubq6srefvttVltb2+rr6ujAdrdSxXzm72Y+83ez8prWl5OQnsiU7yePMcasdbfY2SiVSshkMpSVlXXY+7aHP96PW+Uq/Bg7AkO9nTrknIR0RaZ8P3t0rWhnQF2rCLE8CmxWxgU26lpFiMVQYLOy/h5UM0qIpdlYuwA9AmNAXQ1QU9Zoibh7A68LzqL3dTXwk6zBNuW9dd8I4PlNAI9n7SshpEugwGaOO1eAkpz6wFNqNGA1WrTG+4P6AXhXCKAOQEYT5zu7HYiYC3gObZfLIaS7ocBmjsxvgf+tMX0/Hh+QyHSL2JFb//FCJW7VSvB06EB4uHvcyyORAX/+PyBnP3B2BwU2QlqJAps5nHx0QaZhAJI43ffZyCKyN/o4+f0X6Th85Q569RmK5x7qY7ixouheYBsdT4+jhLQCBTZzhEzVLRbi7+6Aw1fuGK9ACHgCsJECd/OAwizAa5jFzktId0W1op3AvZnhjTT5ENkBAVG69bM7OrBUhHRdFNg6gf4tNdId9Kzu59kduhpWQkizKLB1AgH1bdny71ShplbTOIP/GEBoC5TmAwUnOrh0hHQ9FNg6ARc7EZxthWAMuHzLyF2byFb3rg2gx1FCWoECWyfA4/FaHk2XexzdSY+jhLSAAlsn0b+l0XT9/woI7YCya8CNplryEkIACmydRgDXGb6JOzahFBgwVrdOj6OENIsCWyfhX1+B0OyMVQ1rR7XaDigVIV0TBbZOQj98Ud7tSuM1owDQP1LXe0F5A7h+rANLR0jXQoGtk3BzEEMmFULLgNySSuOZhBJgwJO6dXocJaRJFNg6CV3NaCtG09U/jmbvpMdRQppgVmDbsGEDfH19IZFIoFAocPTo0Wbzb9u2DYGBgZBIJAgKCsLevXsNtjPGsHjxYnh6ekIqlSIyMhKXLl3ith88eBA8Hs/ocuyY7pEsLy/P6PbDhw+bc4lW0WzXKr3+o3Ujg5QXAteOdFDJCOlaTA5sW7ZsQVxcHOLj43HixAkMHToUUVFRBvN5NpSWloaJEydi+vTpyMzMRExMDGJiYnDmzBkuz8qVK/Hpp59i48aNOHLkCOzs7BAVFYWamhoAQHh4OAoLCw2WV155BX5+fhg+fLjB+fbv32+QLyQkxNRLtJr+rZkZ3kYMBEbr1ulxlBDjTJ0CKzQ0lMXGxnKfNRoN8/LyYgkJCUbzT5gwgUVHRxukKRQKNmvWLMYYY1qtlsnlcrZq1Spue2lpKROLxez77783eky1Ws3c3NzYkiVLuLTc3FwGgGVmZpp6SZyOnn7vfn9cvMl85u9mj68+0HzGC/sYi3dkbFUAY5q6DikbIdZmyvfTpDs2tVqNjIwMREZGcml8Ph+RkZFIT083uk96erpBfgCIiori8ufm5qKoqMggj0wmg0KhaPKYu3btwu3btzFt2rRG28aNGwd3d3dERERg165dzV6PSqWCUqk0WKxJ3/sg73YV1HXNvD/r9xgglunGasvvOo/ahHQUkwJbSUkJNBoNPDw8DNI9PDxQVFRkdJ+ioqJm8+t/mnLMr7/+GlFRUejT596gjPb29lizZg22bduGPXv2ICIiAjExMc0Gt4SEBMhkMm7x9vZuMm9H8HAUw0FsA42WNV0zCgA2ImDgU7p1ehwlpJEuVyt6/fp1/PLLL5g+fbpBuqurK+Li4qBQKPDwww9j+fLl+Mc//oFVq1Y1eawFCxagrKyMW65du9bexW8Wj8druWuVHlc7+iOgbaLdGyE9lEmBzdXVFQKBAMXFxQbpxcXFkMvlRveRy+XN5tf/bO0xExMT4eLignHjxrVYXoVCgZycnCa3i8ViODo6GizWFtCaCgQA8BulG4688iZwNa39C0ZIF2JSYBOJRAgJCUFqaiqXptVqkZqairCwMKP7hIWFGeQHgJSUFC6/n58f5HK5QR6lUokjR440OiZjDImJiZgyZQqEQmGL5c3KyoKnp2err68z0Df5aLZrFUCPo4Q0x9SaieTkZCYWi9nmzZtZdnY2mzlzJnNycmJFRUWMMcYmT57M3nvvPS7/oUOHmI2NDVu9ejU7d+4ci4+PZ0KhkJ0+fZrLs3z5cubk5MR+/PFHdurUKfbMM88wPz8/Vl1dbXDu/fv3MwDs3Llzjcq1efNmlpSUxM6dO8fOnTvHPv74Y8bn89mmTZtafW3WrhVljLED54uZz/zdLHLNwZYzX0rR1Y6ufICxutr2LxwhVmTK99PkyVxefPFF3Lp1C4sXL0ZRURGCg4Oxb98+7uV/fn4++Px7N4Lh4eFISkrCwoUL8f7778Pf3x87d+7E4MGDuTzvvvsuKisrMXPmTJSWliIiIgL79u2DRCIxOPfXX3+N8PBwBAYGGi3b0qVLcfXqVdjY2CAwMBBbtmzB888/b+olWpW+M3xuSSVqNVoIBc3cVPuNAqTOQOUt4OohoN+oDiolIZ0bjzEatVBPqVRCJpOhrKzMau/bGGMYHP8LKtUa7I8byTXabdKuN4AT3wAhLwFPf9IhZSTEGkz5fna5WtHuTlcz2soKBAAY9JzuZ/YuQFPXjiUjpOugwNYJ6TvDX2xNYPN9FLB1AarvAHl/tHPJCOkaKLB1QgGtbcsGAAIbYGB90xeqHSUEAAW2TknftarFJh96+sa6534CNLXtVCpCug4KbJ2QfgLlK7cqUadpxZhrPiMAOzeg+i6Q+3s7l46Qzo8CWyfU20kKqVAAtUaLq3eqWt6BHkcJMUCBrRPi83kNBp009XF0N1CnbqeSEdI1UGDrpPSPozmtqUAAAJ9wwM4dqCmlx1HS41Fg66T0FQitavIBAHwB8OAzunV6HCU9HAW2TqpVE7vcjx5HCQFAga3TCqjvfXD5VgU02lb2euv7CGAvB1RlwJUD7Vg6Qjo3CmydVG9nKSRCPtR1WlxrTc0oQI+jhNSjwNZJCfg8POCm71rVygoEABhc33f0/B6gtqYdSkZI50eBrRPTP46a9J6tTyjg4AWolMDl39qpZIR0bhTYOrF7TT5MCGx8PjAoRrdOj6Okh6LA1ondG+XDhEdR4F7t6IW9QG21hUtFSOdHga0T04+me76oHP9KuYia2lbORtV7OODYB1BXADmpLecnpJuhwNaJ+brY4plgL2i0DJ+kXsITa//A/y7danlHehwlPRwFtk6Mx+Nh7YvB2PD3h+DuIEbe7SpM/voo3vw+EzfLW6jx5B5Hf6bHUdLjmBXYNmzYAF9fX0gkEigUChw9erTZ/Nu2bUNgYCAkEgmCgoKwd+9eg+2MMSxevBienp6QSqWIjIzEpUuXDPL4+vqCx+MZLMuXLzfIc+rUKTz66KOQSCTw9vbGypUrzbm8ToXH4yF6iCdS3x6Fl8J9wecBu04WYPTq3/FNel7TjXd7hwAyb6C2EriU0rGFJsTaTJ0CKzk5mYlEIrZp0yZ29uxZNmPGDObk5MSKi4uN5j906BATCARs5cqVLDs7my1cuNDo9HsymYzt3LmTnTx5ko0bN67R9Hs+Pj5syZIlrLCwkFsqKioMpuby8PBgkyZNYmfOnGHff/89k0ql7PPPP2/1tXWG6fdacvp6KRu37n/MZ/5u5jN/N3t63f/YqWulxjP/8oFuer6tL3VsIQlpB6Z8P00ObKGhoSw2Npb7rNFomJeXF0tISDCaf8KECSw6OtogTaFQsFmzZjHGGNNqtUwul7NVq1Zx20tLS5lYLGbff/89l+bj48P+9a9/NVmuf//738zZ2ZmpVCoubf78+WzAgAGtvrauENgYY6xOo2XfpOWywYv3MZ/5u5nfe7tZ/I9nWFm12jDj9eO6wPZPOWOqSusUlhALMeX7adKjqFqtRkZGBiIjI7k0Pp+PyMhIpKenG90nPT3dID8AREVFcflzc3NRVFRkkEcmk0GhUDQ65vLly+Hi4oJhw4Zh1apVqKu7NytTeno6Ro4cCZFIZHCeCxcu4O7du0bLplKpoFQqDZauQMDnYXKYL1LnjcIzwV7QMmBzWh4i1/yO3acKwPQzKno9BDj5ALVVwKVfrVtoQjqQSYGtpKQEGo2GmxxZz8PDA0VFRUb3KSoqaja//mdLx3zzzTeRnJyMAwcOYNasWVi2bBnefffdFs/T8Bz3S0hIgEwm4xZvb+8mr70zcneQ4JO/DcO30xXwc7XDzXIVZidlYmriMVy9XQnwePcqEah2lPQgXaZWNC4uDn/5y18wZMgQvPrqq1izZg3WrVsHlUpl9jEXLFiAsrIybrl27ZoFS9xxIvxd8fOcRzE30h8iGz7+uHgLf/3XH/g09RLUgfWd4i/+AqgrrVtQQjqISYHN1dUVAoEAxcXFBunFxcWQy+VG95HL5c3m1/805ZgAoFAoUFdXh7y8vGbP0/Ac9xOLxXB0dDRYuiqJUIC5kQH4Ze5IPOrvCnWdFv8v5SKeSC5DtX1foK5aF9wI6QFMCmwikQghISFITb3Xml2r1SI1NRVhYWFG9wkLCzPIDwApKSlcfj8/P8jlcoM8SqUSR44cafKYAJCVlQU+nw93d3fuPH/88Qdqa+9NP5eSkoIBAwbA2dnZlMvs0vxc7fDNy6FYN3EY3BzEuHK7CptKgwEAqpM/WLdwhHQUU2smkpOTmVgsZps3b2bZ2dls5syZzMnJiRUVFTHGGJs8eTJ77733uPyHDh1iNjY2bPXq1ezcuXMsPj7eaHMPJycn9uOPP7JTp06xZ555xqC5R1paGvvXv/7FsrKy2OXLl9m3337L3Nzc2JQpU7hjlJaWMg8PDzZ58mR25swZlpyczGxtbbtdcw9TlFWr2eKdp9nYBesZi3dk1fEuLOl/Z1mdRmvtohFisnZt7sEYY+vWrWN9+/ZlIpGIhYaGssOHD3PbRo0axaZOnWqQf+vWrSwgIICJRCI2aNAgtmfPHoPtWq2WLVq0iHl4eDCxWMxGjx7NLly4wG3PyMhgCoWCyWQyJpFI2MCBA9myZctYTU2NwXFOnjzJIiIimFgsZr1792bLly836bq6W2DTO5l/h13/KJCxeEf2xoL32bj1f7LT15to+0ZIJ2XK95PHGGvluNPdn1KphEwmQ1lZWZd+32aMdv9S8P9cjVT2MKar3gKfB7wU7oe4MQGwF9tYu3iEtMiU72eXqRUlbcMfrGv28bjNKbwwWAYtAzYdysXoNQex93Qh6N830p1QYOspPAYBLv7gaVRYNaQA37wcCh8XWxQrVXj9uxOYtvkY8m+3cm4FQjo5Cmw9xX2NdUcGuOGXuSMxZ7Q/RAI+Dl64hb/+63dsO9412/IR0hAFtp5EH9hy9gM1ZZAIBXjrrwHYN/dRhD/gAlWdFu/8cApLd2ejTqO1blkJaQMKbD2J+0DAdQCgUevGaavXz80e305XYM5ofwDA13/m4uX/HEdZdW1TRyKkU6PA1pPwePem57uv7yifz8Nbfw3Ahr8/BIlQ1y3r2X8fwpVbJkwkQ0gnQYGtp3kwRvczJxWoLm20OXqIJ354NRxeMgmu3KrEMxsO4Y+LrRiOnJBOhAJbT+MeCLg/CGhrdbNYGTG4tww/zo5AiI8zymvq8FLiUXz9Zy41CSFdBgW2nqgVQxm5OYiRNEOBF0L6QMuApbuzMf+/p6Cqa+VMWYRYEQW2nkj/OHr5N6Da+CCcACC2EWDl80OwMHog+Dxg6/HrmPTlEdwqN3+oKEI6AgW2nsgtAPAYDGjrgHO7m83K4/HwyqP9kDgtFA4SGxy/ehfPrP8TZwvKzDu3VgPcyACUBebtT0grUGDrqUycd3RUgBt2xo5AP1c7FJTV4PnP0vHz6cLWn6/yNvDnWuDTYcCXjwNrhwC74yjAkXZBneAb6M6d4BspyQHWhwA8AfBODmDbq1W7lVXV4o3kTK6mdM5of8wZ7Q8+n9c4M2O6u7NjXwFntgOa+kdYG6lu4EsAEIiB4S8DEW8BDh6Nj0FIPeoET1rm2h+QBwFMA5z7qdW7yWyF2DR1OF6J8AMAfJJ6CbFJJ1ClvjexDtRVwIlvgC9GAV+NBk5+rwtqnkOBceuBd68AL+0B+obr0o98BnwyFPh1ke7OjpA2oju2BnrUHRsA/G8NkLoE6PcYMGWnybtvPX4NC3ecgVqjxUBPRyQ+7Qz5xSQg61ugpv4dnECsaxT88Cu6SZx5De7sGAOuHAB++xi4cVyXJrIHFK8C4bMBac8Z+Zi0zJTvJwW2BnpcYLtzRffOiycA5l0E7FxNPkRG7k0k/d+XeKb2Z4wUnL63wckHeHg6EPwPwM6l+YMwppse8MDHQOFJXZpYBoTFAo+8Bkh6wP8L0iIKbGbqcYENAD4fBRRmAU+tBYZPa/1+FTeBE/8Bjm8GlNcBAFrGw+8sGALFKxg5diLAF5hWFsaA87uBA8uAm9m6NKkzMGIOEDoTENmZdjzSrVBgM1OPDGx/rgX2xwN+o4Cpu5rPyxiQf1hXGZD9o673AgBIe6F26CQsLQrDN+d1SdNG+OKDJwfCRmDGa1ytFji7HTi4HLh9SZdm56arYBj+MiCUmn5M0uVRYDNTjwxsd/N0L+55fODti4C9W+M8qgrg9Fbg2NdA8Zl76X0e1r07ezAGEErAGMOnqTn41/6LAIBH/V2xfuJDkNkKzSubpg44vQ34fbmunADg4Ak8+jbw0BTARmzecUmX1O61ohs2bICvry8kEgkUCgWOHj3abP5t27YhMDAQEokEQUFB2LvXsI8iYwyLFy+Gp6cnpFIpIiMjcenSJW57Xl4epk+fDj8/P0ilUjzwwAOIj4+HWq02yMPj8Rothw8fNucSew5nX8DrIYBpgXP33bHdugDsfQdYEwjsfksX1GykwLDJwMzfgVf2A0P/BgglAHSNeedE+uOzSQ9BKhTgf5dKEPPvQ8i5aeYIIQIbIHgiMPs48PSngMwbKC8E9s4D1oUAGZsBDQ2tRIwwdaaY5ORkJhKJ2KZNm9jZs2fZjBkzmJOTEysuLjaa/9ChQ0wgELCVK1ey7OxstnDhQqPT78lkMrZz50528uRJNm7cOIPp937++Wf20ksvsV9++YVdvnyZ/fjjj8zd3Z29/fbb3DFyc3MZALZ//35WWFjILWq1utXX1l1nqWrRn58wFu/IWGI0Y3Vqxs7s0K3HO95bPhnGWNoGxqrutOqQZ2+UsfCEVOYzfzcbHL+PHThv/O/DJLU1jB35grHVA+6Va+0QxjK/Y6yutu3HJ51au06/FxoaymJjY7nPGo2GeXl5sYSEBKP5J0yYwKKjow3SFAoFmzVrFmNMN/WeXC5nq1at4raXlpYysVjMvv/++ybLsXLlSubn58d91ge2zMxMUy+J02MD292r9YFCZhg0PnRi7Pu/M5aTyphGY/Jhb5XXsOc/O8R85u9mfu/tZl/+cZlptRaY01RdpQuyKx+4V9ZPQxg7tc2scpKuwZTvp0mPomq1GhkZGYiMjOTS+Hw+IiMjkZ6ebnSf9PR0g/wAEBUVxeXPzc1FUVGRQR6ZTAaFQtHkMQGgrKwMvXo1bi0/btw4uLu7IyIiArt2Nf8yXKVSQalUGiw9klNfoPdwAEz3qGfnBjw6D5h7Gvjbd8ADjwN8099auNqL8d0rj+DF4d7QMuCfe87hnR8sMEKIUAqEvQ7MOQlEfgRIe+kqGf47Hdg4AsjepavoID2WSRNKlpSUQKPRwMPDsOuLh4cHzp8/b3SfoqIio/mLioq47fq0pvLcLycnB+vWrcPq1au5NHt7e6xZswYjRowAn8/Hf//7X8TExGDnzp0YN26c0eMkJCTgo48+auaKe5CxK4CjXwL+fwUGjgNsRBY5rMiGj+XjgxDo6YClu7PxQ8Z1XLlVgWXPBaGfqz1ENm3o/CKyAyLm6mpKj3wOpK3TNRPZOhmQDwEe+wAIiDJsFNwRNHW6HhV19YvBuhqoq7lvXd10fu5zfV6NSvee09kX6OUHOPvp1h3kHX+dnViXmyn3xo0beOKJJ/DCCy9gxowZXLqrqyvi4uK4zw8//DAKCgqwatWqJgPbggULDPZRKpXw9vZuv8J3Zn2G65Z2wOPxMG2EH/q72yP2uxM4kV+KJ9b+D3we0MfZFn6udo0WLycpBMb6nxojcQRGvQOEvgKkbwAOfwYUnQK+fxHo1U/XmwFMdxfHWP26ton1+s9gAEODdWa4buwYmlpd8GFWmAhHH+zuD3i9/HR35D2sBtmkwObq6gqBQIDi4mKD9OLiYsjlcqP7yOXyZvPrfxYXF8PT09MgT3BwsMF+BQUFeOyxxxAeHo4vvviixfIqFAqkpKQ0uV0sFkMs7ln/w63pUX83/Dg7Au9vP42T10tRpdYg/04V8u9U4ff7hh8XCfjo62IY9Hxd7NDPzQ7uDmLwjN2dSJ2BxxcCiteAtE90d6B3rnTQ1TWBJ9AFFRuxrnuZjQiwkdy3LmqcRyDWbePW9dtFgLoCuJOrawJzNxcou64bVODWOd3SuBCAY+/6gOfbOPi1cgCErsSkwCYSiRASEoLU1FTExMQAALRaLVJTUzF79myj+4SFhSE1NRVz587l0lJSUhAWFgYA8PPzg1wuR2pqKhfIlEoljhw5gtdee43b58aNG3jssccQEhKCxMRE8FvxzicrK8sgWBLr83O1w/czHwFjDLfKVbhSUom8kkrkNliu3qmCuk6LnJsVRpuK2IoE8HWxg5+bHfxc6oOeqx36udrB2U6k68L11yVA+JtAQZZuJ179f3j8+ke21q7zWp9fIDQMVAKxrslKe6tTA2XXdEGOC3h599ZrK3W9Q5TXgbz/Nd5fItMFOS7w1a87+eiCnsi+yz3mmvxbj4uLw9SpUzF8+HCEhoZi7dq1qKysxLRpuu44U6ZMQe/evZGQkAAAmDNnDkaNGoU1a9YgOjoaycnJOH78OHfHxePxMHfuXPzzn/+Ev78//Pz8sGjRInh5eXHB88aNG/jLX/4CHx8frF69Grdu3fvXXX/H95///AcikQjDhg0DAGzfvh2bNm3CV199Zf5vh7QbHo8Hd0cJ3B0leKSfYV9SjZahoLQauSWVyLtdiSu3dD9zSypx/W41qtQaZBcqkV3YuLJHJhXCrz7I+braoY9zIBwlQthLbOAgsYGD+N660JxeEZ2RjQhweUC33I8xoPJWg0CXa7heUawbsKAwS7cYw+MDYkddAJQ4AhIn3bpBmuy+tAZ5xY4dE+AbMPlsL774Im7duoXFixejqKgIwcHB2LdvH/fyPz8/3+BuKjw8HElJSVi4cCHef/99+Pv7Y+fOnRg8eDCX591330VlZSVmzpyJ0tJSREREYN++fZBIdA0/U1JSkJOTg5ycHPTp08egPKxB7dfSpUtx9epV2NjYIDAwEFu2bMHzzz9v6iUSKxPwefDuZQvvXrYYCcOeEOo6La7dreLu8hre8RWW1aCsuhZZ10qRda20xfOIbfi6YCcRwl5sA3uxLuDZS2zgIK5Pl9xLd5DYwF4svO+zjXndxjoKjwfYu+sW79DG29WVwN2rjQPe3TygNF9XwcG0QE2pbjGX0O6+gHd/EJTp+gRb6M6QulQ10CO7VHUj1WoN8m7rAp0+4BWW1aBcVYeKmlqU19ShQlWHKrVlJ6SRCgWwl9jAUyaBt7Mt+jhL0aeXLbydpfDuZYveTlJIhCYOCNAZMKarDKkpA2qU9T/LAFXZvXWDdGXjtNrK1p1LZA+8f6PZLKZ8P7tcrSghTZGKBBjo6YiBns3/0ddptKhUaVCuqkWFqk4X8GrqoKzRfa6oqeOCYHlNHcr16dxnXZqqTlf7WV2rQXWtBrfKVTh13fhcEO4OYnj30gU9b2dbePeSoo+zLbydbeHpJOmcj8U8nq7NoFCqa05iDk0toCqvv+NrJgha+B0eBTbS49gI+JDZ8s3vnF9PXadFZX2wU9bU4kZpNa7dqcL1u9W4frcK1+5U49rdKlSpNbhZrsLNchUyrjaeFYzPAzxlUl3Q62XL3fXpHselcHeQtL7pS2cjEOoqIDq45pUeRRugR1FiaYwx3K2q5QLetbtVBuvX71ZDXdd8uzehgIfeTlLujk/uKIVMagMnWxFkUiFktkLdz/qlU979WQA9ihLSSfB4PPSyE6GXnQhDvZ0abddqGUoqVPUBz/BO79rdKhSU1qBWw5B3uwp5t6tadU47kQBOtiI4SoW6ACjVBUAnW2F9mm5dJhVy22RSIRwkNsYn5emCKLARYkV8/r1mLyE+jbfXabQoUtbo7vDuVOHa3WrcKlehrFqNsupalFXXorRK97O8RjehTqVag0p1NW6UVptUFh4PcJTcC3r6YKerMRY2qC22Magt1m/T1yx3hsdmCmyEdGI2Aj76ONuij7Nto/Z+96vTaFFeU4fS+oCnC3pqKBsEv4bbyrg0NWpqtWAM3La2sBMJ7gU6iRAOYv26vi2hPjDeC5b2Ehs87Gu593AU2AjpJmwEfDjbiXS9L0xUU6uB0iAg6n42rA2uUNWioqbh54Y/a1Gr0b2u190xalAMVavPLxHycX7pWJPL3RQKbIQQSIQCSIQCuDtKzD6Gqk7TKPBxn7lmNA2a1DTYbiOg5h6EkE5IbCOA2F4AF3vrDyzRPeuFCSE9GgU2Qki3Q4GNENLtUGAjhHQ7FNgIId0OBTZCSLdDgY0Q0u1QO7YG9AOd9Nj5RQnpxPTfy9YMSESBrYHy8nIA6LlT8BHSBZSXl0MmkzWbh8Zja0Cr1aKgoAAODg7Gp3drQD8H6bVr17rF2G3d7XoAuqauorXXxBhDeXk5vLy8Wpylju7YGuDz+Y0mi2mJo6Njt/kDA7rf9QB0TV1Fa66ppTs1Pao8IIR0OxTYCCHdDgU2M4nFYsTHx0Mstv5IBpbQ3a4HoGvqKtrjmqjygBDS7dAdGyGk26HARgjpdiiwEUK6HQpshJBuhwIbIaTbocBmhg0bNsDX1xcSiQQKhQJHjx61dpHMlpCQgIcffhgODg5wd3dHTEwMLly4YO1iWdTy5cvB4/Ewd+5caxelTW7cuIF//OMfcHFxgVQqRVBQEI4fP27tYplFo9Fg0aJF8PPzg1QqxQMPPIClS5e2qoN7a1BgM9GWLVsQFxeH+Ph4nDhxAkOHDkVUVBRu3rxp7aKZ5ffff0dsbCwOHz6MlJQU1NbWYsyYMaisrLR20Szi2LFj+PzzzzFkyBBrF6VN7t69ixEjRkAoFOLnn39GdnY21qxZA2dnZ2sXzSwrVqzAZ599hvXr1+PcuXNYsWIFVq5ciXXr1lnmBIyYJDQ0lMXGxnKfNRoN8/LyYgkJCVYsleXcvHmTAWC///67tYvSZuXl5czf35+lpKSwUaNGsTlz5li7SGabP38+i4iIsHYxLCY6Opq9/PLLBmnPPfccmzRpkkWOT3dsJlCr1cjIyEBkZCSXxufzERkZifT0dCuWzHLKysoAAL169bJySdouNjYW0dHRBv+/uqpdu3Zh+PDheOGFF+Du7o5hw4bhyy+/tHaxzBYeHo7U1FRcvHgRAHDy5En8+eefGDvWMrPB0+geJigpKYFGo4GHh4dBuoeHB86fP2+lUlmOVqvF3LlzMWLECAwePNjaxWmT5ORknDhxAseOHbN2USziypUr+OyzzxAXF4f3338fx44dw5tvvgmRSISpU6dau3gme++996BUKhEYGAiBQACNRoOPP/4YkyZNssjxKbARTmxsLM6cOYM///zT2kVpk2vXrmHOnDlISUmBRCKxdnEsQqvVYvjw4Vi2bBkAYNiwYThz5gw2btzYJQPb1q1b8d133yEpKQmDBg1CVlYW5s6dCy8vL8tcj0UeaHsIlUrFBAIB27Fjh0H6lClT2Lhx46xTKAuJjY1lffr0YVeuXLF2Udpsx44dDAATCATcAoDxeDwmEAhYXV2dtYtosr59+7Lp06cbpP373/9mXl5eVipR2/Tp04etX7/eIG3p0qVswIABFjk+vWMzgUgkQkhICFJTU7k0rVaL1NRUhIWFWbFk5mOMYfbs2dixYwd+++03+Pn5WbtIbTZ69GicPn0aWVlZ3DJ8+HBMmjQJWVlZEAgE1i6iyUaMGNGoGc7Fixfh4+NjpRK1TVVVVaNRcAUCAbRarWVOYJHw2IMkJyczsVjMNm/ezLKzs9nMmTOZk5MTKyoqsnbRzPLaa68xmUzGDh48yAoLC7mlqqrK2kWzqK5eK3r06FFmY2PDPv74Y3bp0iX23XffMVtbW/btt99au2hmmTp1KuvduzfbvXs3y83NZdu3b2eurq7s3XfftcjxKbCZYd26daxv375MJBKx0NBQdvjwYWsXyWwAjC6JiYnWLppFdfXAxhhjP/30Exs8eDATi8UsMDCQffHFF9YuktmUSiWbM2cO69u3L5NIJKxfv37sgw8+YCqVyiLHp/HYCCHdDr1jI4R0OxTYCCHdDgU2Qki3Q4GNENLtUGAjhHQ7FNgIId0OBTZCSLdDgY0Q0u1QYCOEdDsU2Agh3Q4FNkJIt/P/Ad4EZzF8TCiUAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T04:58:48.282435Z", + "iopub.status.busy": "2024-03-24T04:58:48.282068Z", + "iopub.status.idle": "2024-03-24T05:02:54.805198Z", + "shell.execute_reply": "2024-03-24T05:02:54.804382Z" + }, + "papermill": { + "duration": 246.543926, + "end_time": "2024-03-24T05:02:54.807556", + "exception": false, + "start_time": "2024-03-24T04:58:48.263630", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T05:02:54.843237Z", + "iopub.status.busy": "2024-03-24T05:02:54.842552Z", + "iopub.status.idle": "2024-03-24T05:02:54.864746Z", + "shell.execute_reply": "2024-03-24T05:02:54.863819Z" + }, + "papermill": { + "duration": 0.04212, + "end_time": "2024-03-24T05:02:54.866825", + "exception": false, + "start_time": "2024-03-24T05:02:54.824705", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tvae0.0066440.0357860.001482.8779740.0139170.4571760.0240950.0000024.7272150.0295540.0915550.0384660.1009160.0082077.60519
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "tvae 0.006644 0.035786 0.00148 2.877974 0.013917 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "tvae 0.457176 0.024095 0.000002 4.727215 0.029554 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "tvae 0.091555 0.038466 0.100916 0.008207 7.60519 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T05:02:54.901830Z", + "iopub.status.busy": "2024-03-24T05:02:54.901026Z", + "iopub.status.idle": "2024-03-24T05:02:55.301336Z", + "shell.execute_reply": "2024-03-24T05:02:55.300019Z" + }, + "papermill": { + "duration": 0.42061, + "end_time": "2024-03-24T05:02:55.303873", + "exception": false, + "start_time": "2024-03-24T05:02:54.883263", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T05:02:55.339239Z", + "iopub.status.busy": "2024-03-24T05:02:55.338855Z", + "iopub.status.idle": "2024-03-24T05:07:25.727940Z", + "shell.execute_reply": "2024-03-24T05:07:25.727072Z" + }, + "papermill": { + "duration": 270.40958, + "end_time": "2024-03-24T05:07:25.730480", + "exception": false, + "start_time": "2024-03-24T05:02:55.320900", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_aug_test/tvae/all inf False\n", + "Caching in ../../../../contraceptive/_cache_bs_test/tvae/all inf False\n", + "Caching in ../../../../contraceptive/_cache_synth_test/tvae/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T05:07:25.769004Z", + "iopub.status.busy": "2024-03-24T05:07:25.768138Z", + "iopub.status.idle": "2024-03-24T05:07:25.795186Z", + "shell.execute_reply": "2024-03-24T05:07:25.794448Z" + }, + "papermill": { + "duration": 0.047832, + "end_time": "2024-03-24T05:07:25.797051", + "exception": false, + "start_time": "2024-03-24T05:07:25.749219", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T05:07:25.829695Z", + "iopub.status.busy": "2024-03-24T05:07:25.829132Z", + "iopub.status.idle": "2024-03-24T05:07:25.834346Z", + "shell.execute_reply": "2024-03-24T05:07:25.833429Z" + }, + "papermill": { + "duration": 0.023626, + "end_time": "2024-03-24T05:07:25.836259", + "exception": false, + "start_time": "2024-03-24T05:07:25.812633", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tvae': 0.3903380688599178}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T05:07:25.872376Z", + "iopub.status.busy": "2024-03-24T05:07:25.872030Z", + "iopub.status.idle": "2024-03-24T05:07:26.331834Z", + "shell.execute_reply": "2024-03-24T05:07:26.330819Z" + }, + "papermill": { + "duration": 0.481427, + "end_time": "2024-03-24T05:07:26.334083", + "exception": false, + "start_time": "2024-03-24T05:07:25.852656", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABEtUlEQVR4nO3dd3xb1f0//tfV3pL3iO3YiZ1F9mwSQgJJaUYhKS2EQktSCNA2lNJ88nnQ9PFpIFA+CS2zlObb8iE2+THCCrSFQkIpIRTIICETcJb3npK1xz2/P64kW/GWJV1bej8fDz0kXx1dnWvZb519OMYYAyGEiEgidgYIIYQCESFEdBSICCGio0BECBEdBSJCiOgoEBFCREeBiBAiOgpEhBDRUSAihIiOAhEhRHQUiEhEffbZZ3jwwQfR3t4udlbICEKBiETUZ599hm3btlEgIoNCgYgQIjoKRCRiHnzwQfz3f/83AKCgoAAcx4HjOOh0Olx99dXd0vM8j1GjRuEHP/hB8Nhjjz2GBQsWICUlBWq1GrNmzcIbb7zR4/u9+OKLmDVrFtRqNZKTk3HzzTejqqoqOhdHooqjZUBIpJw6dQo7duzAK6+8gieffBKpqakAgIsXL+Khhx5CTU0NMjMzg+kPHjyIxYsX4/XXXw8Go9zcXFx//fWYNGkS3G439uzZgyNHjuCdd97BqlWrgq995JFH8Nvf/hY33XQTFi9ejKamJjzzzDPQ6XT48ssvYTKZYnrtZIgYIRH0hz/8gQFgZWVlwWOlpaUMAHvmmWdC0v785z9nOp2O2e324LGujxljzO12s8mTJ7NrrrkmeKy8vJxJpVL2yCOPhKQ9ffo0k8lk3Y6T4Y+qZiTqxo0bh+nTp+PVV18NHvP5fHjjjTdw3XXXQa1WB493fdzW1gaz2YxFixbh+PHjweN79+4Fz/O46aab0NzcHLxlZmaiqKgIH330UWwujESMTOwMkMSwdu1a/OY3v0FNTQ1GjRqFAwcOoLGxEWvXrg1J98477+B3v/sdTpw4AZfLFTzOcVzw8fnz58EYQ1FRUY/vJZfLo3MRJGooEJGYWLt2LbZs2YLXX38d9913H1577TUYjUYsX748mOaTTz7B9ddfj6uuugp//vOfkZWVBblcjuLiYrz88svBdDzPg+M4vPfee5BKpd3eS6fTxeSaSORQICIR1bXk0lVBQQHmzp2LV199Fffccw/27t2LNWvWQKlUBtO8+eabUKlU2LdvX8jx4uLikHONHTsWjDEUFBRg3Lhx0bkQElPURkQiSqvVAkCPAxrXrl2LQ4cOYdeuXWhubu5WLZNKpeA4Dj6fL3isvLwcb7/9dki6G264AVKpFNu2bQO7rNOXMYaWlpbIXAyJGeq+JxF19OhRzJ07FytXrsTNN98MuVyO6667DlqtFtXV1cjLy4NOp4NcLkd9fX1Ie86///1vLF26FIsWLcItt9yCxsZGPPvss8jMzMSpU6dCgs6OHTuwZcsWLFiwAGvWrIFer0dZWRneeust3HXXXdi8ebMYl0/CJWKPHYlTDz/8MBs1ahSTSCTduvIXLlzIALANGzb0+Nrnn3+eFRUVMaVSySZMmMCKi4vZAw88wHr6U33zzTfZlVdeybRaLdNqtWzChAls48aNrLS0NFqXRqKESkSEENFRGxEhRHQUiAghoqNARAgRHQUiQojoKBARQkRHgYgQIroRPcWD53nU1tZCr9f3OrWAECIexhg6OjqQnZ0NiaT3cs+IDkS1tbXIzc0VOxuEkH5UVVUhJyen1+dFD0Q1NTW4//778d5778Fut6OwsBDFxcWYPXt2v6/V6/UAhIs0GAzRziohZJAsFgtyc3OD/6u9ETUQtbW1YeHChbj66qvx3nvvIS0tDefPn0dSUtKAXh+ojhkMBgpEhAxj/TWdiBqIHn30UeTm5oYs81BQUCBijgghYhC11+zvf/87Zs+ejRtvvBHp6emYMWMGnnvuuV7Tu1wuWCyWkBshZOQTNRBdunQJO3fuRFFREfbt24ef/exnuPfee/HCCy/0mH779u0wGo3BGzVUExIfRJ19r1AoMHv2bHz22WfBY/feey+OHj2Kzz//vFt6l8sVso5xoCHMbDb32kbEGIPX6w1ZbIsMjFQqhUwmo6ERJGwWiwVGo7HP/1FA5DairKwsTJo0KeTYxIkT8eabb/aYXqlUhiwh2h+32426ujrY7fYh5TORaTQaZGVlQaFQiJ0VEsdEDUQLFy5EaWlpyLFz585h9OjRQz43z/MoKyuDVCpFdnY2FAoFfbMPAmMMbrcbTU1NKCsrQ1FRUZ8D0ggZClED0a9+9SssWLAA//u//4ubbroJR44cwV//+lf89a9/HfK53W43eJ5Hbm4uNBpNBHKbeNRqNeRyOSoqKuB2u6FSqcTOUkRVtthxrLIVOUkazMlPFjs7CU3Ur7g5c+bgrbfewiuvvILJkyfj4YcfxlNPPYVbb701Yu9B3+JDE6+/P7vbi3+cqkVzhxvZJnX/LyBRJfrI6u9+97v47ne/K3Y2SIL5us4Ct5dHul6JLEN8lfRGovj8uiOkH6X1VgDA+Ew9JBJqOxQbBaIEl5+fj6eeekrsbMSU0+NDY4cTAFCQqkWrzY0vyltR2+4QOWeJiwIRSTi17Q4wBpg0cuhVchyraMMn55tR3mwTO2sJiwJRHHC73WJnYUSpbRdKQ6P8jdTpemFsWmOHq9fXkOhKyEDk9vK93rw+fsBpPQNIG44lS5bgnnvuwT333AOj0YjU1FT89re/De50mp+fj4cffhi33XYbDAYD7rrrLgDAf/7zHyxatAhqtRq5ubm49957YbN1fss3Njbiuuuug1qtRkFBAV566aWw8jfSqRUSpOmVwd6ydEMgEDnFzFZCE73XTAzPfnSh1+cKUrVYM2NU8Oe/HrwIj6/nWTA5SWrcOLtzvtuuT8vgcIdOJfnVt8eFlccXXngBd9xxB44cOYIvvvgCd911F/Ly8nDnnXcCAB577DFs3boVDzzwAADg4sWLWL58OX73u99h165daGpqCgazwOoG69evR21tLT766CPI5XLce++9aGxsDCt/I9ms0cmYNbpz3FCSRhg1bnP54PL6oJRJxcpawkrIQDQS5Obm4sknnwTHcRg/fjxOnz6NJ598MhiIrrnmGvzXf/1XMP2GDRtw66234r777gMAFBUV4Y9//CMWL16MnTt3orKyEu+99x6OHDmCOXPmAACef/55TJw4MebXNtyo5FJoFFLY3T6Y7R6kGygQxVpCBqKNVxf2+tzlPbl3XTW217SXzxi5fWHk1lL61re+FTIlZf78+Xj88ceDk3cvX8Hy5MmTOHXqVEh1izEWnOpy7tw5yGQyzJo1K/j8hAkTYDKZIpbnkcDj4yHluG5d9iaNHHa3D212D9JpXFHMJWQgUsgG3jQWrbRDpdVqQ362Wq24++67ce+993ZLm5eXh3PnzsUqa8PamRoz/nO+GVNyjFgyPj143KRRoLbdiXY7NfyLISED0Uhw+PDhkJ8PHTqEoqIiSKU9VxtmzpyJr776CoWFPZf2JkyYAK/Xi2PHjgWrZqWlpWhvb49ovoe7NrsbXp5BLg390pg1OglTc4zB9iISWwnZazYSVFZWYtOmTSgtLcUrr7yCZ555Br/85S97TX///ffjs88+wz333IMTJ07g/Pnz+Nvf/oZ77rkHADB+/HgsX74cd999Nw4fPoxjx45hw4YNUKsTa55Vu90DADCq5SHHU3VKZBnVUMmpfUgMFIiGqdtuuw0OhwNz587Fxo0b8ctf/jLYTd+TqVOn4uOPP8a5c+ewaNEizJgxA1u3bkV2dnYwTXFxMbKzs7F48WLccMMNuOuuu5Cent7rOeNRIBCZNPJ+UpJYoqrZMCWXy/HUU09h586d3Z4rLy/v8TVz5szB/v37ez1nZmYm3nnnnZBjP/7xj4eUz5HExzNYnIFAFFoF8/h4nK4xw+r0YlFRKq1dFWNUIiIJw+LwgDFALuWgVYRWwSQch4PnmnCsog0ODy0rHGsUiEjCaHf424c03VfrlEo4aBVCBaHD6Y153hIdVc2GoQMHDoidhbiklElQlKGDQdVz+5BOJYPV5UWH04sM2q8zpigQkYSRbVL3uRqjTin8O1hdVCKKNaqaEeIXCER2CkQxR4GIJAynx4e+tvHT+BuwbW5qrI41qpqRhLHr0zL4fAw/+tZoJGm7j6DW+Bur7W4qEcUaBSKSEFxeH1weYX0ojbLn0dNj0rT4oT4POhX9W8Qa/cZJQgh0yavk0l7XG9IqZdAq6V9CDNRGRBJCIBDpqbQzLFEgIgmhwz+1o69AxBjDsYpWfHK+CS4vNVjHEn09kIQQKBH1NpgRADiOw6FLrXB7eVyRbaQlY2NI1BLRgw8+CI7jQm4TJkyI3hsyBnjdsb/10WV8ud27dyMlJQUuV+iOEmvWrEmoCaqRNpASEYDgHDQbjSWKKdFLRFdccQX+9a9/BX+WyaKYJZ8H+OTx6J2/N4v+C5ANbMGtG2+8Effeey/+/ve/48YbbwQg7L7x7rvv9jmznvQt06iGl2dI828d1BuNUoY2uwd2GksUU6IHIplMhszMTLGzMWyo1WrccsstKC4uDgaiF198EXl5eViyZIm4mRvBpueaMD3X1G+6wKBGGksUW6IHovPnzyM7OxsqlQrz58/H9u3bkZeX12Nal8sVUmWxWCyDezOpXCidxJp0cItw3XnnnZgzZw5qamowatQolJSUYP369bRGTgyo/Ss0Oj3h7UlHwiNqIJo3bx5KSkowfvx41NXVYdu2bVi0aBHOnDkDvV7fLf327duxbdu28N+Q4wZcRRLTjBkzMG3aNOzevRvXXnstzp49i3fffVfsbI1YXh8Pl5eHRiHtN5h3BiKqmsWSqIFoxYoVwcdTp07FvHnzMHr0aLz22mu44447uqXfsmULNm3aFPzZYrEgNze3W7p4sGHDBjz11FOoqanBsmXL4vY6Y6Ghw4XXjlYhWavAugX5faZVUiAShehVs65MJhPGjRuHCxd63olVqVRCqey7sTFe3HLLLdi8eTOee+457N69W+zsjGiBHjONov/u+PGZeuQkqYMz8UlsDKsBjVarFRcvXkRWVpbYWRGd0WjE97//feh0OqxZs0bs7IxonaOq+2+r0yllyDCoaKpHjIkaiDZv3oyPP/4Y5eXl+Oyzz/C9730PUqkUP/zhD8XM1rBRU1ODW2+9NWFKgdESKBEZaHrHsCXqJ1NdXY0f/vCHaGlpQVpaGq688kocOnQIaWlpYmZLdG1tbThw4AAOHDiAP//5z2JnZ8QbTInI5fXhdLUZbi+PBYWp0c4a8RM1EO3Zs0fMtx+2ZsyYgba2Njz66KMYP3682NkZ8SyDmPDKGPDJ+WYAwNyCZMikw6r1Im5RWXUY6m3fMhKegU7vAIQF9iUcB54xODw+6CkQxQQFIhLXeJ7himwjOpyeAVXNOI6DSi6B3e2D08NDr4pBJkn8B6K+1igm/Rvpvz+JhMPicYNrc1QrpP5ARGOJYiVuy51yufDtZ7fbRc7JyBb4/QV+n4lA5V/+g3Z8jZ24LRFJpVKYTCY0NjYCADQaDc3VGgTGGOx2OxobG2EymSCVjsy1eRxuHxgY1HL/9A7eB9SdBFwWIHMqoEnu9hqVgkZXx1rcBiIAwVn9gWBEBs9kMo3o1RGOV7bhSFkrpueZcPW4NODMXqDFP3K/5hgwcx2gDe2mD8w3c9BSIDET14GI4zhkZWUhPT0dHo9H7OyMOHK5fMSWhAKCPWZKGVB3QghCEhmgMgL2FuCbd4GZtwkTov1mj07C1Bxjn6s5ksiK60AUIJVKR/w/FAlPcAyRQgJUfC4cHLMYSJ8IHP5/gKUWaCsHkguCr+lpzzMSXXHbWE0I0DmqOslRDjjNgEILZM8AlHqhjQgA6k+Ll0ECgAIRiWM8z2ANlIgs54SDmVM6F6rLuEK4bz4nLCPsZ7Z7cLS8Faeq22OY28RGgYjELZvbC54xyJkHSnOZcDAQfADAMEpoK/J5gNay4OF2hxv/Od+Mk9XmGOc4cVEgInErUC3LZA2QMF7oqtd2GdzIcUDKWOFxe0XwcGAbIRd138cMBSISt1RyKWbkmXCFUpjEiqSCkN4xAIBptHDfVt7ldcK/hctL61bHCgUiEreStQosGZ+OiaoW/4GC7olM/o0abM2Aywqgs0Tk9vLw8SN7istIQYGIxDdHG+BoBzhJZ9DpSqHpHNDYUQdAmIEf4KZSUUxQICJxq93uhqOpAgwM0GcCsl5WujRkC/eWWgDCRFmFPxjRNI/YoEBE4tY7p+qw7/PjaLd7AOOo3hPq/VNYOuqDhwKlImonio2EGFlNElOH04ssVwOUJglgyOk9od6/WUNHrbBEI8dh1dQsSDgOyTTKOiYoEJG45PbycLuc0HhaoZCZOqtfPdGmAxIp4HECznZAnYQsozpWWSWgqhmJUx1OD3TuRsgkgExjAlSG3hNLZZ0N1tammOSPhKJAROJSh9MLnbtJaHTuqzQUEBjoaBMCUVWrHUfLW1HVSgvrxQIFIhKXOpxeaN0tQqOzbgDrKWn8JSK7MPjxUrMN/znfjPIWWxRzSQIoEJG41OH0QOMJBKL0/l8QqJr5S0TBXjMP9ZrFAjVWk7iUrZcBWhcMSjmgy+j/BYFAZG8FeB4q/yqNTi+NI4oFCkQkLuWr7UCyRhg5rdT1/wKVSWi09nkBRxuUMmGpECoRxcawqZrt2LEDHMfhvvvuEzsrJB5YG4T7gZSGAGEybJd2IhrQGFvDIhAdPXoUf/nLXzB16lSxs0LiAM8ztDZUwePjwbQDaB8KCLYTNXdWzWiKR0yIHoisVituvfVWPPfcc0hKShI7OyQOdDi9+OJsKY5Xtg2soTpA7d9ayNFGJaIYEz0Qbdy4EatWrcKyZcv6TetyuWCxWEJuhFzObHdD426FUiYFpx/EVkiBPc4crTCo5fj+zBx8f2Yfc9RIxIjaWL1nzx4cP34cR48eHVD67du3Y9u2bVHOFRnpbOZmSJkHCoWqs5QzEGp/idzRBrlUgrwUTXQySLoRrURUVVWFX/7yl3jppZegUqkG9JotW7bAbDYHb1VVVVHOJRmJ7GZhQ02JLhWQDOJPPBCI3HbA64pCzkhvRCsRHTt2DI2NjZg5c2bwmM/nw8GDB/GnP/0JLper215kSqUSSmUva8oQ4ue2CIFIrkvtJ+VlZEqhu99tBxxt+Nqqgc3lxYQsA3RKGukSTaL9dpcuXYrTp0P3k/rJT36CCRMm4P7776cNEUnYvB1NkAFQGtL6TduNOikYiI6UOdFqcyPDoKJAFGWi/Xb1ej0mT54cckyr1SIlJaXbcUIGg7cJa1SrTYPoMQtQJwHmGn/PmRYA9ZzFAoV5ElcYz6NQ54JHpoI2aRA9ZgFdu/DlwhrXLprmEXXDKhAdOHBA7CyQEY7zOjBKywE6HWAYZBsR0NlgbW+F0j+okUpE0Sf6OCJCIsrm38NMZezcWnowunTh0wz82KFAROJKR2sDrC4v3MowR+kHu/BtUEmEAERVs+gbVlUzQoaqrKoSthoztGoFwpq5KFcJ3fheFzRM2HCRqmbRR4GIxBVvh39hs3C67gNURsDaiEI9j5SZuTCo6d8k2ug3TOIKbxXaiMLqug/wByI9bNDTNI+YoDYiEjeY1wWfwwwAMCQPcB2inqiMwr3THIFckYGgEhGJG9b2RvAM8ErVMOiN4Z/IH4jctjaUVpvBM4ZpuabIZJL0iAIRiRu2VmFVRk6TDImEC/9E/kDksbXhXw0NUMolFIiiLKyq2aVLlyKdD0KGzNYuBCKZfggN1UAwEMk8HQCEXWMZY0M7J+lTWIGosLAQV199NV588UU4nc5I54mQsKRLbMhP0WBUdh/73A+EPxDJvXZIeC8Yoy78aAsrEB0/fhxTp07Fpk2bkJmZibvvvhtHjhyJdN4IGRQ9b0aWUY3cnNyhnUimAmQKSDgOWggbLFIgiq6wAtH06dPx9NNPo7a2Frt27UJdXR2uvPJKTJ48GU888QSammj/cBJjPA842oTHmpShnYvjgqUiPQsEIhpdHU1D6r6XyWS44YYb8Prrr+PRRx/FhQsXsHnzZuTm5uK2225DXV1dpPJJSJ+Yow3NFjusXoBXGIZ+QpUJAKALlIhovllUDSkQffHFF/j5z3+OrKwsPPHEE9i8eTMuXryIDz74ALW1tVi9enWk8klIn2ztDTjfaMWRBk4o0QyVv0SkpWkeMRFW9/0TTzyB4uJilJaWYuXKldi9ezdWrlwJiX994IKCApSUlCA/Pz+SeSWkV51d9ylD67oP8AeiKakcxozJQZqeliiOprAC0c6dO3H77bdj/fr1yMrK6jFNeno6nn/++SFljpCBcvgXzJcPZY5ZV/5AlCJxIIWmeURdWIHogw8+QF5eXrAEFMAYQ1VVFfLy8qBQKLBu3bqIZJKQ/rgtQgeJIlKBSOlvZ6JpHjERVhvR2LFj0dzc3O14a2srCgoKhpwpQgaFMXj9k121piHMMevKXyKyWc04XdmCyhZ7ZM5LehRWIOptlKnVah3wHmWERIzbCo/LAYCDLilCJSK5GpDKYLZ78J8zF/F1Pe0qHE2Dqppt2rQJAMBxHLZu3QqNprPu7PP5cPjwYUyfPj2iGSSkP7y1GU6PD06ZESZthNpzOA5QmSCVWKH0WanXLMoGFYi+/PJLAEKJ6PTp01AoFMHnFAoFpk2bhs2bN0c2h4T0x96KcRl6tGlyoVdFcB630gCZlIPCY4PLQwMao2lQn9pHH30EQNgI8emnn4bBEIGBY4QMkcTRgiSNAkl5eUAkuu4DVAZIJRyU3g4qEUVZWF8fxcXFkc4HIeGzCxsqQhPG9kF9URkhk0ig9FlhphJRVA04EN1www0oKSmBwWDADTfc0GfavXv3DjljhAxUS1Mt4HJBJTNCG8kTK6lEFCsDDkRGoxGcf+i80TiE1e8IiSSPE/WNTbA4vUhyqjEhkudWGSGTcFD6rHB7efA8i8yobdLNgANR1+pYpKpmO3fuxM6dO1FeXg4AuOKKK7B161asWLEiIucnCcDeAqfHB7dUgyRjhNssVQbIJBympEowZkZ2ZM9NQoQ1jsjhcMBu7xzgVVFRgaeeegr79+8f1HlycnKwY8cOHDt2DF988QWuueYarF69GmfPng0nWyQBuSyNcPsYHPIkmDRh7OzaF4UenESKFI0MeTqeSkNRFFYgWr16NXbv3g0AaG9vx9y5c/H4449j9erV2Llz54DPc91112HlypUoKirCuHHj8Mgjj0Cn0+HQoUPhZIskIGubMNkV6mQoZdLInlwiAZR64bGLBjRGU9grNC5atAgA8MYbbyAzMxMVFRXYvXs3/vjHP4aVEZ/Phz179sBms2H+/Pk9pnG5XLBYLCE3ktgc/nWq5cYh7GPWF5UB7XY3zlXUwOzwROc9SHiByG63Q68Xvin279+PG264ARKJBN/61rdQUVExqHOdPn0aOp0OSqUSP/3pT/HWW29h0qRJPabdvn07jEZj8JabO8QlQcmIF5jsqjZGaGrH5VRGVLc78OX5CjR10Prs0RL24vlvv/02qqqqsG/fPlx77bUAgMbGxkEPchw/fjxOnDiBw4cP42c/+xnWrVuHr776qse0W7ZsgdlsDt6qqqrCyT6JFz4PPFZheVhtUmZ03kMpNFgrvVY4aZXGqAlrQOPWrVtxyy234Fe/+hWWLl0arErt378fM2bMGNS5FAoFCgsLAQCzZs3C0aNH8fTTT+Mvf/lLt7RKpRJKJS1QRfzsrRidrEaKSQ95ZvSqZjSWKPrCCkQ/+MEPcOWVV6Kurg7Tpk0LHl+6dCm+973vDSlDPM/D5XIN6RwkQdhboJJLoTJmA1pF/+nD4R9LpPBZaQH9KAp7hmBmZiYyM0OLw3Pnzh3UObZs2YIVK1YgLy8PHR0dePnll3HgwAHs27cv3GyRRGL3r4kV6akdXSmNkEokUHqtNPE1isIKRDabDTt27MCHH36IxsZG8HxokXWgO8E2NjYGd/swGo2YOnUq9u3bh29/+9vhZIskGHNLPaztDsjStYjQcmjd+Qc1SpkHdpcjWu+S8MIKRBs2bMDHH3+MH//4x8jKygpO/RgsWtOaDIW5uQ41rXa4zIroBSKpHJxSC8AO3tEerXdJeGEFovfeew/vvvsuFi5cGOn8EDIwvA/uDmHWvTopamEIAGA0pWAc7wDLjPCASRIUViBKSkpCcnJypPNCyMDZW+Byu+GTyJFkGuLOrv3QGJKhcTUBMupEiZawxhE9/PDD2Lp1a8h8M0JiytYEh8cHuzwFydooD+lQBXb0aI/u+ySwsEpEjz/+OC5evIiMjAzk5+dDLg+dbHj8+PGIZI6Q3rja6+HxMdjVyTBpotR17+dVGGC2uWCrrUdeYVTfKmGFFYjWrFkT4WwQMjj2tjrhgS4NCtmQdk7vl0emw7kGK6xtNcihNYmiIqxA9MADD0Q6H4QMirOtHgCgNEa3oRoAFNok4b28Vrh9PFQSarSOtLC/Strb2/F///d/2LJlC1pbWwEIVbKampqIZY6QHnkcyFC6MTXHiFmTxkX97aQaE6QcIOcdcDmpwToawioRnTp1CsuWLYPRaER5eTnuvPNOJCcnY+/evaisrAyuVURIVFgbIeE4aA0p0Cabov9+MhU4uRJwu+C2twH6CO2dRoLCKhFt2rQJ69evx/nz50N2dl25ciUOHjwYscwR0iObsPQHdFGa6Ho5jgOvEHrOPDZzbN4zwYQViI4ePYq777672/FRo0ahvr5+yJkipC++jgZcarLinFUFH9/z9ueRxpRCIPLR6OqoCCsQKZXKHldHPHfuHNLSorRAFSF+ztYaNHS48EWLMqL7KfbJP5bIa2uP0RsmlrAC0fXXX4+HHnoIHo+wdCbHcaisrMT999+P73//+xHNICEhfB44/cvDykzhz3McrIKcbIzL0CFd6Y7J+yWasALR448/DqvVirS0NDgcDixevBiFhYXQ6/V45JFHIp1HQjpZG+F0e+CRqKE3xG6aUUZaOlK0Smh5a8zeM5GE1WtmNBrxwQcf4NNPP8XJkydhtVoxc+ZMLFu2LNL5I8OI3e3F4UutmD82BSq5MJaGMRazUgkAoKMeDo8PVmUGknUxXK1T5d9U1EkbNkTDoAMRz/MoKSnB3r17UV5eDo7jUFBQgMzMzNj/UZKYOnSpBSerzLA4Pbh+WjbO1lrwVa0FN8wcBZk0uqObgzrq4HD7YFOloShaqzL2wMw08NpckDlbYeR5YashEjGD+m0yxnD99ddjw4YNqKmpwZQpU3DFFVegoqIC69evH/IysWT4crh9OFsjlAZm5iXB42P4/GILatodOFsbu1ICs9QKJSJFKpJjGIiqrBxKG2yoa7cB7o6YvW+iGFSJqKSkBAcPHsSHH36Iq6++OuS5f//731izZg12796N2267LaKZJOI739gBL8+QblAiJ0kNjuMwOz8JB0qbcLK6HVNzjNEvDXtdcHc0wcczOBRpUZ/s2pVSLoNbpoOPdwjVs0BVjUTEoEpEr7zyCn7zm990C0IAcM011+DXv/41XnrppYhljgwf5xuERtrxGfpgwJmUbYBCJkGL1Y3qthgso2qpgVIqwZwJBbhx4URIYzj5VCmTwiXVwcszwEmDGiNtUIHo1KlTWL58ea/Pr1ixAidPnhxypsjw4vT4goGmMF0XPK6USTE+Q9ho85v6GFRXzNUAAFlSLtL0sd1WSimXwCXTCwMoafvpiBtUIGptbUVGRu+znTMyMtDW1jbkTBER8TxQeQg4sxdo/BoAUNVqB88YUnWKbtWh8ZlCIDrf2BH9Uc7+QARjTnTfpwdKmaRLiYgCUaQNqo3I5/NBJuv9JVKpFF6vd8iZIiI6vw+oPSE8bioFvE7YfAWQSznkJHWf7DnKpIZOKYPV5UVFiw1j0nTd0kQE7wMsNShrtqE6SY0pSe7YthHJpP42Igbe0R7+shWkR4MKRIwxrF+/vtfdVmljxBHOXC0EIY4DkscALReBCx9i+ty7MGVUITy+7judSiQcJmYZ0O5wQ62I4jo9HfVgPg/q7By+aJRgYmFs5pgFKGUSuGRCkPXazYhdCEwMgwpE69at6zcN9ZiNYFVHhPvMKcD4lcCX/x9grgGqjkBatAzSXhYEu7IoihscBrRXwO3l0SbPgEQigUkt7/81ESSRcJg/aQzSOR2kbjPAmBCwSUQMKhAVFxdHKx9EbG4b0HJBeJwzR/gnG70Q7NSr4OpPAgWLAFlsG4hDtF6Cw+ODWZUDk0YeuwGUXUzMzwEqlQDzAR4HoKB1iSKFqrpE0FQqtMPoMzvX+Ukeg0qnGifKGnHu9NF+T9Fqc+NiUxTmYnmcgLkGDrcP7ercmA5kDCGVAQqt8Jh6ziJK1EC0fft2zJkzB3q9Hunp6VizZg1KS0vFzFLiavVvE57aZelVjkO1YiwcHh/krd/0+fLadgde+Kwc+882RL73rK0cYDzMnAEumR4pIgWierMTDW4FnB4fjSWKMFED0ccff4yNGzfi0KFD+OCDD+DxeHDttdfCZrOJma3Ew/uEf3ZAaKTu4pIkTzjsrhOqb73INKigUUj9Y44ivN9d60UAQL0sGwCQEsvJrl0cLmvB0XoeFqeHuvAjLKzZ95Hy/vvvh/xcUlKC9PR0HDt2DFdddZVIuUpAllrA5xHaPPSZwcMOtw/NXg2yFKnQKnxA8zkge0aPp5BIOBSm63Cq2ozzDVaMTtFGJm+8D2gqBQNDszIPYIj5YMYApUwCm1QHL19PJaIIEzUQXc5sFj7c3razdrlcIUMEelolkoTB4t95xZgT0hPU2OEEALhNYyGTXBC683sJRACCgehikxXX8OmR2f+rtQzwusAp9fjBtxbA4WVQRnkfs94oZVK0yXTw+RjgokAUScOmsZrnedx3331YuHAhJk+e3GOa7du3w2g0Bm+5ubkxzmWcCoxYNoSOWG7sEIK+It3fbtRWDvh6H7Cak6SBSi6F3e1DTXuE5p41nBHu0yYCEgnUCqloGxzS6OroGTaBaOPGjThz5gz27NnTa5otW7bAbDYHb1VVVTHMYZxirEuJaFTIUy1WIRAZ0nIApU6ovpkrez2VVMJhbJpQJbvQGIHeM5dVqA4CQGbPX06xJMw308HH81Q1i7BhEYjuuecevPPOO/joo4+Qk9P7PCKlUgmDwRByI0PkaAPcdkAiBXSZIU8ZVHKk6ZVI1Ss7G7FbLvV5uiL/JNiKFhsYG2LvWd1JoY3IkI19lcBbX1ajzhyDWf69UMqkcMn0QonI4wC8tH51pIjaRsQYwy9+8Qu89dZbOHDgAAoKCsTMTmLq8G//pEsXxsl0saAwFQsKA6OmC4G6U/4erN6XBM5L1uD66dkYnawZ2vpEXhdQ7R+7NGoWqs7b0eH0Yl5BSvjnHCKlTAKfRAk3/KO6XRZAFoNR5QlA1BLRxo0b8eKLL+Lll1+GXq9HfX096uvr4XCI962XcGyNwr2unz3kk/IBTgLYW4VSVC+E6plu6COfqw4LpQ5NChxJ49Hh9ILjgBSdeLO80g0qfHtSBkZl+n9XVD2LGFED0c6dO2E2m7FkyRJkZWUFb6+++qqY2Uos1kAgCt011ePjwXcdmChTdi6/0dp39SyA51noOQaTp8pDwuOCq9DQIVSBTGo5lLIoTqzth1Etx+RRRqSk+H9XNLo6YkSvmhGRWYU9wi4vEZ2oasehiy2YkZfUOak1eQzQXim0E42a1edpT1ebcaS8FYuKUjHO3240IG47cPZtoW0otQhIG4+6S60AgEyjeuDniSb/ZotUIoqcYdFYTUTitgk9UwCgDd2ht8XqhpdnkEu7tPMEGqzby/vsxgeADpcHFocHJyrbB56fjgbgxEuAvQVQ6oFxywGOQ71FqKpnGVUDP1cU8DxDZYsdlTa58CVKXfgRQ4EokQWqZeqkbjPrW21CdSikTUaX7u/G9wLmvodOTM0xQcJxqGl39N/TxfNAxefA8RcAW7MwsXTazYBSB8YY6szCwEqxAxEA7P2yGgcqnPDQ2tURRYEokfXSPsQYQ6tNGEOUrO0SoAILpgHB+V+90SllmJglVMk+Od/cezXc0SaUgi4d6KyOzbkD0ArVQZeXR4ZeBa1SilSR5pgFSCQcVHIpXDIdvD6e2ogiaFhN8SAx1kv7kMXhhcfHIJVw3RcgSx7j78Yv6/f03xqbgtL6DtS0OXCh0RocYxTUfB74+h9CV71MARR+W1iUrUu3v0ouxfdn5QybzTvVcik6pHp4fAxwdQilOdpsccjoN5jIbD2XiFr8paEkraL7dIqkfCFQ2Jr7rZoYVHLMGp0EAPjwm0bYXF3alRq+As68KQQh4yhg9h1A1tReVz0cDkEIEAKRR6qBl3HCqHQqFUUEBaJExfsAW4vw+LKG6mD7UE/r/sjVgME/FWQA3fhzC5KRqlfC6fF1thW1lgklIcaEEtD0WwG1qdtrGWOwuobXZgxqhRTgODil/k0CKBBFBAWiRGVvBRgvVIku27XUpJGjMF2HnKReusuD0z36bicCAJlUgpWTM3H9tGwUpuuFUtRXfxPeO+MKYMIqYXpJD5o6XHju4CW8cqRy2Az1UMuFvDoCgaiPwZ1k4KiNKFHZm4V7TWq36lBhul4IGr1JHgOUHRRm4/O+XgNJQIpOKSxmxhhwbj+cDiukhizIx6/ocwH6wMRZrVI2fKpm/p1K7FIDgGYKRBFCJaJEZWsS7i+rlg2IPlPoYvd5Old2HIjm83A3nsPZOhvecs2CxdN7UsYYzjUIu8eOy4jSXmlhGJumw7KJGRiVJawWCXuruBmKExSIElUvgcjj49Hh9PRdFeI4IG288Ni/G2y/vG7gwr/g8fGoM0xBjUuDV49UodHi7DF5g8WFNrsHMgmHgtQIrfYYAZlGFabkGJGSliUcoBJRRFAgSlQ2f9VMGzqbvabNgf/7pAx7jvaz1lP6ROG+ubTfUdYAgMrPAKcZWkMyFi1bjVSdAlaXF68fq+5x548vK4V/8KIMnajzy3ql8a8i6mgVqpxkSCgQJSKft/Ob/PKpHf4eM72qn+ZDY64wDcPr7ndwI2wtnZs3Fn4bRp0WN87OxegUDdxeHv84WRsMPADQaHGi1F8tm5mXNPDrigGPj0dlix0XLFJhNQKfVxhPRIaEAlEisrcI3+JyFaAIbX8JdN33u3cYxwEZk4THtSd6T8cYcH6/0KidMlYYOQ1hoOLq6aMwZZQRjAEHSptwoqodAHChyQrGgPGZeqQbxJ/W0ZXd7cObx6vx3tlGsMDkV6qeDRkFokQUaB/qoccsMLUjRTuA6RTZM4TXt17qvdG24azQoC2RAYXLQt5PKuGwdGI6FhWlIlmrwHj/yOv5Y1KwsDAVi8eF0ZAeZYHuey/P4FOahIMOarAeKgpEiSjQdX9ZtYwxFqyaDWg3VXUSkDxWeFz5effn3Xbgwr+Ex/kLO9tVuuA4DrPzk3HLvLxg1zjHcZhbkAytcviNLpFLOcj8o81dcpNwkEpEQ0aBKBHZeg5EdrcPLg8PjgOSNPIeXtiD0QuE+/rTwjIeAYwB594TVlnUpgK58/o8jVyEvezDwXFcMGA65f6qGXXhD9nI+PRJZAW77kN7zALtQ0a1fOBLvRpHAekThMDz9d+FfeoZE2bTN50TBjv2MXp6JFIFR1f7R6RTiWjIhl/Zl0SX1905WfWyEpFWKcPs/KTBl04Klwl7o9magS+eB+SazkX5i74NGLIjkPHhI9BOZJUGGqvbheA7TEZ/j0RUIko0gR4zhUYYHd1FslaBRUVp+NaYQe6UodQDU24S7p0WIQhJZMD45X3uDDtSaZX+QASNUNLjvbRI2hBRiSjRDGVqR1/0GcDcu4QxRT43kFTQubZznJmUZUROkgaZRhXQlCSUBO0tPa4gQAaGAlGi6dp1f5madgdMajk0Cml4k0xlis4R13EsL0XT+YMmpTMQpYwVL1MjHFXNEo09sAZRaCByenx47WgV/nrwEtw+XoSMjVCB32OgJ5KEhUpEiaaXqllrl6kdw3Ju1zDi9vKoNzvh9vEoDJQsA79XEhYKRInEbe/cAueyQNRs9Y+oFnEn1ZGiw+nBm8eroZRLUDjb/3u0N1PP2RCIWjU7ePAgrrvuOmRnZ4PjOLz99ttiZif+BdaoVpuEeWZdBAJRmm54ze0ajgIjvl0eHh6lSZj86nXT5NchEDUQ2Ww2TJs2Dc8++6yY2UgcvWwfBADN/m2dU/VUIuqPUiYJTvOweyFMdQGoejYEolbNVqxYgRUrVoiZhcQSGGR42fZBjDE0+UtEYu8dNhJwHAeNUgaLwwObywujNlXoBKCes7CNqDYil8sFl8sV/NlioR0UBiW4j1lmyGGLwwu3l4dUwiFJQyWigdAqpLA4PLC7vULPWVMp9ZwNwYjqvt++fTuMRmPwlpubK3aWRg6ft3Ny5mVVM6Vcgm9PysC3xqRAevk+ZqRHgXYim8vXOSaLqmZhG1GBaMuWLTCbzcFbVVU/y5mSTrYmYQsfuVqYitGFSi7F5FFGzC3ovkwH6VlgmofN5e3sgbQ10bKxYRpRVTOlUgmlktowwmLt0j5EXcxDNi5Dj3S9ChkGFaCRC3PrfB5hJn4P6y6Rvo2oEhEZAkutcG/I6vbUmRoz6s1O8Dx9mw9UTpIGk0cZkaZXAhIJoPOXigIdAmRQRA1EVqsVJ06cwIkTJwAAZWVlOHHiBCorK8XMVnwKBqKckMMOtw8ffNWAV45U0tSOoQh0AFgpEIVD1KrZF198gauvvjr486ZNmwAA69atQ0lJiUi5ikMeR2ePzmUlonr/vmLJWkVwwS/SP55nqGl3wOL0YFKWAZzePyQiMFaLDIqogWjJkiXDZk/zuBYoDamTuq1BVG8WAlHGMNstYyR483g1GANGp2ihC5SIOuppqkcYqI0oEQSrZd1XSmzsEAJRppEC0WBIJBx0/i78DqdH6DnjJELp00Xj2waLAlEiMPuHORhHhRxmjAVLRJlUIho0g0rYYKDD6QWkss41wLtuIkAGhAJRvPN5hPWkAWHVxC4sDi/sbh+kEg6pNOt+0AK74XY4Pf4D/hKnpUakHI1cFIjinblK2GVVZeicnOlX1WYHIJSGBrxrBwnS+0tEFqdXOBAocVIgGrQRNaCRhKGtXLhPyu/WgDohUw+jeoD7l5FuDGrh38ds95eIAkMjLHVC8I+jLZSijQJRvGu+INwn5Xd7SiaVIDdZ0+04GZjABOF2u7CECjTJwjpPHqfQe3ZZmxzpHQWieBZY1F0i7dwamkRMml6JZRMzkKT1lyo5TigVtVwQqmcUiAaMGgbiWVOpcJ+U321Fxm/qLfj3Nw2oMztin684oZJLMSVH2FooyOivngU6CMiAUIkoXjEGNH4lPE4d1+3pr2otqGixw6CSI8uojnHm4ligFGSuooGNg0AlongV2AJaKgPSJoQ85fL6UN0mlITGpOnEyF3caLW5cabGjGp/DyT02YBULmxUQNM9BowCUbyqOyHcp0/qVi0rb7bDxzMkaxVI1tL4oaH4ps6CD75qwNd1/oXzpTLANFp43FYmXsZGGApE8cjRDjT4q2U97D3/dZ0wBaEwnUpDQ5WqF9bHCuyCAgBI9g8cbaVANFAUiOJRxWfCaozJBd3ml1ldXpS32AAAk7Lic2/6WErzbzbQ3OHqXM8pMILdXC2MbCf9okAUbyx1QP0p4fHohd2ePl1tBmNAtkmFJKqWDZlJI4dCJoGXZ2jrOp5IZQR4L9B6SdwMjhAUiOIJ7wPOvSf01mRMAkzdNxfQKWXQKWWYkZfUwwnIYHFc5zy9wJZM4DggbbzwuOkbkXI2slAgiicXPxJmfstVwNhrekwyJceInyzMRyH1lkVMmr+dqNHSpZ0o0FPZckHYQYX0iQJRvKg7BVQfFR5P+G63nTq6LkAnk0ogoW2DIibTIIzDqm3vMjjUkC18Bl43Vc8GgAJRPGi5CJS+JzwePR9ILQp52uX14fUvqnGugfZmj4b8VA3WzBiFNTO6TOngOCB9ovC47qQ4GRtBKBCNdO2VwNm9Qi9Z5mSgYHHI0w63D29/WYOadgc++qYRLq9PpIzGL41ChoJUbfc1vwNDJ1ovCkMqSK8oEI1k7ZXAqdeENoiUscD4lcEpBYwxXGjswEuHK1Db7oRSLsGaGaOglNHSFDGjSRbm+TEG1H4pdm6GNZprNlK1VQCn/UEouQC44nuAROoPQFYcLmtFU4fQeJqkkWPV1OxgoyqJPKfHh+OVbWjqcOH6adngAnPMRs0U1oSqPQ7kzgMUtOxKTygQjUQhQWgMMPkGYX4TAIfHh31n6+HxMShkEkzPNWFOfjIUMir8RpOE43CsvA1enqHJ6kK63j+tJnUcoEsX5p1VHQbGXt33iRIU/XWONE2lndWx5DHA5O+j0d7Z7qNRyDBrdDLmjUnGHVcWYGFhKgWhGFDIJBidKmzVFJx3BghV5YKrhMfVXwD2VhFyN/zRX+hIUnMMOPuWMGI3tQgNo1dh78l6vHSoEjVduo7nj03BgrGptGFijE0ZZQQgbOEd0imQUihUn3kv8M27wsBTEoIC0Ujg8wDf/BM4tx9gDNbkyXiXn4+Xj9aiosUOqYQLtgcR8eSnaJCiU8Dt5XGsoq3zCY4Dxi0Xqs/mauDCv4QGbBI0LALRs88+i/z8fKhUKsybNw9HjhwRO0vDR3sVcKwEqDsJp5fHEW4y/q9xHM412sBxwMQsA9bNz8f0XJPYOU14HMdh/hhhb7MvytuCm1cCANQmYOL1wuOa48C5fVQy6kL0QPTqq69i06ZNeOCBB3D8+HFMmzYN3/nOd9DYmMCLSjEmfHOe2Qt8+aKwwJlCg/ekS/CpZzwYOIxJ0+LWeaOxfHImjBraiWO4KEzXYUyaFj6e4cA3TaFbqqeNA8YvF0pItV8Cx18QvmgIOCby5vPz5s3DnDlz8Kc//QkAwPM8cnNz8Ytf/AK//vWv+3ytxWKB0WiE2WyGwTBCl7TgecBjB9xWeCwNcLfWwN10Hk5LC6wuL/JTdZDnzAAKFuNEgxuXmqxYMDaVtogexpweHw6UNmJRURq0ys4th+QyDmq5FFzzOaD0n8JuHwCgSwNSigB9FqBNBRRaQKqIi2VmB/o/Kmr3vdvtxrFjx7Bly5bgMYlEgmXLluHzzz/vlt7lcsHl6mwLsVgGuMd4axlw8cNu9fKadgdqWoW1eRgADgwMCKablGUI7uZZZ3ag0r+OT+B54TWC8Zk6mPx7hDVYnChrFtJyjAXOKpyfAeMy9UjWyAHG0GK24FKTDYwx+Lpkj+ekaNYUghUsxbgxYwAA03LUVAUbAVRyKZZPzgo59s8zdag3OyGTcJDLZFDzyzDKfAwpHaVQyyyYYG0Kpv2m3gKnj4NPIgfjJGCcVLhBAqWMw/TczpUTztaaYXX5q3iXxS25RIKZeabgz1/VdwjbYwOYV5A89AudcpOwcWcEiBqImpub4fP5kJGREXI8IyMD33zTffmE7du3Y9u2bYN/I68L6PJBBzCrHT5b77tYMIcPYP5qj8MB3mHvnibwwMUAiX99H5cTzGULfb7rY7cMkAt/PDwDvDzgkajhUJjgUqbAY8iDIn0M0g16JHdZRZGLg2/IROT18XC4feA4wMszeN0+OKBEq3YBZKqZKOBqMSHTI4w1crTB6eHh8PgAuLudi5dJAFtn2xJvNYN39Ty7n5dygK3zOdZhBu/0CvHKGoH2KcYP/Rx+I2pA45YtW7Bp06bgzxaLBbm53dfc6caUC0y7WXgc/GfmYHB6kef0hpaA/T9wABQ6JeCfEmFw+1Dg9HY+zwmpOP/XkEotB/zd5QYPjyKXF+A6gw/HdTbHaVVy/3k56CHHOF4OmVQGtUIKuZSjgBNnZFIJbr+yAF4fD5vLBw/Pw+tj8PLCP7JcOg4wdFa1Myab4XPZhd5SngfHfELXPwCpBKFpxzqR7O25dUUiYZeldcEUSGuKQNVeoR36OfxEDUSpqamQSqVoaGgIOd7Q0IDMzMxu6ZVKJZTKMKYpKLSd6wh3YfDfBkLrvw2E2n8bCJX/RuKfTCqBUdN//1BWshGAcUDnTB/E+napEaiNRYuovWYKhQKzZs3Chx9+GDzG8zw+/PBDzJ8/X8ScEUJiSfSq2aZNm7Bu3TrMnj0bc+fOxVNPPQWbzYaf/OQnYmeNEBIjogeitWvXoqmpCVu3bkV9fT2mT5+O999/v1sDNiEkfok+jmgo4mIcESFxbKD/o6KPrCaEEApEhBDRUSAihIhO9MbqoQg0bw14qgchJKYC/5v9NUWP6EDU0SGshDeg0dWEENF0dHTAaOx9kOaI7jXjeR61tbXQ6/XDclpEYApKVVVVXPbqxfP1xfO1AbG7PsYYOjo6kJ2dDYmk95agEV0ikkgkyMnJETsb/TIYDHH5xxwQz9cXz9cGxOb6+ioJBVBjNSFEdBSICCGio0AURUqlEg888EB4KwaMAPF8ffF8bcDwu74R3VhNCIkPVCIihIiOAhEhRHQUiAghoqNARAgRHQWiQRrsrrSvv/46JkyYAJVKhSlTpuCf//xnyPOMMWzduhVZWVlQq9VYtmwZzp8/H81L6NVgru25557DokWLkJSUhKSkJCxbtqxb+vXr14PjuJDb8uXLo30ZvRrM9ZWUlHTLu0oVurr4SP3slixZ0u3aOI7DqlWrgmli/tkxMmB79uxhCoWC7dq1i509e5bdeeedzGQysYaGhh7Tf/rpp0wqlbLf//737KuvvmL/8z//w+RyOTt9+nQwzY4dO5jRaGRvv/02O3nyJLv++utZQUEBczgcsbosxtjgr+2WW25hzz77LPvyyy/Z119/zdavX8+MRiOrrq4Oplm3bh1bvnw5q6urC95aW1tjdUkhBnt9xcXFzGAwhOS9vr4+JM1I/exaWlpCruvMmTNMKpWy4uLiYJpYf3YUiAZh7ty5bOPGjcGffT4fy87OZtu3b+8x/U033cRWrVoVcmzevHns7rvvZowxxvM8y8zMZH/4wx+Cz7e3tzOlUsleeeWVKFxB7wZ7bZfzer1Mr9ezF154IXhs3bp1bPXq1ZHOalgGe33FxcXMaDT2er54+uyefPJJptfrmdVqDR6L9WdHVbMBCuxKu2zZsuCxvnalBYDPP/88JD0AfOc73wmmLysrQ319fUgao9GIefPm9XrOaAjn2i5nt9vh8XiQnBy6Z82BAweQnp6O8ePH42c/+xlaWloimveBCPf6rFYrRo8ejdzcXKxevRpnz54NPhdPn93zzz+Pm2++GVpt6IZZsfzsKBANUF+70tbX1/f4mvr6+j7TB+4Hc85oCOfaLnf//fcjOzs75B9i+fLl2L17Nz788EM8+uij+Pjjj7FixQr4fBHYZXQQwrm+8ePHY9euXfjb3/6GF198ETzPY8GCBaiurgYQP5/dkSNHcObMGWzYsCHkeKw/uxE9+54MDzt27MCePXtw4MCBkAbdm2++Ofh4ypQpmDp1KsaOHYsDBw5g6dKlYmR1wObPnx+yt96CBQswceJE/OUvf8HDDz8sYs4i6/nnn8eUKVMwd+7ckOOx/uyoRDRAg92VFgAyMzP7TB+4H8w5oyGcawt47LHHsGPHDuzfvx9Tp07tM+2YMWOQmpqKCxcuDDnPgzGU6wuQy+WYMWNGMO/x8NnZbDbs2bMHd9xxR7/vE+3PjgLRAIWzK+38+fND0gPABx98EExfUFCAzMzMkDQWiwWHDx+O6U634e64+/vf/x4PP/ww3n//fcyePbvf96murkZLSwuysrIiku+BisSOwj6fD6dPnw7mfaR/doAwtMTlcuFHP/pRv+8T9c8uZs3icWDPnj1MqVSykpIS9tVXX7G77rqLmUymYLfuj3/8Y/brX/86mP7TTz9lMpmMPfbYY+zrr79mDzzwQI/d9yaTif3tb39jp06dYqtXrxatC3gw17Zjxw6mUCjYG2+8EdLF29HRwRhjrKOjg23evJl9/vnnrKysjP3rX/9iM2fOZEVFRczpdMb02sK5vm3btrF9+/axixcvsmPHjrGbb76ZqVQqdvbs2WCakfrZBVx55ZVs7dq13Y6L8dlRIBqkZ555huXl5TGFQsHmzp3LDh06FHxu8eLFbN26dSHpX3vtNTZu3DimUCjYFVdcwd59992Q53meZ7/97W9ZRkYGUyqVbOnSpay0tDQWl9LNYK5t9OjRDEC32wMPPMAYY8xut7Nrr72WpaWlMblczkaPHs3uvPPObmNxYmkw13ffffcF02ZkZLCVK1ey48ePh5xvpH52jDH2zTffMABs//793c4lxmdHy4AQQkRHbUSEENFRICKEiI4CESFEdBSICCGio0BECBEdBSJCiOgoEBFCREeBiBAiOgpEZEQpKSmByWQK/vzggw9i+vTpwZ/Xr1+PNWvWxDxfZGgoEJEeBdYs/ulPf9rtuY0bN4LjOKxfvz4kfaQDQH5+Pp566qmQY2vXrsW5c+d6fc3TTz+NkpKS4M9LlizBfffdF9F8kcijQER6lZubiz179sDhcASPOZ1OvPzyy8jLyxMlT2q1Gunp6b0+bzQaQ0pMZGSgQER6NXPmTOTm5mLv3r3BY3v37kVeXh5mzJgxpHP3VFJZs2ZNsJS1ZMkSVFRU4Fe/+lVwFwmge9Xscl1LZuvXr8fHH3+Mp59+OniOsrIyFBYW4rHHHgt53YkTJ8BxXMzXSiICCkSkT7fffjuKi4uDP+/atQs/+clPov6+e/fuRU5ODh566CHU1dWhrq5u0Od4+umnMX/+fNx5553Bc+Tl5XW7JgAoLi7GVVddhcLCwkhdAhkECkSkTz/60Y/wn//8BxUVFaioqMCnn346oIW0hio5ORlSqRR6vR6ZmZlhrXpoNBqhUCig0WiC55BKpVi/fj1KS0uDe395PB68/PLLuP322yN9GWSAaM1q0qe0tDSsWrUKJSUlYIxh1apVSE1NFTtbQ5KdnY1Vq1Zh165dmDt3Lv7xj3/A5XLhxhtvFDtrCYtKRKRft99+O0pKSvDCCy9ErNQgkUhw+VJYHo8nIuceiA0bNgQb4ouLi7F27VpoNJqYvT8JRYGI9Gv58uVwu93weDz4zne+E5FzpqWlhbT7+Hw+nDlzJiSNQqEY8vY1vZ1j5cqV0Gq12LlzJ95//32qlomMqmakX1KpFF9//XXwcW/MZjNOnDgRciwlJQW5ubnd0l5zzTXYtGkT3n33XYwdOxZPPPEE2tvbQ9Lk5+fj4MGDuPnmm6FUKsOqEubn5+Pw4cMoLy+HTqdDcnIyJBJJsK1oy5YtKCoqiumC96Q7KhGRATEYDDAYDH2mOXDgAGbMmBFy27ZtW49pb7/9dqxbtw633XYbFi9ejDFjxuDqq68OSfPQQw+hvLwcY8eORVpaWlj53rx5M6RSKSZNmoS0tDRUVlYGn7vjjjvgdrtj0gtI+kZrVpOE9cknn2Dp0qWoqqrqtlMqiS0KRCThuFwuNDU1Yd26dcjMzMRLL70kdpYSHlXNSMJ55ZVXMHr0aLS3t+P3v/+92NkhoBIRIWQYoBIRIUR0FIgIIaKjQEQIER0FIkKI6CgQEUJER4GIECI6CkSEENFRICKEiO7/ByYlU2ZS1c7qAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T05:07:26.370391Z", + "iopub.status.busy": "2024-03-24T05:07:26.369764Z", + "iopub.status.idle": "2024-03-24T05:07:26.797880Z", + "shell.execute_reply": "2024-03-24T05:07:26.796993Z" + }, + "papermill": { + "duration": 0.44865, + "end_time": "2024-03-24T05:07:26.799989", + "exception": false, + "start_time": "2024-03-24T05:07:26.351339", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABDG0lEQVR4nO2deZxT5b3/Pyf7MllmX5gVBhBkF6GAKAhVFhVqr0ulFdyreHspP15V7m2xai3YqpW2XtpaYeQKUq2ot9cqIgpURFaHVfbZ9zWZ7Mt5fn+cJDNh9kySM5N8369XXsmcPDn5nsnMJ9/neb4LxxhjIAiCEBGJ2AYQBEGQEBEEITokRARBiA4JEUEQokNCRBCE6JAQEQQhOiREBEGIDgkRQRCiIxPbgIHA8zyqq6uh0+nAcZzY5hAEcRWMMbS1tSErKwsSSfd+z5AWourqauTk5IhtBkEQvVBRUYHs7OxunxddiKqqqvDUU0/h448/hs1mQ2FhIbZs2YKpU6f2+lqdTgdAuEi9Xh9pUwmC6Cdmsxk5OTmB/9XuEFWIWlpaMGvWLMydOxcff/wxUlNTcfHiRSQmJvbp9f7pmF6vJyEiiEFMb0snogrRiy++iJycHGzZsiVwrKCgQESLCIIQA1F3zf73f/8XU6dOxV133YW0tDRMnjwZr7/+erfjnU4nzGZz0I0giKGPqEJ05coVbNq0CSNHjsSuXbvw+OOP4yc/+QnefPPNLsevX78eBoMhcKOFaoKIDTgx6xEpFApMnToVX331VeDYT37yExw5cgQHDx7sNN7pdMLpdAZ+9i+EmUymbteIGGPweDzwer3hvwAigFQqhUwmozAKIgiz2QyDwdDj/ygg8hpRZmYmxo4dG3RszJgxeO+997ocr1QqoVQq+3x+l8uFmpoa2Gy2AdlJ9A2NRoPMzEwoFAqxTSGGGKIK0axZs3D+/PmgYxcuXEBeXt6Az83zPEpKSiCVSpGVlQWFQkHf1hGCMQaXy4WGhgaUlJRg5MiRPQavEcTViCpEP/3pTzFz5kz8+te/xt13343Dhw/jL3/5C/7yl78M+Nwulws8zyMnJwcajSYM1hI9oVarIZfLUVZWBpfLBZVKJbZJvVLeZMOx8mZkJ2pwfX6S2ObENaJ+bV1//fV4//338fbbb2PcuHF4/vnn8eqrr2LZsmVhew/6Zo4eQ+l3bXN58I+T1WhscyHLqBbbnLhH9Mjq2267DbfddpvYZhBxxrc1Zrg8PNJ0SmTqB7/3FusMna8wgggj52stAIDRGTpIJLR2KDYkRETc4XB7Ud/mAAAUpGjRbHXhaGkzqlvtIlsWv5AQDULmzJmDVatWiW1GzFLdagdjgFEjh04lx7GyFvzrYiNKG61imxa3kBARcUd1q+ANDfMtUqfphNi0+jZnt68hIktcCpHLw3d783j5Po9192Fsf1mxYgX27duHjRs3guM4cByH7OxsbNq0KWjcN998A4lEgrKyMgDAK6+8gvHjx0Or1SInJwdPPPEELBZL0Gu+/PJLzJ49G2q1Gjk5OfjJT34CqzX+vAC1QoJUnTKwW5am9wuRQ0yz4hrRd83E4LUvLnX7XEGKFksnDwv8/Jf9l+H2dp0Fk52oxl1T2/PdNh8ogd0VnEry0++O6pdtGzduxIULFzBu3Dg899xzAICXX34Z27dvx+OPPx4Yt23bNsyaNSsQ/CmRSPD73/8eBQUFuHLlCp544gn87Gc/w3//938DAC5fvowFCxbgV7/6FTZv3oyGhgY8+eSTePLJJ4OqH8QD1+Ul4bq89rihRI0QCW51euH0eKGUScUyLW6JS49oMGMwGKBQKKDRaJCRkYGMjAwsW7YMBw4cQHl5OQAhanzHjh1B8VarVq3C3LlzkZ+fj5tvvhm/+tWv8M477wSeX79+PZYtW4ZVq1Zh5MiRmDlzJn7/+99j69atcDji2xNQyaXQKATxMdncIlsTn8SlR7RybmG3z129k/vojSO6HXt1xsiDsyJTS2nSpEkYM2YMtm/fjqeffhr79u1DfX097rrrrsCYzz77DOvXr8e5c+dgNpvh8XjgcDhgs9mg0Whw4sQJnDx5Etu2bQu8hjEWSIUZM2ZMRGwfbLi9PKQc12nL3qiRw+byosXmRhrFFUWduPSIFDJJtzeZVNLnsfI+jA0Xy5Ytw/bt2wEA27dvx4IFC5CcnAwAKC0txW233YYJEybgvffew7Fjx/Daa68BEFJdAMBiseCxxx5DcXFx4HbixAlcvHgRI0Z0L7axxukqE1774hL2nq8POm70Tc9abS4xzIp74tIjGuwoFIpOZUvuu+8+/PznP8exY8fw97//HX/6058Czx07dgw8z+Pll18OpFl0nJYBwJQpU3D27FkUFnbvDcYDLTYXPDzr9CVyXV4iJmQbAutFRHSJS49osJOfn49Dhw6htLQUjY2N4Hke+fn5mDlzJh566CF4vV7ccccdgfGFhYVwu934wx/+gCtXruB//ud/goQKAJ566il89dVXePLJJ1FcXIyLFy/iww8/xJNPPhntyxOVVt8akEEtDzqekqBEpkENlZwWqsWAhGgQsmbNGkilUowdOxapqamBReply5bhxIkT+N73vge1uj1Rc+LEiXjllVfw4osvYty4cdi2bRvWr18fdM4JEyZg3759uHDhAmbPno3Jkydj3bp1yMrKiuq1iY1fiIwaeS8jiWgiaoXGgdJT9TeHw4GSkhIUFBQMiZIUscBg/517eYY/fH4RjAGP3DgcCcr2lQm3l8epKhMsDg9mj0yh2lVhoq8VGskjIuIGs90NxgC5lINWETwFk3Ac9l9owLGyFtjdVFY42pAQEXFDq923PqTpXK1TKuGgVQgeUpvDE3Xb4h0SIiJuUMokGJmegLykrit2JqhIiMSCtu+JuCHLqO6xGqN/zcjiJCGKNuQREYQPvxDZSIiiDgkRETc43F70tEnszzezumixOtrQ1IyIGzYfKIHXy/DD7+QhUds5glrjW6y2ucgjijYkRERc4PR44XQL9aE0yq6jp4enavEDXW5g0ZqIHvQbJ+IC/06YSi7ttt6QVimDVkn/EmJAa0REXOAXIh15O4MSEqJByGArnj/Y7AmFNocQzNiTEDHGcKysGf+62ACnhxasowkJUYzir0NECPg9Ir2q+2RXjuPw9ZVmHC1tgdVJQhRNRBWiX/7yl4EC8f7bNddcE7k3ZAzwuKJ/60decVfF8y9fvoyHHnoIBQUFUKvVGD16NDZu3NjpdUuXLsULL7yArKwsjB49GgDw1VdfYdKkSVCpVJg6dSo++OADcByH4uLiwGtPnz6NhQsXIiEhAenp6fjRj36ExsbGbu0pLS0d8EcRbfriEQEI5KBZKZYoqog+Yb722mvx2WefBX6WySJoktcN/OvlyJ2/O2b/P0DWt4JbXRXPT0xMRHZ2Nt59910kJyfjq6++wqOPPorMzEzcfffdgdfu2bMHer0eu3fvBiBkPt9+++1YtGgRtm/fjrKysk5TrNbWVtx88814+OGH8bvf/Q52ux1PPfUU7r77bnz++edd2pOamhqGX0p0yTCo4eEZUn2tg7pDo5ShxeaGjWKJooroQiSTyZCRkSG2GYOGq4vn+3n22WcDjwsKCnDw4EG88847QUKk1Wrx17/+FQqFIHp/+tOfwHEcXn/9dahUKowdOxZVVVV45JFHAq/54x//iMmTJ+PXv/514NjmzZuRk5ODCxcuYNSoUV3aM9SYlGPEpBxjr+P8QY0USxRdRBeiixcvIisrCyqVCjNmzMD69euRm5vb5Vin0wmns70Jntls7t+bSeWCdxJtpAMvwvXaa69h8+bNKC8vh91uh8vlwqRJk4LGjB8/PiBCAHD+/HlMmDAhqDbQtGnTgl5z4sQJfPHFF0hISOj0npcvX8aoUf1rhzTUUfsqNDrc/e9JR4SOqEI0ffp0FBUVYfTo0aipqcGzzz6L2bNn4/Tp09DpdJ3Gr1+/Psgz6Dcc1+cp0mBix44dWLNmDV5++WXMmDEDOp0Ov/3tb3Ho0KGgcVqttt/ntlgsuP322/Hiiy92ei4zMzNkmwcTHi8Pp4eHRiHtteBZuxDR1CyaiCpECxcuDDyeMGECpk+fjry8PLzzzjt46KGHOo1fu3YtVq9eHfjZbDYjJyen07ihztXF8w8cOICZM2fiiSeeCBy7fPlyr+cZPXo03nrrLTidTiiVwtrIkSNHgsZMmTIF7733HvLz87tdn+uqmP9Qoq7NiXeOVCBJq8Dymfk9jlWSEInCoNq+NxqNGDVqFC5d6roTq1KphF6vD7rFIlcXzx85ciSOHj2KXbt24cKFC/jFL37RSVC64r777gPP83j00Ufx7bffYteuXXjppZcAIOAZrFy5Es3NzfjBD36AI0eO4PLly9i1axceeOCBgPh0Vcx/KOHfMdMoei+MPzpDh/um5+LGUUNvQX4oM6iEyGKx4PLlyzEzJQiVq4vn33rrrbjzzjtxzz33YPr06WhqagryjrpDr9fjH//4B4qLizFp0iT813/9F9atWwcAgXWjrKwsHDhwAF6vF7fccgvGjx+PVatWwWg0BloTdVfMf6jQHlXd+1pdglKGdL2KUj2ijKjF89esWYPbb78deXl5qK6uxjPPPIPi4mKcPXu2T1vEVDy//2zbtg0PPPAATCZTUCeQcDBYf+efn6vDiQoTphckYWZhitjmxBV9LZ4vquxXVlbiBz/4AZqampCamoobbrgBX3/99ZCMUxmsbN26FcOHD8ewYcNw4sSJQIxQuEVoMNMfj8jp8eJUpQkuD0+iFUVEFaIdO3aI+fZxQW1tLdatW4fa2lpkZmbirrvuwgsvvCC2WVHF3I+EV8aAf10UosqnFSR1akFORAaaCMc4P/vZz/Czn/1MbDNEpa/pHYBQYF/CceAZg93thY6EKCqQEBExDc8zXJtlQJvD3aepGcdxUMklsLm8cLh56AbPUldME/NCNIQb2Q45BuPvWiLhcFM/t+LVCqlPiCiWKFrErN8plwvffjabTWRL4gf/79r/ux+qqHwVHKnja/SIWY9IKpXCaDSivr4eAKDRaKifeYRgjMFms6G+vh5GoxFSae+Bg9HC7vKCgUEt96V38F6g5gTgNAMZEwBNUqfXqBQUXR1tYlaIAASyxf1iREQWo9E46DL0j5e34HBJMyblGjF3VCpweifQ5IvcrzoGTFkOaIO36f35ZnYqBRI1YlqIOI5DZmYm0tLS4Ha7xTYnppHL5YPKE/IT2DFTyoCaYkGEJDJAZQBsTcC5j4Ap9wsJ0T6m5iViQrahx2qORHiJaSHyI5VKB+U/CRF5AjFECglQdlA4OPwmIG0McOhPgLkaaCkFkgoCr+mq5xkRWWJ2sZoggPao6kR7KeAwAQotkDUZUOqENSIAqD0lnoEEABIiIobheQaL3yMyXxAOZoxvL1SXfq1w33hBKCPsw2Rz40hpM05WtkbR2viGhIiIWawuD3jGIGduKE0lwkG/+ACAfpiwVuR1A80lgcOtdhe+vNiIE5WmKFscv5AQETGLf1qWweogYbywVa/tENzIcUDyCOFxa1ngsL8TrJO276MGCRERs6jkUkzONeJapZDEisSCoN0xAIAxT7hvKe3wOuHfwukZWgXghjIkRETMkqRVYM7oNIxRNfkOFHQeZPQ1arA2Ak4LgHaPyOXh4eUHX9pKLEJCRMQ29hbA3gpwknbR6YhC0x7Q2FYDQMjA9+MirygqkBARMUurzQV7QxkYGKDLAGTdNFfUZwn35moAQqKswidGlOYRHUiIiJjl/07WYNfB42i1uQHDsO4H6nxpKW21gUN+r4jWiaJDXERWE/FJm8ODTGcdlEYJoM/ufqDO16yhrVoo0chxWDwhExKOQxJFWUcFEiIiJnF5eLicDmjczVDIjO3Tr67QpgESKeB2AI5WQJ2ITEP81PQeDNDUjIhJ2hxuJLjqIZMAMo0RUPXQA08qa1+wtjRExT4iGBIiIiZpc3iQ4GoQFp178ob8+AMdrYIQVTTbcKS0GRXNVFgvGpAQETFJm8MDratJWHRO6EONJI3PI7IJwY9XGq348mIjSpusEbSS8ENCRMQkbQ43NG6/EKX1/gL/1MznEQV2zdy0axYNaLGaiEmydDJA64ReKQcS0nt/gV+IbM0Az0Plq9Lo8FAcUTQgISJikny1DUjSCJHTyoTeX6AyCovWXg9gb4FSJpQKIY8oOgyaqdmGDRvAcRxWrVoltilELGCpE+774g0BQjJsh3UiCmiMLoNCiI4cOYI///nPmDBhgtimEDEAzzM011XA7eXBtH1YH/ITWCdqbJ+aUYpHVBBdiCwWC5YtW4bXX38diYmJYptDxABtDg+OnjmP4+UtfVuo9qP2tRayt5BHFGVEF6KVK1di8eLFmD9/fq9jnU4nzGZz0I0grsZkc0HjaoZSJgWn60d7I3+PM3sz9Go5vj8lG9+f0kOOGhE2RF2s3rFjB44fP44jR470afz69evx7LPPRtgqYqhjNTVCytxQKFTtXk5fUPs8cnsL5FIJcpM1kTGQ6IRoHlFFRQX+4z/+A9u2bYNKperTa9auXQuTyRS4VVRURNhKYihiMwkNNSUJKYCkH3/ifiFy2QCPMwKWEd0hmkd07Ngx1NfXY8qUKYFjXq8X+/fvxx//+Ec4nc5OvciUSiWUym5qyhCED5dZECJ5QkovI69CphS2+102wN6Cby0aWJ0eXJOpR4KSIl0iiWi/3Xnz5uHUqeB+Ug888ACuueYaPPXUU9QQkQgZT1sDZACU+tRex3ZCnRgQosMlDjRbXUjXq0iIIoxov12dTodx48YFHdNqtUhOTu50nCD6A28ValSrjf3YMfOjTgRMVb6dMy0A2jmLBiTzREzBeB6FCU64ZSpoE/uxY+an4xa+XKhx7aQ0j4gzqIRo7969YptADHE4jx3DtByQkADo+7lGBLQvWNuaofQFNZJHFHlEjyMiiLBi9fUwUxnaW0v3hw5b+JSBHz1IiIiYoq25DhanBy5liFH6gS18K1QSQYBoahZ5BtXUjCAGSklFOaxVJmjVCoSUuShXCdv4Hic0TGi4SFOzyENCRMQUnjZfYbNQtu79qAyApR6FOh7JU3KgV9O/SaSh3zARU/AWYY0opK17Pz4h0sEKHaV5RAVaIyJiBuZxwms3AQD0SX2sQ9QVKoNw7zCFwSqiL5BHRMQMltZ68AzwSNXQ6wyhn8gnRC5rC85XmsAzhok5xvAYSXQJCRERM1ibhaqMnCYJEgkX+ol8QuS2tuCzujoo5RISoggT0tTsypUr4baDIAaMtVUQIpluAAvVQECIZO42AELXWMbYwM5J9EhIQlRYWIi5c+firbfegsPhCLdNBBESaRIr8pM1GJbVQ5/7vuATIrnHBgnvAWO0hR9pQhKi48ePY8KECVi9ejUyMjLw2GOP4fDhw+G2jSD6hY43IdOgRk52zsBOJFMBMgUkHActhAaLJESRJSQhmjRpEjZu3Ijq6mps3rwZNTU1uOGGGzBu3Di88soraGig/uFElOF5wN4iPNYkD+xcHBfwinTML0QUXR1JBrR9L5PJcOedd+Ldd9/Fiy++iEuXLmHNmjXIycnB/fffj5qamnDZSRA9wuwtaDTbYPEAvEI/8BOqjACABL9HRPlmEWVAQnT06FE88cQTyMzMxCuvvII1a9bg8uXL2L17N6qrq7FkyZJw2UkQPWJtrcPFegsO13GCRzNQfB6RltI8okJI2/evvPIKtmzZgvPnz2PRokXYunUrFi1aBImvPnBBQQGKioqQn58fTlsJolvat+6TB7Z178cnRONTOAwfno1UHZUojiQhCdGmTZvw4IMPYsWKFcjMzOxyTFpaGt54440BGUcQfcXuK5gvH0iOWUd8QpQssSOZ0jwiTkhCtHv3buTm5gY8ID+MMVRUVCA3NxcKhQLLly8Pi5EE0Rsus7BBogiXECl960yU5hEVQlojGjFiBBobGzsdb25uRkFBwYCNIoh+wRg8vmRXrXEAOWYd8XlEVosJp8qbUN5kC895iS4JSYi6izK1WCx97lFGEGHDZYHbaQfAISExTB6RXA1IZTDZ3Pjy9GV8W0tdhSNJv6Zmq1evBgBwHId169ZBo2mfO3u9Xhw6dAiTJk0Kq4EE0Ru8pREOtxcOmQFGbZjWczgOUBkhlVig9Fpo1yzC9EuIvvnmGwCCR3Tq1CkoFIrAcwqFAhMnTsSaNWvCayFB9IatGaPSdWjR5ECnCmMet1IPmZSDwm2F000BjZGkX5/aF198AUBohLhx40bo9WEIHCOIASKxNyFRo0Bibi4Qjq17Pyo9pBIOSk8beUQRJqSvjy1btoTbDoIIHZvQUBGaENoH9YTKAJlEAqXXAhN5RBGlz0J05513oqioCHq9HnfeeWePY3fu3DlgwwiirzQ1VANOJ1QyA7ThPLGSPKJo0WchMhgM4Hyh8wbDAKrfEUQ4cTtQW98As8ODRIca14Tz3CoDZBIOSq8FLg8PnmfhidomOtFnIeo4HQvX1GzTpk3YtGkTSktLAQDXXnst1q1bh4ULF4bl/EQcYGuCw+2FS6pBoiHMa5YqPWQSDuNTJBg+OSu85yaCCCmOyG63w2ZrD/AqKyvDq6++ik8//bRf58nOzsaGDRtw7NgxHD16FDfffDOWLFmCM2fOhGIWEYc4zfVweRns8kQYNSF0du0JhQ6cRIpkjQy5CTx5QxEkJCFasmQJtm7dCgBobW3FtGnT8PLLL2PJkiXYtGlTn89z++23Y9GiRRg5ciRGjRqFF154AQkJCfj6669DMYuIQywtQrIr1ElQyqThPblEAih1wmMnBTRGkpArNM6ePRsA8Pe//x0ZGRkoKyvD1q1b8fvf/z4kQ7xeL3bs2AGr1YoZM2Z0OcbpdMJsNgfdiPjG7qtTLTcMoI9ZT6j0aLW5cKGsCia7OzLvQYQmRDabDTqd8E3x6aef4s4774REIsF3vvMdlJWV9etcp06dQkJCApRKJX784x/j/fffx9ixY7scu379ehgMhsAtJ2eAJUGJIY8/2VVtCFNqx9WoDKhsteObi2VoaKP67JEi5OL5H3zwASoqKrBr1y7ccsstAID6+vp+BzmOHj0axcXFOHToEB5//HEsX74cZ8+e7XLs2rVrYTKZAreKiopQzCdiBa8bbotQHlabmBGZ91AKC9ZKjwUOqtIYMUIKaFy3bh3uu+8+/PSnP8W8efMCU6lPP/0UkydP7te5FAoFCgsLAQDXXXcdjhw5go0bN+LPf/5zp7FKpRJKJRWoInzYmpGXpEayUQd5RuSmZhRLFHlCEqJ/+7d/ww033ICamhpMnDgxcHzevHn43ve+NyCDeJ6H0+kc0DmIOMHWBJVcCpUhC9Aqeh8fCr5YIoXXQgX0I0jIGYIZGRnIyAh2h6dNm9avc6xduxYLFy5Ebm4u2trasH37duzduxe7du0K1SwinrD5amKFO7WjI0oDpBIJlB4LJb5GkJCEyGq1YsOGDdizZw/q6+vB88Eua187wdbX1we6fRgMBkyYMAG7du3Cd7/73VDMIuIMU1MtLK12yNK0CFM5tM74ghqlzA2b0x6pd4l7QhKihx9+GPv27cOPfvQjZGZmBlI/+gvVtCYGgqmxBlXNNjhNisgJkVQOTqkFYANvb43Uu8Q9IQnRxx9/jI8++gizZs0Ktz0E0Td4L1xtQta9OjFiMgQAMBiTMYq3g2WEOWCSCBCSECUmJiIpKSncthBE37E1welywSuRI9E4wM6uvaDRJ0HjbABktIkSKUKKI3r++eexbt26oHwzgogq1gbY3V7Y5MlI0kY4pEPl7+jRGtn3iWNC8ohefvllXL58Genp6cjPz4dcHpxsePz48bAYRxDd4WythdvLYFMnwaiJ0Na9D49CD5PVCWt1LXILI/pWcUtIQrR06dIwm0EQ/cPWUiM8SEiFQjagzum94pYl4EKdBZaWKmRTTaKIEJIQPfPMM+G2gyD6haOlFgCgNER2oRoAFNpE4b08Fri8PFQSWrQONyF/lbS2tuKvf/0r1q5di+bmZgDClKyqqipsxhFEl7jtSFe6MCHbgOvGjor420k1Rkg5QM7b4XTQgnUkCMkjOnnyJObPnw+DwYDS0lI88sgjSEpKws6dO1FeXh6oVUQQEcFSDwnHQatPhjbJGPn3k6nAyZWAywmXrQXQhal3GhEgJI9o9erVWLFiBS5evBjU2XXRokXYv39/2IwjiC6xCqU/kBChRNer4TjwCmHnzG01Rec944yQhOjIkSN47LHHOh0fNmwYamtrB2wUQfSEt60OVxosuGBRwct33f483DClIEReiq6OCCEJkVKp7LI64oULF5CaGqECVQThw9Fchbo2J442KcPaT7FHfLFEHmtrlN4wvghJiO644w4899xzcLuF0pkcx6G8vBxPPfUUvv/974fVQIIIwuuGw1ceVmYMPc+xvxRkZ2FUegLSlK6ovF+8EZIQvfzyy7BYLEhNTYXdbsdNN92EwsJC6HQ6vPDCC+G2kSDasdTD4XLDLVFDp49emlF6ahqStUpoeUvU3jOeCGnXzGAwYPfu3Thw4ABOnDgBi8WCKVOmYP78+eG2jxhE2FweHLrSjBkjkqGSC7E0jLGoeSUAgLZa2N1eWJTpSEqIYrVOla+pqIMaNkSCfgsRz/MoKirCzp07UVpaCo7jUFBQgIyMjOj/URJR5esrTThRYYLZ4cYdE7NwptqMs9Vm3DllGGTSyEY3B2irgd3lhVWVipGRqsrYBSamgcfqhMzRDAPPC62GiLDRr98mYwx33HEHHn74YVRVVWH8+PG49tprUVZWhhUrVgy4TCwxeLG7vDhTJXgDU3IT4fYyHLzchKpWO85UR89LYOZqwSNSpCApikJUYeFwvs6KmlYr4GqL2vvGC/3yiIqKirB//37s2bMHc+fODXru888/x9KlS7F161bcf//9YTWSEJ+L9W3w8AxpeiWyE9XgOA5T8xOx93wDTlS2YkK2IfLesMcJV1sDvDyDXZEa8WTXjijlMrhkCfDydmF65p+qEWGhXx7R22+/jf/8z//sJEIAcPPNN+Ppp5/Gtm3bwmYcMXi4WCcs0o5O1wUEZ2yWHgqZBE0WFypbolBG1VwFpVSC668pwF2zxkAaxeRTpUwKpzQBHp4BDgpqDDf9EqKTJ09iwYIF3T6/cOFCnDhxYsBGEYMLh9sbEJrCtITAcaVMitHpQqPNc7VRmK6YKgEAssQcpOqi21ZKKZfAKdMJAZTUfjrs9EuImpubkZ7efbZzeno6WlpaBmwUISI8D5R/DZzeCdR/CwCoaLaBZwwpCYpO06HRGYIQXaxvi3yUs0+IYMiO7Pt0gVIm6eARkRCFm36tEXm9Xshk3b9EKpXC4/EM2ChCRC7uAqqLhccN5wGPA1ZvAeRSDtmJnZM9hxnVSFDKYHF6UNZkxfDUhE5jwgLvBcxVKGm0ojJRjfGJruiuEcmkvjUiBt7eGnrZCqJL+iVEjDGsWLGi226r1BhxiGOqFESI44Ck4UDTZeDSHkya9ijGDyuE29u506lEwmFMph6tdhfUigjW6WmrBfO6UWPjcLRegjGF0ckx86OUSeCUCSLrsZkQPQmMD/olRMuXL+91DO2YDWEqDgv3GeOB0YuAb/4HMFUBFYchHTkf0m4Kgt0wMoINDv20lsHl4dEiT4dEIoFRLe/9NWFEIuEwY+xwpHEJkLpMAGOCYBNhoV9CtGXLlkjZQYiNywo0XRIeZ18v/JPlzQI7+TdwtSeAgtmALLoLxEE0X4Hd7YVJlQ2jRh69AMoOjMnPBsqVAPMCbjugoLpE4ULUqe769etx/fXXQ6fTIS0tDUuXLsX58+fFNCl+aTgvrMPoMtrr/CQNR7lDjeKSelw4daTXUzRbXbjcEIFcLLcDMFXB7vKiVZ0T1UDGIKQyQKEVHtPOWVgRVYj27duHlStX4uuvv8bu3bvhdrtxyy23wGq1imlWfNLsaxOe0qH0KsehUjECdrcX8uZzPb68utWON78qxadn6sK/e9ZSCjAeJk4Pp0yHZJGEqNbkQJ1LAYfbS7FEYSakpNdw8cknnwT9XFRUhLS0NBw7dgw33nijSFbFIbxX+GcHhEXqDlyR5KIQQJKrRpi++T2Cq8jQq6BRSGFzeVHZYkNectfjQqL5MgCgVpYFAEiOZrJrBw6VNEFay2Oa1g0VbeGHlUG1C2kyCd8y1EU2ypirAa9bWPPQZQQO211eNHo0sCpSoFVIgMYL3Z5CIuECwY7+KOywwHuBhvNgYGhU5gJA1IMZ/QTHEpFHFE5E9Yg6wvM8Vq1ahVmzZmHcuHFdjnE6nUEhAl1ViSRCwOzrvGLIDtoJqm9zAABcxhGQSS4J2/lZk7s9TWFaAk5WmnC5wYKb+bTw9P9qLgE8TnBKHf7tOzNh9zAoI9zHrDuUMilaZAnwehngJCEKJ4PGI1q5ciVOnz6NHTt2dDtm/fr1MBgMgVtOTk4ULYxh/BHL+uCI5fo2QfQVab51o5ZSwNt9wGp2ogYquTA9q2oNU+5Z3WnhPnUMIJFArZCK1uCQoqsjx6AQoieffBL/93//hy+++ALZ2d2H769duxYmkylwq6ioiKKVMQpjHTyiYUFPNVkEIdKnZgPKBGH6Zirv9lRSCYcRqcLa0KX6MEzPnJb26WBG115yNBHyzRLg5XmamoUZUYWIMYYnn3wS77//Pj7//HMUFBT0OF6pVEKv1wfdiAFibwFcNkAiBRIygp7Sq+RI1SmRolO2L2I3XenxdCN9SbBlTVYwNsDds5oTwhqRPgu7yoH3v6lEjSkKWf7doJRJ4ZTpBI/IbQc8VL86XIi6RrRy5Ups374dH374IXQ6XaAVkcFggFqtFtO0+KHN1/4pIU2Ik+nAzMIUzCz0R00XAjUnfTtY3ZcEzk3S4I5JWchL0gysPpHHCVT6YpeGXYeKiza0OTyYXpAc+jkHiFImgVeihAu+qG6nGZBFIao8DhDVI9q0aRNMJhPmzJmDzMzMwO1vf/ubmGbFF9Z64T6hlx7yifkAJwFszYIX1Q3C9Cxh4JHPFYcEr0OTDHviaLQ5POA4IDlBvCyvNL0K3x2bjmEZvt8VTc/Chqge0YBdd2LgWPxCFNw11e3lIeW49oVhmVLYVWstF4Ifh13X66l5X2BjvxeXLfVCKRIAKLgRdW3CFMiolkMpi2BibS8Y1HIYhhmApjSgyUzR1WFkUCxWEyJiEXqEXe0RFVe04rUvLuHLi43tB/u4TgQApypN2PJVKS71N+XDZQPOfCCsDaWMBFJHo8YkhBFkGAbJdN3XbJE8ovBBQhTPuKzCzhQAaIM79DZZXPDwDHJpB2/GL0StpT1u4wNAm9MNs92N4vLWvtvTVgcUbwNsTYBSB4xaAHAcas3CAnWmQdX3c0UAnmcob7Kh3CoXvHnawg8bJETxjH9apk7slFnfbBWmQ0FrMglpvm18D2DqOXRiQrYREo5DVau9950ungfKDgLH3wSsjUIaycR7AWUCGGMBj0hsIQKAnd9UYm+ZA26Krg4rJETxTDfrQ4wxNFuFGKIkbQeB8hdMAwL5X92RoJRhTKawlf+vi43drwfaWwQv6Mre9unY9Q8BWmE3yunhka5TQauUIkWkHDM/EgkHlVwKpywBHi9Pa0RhZNCkeBAi0M36kNnugdvLIJVwnQuQJQ33beOX9Hr674xIxvnaNlS12HGp3hKIMQrQeBH49h/CVr1MARR+VyjK1mHbXyWX4vvXZQ+a5p1quRRtUh3cXgY42wRvjpotDhj6DcYz1q49oiafN5SoVXTe8UrMF4TC2tjr1ESvkuO6vEQAwJ5z9bA6O6wr1Z0FTr8niJBhGDD1ISBzQrdVDweDCAGCELmlGngYJ0Slk1cUFkiI4hXeC1ibhMdXLVQH1oe6qvsjVwN6XypIc++7Z9MKkpCiU8Lh9ravFTWXCJ4QY4IHNGkZoDZ2ei1jDBbn4GrGoFZIAY6DQ+prEkBCFBZIiOIVWzPAeGFKdFXXUqNGjsK0BGQndrNdHtjG73mdCABkUgkWjcvAHROzUJimE7yosx8K751+LXDNYiG9pAsa2px4ff8VvH24fNDEnKnlgq12vxD1ENxJ9B1aI4pXbL74IE1Kp+lQYZpOEI3uSBoOlOwXsvF5b7dC4ic5QSkUM2MMuPApHHYLpPpMyEcv7LEAvT9xVquUDZ6pma9TiU2qB9BIQhQmyCOKV6wNwv1V07I+ocsQtti97vbKjn2h8SJc9RdwpsaK953XwezufihjDBfqhO6xo9Ij1CstBEakJmD+mHQMyxSqRcLWLK5BMQIJUbzSjRC5vTzaHO6ep0IcB6SOFh77usH2iscFXPoMbi+PGv14VDk1+NvhCtSbHV0OrzM70WJzQybhUJASxrKzAyTDoML4bAOSUzOFA+QRhQUSonjF6puaaYOz2ata7Pjrv0qw40gvtZ7Sxgj3jed7jbIGAJR/BThM0OqTMHv+EqQkKGBxevDuscouO398Uy78g49MTxA1v6xbNL5yxvZmYcpJDAgSonjE62n/Jr86tcO3Y6ZT9bJ8aMgR0jA8rl6DG2Ftam/eWPhdGBK0uGtqDvKSNXB5ePzjRHVAeACg3uzAed+0bEpuYt+vKwq4vTzKm2y4ZJYK1Qi8HiGeiBgQJETxiK1J+BaXqwBF8PqLf+u+195hHAekjxUeVxd3P44x4OKnwqJ28gghchpCoOKSScMwfpgBjAF7zzeguKIVAHCpwQLGgNEZOqTpxU/r6IjN5cV7xyvx8Zl6MH/yK03PBgwJUTziXx/qYsfMn9qRrO1DOkXWZOH1zVe6X7StOyMsaEtkQOH8oPeTSjjMG5OG2SNTkKRVYLQv8nrG8GTMKkzBTaNCWEiPMP7tew/P4FUahYN2WrAeKCRE8Yh/6/6qaRljLDA161M3VXUikDRCeFx+sPPzLhtw6TPhcf6s9nWVDnAch6n5Sbhvem5ga5zjOEwrSIJWOfiiS+RSDjJftLlTbhQOkkc0YEiI4hFr10Jkc3nhdPPgOCBRI+/ihV2QN1O4rz0llPHwwxhw4WOhyqI2BciZ3uNp5CL0sg8FjuMCgumQ+6ZmtIU/YIbGp0+El8DWffCOmX99yKCW973Uq2EYkHaNIDzf/q/Qp54xIZu+4YIQ7NhD9PRQRBWIrvZFpJNHNGAGn+9LRBaPqz1Z9SqPSKuUYWp+Yv+9k8L5Qm80ayNw9A1Armkvyj/yu4A+KwyGDx7860QWqX+xulUQ30ES/T0UIY8o3vDvmCk0nfrYJ2kVmD0yFd8Z3s9OGUodMP5u4d5hFkRIIgNGL+ixM+xQRav0CRE0gqfHe6hI2gAhjyjeGEhqR0/o0oFpjwoxRV4XkFjQXts5xhibaUB2ogYZBhXQkCh4gramLisIEH2DhCje6Lh1fxVVrXYY1XJoFNLQkkxlivaI6xgmN1nT/oMmuV2IkkeIZ9QQh6Zm8YbNX4MoWIgcbi/eOVKBv+y/ApeXF8GwIYr/92ht7Hkc0SPkEcUb3UzNmjukdgzK3K5BhMvDo9bkgMvLo9DvWfp/r0RIkBDFEy5bewucq4So0eKLqBaxk+pQoc3hxnvHK6GUS1A41fd7tDXSztkAEHVqtn//ftx+++3IysoCx3H44IMPxDQn9vHXqFYbhTyzDviFKDVhcOV2DUb8Ed9ONw+30igkv3pclPw6AEQVIqvViokTJ+K1114T04z4oZv2QQDQ6GvrnKIjj6g3lDJJIM3D5oGQ6gLQ9GwAiDo1W7hwIRYuXCimCfGFP8jwqvZBjDE0+DwisXuHDQU4joNGKYPZ7obV6YFBmyJsAtDOWcgMqTUip9MJp9MZ+Nlspg4K/SLQxywj6LDZ7oHLw0Mq4ZCoIY+oL2gVUpjtbthcHmHnrOE87ZwNgCG1fb9+/XoYDIbALScnR2yThg5eT3ty5lVTM6Vcgu+OTcd3hidDenUfM6JL/OtEVqe3PSaLpmYhM6SEaO3atTCZTIFbRUUv5UyJdqwNQgsfuVpIxeiASi7FuGEGTCvoXKaD6Bp/mofV6WnfgbQ2UNnYEBlSUzOlUgmlktYwQsLSYX2ItpgHzKh0HdJ0KqTrVYBGLuTWed1CJn4XdZeInhlSHhExAMzVwr0+s9NTp6tMqDU5wPP0bd5XshM1GDfMgFSdEpBIgASfV+TfECD6hahCZLFYUFxcjOLiYgBASUkJiouLUV5eLqZZsUlAiLKDDttdXuw+W4e3D5dTasdA8G8AWEiIQkHUqdnRo0cxd+7cwM+rV68GACxfvhxFRUUiWRWDuO3tOzpXeUS1vr5iSVpFoOAX0Ts8z1DVaofZ4cbYTD04nS8kwh+rRfQLUYVozpw5g6aneUzj94bUiZ1qENWaBCFKH2TdMoYC7x2vBGNAXrIWCX6PqK2WUj1CgNaI4oHAtKxzpcT6NkGIMgwkRP1BIuGQ4NvCb3O4hZ0zTiJ4n06Kb+svJETxgMkX5mAYFnSYMRbwiDLII+o3epXQYKDN4QGksvYa4B2bCBB9goQo1vG6hXrSgFA1sQNmuwc2lxdSCYcUyrrvN/5uuG0Ot++Az+M0V4lk0dCFhCjWMVUIXVZV+vbkTB8VLTYAgjfU564dRACdzyMyOzzCAb/HSULUb4ZUQCMRAi2lwn1ifqcF1GsydDCo+9i/jOiEXi38+5hsPo/IHxphrhHEP4ZaKEUaEqJYp/GScJ+Y3+kpmVSCnCRNp+NE3/AnCLfahBIq0CQJdZ7cDmH37Ko1OaJ7SIhiGX9Rd4m0vTU0ETZSdUrMH5OORK3Pq+Q4wStquiRMz0iI+gwtDMQyDeeF+8T8ThUZz9Wa8fm5OtSY7NG3K0ZQyaUYny20Fgpg8E3P/BsERJ8gjyhWYQyoPys8ThnV6emz1WaUNdmgV8mRaVBH2bgYxu8FmSoosLEfkEcUq/hbQEtlQOo1QU85PV5Utgie0PDUBDGsixmarS6crjKh0rcDCV0WIJULjQoo3aPPkBDFKjXFwn3a2E7TstJGG7w8Q5JWgSQtxQ8NhHM1Zuw+W4dva3yF86UywJgnPG4pEc+wIQYJUSxibwXqfNOyLnrPf1sjpCAUppE3NFBSdEJ9LH8XFABAki9wtJmEqK+QEMUiZV8J1RiTCjrll1mcHpQ2WQEAYzNjszd9NEn1NRtobHO213PyR7CbKoXIdqJXSIhiDXMNUHtSeJw3q9PTpypNYAzIMqqQSNOyAWPUyKGQSeDhGVo6xhOpDADvAZqviGvgEIGEKJbgvcCFj4XdmvSxgLFzc4EEpQwJShkm5yZ2cQKiv3Bce56evyUTOA5IHS08bjgnkmVDCxKiWOLyF0Lmt1wFjLi5yyHjsw14YFY+Cmm3LGyk+taJ6s0d1on8O5VNl4QOKkSPkBDFCjUngcojwuNrbuvUqaNjATqZVAIJtQ0KGxl6IQ6rurVDcKg+S/gMPC6anvUBEqJYoOkycP5j4XHeDCBlZNDTTo8X7x6txIU66s0eCfJTNFg6eRiWTu6Q0sFxQNoY4XHNCXEMG0KQEA11WsuBMzuFXbKMcUDBTUFP211efPBNFapa7fjiXD2cHq9IhsYuGoUMBSnazjW//aETzZeFkAqiW0iIhjKt5cDJd4Q1iOQRwOhFgZQCxhgu1bdh26EyVLc6oJRLsHTyMChlVJoiamiShDw/xoDqb8S2ZlBDuWZDlZYy4JRPhJIKgGu/B0ikPgGy4FBJMxrahMXTRI0ciydkBRZVifDjcHtxvLwFDW1O3DExC5w/x2zYFKEmVPVxIGc6oKCyK11BQjQUCRKh4cC4O4X8JgB2txe7ztTC7WVQyCSYlGPE9flJUMjI+Y0kEo7DsdIWeHiGBosTaTpfWk3KKCAhTcg7qzgEjJjb84niFPrrHGo0nG+fjiUNB8Z9H/W29nUfjUKG6/KSMH14Eh66oQCzClNIhKKAQiZBXorQqimQdwYIU+WCG4XHlUcBW7MI1g1+6C90KFF1DDjzvhCxmzISdXmLsfNELbZ9XY6qDlvHM0YkY+aIFGqYGGXGDzMAEFp4B20KJBcK02feA5z7SAg8JYIgIRoKeN3AuX8CFz4FGIMlaRw+4mdg+5FqlDXZIJVwgfUgQjzykzVITlDA5eFxrKyl/QmOA0YtEKbPpkrg0mfCAjYRYFAI0WuvvYb8/HyoVCpMnz4dhw8fFtukwUNrBXCsCKg5AYeHx2FuHP5aPwoX6q3gOGBMph7LZ+RjUo5RbEvjHo7jMGO40NvsaGlLoHklAEBtBMbcITyuOg5c2EWeUQdEF6K//e1vWL16NZ555hkcP34cEydOxK233or6+jguKsWY8M15eifwzVtCgTOFBh9L5+CAezQYOAxP1WLZ9DwsGJcBg4Y6cQwWCtMSMDxVCy/PsPdcQ3BL9dRRwOgFgodU/Q1w/E3hi4YAx0RuPj99+nRcf/31+OMf/wgA4HkeOTk5+Pd//3c8/fTTPb7WbDbDYDDAZDJBrx+iJS14HnDbAJcFbnMdXM1VcDVchMPcBIvTg/yUBMizJwMFN6G4zoUrDRbMHJFCLaIHMQ63F3vP12P2yFRole0th+QyDmq5FFzjBeD8P4VuHwCQkAokjwR0mYA2BVBoAakiJsrM9vV/VNTte5fLhWPHjmHt2rWBYxKJBPPnz8fBgwc7jXc6nXA629dCzOY+9hhvLgEu7+k0L69qtaOqWajNwwBwYGBAYNzYTH2gm2eNyY5yXx0f//PCawRGZyTA6OsRVmd2oKRRGMsx5j+rcH4GjMrQIUkjBxhDk8mMKw1WMMbg7WAez0nRqCkEK5iHUcOHAwAmZqtpCjYEUMmlWDAuM+jYP0/XoNbkgEzCQS6TQc3PxzDTMSS3nYdaZsY1lobA2HO1Zji8HLwSORgnAeOkwg0SKGUcJuW0V044U22Cxemb4l2lW3KJBFNyjYGfz9a2Ce2xAUwvSBr4hY6/W2jcGQZEFaLGxkZ4vV6kp6cHHU9PT8e5c53LJ6xfvx7PPvts/9/I4wQ6fNB+mMUGr7X7LhbM7gWYb9pjt4O32zqP8T9wMkDiq+/jdIA5rcHPd3zskgFy4Y+HZ4CHB9wSNewKI5zKZLj1uVCkDUeaXoekDlUUuRj4hoxHPF4edpcXHAd4eAaPyws7lGjWzoRMNQUFXDWuyXALsUb2FjjcPOxuLwBXp3PxMglgbV9b4i0m8M6us/t5KQdY259jbSbwDo+gV5YwrE8xfuDn8DGkAhrXrl2L1atXB342m83Iyelcc6cTxhxg4r3C48A/Mwe9w4NchyfYA/b9wAFQJCgBX0qE3uVFgcPT/jwnjOJ8X0MqtRzwbZfr3TxGOj0A1y4+HNe+HKdVyX3n5aCDHKN4OWRSGdQKKeRSjgQnxpBJJXjwhgJ4vDysTi/cPA+Pl8HDC//IcukoQN8+1U4fZ4LXaRN2S3keHPMKW/8ApBIEjx3hQJKn69UViYRdNdYJo3+sMQxTe4V24OfwIaoQpaSkQCqVoq6uLuh4XV0dMjIyOo1XKpVQKkNIU1Bo2+sId0Dvu/UFre/WF9S+W19Q+W5E7COTSmDQ9L4/lJlkAGDo0znT+lHfLiUMs7FIIequmUKhwHXXXYc9e/YEjvE8jz179mDGjBkiWkYQRDQRfWq2evVqLF++HFOnTsW0adPw6quvwmq14oEHHhDbNIIgooToQnTPPfegoaEB69atQ21tLSZNmoRPPvmk0wI2QRCxi+hxRAMhJuKICCKG6ev/qOiR1QRBECREBEGIDgkRQRCiI/pi9UDwL2/1OdWDIIio4v/f7G0pekgLUVubUAmvT9HVBEGIRltbGwyG7oM0h/SuGc/zqK6uhk6nG5RpEf4UlIqKipjc1Yvl64vlawOid32MMbS1tSErKwsSSfcrQUPaI5JIJMjOzhbbjF7R6/Ux+cfsJ5avL5avDYjO9fXkCfmhxWqCIESHhIggCNEhIYogSqUSzzzzTGgVA4YAsXx9sXxtwOC7viG9WE0QRGxAHhFBEKJDQkQQhOiQEBEEITokRARBiA4JUT/pb1fad999F9dccw1UKhXGjx+Pf/7zn0HPM8awbt06ZGZmQq1WY/78+bh48WIkL6Fb+nNtr7/+OmbPno3ExEQkJiZi/vz5ncavWLECHMcF3RYsWBDpy+iW/lxfUVFRJ9tVquDq4kP1s5szZ06na+M4DosXLw6Mifpnx4g+s2PHDqZQKNjmzZvZmTNn2COPPMKMRiOrq6vrcvyBAweYVCplv/nNb9jZs2fZz3/+cyaXy9mpU6cCYzZs2MAMBgP74IMP2IkTJ9gdd9zBCgoKmN1uj9ZlMcb6f2333Xcfe+2119g333zDvv32W7ZixQpmMBhYZWVlYMzy5cvZggULWE1NTeDW3NwcrUsKor/Xt2XLFqbX64Nsr62tDRozVD+7pqamoOs6ffo0k0qlbMuWLYEx0f7sSIj6wbRp09jKlSsDP3u9XpaVlcXWr1/f5fi7776bLV68OOjY9OnT2WOPPcYYY4zneZaRkcF++9vfBp5vbW1lSqWSvf322xG4gu7p77VdjcfjYTqdjr355puBY8uXL2dLliwJt6kh0d/r27JlCzMYDN2eL5Y+u9/97ndMp9Mxi8USOBbtz46mZn3E35V2/vz5gWM9daUFgIMHDwaNB4Bbb701ML6kpAS1tbVBYwwGA6ZPn97tOSNBKNd2NTabDW63G0lJwT1r9u7di7S0NIwePRqPP/44mpqawmp7Xwj1+iwWC/Ly8pCTk4MlS5bgzJkzgedi6bN74403cO+990KrDW6YFc3PjoSoj/TUlba2trbL19TW1vY43n/fn3NGglCu7WqeeuopZGVlBf1DLFiwAFu3bsWePXvw4osvYt++fVi4cCG83jB0Ge0HoVzf6NGjsXnzZnz44Yd46623wPM8Zs6cicrKSgCx89kdPnwYp0+fxsMPPxx0PNqf3ZDOvicGBxs2bMCOHTuwd+/eoAXde++9N/B4/PjxmDBhAkaMGIG9e/di3rx5YpjaZ2bMmBHUW2/mzJkYM2YM/vznP+P5558X0bLw8sYbb2D8+PGYNm1a0PFof3bkEfWR/nalBYCMjIwex/vv+3POSBDKtfl56aWXsGHDBnz66aeYMGFCj2OHDx+OlJQUXLp0acA294eBXJ8fuVyOyZMnB2yPhc/OarVix44deOihh3p9n0h/diREfSSUrrQzZswIGg8Au3fvDowvKChARkZG0Biz2YxDhw5FtdNtqB13f/Ob3+D555/HJ598gqlTp/b6PpWVlWhqakJmZmZY7O4r4ego7PV6cerUqYDtQ/2zA4TQEqfTiR/+8Ie9vk/EP7uoLYvHADt27GBKpZIVFRWxs2fPskcffZQZjcbAtu6PfvQj9vTTTwfGHzhwgMlkMvbSSy+xb7/9lj3zzDNdbt8bjUb24YcfspMnT7IlS5aItgXcn2vbsGEDUygU7O9//3vQFm9bWxtjjLG2tja2Zs0advDgQVZSUsI+++wzNmXKFDZy5EjmcDiiem2hXN+zzz7Ldu3axS5fvsyOHTvG7r33XqZSqdiZM2cCY4bqZ+fnhhtuYPfcc0+n42J8diRE/eQPf/gDy83NZQqFgk2bNo19/fXXgeduuukmtnz58qDx77zzDhs1ahRTKBTs2muvZR999FHQ8zzPs1/84hcsPT2dKZVKNm/ePHb+/PloXEon+nNteXl5DECn2zPPPMMYY8xms7FbbrmFpaamMrlczvLy8tgjjzzSKRYnmvTn+latWhUYm56ezhYtWsSOHz8edL6h+tkxxti5c+cYAPbpp592OpcYnx2VASEIQnRojYggCNEhISIIQnRIiAiCEB0SIoIgRIeEiCAI0SEhIghCdEiICIIQHRIiYkhRVFQEo9EY+PmXv/wlJk2aFPh5xYoVWLp0adTtIgYGCRHRJf5SoT/+8Y87Pbdy5UpwHIcVK1YEjQ+3AOTn5+PVV18NOnbPPffgwoUL3b5m48aNKCoqCvw8Z84crFq1Kqx2EeGHhIjolpycHOzYsQN2uz1wzOFwYPv27cjNzRXFJrVajbS0tG6fNxgMQR4TMTQgISK6ZcqUKcjJycHOnTsDx3bu3Inc3FxMnjx5QOfuylNZunRpwMuaM2cOysrK8NOf/jRQvB3oPDW7mo6e2YoVK7Bv3z5s3LgxcI6SkhIUFhbipZdeCnpdcXExOI6LeokSQoCEiOiRBx98EFu2bAn8vHnzZjzwwAMRf9+dO3ciOzsbzz33HGpqalBTU9Pvc2zcuBEzZszAI488EjhHbm5up2sCgC1btuDGG29EYWFhuC6B6AckRESP/PCHP8SXX36JsrIylJWV4cCBA32qXzNQkpKSIJVKodPpkJGREVKxMYPBAIVCAY1GEziHVCrFihUrcP78+UDLHbfbje3bt+PBBx8M92UQfYRKxRI9kpqaisWLF6OoqAiMMSxevBgpKSlimzUgsrKysHjxYmzevBnTpk3DP/7xDzidTtx1111imxa3kEdE9MqDDz6IoqIivPnmm2HzGiQSCa6uQON2u8Ny7r7w8MMPBxbit2zZgnvuuQcajSZq708EQ0JE9MqCBQvgcrngdrtx6623huWcqampQes+Xq8Xp0+fDhqjUCgG3DWiu3MsWrQIWq0WmzZtwieffELTMpGhqRnRK1KpFN9++23gcXeYTCYUFxcHHUtOTkZOTk6nsTfffDNWr16Njz76CCNGjMArr7yC1tbWoDH5+fnYv38/7r33XiiVypCmhPn5+Th06BBKS0uRkJCApKQkSCSSwFrR2rVrMXLkyKjWmSY6Qx4R0Sf0ej30en2PY/bu3YvJkycH3Z599tkuxz744INYvnw57r//ftx0000YPnw45s6dGzTmueeeQ2lpKUaMGIHU1NSQ7F6zZg2kUinGjh2L1NRUlJeXB5576KGH4HK5orILSPQMlYol4pZ//etfmDdvHioqKjo1KCSiCwkREXc4nU40NDRg+fLlyMjIwLZt28Q2Ke6hqRkRd7z99tvIy8tDa2srfvOb34htDgHyiAiCGASQR0QQhOiQEBEEITokRARBiA4JEUEQokNCRBCE6JAQEQQhOiREBEGIDgkRQRCiQ0JEEITo/H97x8xjWx20WgAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T05:07:26.837690Z", + "iopub.status.busy": "2024-03-24T05:07:26.836863Z", + "iopub.status.idle": "2024-03-24T05:07:27.060396Z", + "shell.execute_reply": "2024-03-24T05:07:27.059427Z" + }, + "papermill": { + "duration": 0.244584, + "end_time": "2024-03-24T05:07:27.062488", + "exception": false, + "start_time": "2024-03-24T05:07:26.817904", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEpCAYAAADCh6TFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAluUlEQVR4nO3de1xUZf4H8A8DzHBHEbmI2HhZRStBYUVc27JE3LayrTVeWYHkWqvSWrNsxm4La1bUZkptKJtFlmWyW7a2XbyE0ppRKqSLFzAUxRQQVO45DDPP7w9/nHacAQYbOHOGz/v14oXnmeec+R6f4cM5D2fOuAghBIiIFEYldwFERFeD4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEiiR7eOXk5ECr1cLDwwOxsbHYu3dvt/0bGhqwZMkShIaGQqPRYOzYsfjkk0/6qVoichRucj55fn4+dDodcnNzERsbi+zsbCQkJKC8vBxBQUEW/dvb2xEfH4+goCC89957CAsLw6lTpzBo0CCbn9NkMuHs2bPw9fWFi4uLHfeGiOxBCIHm5mYMGzYMKlU3x1dCRlOmTBFLliyRlo1Goxg2bJjIysqy2n/t2rVi1KhRor29/aqf8/Tp0wIAv/jFLwf/On36dLc/yy5CyHNXifb2dnh5eeG9997DnXfeKbUnJyejoaEBW7ZssVjn1ltvRUBAALy8vLBlyxYMHToU8+bNw7Jly+Dq6mr1efR6PfR6vbTc2NiIESNGoLKyEr6+vnbfL0dhMBiwa9cuzJgxA+7u7nKXQ3YwUMa0ubkZI0eORENDA/z9/bvsJ9tpY319PYxGI4KDg83ag4ODUVZWZnWdEydOYOfOnbjvvvvwySefoKKiAosXL4bBYEBmZqbVdbKysrB8+XKL9qKiInh5ef34HXFgXl5e+Prrr+Uug+xoIIxpW1sbAPQ4rSPrnFdvmUwmBAUF4dVXX4Wrqyuio6Nx5swZvPDCC12GV3p6OnQ6nbTc1NSE8PBwzJo1C35+fv1Ver8zGAzYsWMH4uPjnfq39EAyUMa0qanJpn6yhVdgYCBcXV1RW1tr1l5bW4uQkBCr64SGhsLd3d3sFHH8+PGoqalBe3s71Gq1xToajQYajcai3d3d3alfAJ0Gyn4OJM4+prbum2yXSqjVakRHR6OgoEBqM5lMKCgoQFxcnNV1fvazn6GiogImk0lqO3bsGEJDQ60GFxE5L1mv89LpdFi3bh3efPNNHD16FIsWLUJraytSUlIAAElJSUhPT5f6L1q0CBcuXMDSpUtx7NgxfPzxx3j22WexZMkSuXaBiGQi65xXYmIi6urqkJGRgZqaGkRFRWHr1q3SJH5VVZXZdR7h4eHYtm0bHnvsMUycOBFhYWFYunQpli1bJtcuEJFMZJ+wT01NRWpqqtXHCgsLLdri4uLw1Vdf9XFVROToZA8vIrLU1tZmcclQy/d6fFl6HIMD98PH84c/QkVERDj9ZT/WMLyIHFBZWRmio6OtPvbXK5aLi4sxefLkvi/KwTC8iBxQREQEiouLzdrKqxug+2cpVs29HuNCB5n1HYgYXkQOyMvLy+JoSnXqPDS7v8f46yIRdc0QmSpzHLLfEoeI6GowvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkXgbaCdx5afNdPVJM8DA/bQZci4MLyfR1afNXPlJM8DA/bQZci4MLydx5afNdPVJM519iZSO4eUkrvy0GX7SDDk7TtgTkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEiuQQ4ZWTkwOtVgsPDw/ExsZi7969XfZdv349XFxczL48PDz6sVoicgSyh1d+fj50Oh0yMzNRUlKCyMhIJCQk4Ny5c12u4+fnh+rqaunr1KlT/VgxETkC2cNr1apVWLhwIVJSUjBhwgTk5ubCy8sLeXl5Xa7j4uKCkJAQ6Ss4OLgfKyYiRyDr24Pa29tRXFyM9PR0qU2lUmHmzJkoKirqcr2WlhZcc801MJlMmDx5Mp599llce+21Vvvq9Xro9XppuampCQBgMBhgMBjstCeOp6OjQ/ruzPs5kAyUMbV132QNr/r6ehiNRosjp+DgYLPbu/yvcePGIS8vDxMnTkRjYyNWrlyJadOm4fDhwxg+fLhF/6ysLCxfvtyiffv27U59W5jTLQDghq+++gpnDsldDdnDQBnTtrY2m/op7o3ZcXFxiIuLk5anTZuG8ePH4+9//ztWrFhh0T89PR06nU5abmpqQnh4OGbNmgU/P79+qVkOB6suAKX7MXXqVESOCJC7HLKDgTKmnWdHPZE1vAIDA+Hq6ora2lqz9traWoSEhNi0DXd3d0yaNAkVFRVWH9doNNBoNBbt7u7ucHd3733RCuHm5iZ9d+b9HEgGypjaum+yTtir1WpER0ejoKBAajOZTCgoKDA7uuqO0WhEaWkpQkND+6pMInJAsp826nQ6JCcnIyYmBlOmTEF2djZaW1uRkpICAEhKSkJYWBiysrIAAE899RSmTp2KMWPGoKGhAS+88AJOnTqF3/zmN3LuBhH1M9nDKzExEXV1dcjIyEBNTQ2ioqKwdetWaRK/qqoKKtUPB4gXL17EwoULUVNTg8GDByM6OhpffvklJkyYINcuEJEMZA8vAEhNTUVqaqrVxwoLC82WV69ejdWrV/dDVUTkyGS/SJWI6GowvIhIkRheRKRIDC8iUiSHmLCnq1NZ34pWfYfVx47XtUrfOy9utMZb44aRgd59Uh/1TnfjCdg2pgNpPF2EEELuIvpTU1MT/P390djYqOi3B1XWt2LGykK7bGtX2k0D5gXvqDieP7D1Z5RHXgrV+Rs6OzEKY4J8LB//Xo+PCotw201x8Pa0fHsUAFSca8Gj+Qe6/W1P/aOn8QR6HtOBNp4ML4UbE+SD68L8LdoNBgNqhgKTrxns1O+DczZdjSfAMb0SJ+yJSJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEi8WaECqU3XoLK4wwqm8qh8rC882ZHRwfOdpzF0QtHu7zfeWVTC1QeZ6A3XgJg/QZ4RI6K4aVQZ1tPwXvk3/DHvd33W7N1TbePe48EzrZGIRrBdqyOqO8xvBRqmPc1aK18BC8lRmG0lXued3R0YM8Xe/Cz6T/r8sjr+LkWLM0/gGEzrunrconsjuGlUBpXD5guhWGk3zhMGGL9HvaVbpUYHzC+y/udmy41wnSpDhpXj74ul8juGF5EDqCnOUyg53nMgTaHyfAicgC2zmEC3c9jDqQ5TIYXkQPoaQ4T6Hkec6DNYTK8iBxAT3OYQM/zmANtDrPXF6meOHGiL+ogIuqVXofXmDFjMGPGDLz99tu4dOmSXYrIycmBVquFh4cHYmNjsXevDSf+ADZt2gQXFxfceeeddqmDiJSj1+FVUlKCiRMnQqfTISQkBA8//LDNYWNNfn4+dDodMjMzUVJSgsjISCQkJODcuXPdrnfy5EmkpaXhhhtuuOrnJiLl6nV4RUVF4aWXXsLZs2eRl5eH6upqTJ8+Hddddx1WrVqFurq6Xm1v1apVWLhwIVJSUjBhwgTk5ubCy8sLeXl5Xa5jNBpx3333Yfny5Rg1alRvd4GInMBVT9i7ubnhrrvuwi9/+UusWbMG6enpSEtLwx//+Efcc889eP755xEaGtrtNtrb21FcXIz09HSpTaVSYebMmSgqKupyvaeeegpBQUFYsGABdu/e3e1z6PV66PV6abmpqQnA5clPg8Fgy646pI6ODum7tf3obOtuH3vaBvUfW8aipzF1lvG0tfarDq/9+/cjLy8PmzZtgre3N9LS0rBgwQJ89913WL58OebMmdPj6WR9fT2MRiOCg82vSQkODkZZWZnVdb744gu8/vrrOHDggE11ZmVlYfny5Rbt27dvh5eXl03bcESnWwDADV988QVOWf/LOgBgx44dP3ob1Pd6MxZdjamzjGdbW5tN/XodXqtWrcIbb7yB8vJy3HrrrXjrrbdw6623QqW6fAY6cuRIrF+/Hlqttreb7lFzczMeeOABrFu3DoGBgTatk56eDp1OJy03NTUhPDwcs2bNgp+fn91r7C+HzzZhZelXmD59Oq4dZrkfBoMBO3bsQHx8fJdvD+ppG9R/bBmLnsbUWcaz8+yoJ70Or7Vr1+LBBx/E/PnzuzwtDAoKwuuvv97jtgIDA+Hq6ora2lqz9traWoSEhFj0P378OE6ePInbb79dajOZTAAun8aWl5dj9OjRZutoNBpoNBqLbbm7u3f5Q60EnRcpurm5dbsf3e2nrdugvtebsehqTJ1lPG2tvdfhtWPHDowYMUI60uokhMDp06cxYsQIqNVqJCcn97gttVqN6OhoFBQUSJc7mEwmFBQUIDU11aJ/REQESktLzdqefPJJNDc346WXXkJ4eHhvd4eIFKrX4TV69GhUV1cjKCjIrP3ChQsYOXIkjEZjr7an0+mQnJyMmJgYTJkyBdnZ2WhtbUVKSgoAICkpCWFhYcjKyoKHhweuu+46s/UHDRoEABbtROTceh1eQgir7S0tLfDw6P3bEhITE1FXV4eMjAzU1NQgKioKW7dulSbxq6qqLI7yiIhsDq/OSW8XFxdkZGSY/aXOaDTi66+/RlRU1FUVkZqaavU0EQAKCwu7XXf9+vVX9ZxEpGw2h9c333wD4PKRV2lpKdRqtfSYWq1GZGQk0tLS7F8hEZEVNofXrl27AAApKSl46aWXFH2ZAREpX6/nvN54442+qIOIqFdsCq+77roL69evh5+fH+66665u+27evNkuhRERdcem8PL394eLi4v0byIiudkUXv97qsjTRiJyBLyAiogUyaYjr0mTJkmnjT0pKSn5UQUREdnCpvDibZaJyNHYFF6ZmZl9XQcRUa9wzouIFMmmI6+AgAAcO3YMgYGBGDx4cLfzXxcuXLBbcUREXbEpvFavXg1fX1/p37ZO3hMR9RWbwut/byw4f/78vqqFiMhmvZ7zcnV1tfqZiufPn4erq6tdiiIi6kmvw6urmxHq9Xqz2+QQEfUlm+8q8fLLLwO4fDPC1157DT4+P3y2ktFoxH/+8x9ERETYv0IiIitsDq/Vq1cDuHzklZuba3aKqFarodVqkZuba/8KiYissDm8KisrAQAzZszA5s2bMXjw4D4rioioJ72+GWHnHVWJiOTU6/B68MEHu308Ly/vqoshIrJVr8Pr4sWLZssGgwGHDh1CQ0MDbr75ZrsVRkTUnV6H1wcffGDRZjKZsGjRIowePdouRRER9cQub8xWqVTQ6XTSXySJiPqa3e4qcfz4cXR0dNhrc0RE3er1aWPnJ2d3EkKguroaH3/8sdl7IImI+lKvw6vzk7M7qVQqDB06FC+++GKPf4kkIrIXXudFRIrEO6kSkSIxvIhIkRheRKRIDC8iUiS7hdd3332Hhx56yF6bIyLqlt3C6/z583j99devat2cnBxotVp4eHggNjYWe/fu7bLv5s2bERMTg0GDBsHb2xtRUVHYsGHD1ZZNRArV60sl7C0/Px86nQ65ubmIjY1FdnY2EhISUF5ejqCgIIv+AQEB+NOf/oSIiAio1Wp89NFHSElJQVBQEBISEmTYA3l8bzACAA6dabT6eOv3euyvA0JOXYS3p8Zqn4pzLX1WH1Ffkz28Vq1ahYULFyIlJQUAkJubi48//hh5eXl44oknLPrfdNNNZstLly7Fm2++iS+++GJAhdfx/w+eJzaXdtPLDRsq9vW4LW+N7C8Dol6T9VXb3t6O4uJipKenS20qlQozZ85EUVFRj+sLIbBz506Ul5fj+eeft9pHr9dDr9dLy01NTQAu38rHYDD8yD2Qz4yxQ/DMnAkYNdQbnu6Wn9p0rKYRj39wFH/91XiMDfHvcjveGlcM91cr+v/CGTR/f/k1erDqQpfvEW69dPloOvBEHbw9LI+mK+paAQAdHR2KHk9ba7c5vO66665uH29oaLB1U5L6+noYjUYEBwebtQcHB6OsrKzL9RobGxEWFga9Xg9XV1esWbMG8fHxVvtmZWVh+fLlFu3bt2+Hl5dXr2t2JD4ArHwKHQDg8oGZG85VlEJT0/12jti5Luq9oloXAK7405aeRsMNGyq+6bbHvqIvcMrTbqX1u7a2Npv62Rxe/v5d//bufDwpKcnWzf0ovr6+OHDgAFpaWlBQUACdTodRo0ZZnFICQHp6utmbyZuamhAeHo5Zs2bBz8+vX+qVw8GqC0DpfkydOhWRIwLkLod6MLW1HdcfPdflkTRg29G0t8YV2iHefVlqn+s8O+qJzeH1xhtvXHUxXQkMDISrqytqa2vN2mtraxESEtLleiqVCmPGjAEAREVF4ejRo8jKyrIaXhqNBhqN5SG2u7s73N3df9wOODA3NzfpuzPvp7MIHuSO++JG2tR3bIg/oq4Z0scVycfW16usF6mq1WpER0ejoKBAajOZTCgoKEBcXJzN2zGZTGbzWkTk/Gw+8rL1dje9/QAOnU6H5ORkxMTEYMqUKcjOzkZra6v018ekpCSEhYUhKysLwOU5rJiYGIwePRp6vR6ffPIJNmzYgLVr1/bqeYlI2WwOr/Xr1+Oaa67BpEmTIISwWwGJiYmoq6tDRkYGampqEBUVha1bt0qT+FVVVVCpfjhAbG1txeLFi/Hdd9/B09MTERERePvtt5GYmGi3mojI8dkcXosWLcK7776LyspKpKSk4P7770dAgH0mglNTU5Gammr1scLCQrPlp59+Gk8//bRdnpeIlMvmOa+cnBxUV1fj8ccfx7///W+Eh4fjnnvuwbZt2+x6JEZEZIteTdhrNBrce++92LFjB44cOYJrr70WixcvhlarRUsL32pCRP3nqv/aqFKp4OLiAiEEjEajPWsiIupRr8JLr9fj3XffRXx8PMaOHYvS0lK88sorqKqqgo+PT1/VSERkweYJ+8WLF2PTpk0IDw/Hgw8+iHfffReBgYF9WRsRUZdsDq/c3FyMGDECo0aNwueff47PP//car/NmzfbrTgioq7YHF5JSUlwcXHpy1qIiGzWq4tUiYgcBT+Ag4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUySHCKycnB1qtFh4eHoiNjcXevXu77Ltu3TrccMMNGDx4MAYPHoyZM2d225+InJPs4ZWfnw+dTofMzEyUlJQgMjISCQkJOHfunNX+hYWFuPfee7Fr1y4UFRUhPDwcs2bNwpkzZ/q5ciKSk+zhtWrVKixcuBApKSmYMGECcnNz4eXlhby8PKv933nnHSxevBhRUVGIiIjAa6+9BpPJhIKCgn6unIjk5Cbnk7e3t6O4uBjp6elSm0qlwsyZM1FUVGTTNtra2mAwGBAQEGD1cb1eD71eLy03NTUBAAwGAwwGw4+o3rF1dHRI3515PweSgTKmtu6brOFVX18Po9GI4OBgs/bg4GCUlZXZtI1ly5Zh2LBhmDlzptXHs7KysHz5cov27du3w8vLq/dFK8TpFgBww1dffYUzh+SuhuxhoIxpW1ubTf1kDa8f67nnnsOmTZtQWFgIDw8Pq33S09Oh0+mk5aamJmmezM/Pr79K7XcHqy4ApfsxdepURI6wflRKyjJQxrTz7KgnsoZXYGAgXF1dUVtba9ZeW1uLkJCQbtdduXIlnnvuOXz22WeYOHFil/00Gg00Go1Fu7u7O9zd3a+ucAVwc3OTvjvzfg4kA2VMbd03WSfs1Wo1oqOjzSbbOyff4+Liulzvr3/9K1asWIGtW7ciJiamP0olIgcj+2mjTqdDcnIyYmJiMGXKFGRnZ6O1tRUpKSkAgKSkJISFhSErKwsA8PzzzyMjIwMbN26EVqtFTU0NAMDHxwc+Pj6y7QcR9S/ZwysxMRF1dXXIyMhATU0NoqKisHXrVmkSv6qqCirVDweIa9euRXt7O37961+bbSczMxN/+ctf+rN0IpKR7OEFAKmpqUhNTbX6WGFhodnyyZMn+74gInJ4sl+kSkR0NRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFYngRkSIxvIhIkWQPr5ycHGi1Wnh4eCA2NhZ79+7tsu/hw4dx9913Q6vVwsXFBdnZ2f1XKFE/amtrQ0lJidnX0UMHoa+pwNFDB83a29ra5C5XFm5yPnl+fj50Oh1yc3MRGxuL7OxsJCQkoLy8HEFBQRb929raMGrUKMydOxePPfaYDBU7rra2NpSVlUnL5dUN//9C94Tp/CCzvhEREfDy8urnCqk3ysrKEB0dbfWxeW+aLxcXF2Py5Mn9UJVjcRFCCLmePDY2Fj/96U/xyiuvAABMJhPCw8PxyCOP4Iknnuh2Xa1Wi0cffRSPPvpor56zqakJ/v7+aGxshJ+f39WW7nBKSkq6fLFfaaC+2JXkyl9GANDyvR4f7yrCL2fEwcdTI7U72y8jW39GZTvyam9vR3FxMdLT06U2lUqFmTNnoqioyG7Po9frodfrpeWmpiYAgMFggMFgsNvzyG306NH4+uuvpeWW7/XYtnsfEm74qdkLvbOvM+27M3J3d8f1119v1mYwGHCx/hxiJkXC3d3d4jFnYeu+yBZe9fX1MBqNCA4ONmsPDg62+I3zY2RlZWH58uUW7du3b3eq31bWTLt+NJobLqC5wby9urpalnrIPnbs2CF3CX3K1jk8Wee8+kN6ejp0Op203NTUhPDwcMyaNcupThuvZDAYsGPHDsTHx1v8liZlGihj2nl21BPZwiswMBCurq6ora01a6+trUVISIjdnkej0UCj0Vi0u7u7O/ULoNNA2c+BxNnH1NZ9k+1SCbVajejoaBQUFEhtJpMJBQUFiIuLk6ssIlIIWU8bdTodkpOTERMTgylTpiA7Oxutra1ISUkBACQlJSEsLAxZWVkALk/yHzlyRPr3mTNncODAAfj4+GDMmDGy7QcR9T9ZwysxMRF1dXXIyMhATU0NoqKisHXrVmkSv6qqCirVDweHZ8+exaRJk6TllStXYuXKlbjxxhtRWFjY3+UTkYxkn7BPTU1Famqq1ceuDCStVgsZL0sjIgci+9uDiIiuhuxHXv2t88jN1j/HKpXBYEBbWxuampqc+i9TA8lAGdPOn82ezrIGXHg1NzcDAMLDw2WuhIi609zcDH9//y4fl/W9jXIwmUw4e/YsfH194eLiInc5fabzYtzTp0879cW4A8lAGVMhBJqbmzFs2DCzP9hdacAdealUKgwfPlzuMvqNn5+fU7/QB6KBMKbdHXF14oQ9ESkSw4uIFInh5aQ0Gg0yMzOtvq+TlIljam7ATdgTkXPgkRcRKRLDi4gUieFFRIrE8CIiRWJ4KcBNN93U609JIsfgaGPnaPX8GAwvIgfX3t4udwmOSZBDS05OFgDMvsLCwsSaNWvM+pWUlAgXFxdx8uRJIYQQL774orjuuuuEl5eXGD58uFi0aJFobm42W2f37t1i+vTpwsPDQwwfPlw88sgjoqWlpd/2zdlZG7uKigrx4IMPCq1WKzw8PMTYsWNFdna2xXpz5swRTz/9tAgNDRVarVYIIcSePXtEZGSk0Gg0Ijo6WnzwwQcCgPjmm2+kdUtLS8Xs2bOFt7e3CAoKEvfff7+oq6vrsp7Kysr++u+wO4aXg2toaBBxcXFi4cKForq6WlRXV4u0tDQxffp0s36///3vzdpWr14tdu7cKSorK0VBQYEYN26cWLRokfR4RUWF8Pb2FqtXrxbHjh0Te/bsEZMmTRLz58/vt31zdtbG7tKlSyIjI0Ps27dPnDhxQrz99tvCy8tL5OfnS+slJycLHx8f8cADD4hDhw6JQ4cOicbGRhEQECDuv/9+cfjwYfHJJ5+IsWPHmoXXxYsXxdChQ0V6ero4evSoKCkpEfHx8WLGjBld1tPR0SHHf41dMLwU4MYbbxRLly6Vlr/55hvh4uIiTp06JYQQwmg0irCwMLF27dout/HPf/5TDBkyRFpesGCBeOihh8z67N69W6hUKvH999/bdwcGsCvHzpolS5aIu+++W1pOTk4WwcHBQq/XS21r164VQ4YMMRubdevWmYXXihUrxKxZs8y2ffr0aQFAlJeX21yPUnDOS4GioqIwfvx4bNy4EQDw+eef49y5c5g7d67U57PPPsMtt9yCsLAw+Pr64oEHHsD58+elD/Q8ePAg1q9fDx8fH+krISEBJpMJlZWVsuzXQJGTk4Po6GgMHToUPj4+ePXVV1FVVWXW5/rrr4darZaWy8vLMXHiRHh4eEhtU6ZMMVvn4MGD2LVrl9mYRkREAACOHz/eh3skD4aXQt13331SeG3cuBGzZ8/GkCFDAAAnT57EbbfdhokTJ+L9999HcXExcnJyAPww+dvS0oKHH34YBw4ckL4OHjyIb7/9FqNHj5ZnpwaATZs2IS0tDQsWLMD27dtx4MABpKSkWEzKe3t793rbLS0tuP32283G9MCBA/j222/x85//3F674DAG3P28lEitVsNoNJq1zZs3D08++SSKi4vx3nvvITc3V3qsuLgYJpMJL774onQzt3/84x9m60+ePBlHjhzhR8b1sSvHbs+ePZg2bRoWL14stdlyVDRu3Di8/fbb0Ov10huz9+3bZ9Zn8uTJeP/996HVauHmZv1H29prSal45KUAWq0WX3/9NU6ePIn6+nqYTCZotVpMmzYNCxYsgNFoxB133CH1HzNmDAwGA/72t7/hxIkT2LBhg1m4AcCyZcvw5ZdfIjU1VfrtvGXLli4/yYmuzpVj95Of/AT79+/Htm3bcOzYMfz5z3+2CCFr5s2bB5PJhIceeghHjx7Ftm3bsHLlSgCQ7gi8ZMkSXLhwAffeey/27duH48ePY9u2bUhJSZECy9prSbHknnSjnpWXl4upU6cKT09Psz9vr1mzRgAQSUlJFuusWrVKhIaGCk9PT5GQkCDeeustAUBcvHhR6rN3714RHx8vfHx8hLe3t5g4caJ45pln+mmvBoYrx66srEzMnz9f+Pv7i0GDBolFixaJJ554QkRGRkrrdF4qcaU9e/aIiRMnCrVaLaKjo8XGjRulbXY6duyY+NWvfiUGDRokPD09RUREhHj00UeFyWSyWo+SL5XgLXGIFOqdd95BSkoKGhsb4enpKXc5/Y5zXkQK8dZbb2HUqFEICwvDwYMHsWzZMtxzzz0DMrgAhheRYtTU1CAjIwM1NTUIDQ3F3Llz8cwzz8hdlmx42khEisS/NhKRIjG8iEiRGF5EpEgMLyJSJIYXESkSw4v61Pz58+Hi4gIXFxe4u7sjODgY8fHxyMvL69VbU9avX49Bgwb1XaFdmD9/Pu68885+f17qGcOL+tzs2bNRXV2NkydP4tNPP8WMGTOwdOlS3Hbbbejo6JC7PFIqed+dRM6uq/fpFRQUCABi3bp1Qojub1u9a9cui9sXZ2ZmCiGEeOutt0R0dLTw8fERwcHB4t577xW1tbXS81y4cEHMmzdPBAYGCg8PDzFmzBiRl5cnPV5VVSXmzp0r/P39xeDBg8Udd9whvd8vMzPT4nl37drVJ/9P1Hs88iJZ3HzzzYiMjMTmzZsBACqVCi+//DIOHz6MN998Ezt37sTjjz8OAJg2bRqys7Ph5+eH6upqVFdXIy0tDQBgMBiwYsUKHDx4EP/6179w8uRJzJ8/X3qeP//5zzhy5Ag+/fRTHD16FGvXrkVgYKC0bkJCAnx9fbF7927s2bMHPj4+mD17Ntrb25GWloZ77rlHOnKsrq7GtGnT+vc/iromd3qSc+vqyEsIIRITE8X48eOtPnblbavfeOMN4e/v3+Pz7du3TwCQjtpuv/12kZKSYrXvhg0bxLhx46Q7LgghhF6vF56enmLbtm091k/y4pEXyUYIId2LqqfbVneluLgYt99+O0aMGAFfX1/ceOONACDdVnnRokXYtGkToqKi8Pjjj+PLL7+U1j148CAqKirg6+sr3TY5ICAAly5dcsrbJjsbhhfJ5ujRoxg5cqRNt622prW1FQkJCfDz88M777yDffv24YMPPjBb7xe/+AVOnTqFxx57DGfPnsUtt9winXK2tLQgOjra4rbJx44dw7x58/p47+nH4l0lSBY7d+5EaWkpHnvsMZtuW23t9sVlZWU4f/48nnvuOYSHhwMA9u/fb/FcQ4cORXJyMpKTk3HDDTfgD3/4A1auXInJkycjPz8fQUFB8PPzs1qnM9022dnwyIv6nF6vR01NDc6cOYOSkhI8++yzmDNnDm677TYkJSXZdNtqrVaLlpYWFBQUoL6+Hm1tbRgxYgTUarW03ocffogVK1aYrZeRkYEtW7agoqIChw8fxkcffYTx48cDuPwhJoGBgZgzZw52796NyspKFBYW4ne/+x2+++476Xn/+9//ory8HPX19TAYDP3zn0Y9k3vSjZzb/35Ks5ubmxg6dKiYOXOmyMvLE0ajUepny22rf/vb34ohQ4aYXSqxceNGodVqhUajEXFxceLDDz+0+CzD8ePHC09PTxEQECDmzJkjTpw4IW2zurpaJCUlicDAQKHRaMSoUaPEwoULRWNjoxBCiHPnzkm3ygYvlXAovJ8XESkSTxuJSJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKdL/AbWFSnfawRx/AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T05:07:27.099990Z", + "iopub.status.busy": "2024-03-24T05:07:27.099677Z", + "iopub.status.idle": "2024-03-24T05:07:27.364047Z", + "shell.execute_reply": "2024-03-24T05:07:27.363099Z" + }, + "papermill": { + "duration": 0.285602, + "end_time": "2024-03-24T05:07:27.366158", + "exception": false, + "start_time": "2024-03-24T05:07:27.080556", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABRzElEQVR4nO2deXhT1daH3wxNOrfQlrZAoUwyTwJFcMChCIoi6L0i8gFWBJXL1WtVBEQGFYsyWAcEEQFREHDmegWECigylzIPylimQkuhIyRNcr4/0qRJm7RJmzZJu9/nyUN7ss85Oyn5Za21115LJkmShEAgEHgZcndPQCAQCCqDEC+BQOCVCPESCAReiRAvgUDglQjxEggEXokQL4FA4JUI8RIIBF6JEC+BQOCVKN09gZrGYDBw8eJFgoKCkMlk7p6OQCAohSRJ5OXl0bBhQ+Ry+/ZVnROvixcvEhMT4+5pCASCCjh37hyNGze2+3ydE6+goCDA+MYEBwe7eTYCgaA0ubm5xMTEmD+r9qhz4mVyFYODg4V4CQQeTEVhHRGwFwgEXokQL4FA4JUI8RIIBF5JnYt5CbwLSZLQ6XTo9Xp3T0XgIhQKBUqlssqpSkK8BB6LVqvl0qVLFBYWunsqAhfj7+9PdHQ0KpWq0tcQ4iXwSAwGA6dPn0ahUNCwYUNUKpVIKq4FSJKEVqslMzOT06dP06pVq3ITUctDiJcdDl/MYcJ3Bwnx8+GrZ3q6ezp1Dq1Wi8FgICYmBn9/f3dPR+BC/Pz88PHx4ezZs2i1Wnx9fSt1HSFedpDLZBy8kEP9gMqbtYKqU9lvZYFn44q/q/ifYYewQKNoXSvUotMb3DwbgUBQGiFedqjvr0ImA0mC7EKtu6cjEAhKIcTLDkqFnPr+Ruvrar4QL4HAGZ566ikGDRpUrfcQ4lUOJtcxK1/j5pkIvIm7776b//znP+6eRq1HiFc5hAeqASFegrpJUVGRu6dQLkK8ysEkXsJtdD+SJFGo1bnl4UxT+aeeeootW7bwwQcfIJPJkMlkNG7cmPnz51uNS0tLQy6Xc/bsWQDmzp1Lx44dCQgIICYmhrFjx5Kfn291ztatW7nzzjvx8/MjJiaGF154gYKCAofmdenSJQYMGICfnx/NmjVjxYoVxMbGkpycbB4jk8mYP38+AwcOJCAggBkzZqDX6xk1ahTNmjXDz8+P1q1b88EHH1hdW6/Xk5iYSGhoKGFhYYwfP96p96yyiFSJcjC5jZnC8nI7N4r0tJuy3i33PvJmP/xVjn1UPvjgA/766y86dOjAm2++CcCcOXNYsWIFzz//vHnc8uXLuf3222natClgTB348MMPadasGadOnWLs2LGMHz+eTz75BICTJ0/Sv39/3n77bRYvXkxmZibjxo1j3LhxLFmypMJ5jRgxgqysLDZv3oyPjw+JiYlcuXKlzLhp06Yxc+ZMkpOTUSqVGAwGGjduzDfffENYWBjbtm1jzJgxREdH8/jjj5tf39KlS1m8eDFt27Zlzpw5/PDDD9x7770OvWeVRYhXOZjdxjxheQkcIyQkBJVKhb+/P1FRUQAMGzaMOXPmkJ6eTpMmTTAYDKxcuZLJkyebz7OMkcXGxvL222/z3HPPmcUrKSmJYcOGmce1atWKDz/8kD59+jB//vxyEz2PHTvGxo0b2b17N927dwdg0aJFtGrVqszYJ598koSEBKtj06dPN//crFkztm/fzurVq83ilZyczMSJE3n00UcBWLBgAevXV/8XjRCvcggvtryuFgjLy934+Sg48mY/t927KnTp0oW2bduyYsUKJkyYwJYtW7hy5Qr//Oc/zWM2btxIUlISx44dIzc3F51Ox82bNyksLMTf35/9+/dz4MABli9fbj5HkiTzNqq2bdvavf/x48dRKpXceuut5mMtW7akXr16ZcaaxM2SefPmsXjxYtLT07lx4wZarZYuXboAkJOTw6VLl+jZs2QXilKppHv37tXuOgrxKgcRsPccZDKZw66bJzJs2DCzeK1YsYL+/fsTFhYGwJkzZ3jooYd4/vnnmTFjBvXr12fr1q2MGjUKrVaLv78/+fn5PPvss7zwwgtlrt2kSROXzTMgIMDq95UrV/LKK68wZ84cevXqRVBQELNmzWLnzp0uu2dl8d7/DTWACNgLKoNKpSpTwufJJ59k8uTJpKam8u2337JgwQLzc6mpqRgMBubMmWPeNrN69Wqr82+99VaOHDlCy5YtnZ5P69at0el0pKWl0a1bNwBOnDjBtWvXKjz3zz//pHfv3owdO9Z87OTJk+afQ0JCiI6OZufOndx1110A6HQ6UlNTrSy96kCsNpaDKWB/NV9bI6sngtpBbGwsO3fu5MyZM2RlZWEwGIiNjaV3796MGjUKvV7PwIEDzeNbtmxJUVERH330EadOneLLL7+0EjeA1157jW3btjFu3Dj27dvH33//zU8//cS4ceMqnE+bNm2Ij49nzJgx7Nq1i7S0NMaMGYOfn1+FlTpatWrFnj17WL9+PX/99RdvvPEGu3fvthrz4osvMnPmTH788UeOHTvG2LFjuX79uuNvWCUR4lUOJstLqzeQe0Pn5tkIvIVXXnkFhUJBu3btiIiIID09HTC6jvv372fw4MH4+fmZx3fu3Jm5c+fy7rvv0qFDB5YvX05SUpLVNTt16sSWLVv466+/uPPOO+natStTpkyhYcOGDs1p2bJlREZGctdddzF48GBGjx5NUFBQhRUdnn32WR599FGGDBlCz549uXr1qpUVBvDyyy8zfPhwRo4caXYtBw8e7NC8qoJMqmMmRW5uLiEhIeTk5DjUPajD1PXka3SkvNyHFhGBNTBDAcDNmzc5ffo0zZo1q3TJFIF9zp8/T0xMDBs3buS+++6r8fuX9/d19DMqYl4VEB6oIl+jIytPI8RL4LX89ttv5Ofn07FjRy5dusT48eOJjY01x6m8EeE2VoA5aF8ggvYCz+SPP/4gMDDQ7gOMW30mTZpE+/btGTx4MBEREeaEVW9FWF4VIDZnCzyd7t27s2/fvnLH9OvXj3793JMnV10I8aqAkix7IV4Cz8TPz69SKRTejnAbK8AsXsJtFAg8CiFeFWDaIiQsL4HAs/AI8Zo3bx6xsbH4+vrSs2dPdu3a5dB5K1euRCaTVWvFRrFFSCDwTNwuXqtWrSIxMZGpU6eyd+9eOnfuTL9+/WyW67DkzJkzvPLKK9x5553VOr8wsdooEHgkbhevuXPnMnr0aBISEmjXrh0LFizA39+fxYsX2z1Hr9czbNgwpk+fTvPmzat1fsJtFAg8E7eKl1arJTU1lfj4ePMxuVxOfHw827dvt3vem2++SYMGDRg1alSF99BoNOTm5lo9nCE8yGh5FWj13NDqKxgtEFQNW9VNf/zxR7fNx5Nxa6pEVlYWer2eyMhIq+ORkZEcO3bM5jlbt27l888/rzCvxURSUpJVMTVnCVIrUSnkaPUGsvI1xNQX3ZsFNcelS5ds1t0SeIDb6Ax5eXkMHz6czz77jPDwcIfOmThxIjk5OebHuXPnnLqnTCYrcR1F0F5Qw0RFRaFWq906B0mS0Ok8rzCBW8UrPDwchULB5cuXrY5fvnzZXELXkpMnT3LmzBkefvhhlEolSqWSZcuWsWbNGpRKpVWdIRNqtZrg4GCrh7OEibpe7keSQFvgnoeTtQvy8vIYNmwYAQEBREdH8/7771e6HZql23jmzBlkMhnff/8999xzD/7+/nTu3LlMiKWiRh1ffvkl3bt3JygoiKioKJ588kmrBbLNmzcjk8lYu3Yt3bp1Q61Ws3XrVqfnXt241W1UqVR069aNlJQUc7qDwWAgJSXFZp2iNm3acPDgQatjkydPJi8vjw8++ICYmJhqmaewvDyAokJ4x7HyLy5n0kVQBVQ8rpjExET+/PNP1qxZQ2RkJFOmTGHv3r3m0slV5fXXX2f27Nm0atWK119/naFDh3LixAnzF3hFjTqKiop46623aN26NVeuXCExMZGnnnqKX375xeo+EyZMYPbs2TRv3twjXVe3bw9KTExk5MiRdO/enbi4OJKTkykoKDA3ARgxYgSNGjUiKSkJX19fOnToYHV+aGgoQJnjrkRszhY4Sl5eHl988QUrVqwwl5pZsmSJw3W3HOGVV15hwIABgLE5Rvv27Tlx4gRt2rRxqFHH008/bb5W8+bN+fDDD+nRowf5+fnmjdxgXBjr27evy+btatwuXkOGDCEzM5MpU6aQkZFBly5dWLdunTmIn56ebi6N6y5MbmOmSJdwHz7+RgvIXfd2kFOnTlFUVERcXJz5WEhICK1bt3bZdDp16mT+OTo6GoArV67Qpk0bhxp1pKamMm3aNPbv38+1a9cwGAyA8bPWrl0783m2mnF4Em4XL8Bs1tpi8+bN5Z67dOlS10+oFMJt9ABkMqdct9qMZRkbUxlnkwBV1KijoKDAXGFi+fLl5kqv/fr1Q6u19ixKN+PwNDxCvDydiCARsBc4RvPmzfHx8WH37t3mrj45OTn89ddfNVL4r6JGHQcPHuTq1avMnDnTHCPes2dPtc+rOvCqVAl3ERYg9jcKHCMoKIiRI0fy6quvsmnTJg4fPsyoUaOQy+UVNrtwBRU16mjSpAkqlcrc7GPNmjW89dZb1T6v6kCIlwOEB5mazwrLS1Axc+fOpVevXjz00EPEx8dz++2307Zt2xqpxV9Ro46IiAiWLl3KN998Q7t27Zg5cyazZ8+u9nlVB6IBhwNk5mnoMWMjMhn8/fYDKBVC86ub2tSAo6CggEaNGjFnzhyHtrTVBUQDjhqifoAKuQwMEmQXaGkQ7N0fJkH1kpaWxrFjx4iLiyMnJ4c333wTgEceecTNM6tdCPFyAIVcRv0AFVn5WrLyhXgJKmb27NkcP37cnIj9xx9/cPToUR544AG75+Tn59fgDL0fIV4OEhagLhYvEbQXlE/Xrl1JTU0tc/zGjRsOFxQQVIwQLwcJD1Jx/LJYcRRUnrraKKO6EJFnBzGlS4hcr5qljq0n1Rlc8XcV4uUgopZ9zWLKIi8sLHTzTATVgenvWpWmt8JtdBBTrleWsLxqBIVCQWhoqLlUi7+/f40keQqqF0mSKCws5MqVK4SGhqJQKCp9LSFeDhIusuxrHFNNt4qasQi8j9DQUJs1+5xBiJeDlFheQrxqCplMRnR0NA0aNKCoqMjd0xG4CB8fnypZXCaEeDlIuKim6jYUCoVL/rMLahciYO8gJf0bNWIFTCDwAIR4OUhYgNFtLNJL5N7wvGYEAkFdQ4iXg/j6KAhSG73sTBH3EgjcjhAvJzA1oBVBe4HA/QjxcgJTOWgRtBcI3I8QLycQFVUFAs9BiJcTiFwvgcBzEOLlBCWWl3AbBQJ3I8TLCUTAXiDwHIR4OUGEOWAvxEsgcDdV2h6Un59vbnZpwtGmFt5IWKBwGwUCT8Fpy+v06dMMGDCAgIAAQkJCqFevHvXq1SM0NJR69epVxxw9BlHTSyDwHJy2vP7v//4PSZJYvHgxkZGRdarGkinPq1Crp1Crw18l9rULBO7C6U/f/v37SU1NpXXr1tUxH48mUK1EpZSj1Rm4mq/Fv74QL4HAXTjtNvbo0YNz585Vx1w8HplMRoRwHQUCj8Bp02HRokU899xzXLhwgQ4dOpSpQd2pUyeXTc4TCQtUceH6DRG0FwjcjNPilZmZycmTJ0lISDAfk8lkSJKETCZDr9e7dIKehgjaCwSegdPi9fTTT9O1a1e+/vrrOhewB8vN2UK8BAJ34rR4nT17ljVr1tTZ5pki10sg8AycDtjfe++97N+/vzrm4hWY3EZRkFAgcC9OW14PP/wwL730EgcPHqRjx45lAvYDBw502eQ8EeE2CgQeguQkMpnM7kMulzt7OUmSJOnjjz+WmjZtKqnVaikuLk7auXOn3bHfffed1K1bNykkJETy9/eXOnfuLC1btszhe+Xk5EiAlJOTU6m5bv07U2r62s/SfXM2V+p8gUBQPo5+Rp12Gw0Gg91HZVYaV61aRWJiIlOnTmXv3r107tyZfv362W00Wr9+fV5//XW2b9/OgQMHSEhIICEhgfXr1zt978pQ0gJNWF4CgTtxSryKiopQKpUcOnTIZROYO3cuo0ePJiEhgXbt2rFgwQL8/f1ZvHixzfF33303gwcPpm3btrRo0YIXX3yRTp06sXXrVpfNqTzCit3Ga4VFFOkNFYwWCATVhVPi5ePjQ5MmTVyWy6XVaklNTSU+Pr5kQnI58fHxbN++vcLzJUkiJSWF48ePc9ddd9kco9FoyM3NtXpUhXr+KuTF2SHZBWLFUSBwF067ja+//jqTJk0iOzu7yjfPyspCr9cTGRlpdTwyMpKMjAy75+Xk5BAYGIhKpWLAgAF89NFH9O3b1+bYpKQkQkJCzI+YmJgqzVkhl1Ff1LIXCNyO06uNH3/8MSdOnKBhw4Y0bdqUgIAAq+f37t3rssnZIygoiH379pGfn09KSgqJiYk0b96cu+++u8zYiRMnkpiYaP49Nze3ygIWHqgiK18jcr0EAjfitHgNGjTIZTcPDw9HoVBw+fJlq+OXL18mKirK7nlyudycJNulSxeOHj1KUlKSTfFSq9Wo1WqXzRlMQfs8EbQXCNyI0+I1depUl91cpVLRrVs3UlJSzKJoMBhISUlh3LhxDl/HYDCg0dSckJiC9sJtFAjcR6ULUqWmpnL06FEA2rdvT9euXSt1ncTEREaOHEn37t2Ji4sjOTmZgoIC88bvESNG0KhRI5KSkgBjDKt79+60aNECjUbDL7/8wpdffsn8+fMr+1KcJlxsERII3I7T4nXlyhWeeOIJNm/eTGhoKADXr1/nnnvuYeXKlURERDh1vSFDhpCZmcmUKVPIyMigS5curFu3zhzET09PRy4vWVcoKChg7NixnD9/Hj8/P9q0acNXX33FkCFDnH0plUZUlhAI3I9MkiTJmROGDBnCqVOnWLZsGW3btgXgyJEjjBw5kpYtW/L1119Xy0RdRW5uLiEhIeTk5FS6WcjqPecY/+0B7rolgmVPx7l4hgJB3cbRz6jTlte6devYuHGjWbgA2rVrx7x587j//vsrN1svw1xNNU9YXgKBu6jU9qDSm7HBmMBaug1abcW8RahAiJdA4C4qVRLnxRdf5OLFi+ZjFy5c4KWXXuK+++5z6eQ8lTBzZQktBoNTXrdAIHARTovXxx9/TG5uLrGxsbRo0YIWLVrQrFkzcnNz+eijj6pjjh6HSbx0Boncm0Vuno1AUDdxOuYVExPD3r172bhxI8eOHQOgbdu2VvsTaztqpYIgXyV5N3Vk5WsI9Ve5e0oCQZ2jUnleMpmMvn372t1PWBeICFSTd1NHZp6Wlg3cPRuBoO5RKfFKSUkhJSWFK1eulAnS2ytlU9sID1RzKqtABO0F1ULezSKCfMsujAlKcDrmNX36dO6//35SUlLIysri2rVrVo+6gnmLkEiXELiYBVtO0nHar/xy8JK7p+LROG15LViwgKVLlzJ8+PDqmI/XUJIuIbYICVzLzLXGWPLirad5sGO0m2fjuThteWm1Wnr37l0dc/EqxOZsQXUjl9etnqjO4rR4PfPMM6xYsaI65uJVmFug5QnLS1A9BKorXTehTuD0u3Pz5k0WLlzIxo0b6dSpU5ls+7lz57pscp6MyLIXVDcBQrzKxel358CBA3Tp0gWgTCMOmazumLnhwm0UVAOWOzYCVAo3zsTzcVq8Nm3aVB3z8DpKWqAJt1HgOgq0OvPPwvIqH6djXgIj4UFG8SrU6im0+A8nEFSFAk1JZy6V0nUfzwPnr3PoQo7LrucJCPGqJAEqBeri/1xZImgvcBH5Gtd/EWp0ep5YuIOhn+1AV4t6jQrxqiQymaykoqoI2gtcREE1iNcNrZ5CrZ68mzoKi1zTc9UTEOJVBcJFlr3AxVSHeOksFgFu1mXx+v3339Hpyr7BOp2O33//3SWT8hZEIw6Bq7F0G50r0G4fvYV4aYrqsNt4zz332OyWnZOTwz333OOSSXkLJSuOwvISuIaCalj8EZZXMZIk2cznunr1apnu2bUdsUVI4GryNa4XF4OVeNUey8vhRJJHH30UMAaqn3rqKasu1Hq9ngMHDtS5PY8lAXvhNgpcQ7XHvHS1x/JyWLxCQkIAo+UVFBSEn5+f+TmVSsVtt93G6NGjXT9DD0aUxRG4GkvxknBN0EtvUXOvNrmNDovXkiVLAIiNjeWVV16pcy6iLSJE81mBi6mOPC9dXXcbTUydOrU65uGVmLLsRU0vgauoFrdRLwL2AFy+fJnhw4fTsGFDlEolCoXC6lGXCAswuo3XC4soqkWZywL3UVANAXt9LV1tdNryeuqpp0hPT+eNN94gOjq6TlWSKE09fxUKuQy9QSK7QEtksK+7pyTwcqzcRhfleVkH7GvPl6zT4rV161b++OMPc1mcuoxcLqN+gIrMPA2ZeRohXoIqUx1uo0GyTFKtPZaX025jTEwMkqtSf2sBJtdRBO0FrqBaAvYi5mUkOTmZCRMmcObMmWqYjvcRESTqeglch2WGvatMBL1YbTQyZMgQCgsLadGiBf7+/mXKQNvaOlSbEZaXwJVUR8BeV9fzvEwkJydXwzS8l3CR6yVwIdUR89LX9Qx7EyNHjqyOeXgt4cJtFDiBJEl8kPI3baOD6dc+yuo5nd6AphpWA2trkmql6nmdPHmSyZMnM3ToUK5cuQLA2rVrOXz4sEsn5w2Y3MZMYXkJHOCPv7NI3vg3z36ZWua50i6jqxbGrEri1KJUCafFa8uWLXTs2JGdO3fy/fffk5+fD8D+/fvrZPa9sLwEzlBeeCG/mnoh1NYkVafFa8KECbz99tts2LABlUplPn7vvfeyY8eOSk1i3rx5xMbG4uvrS8+ePdm1a5fdsZ999hl33nkn9erVo169esTHx5c7vroJDxAxL4HjKMrpgl0d8S4Q4mXm4MGDDB48uMzxBg0akJWV5fQEVq1aRWJiIlOnTmXv3r107tyZfv36md3R0mzevJmhQ4eyadMmtm/fTkxMDPfffz8XLlxw+t6uIDzIKOBXC7RWdZMEAlvIy9mRUjrHy1XplDpRSdVIaGgoly5dKnM8LS2NRo0aOT2BuXPnMnr0aBISEmjXrh0LFizA39+fxYsX2xy/fPlyxo4dS5cuXWjTpg2LFi3CYDCQkpLi9L1dQVix5aU3SOTcKHLLHATeg3ssL4tUiVq02ui0eD3xxBO89tprZGRkIJPJMBgM/Pnnn7zyyiuMGDHCqWtptVpSU1OJj48vmZBcTnx8PNu3b3foGoWFhRQVFVG/fn2n7u0qVEo5wb7GRVvhOgoqwtLyKm2pV5d4iTLQxbzzzju0adOGmJgY8vPzadeuHXfddRe9e/dm8uTJTl0rKysLvV5PZGSk1fHIyEgyMjIcusZrr71Gw4YNrQTQEo1GQ25urtXD1ZiC9qIRh6AilBaWV5HB2oVztgR0vkbHzwcuVih6IsO+GJVKxWeffcYbb7zBoUOHyM/Pp2vXrrRq1ao65lcuM2fOZOXKlWzevBlfX9ubopOSkpg+fXq1ziM8UM2pzAJheQkqxNJtLNJLqC0+gaVFqKKQ139WprHx6BUGdIxm3rBb7Y6rrXsbnRYvE02aNKFJkyZVunl4eDgKhYLLly9bHb98+TJRUVF2zjIye/ZsZs6cycaNG+nUqZPdcRMnTiQxMdH8e25uLjExMVWad2nCRSMOgYPILcVLZ4CSVhBOb8reeNS4qPW/g5eYV844y6oSdU68EhMTeeuttwgICLASAlvMnTvX4ZurVCq6detGSkoKgwYNAjAH38eNG2f3vPfee48ZM2awfv16unfvXu491Gq1VbOQ6qCkBZpwGwXlYxmuL+021kjMqxYlqTokXmlpaRQVFZl/tkdlChMmJiYycuRIunfvTlxcHMnJyRQUFJCQkADAiBEjaNSoEUlJSQC8++67TJkyhRUrVhAbG2uOjQUGBhIYGOj0/V1BmMj1EjiIpRWkr6GAveV9tDoDBoNkZQF6Kw6J16ZNm2z+7AqGDBlCZmYmU6ZMISMjgy5durBu3TpzED89PR25vGRdYf78+Wi1Wv7xj39YXWfq1KlMmzbNpXNzFFOulxAvQUVYilfpPC5TwF4mMz7nsjwvvfWFNDoDfirvL9le6ZiXKxk3bpxdN3Hz5s1Wv3tiHbGSyhLCbRSUj6WnWFqbTJZXoEpJngutMH0p9/Rmkb7uiJep4awjfP/995WejLciAvYCRzGUY06ZChEGqG2L17nsQlbsSiehdywNnCg5rivlntaWRFWH8rxCQkLMj+DgYFJSUtizZ4/5+dTUVFJSUsyNaesaImAvcBRrt9FaVEyrjYHFSc+lm84u3XaG+ZtP8k3q+TLX3Zt+jUw7zY9Lx9ZqS66XQ5aXqeEsGJNCH3/8cRYsWGBudabX6xk7dizBwcHVM0sPJ6xYvG4U6SnQ6AhQe4Q3LvBALHWktBFmchuDTOJV6vlrhcYvx0Ib1Sce/WQbPgoZf894sMxzZSyvWpIu4XSG/eLFi3nllVesejQqFAoSExPt7kes7QSoFPj6GN9K4ToKyqNct7E4YB/k62NzrEnc7O3/L9LbfqKs5VVHxUun03Hs2LEyx48dO4bBUDvMUWeRyWQiaC9wiNJCYkl+KctLb5BIWnuUeZtOAFCoNYqOs9VL6rTbaElCQgKjRo3i5MmTxMXFAbBz505mzpxpzs2qi4QFqjl/7YawvATlItlxGyVJMltWwcWW19mrhWw9YSwz9XyfFubnyxNAW9TWgL3T4jV79myioqKYM2eOuTROdHQ0r776Ki+//LLLJ+gtRBSvOIqgvaA8rAL2FgF5jc5gFplgP+PHMvdmSYklvSSZLS+9kwlgpVMlakvjWafFSy6XM378eMaPH2+u0FBXA/WWiC5CAkewtJosNcgyu95keWkttvLoDZI5lcLZ5NWyAfs66jZaIkSrhDCR6yVwACtX0eK4KVjv6yPHR2HcuqPVlxKv4jHOuo21NWBfKfH69ttvWb16Nenp6Wi11m7S3r17XTIxb6M25nrtOZPNsu1nmTygrVNJkQL72FttNOd4qZXmgoWWlpfOUBITc9ZtFKkSxXz44YckJCQQGRlJWloacXFxhIWFcerUKR544IHqmKNXYMr1qk0t0P6xYDtr9l/kte8OuHsqtQa9nSRVy+x6W+Kl1ZX0dCyvJZqt50qvTtaWyhJOi9cnn3zCwoUL+eijj1CpVIwfP54NGzbwwgsvkJOTUx1z9Apq8xahs1cL3T2FWoPBjttosrwCVEpzwULLHot5lsH7ctzG0laW5TFTIQm3WV4613olTotXeno6vXv3BsDPz4+8vDwAhg8fztdff+3SyXkTEbXQbTRRXmKlwDkkO1UlCizdxmKVsRSZvJslAX19OYZT6QoSxvHGY6adHzUesNcXwe+z4ZPbQJPnsss6LV5RUVFkZ2cDxmqqpl6Np0+fdlmHX2/E5Dbm3CiyMvdrA6Kjm+uwZzWZxCtArTBbSJaWl2XaRHmfM60NZTNZXgEqk3jVoOV16QB8dg/89hZkn4T9K112aafF695772XNmjWAMWH1pZdeom/fvgwZMsRmP8e6Qqifj9nczy6oXdaXsLxch7V2lfxiquUVoFaisFHUM/eGheUlWbuBluhsiJdpw7ZpRVxTE0mqOg389rZRuDIOgl89ePQz6PGMy27h9GrjwoULzduA/vWvfxEWFsa2bdsYOHAgzz77rMsm5m3I5TLqB6jIzNOQla8hKqT2rM5JEpy9WsCB8zk81Cm6UhVzBUaccRstybUR85LLZGW+WErvb8y9WcSxDGM+5m3Nwzh8Mbf6G8+e3wM//Qsyi7cRtnsEHpwNgQ1cehunxEun0/HOO+/w9NNP07hxY8DYx/GJJ55w6aS8lfBANZl5Gq9dcbySd5PXvj3AsJ5NiW9X0o5OkiT6zNps/BkY2Lmheyboofx27DJqpYLbW4ZXONYqSdXiuOVqoy3LK8/KbTT+a+s7pKiU5ZV69hqSBE3q+9M0zB+oxu1B2kLY/A5snweSAQIiYMAco3hVA065jUqlkvfeew+drnpqbXs74V6+Rejtn4+y6XgmzyzbY3Xc0tXZcya7hmfl2VzN1/D00j0MW7TToeRRW0POXi1gzb6LQHGqhI1PpZXbWHwRGWXVq7R4mf5ePWLr46s0VoKploD9mT9hwe2w7SOjcHV6Av61q9qECyrhNt53331s2bKF2NjYapiOd+PtW4TsFbMrr+56XeeqRXzT6BKW71Lbei9NVi1AoFph1VXbROl9jmDb8iqdKrH79DUA4prVQ11ctsmlAXtNHmycDrs/M/4e1BAeToZb+rnuHnZwWrweeOABJkyYwMGDB+nWrRsBAQFWzw8cONBlk3M7G6dBm4ehcTeHhpdYXt4pXvZCWWK10T6lLZ2KMFi5jWXf2AC10qoxrYncGyXiZbCIeZXGcqVbo9Oz7/x1wGh5/X0lH3CheJ38Dda8CDnpxt9vHQn3vwW+NVNR2WnxGjt2LGC7P6NMJkOvrx1bDzj6X9j6Pvz5AfT6F9zzOvj4lXuKt9f0svVhgFJB5gr7ONctLPOqHHlnyqukCqBSyG2vNlrkeRkctLwOnM9BqzMQHqiiWXgA567dAFzgNt64Dr++DmlfGX8PbQIDP4Lmd1ftuk7idKqEwWCw+6g1wgXQpDd0fNzov2/7CObfDme3lXtKmJe4jV/uOMvrPxwsky9kz/KyHCXcRmt0dqpE2KOitBOFXGZztdEqw774Era+bCxTJXadNsa7ujetj0wmw1dZ7DZWJWB/7BeY17NYuGTQ8zl4fnuNCxdUQrzqDAFh8NhnMHQlBEUbE+yWPAC/vAqafJunlGwR8mzL640fD7F8Zzp//J3l0HjrGlQCS2zlVTmKLR2Ty2S2Y14WAXuT22jry8YySXW3KVjfrD4Avj7GgH2lUiUKrsK3o2DlUMjPgLCWkLAWHngX1O5p9uyw23jjxg1SUlJ46KGHAJg4cSIaTYmFoVAoeOutt/D1rT35TQC0fgCa9IJfJ0Pal7BrIRxfBwM/hBb3WA31toD9iMW7+GP8PcTUNy6h28vfMjhpXdQl9BXEsCz580QWaw9dKne8TAYKW6uNFpaXQbIf8zK5sXqDROqZ4mB9rLV4ORXzkiQ4/IPxS7swC2Ry6P1vuHtihWGU6sZhy+uLL77g008/Nf/+8ccfs23bNtLS0khLS+Orr75i/vz51TJJt+MXCo98DMN/gJAmxgDll4Pgp3FG/78Yk3hlF2idrjPuLqb8dMj8s70O8NaC5R2vq6pk5Nxk5a70Cj/oRU4I+7BFO/nrconVbmu8Qm7P8iqbpFpentexjFzyNDoCVAraRgcBmJvEOCxeeRmw6v/g2wSjcDVoB89shL5vul24wAnxWr58OWPGjLE6tmLFCjZt2sSmTZuYNWsWq1evdvkEPYoW98LY7RBX/D6kfWncbHp8LQD1A4xuo94gcd3iP5snYzlPe4v8dXF70EMf/cGE7w8y59fj5Y6ritsIZcvVyGUym6uNBdoSwTEH7G1cz5Rhv7s43nVr03ooi005s+VV0d5bSYJ9K2BeHBz7GeRK6DMBxmyBRo6tvNcEDovXiRMn6Nixo/l3X19f5BbZdHFxcRw5csS1s/NE1IHw4Cyjv1+/BeRdgq+fgO+eQaW5RoifsYSvt7iOlvEPe25j3ZOukrjl5uOZ5Y6zVYLGGYpK1ZeXy+yv+powlBOwN1leu0u5jIA5SVVvkOyneFw/B8v/AT8+DzdzILqLUbTumQhKlSMvqcZwWLyuX79uFePKzMy0SlQ1GAxWz9d6mvaG5/+E3i8Y4wAHv4F5cTyq3g1I3iNeFitPjlheph/PZRfWug3otqhImuzVpHfo2lLZvYgyOwF7W/e09WWjMxiQJIldpYL1gDlJFWy4jgYD7P7c6Emc2AgKNcRPg2dSIKqDMy+rxnBYvBo3bsyhQ4fsPn/gwAHzfsc6g4+fMSlv1EaIaAuFWUy9+R4LfJLJy7pQo1PJuVHE7jPZTpclsiy7Ys/ysswLkiRjEu6d723i1rc2VG6ytQhnk1QtkZAo0pW2vGy7jZaUl+dVpJM4e7WQzDwNPgoZXWJCzc+plXLzOVa5XtmnYNlA+F8iaPMhpic8txXueAkUntv93WHxevDBB5kyZQo3b94s89yNGzeYPn06AwYMcOnkvIbG3eDZLdDnNfQo6K/Yzd0bHoJ9X9fY8txDH/3BP4vLNjuDtXhVPF5C4liG6wrKeTvWSarO/61Li5/czmqjJSWrjTauZzCYUyQ6NQ41x7nA+OWkVloE7Q162P4JfNIbzvwBPv7Q/11jSCTiFqdfS03jsKxOmjSJ1atX07p1a8aNG8cttxhf3PHjx/n444/R6XRMmjSp2ibq8SjVcM8kPs1oy11Hp9FBdwZ+fA4OfWfc6xVSvVbpuWxj9vR/91/ikS6NHD7P2R5+kuR89xpvpiJL1lG30dZ1JMn6ywOMlldFJYdMemcz5qUzsP/cdcC4Jag0vj4KbhYZMFw5Bt+/Bud3GZ+IvdOYJV+/Wbn39iQcFq/IyEi2bdvG888/z4QJE8x/DJlMRt++ffnkk0+IjIys4Cq1H12DDgza/yYfNd3KA1lL4cQGmHeb0b3s9pRj5k1V7m9wzo2x/PDkFJasPL6/4S+75zjbvaY2Uzrgbg9bb5lEWctLJsPm9iBLzEmqNp7TGSRzsL5HbL0yzwcoJJ5U/ETM6h/AoAVVUI3933Q1Tjm0zZo1Y926dWRnZ3PixAkAWrZsSf36ZRW+rhIeqEaHku/8H+eB554x5oKd3wU//8dohVXzt1tFVlFpC8AkXskb/zIHeQE+SPnb9vmUXd6vzTgVsMcYe7yce5NbIoMqvI4kSWUC9s7FvMqOu3j9JqezCpDJjNuCrMg4yGL9BFr7nAID0Op+eOj9avcKqotKbQ+qX78+cXFxxMXFCeEqhWmLUGa+FiJaw9ProF8SKP2McYX5vWHHfGO8oRqoKIBsT9ySN9oWq9LUNbexIizFR5IkeielcP/7v3PogmOdtMrGvBxYbSwnYL/91FUAWkcGEeJvTNsxlmSeAQvvprXhFNelAI73mg1PrvZa4QKxt9HlmDdnm2pjyRXQayyM3WaMKxQVwroJxn2SmfZds8pSkbBUVXckpLpVIqeC16q3cBslSpJJf//bOj/MVsxr9Z7zNmJeVGx5lVMS5+glY8lnc7zrfCp82gd+fw8MOrapetNXM4tzMQO9zk0sjRAvF2NugVagsf4PW785jFhjNNNVQXBuJyy4A/6YC3rXVaYt7YaUpsrZ8lLtzrjXGyT2FQe8HcHe+136LbI16utd6aSlX7M6ZszzqmCO5aw2mujZxB9+fQM+j4fMo8aSzP/8gk+jppNJKNmF3p+j53bxmjdvHrGxsfj6+tKzZ0927dpld+zhw4d57LHHiI2NRSaTkZycXHMTdRBTh5abRQarLR0AyOXQ/WnjFqOW8aDXQMp0WHQfZNjPoXOGigL2rtCd2uw2zlp/nEHz/nR4vOOrjbaPny+usWVCLsNmSRxLTH9ie6uSPWTH6P/7Y7DtQ2NJp46Pw9id0H4QUcHGwgkZOWVTnrwNt4rXqlWrSExMZOrUqezdu5fOnTvTr18/rly5YnN8YWEhzZs3Z+bMmURFRdXwbB0jQK3Erzi3xm5F1dAYGPYtDJpvrDp5aR8s7AOb3qlyV2FbTUctcYXVVJstrwVbTlr9LgGFWh1zN/zFkYu5ZcZb7W20KjRo/R7ZywEr4zbKZXZXG03upL3335+bTFMu5Rv1myivnzKWchq6yljaKSAMwNzV6pIQr6oxd+5cRo8eTUJCAu3atWPBggX4+/uzePFim+N79OjBrFmzeOKJJ1Cr1TU8W8cJDzLV9Spni5BMBl2eNDYpaPMQGHSw5V2jiF1IrfS9K9prV1XhkVxwDW9j7q9/8WHK3zz44R9lniuy836XcRvtvGWl8+zKi3kF+RqTA0zWnuXf4Xb5QX5Vj+cp5a/GA7eOgLE7oHV/q2uYxOtyrhCvSqPVaklNTSU+Pr5kMnI58fHxbN++3WX30Wg05ObmWj2qm7AAo7Bm5jlgRQVFwZCv4B9LwD8crhyBRfGwYQoU3aj4/FJUVOWgygF7ScLJVDKvRpIkDl20v3Jor56Xo29zactLJpPZjaMHqo3iZZCM8zp7tZBgCpipXMhyVRKNZVmcl8LZ1ONTY0qOX2iZawjLywVkZWWh1+vLJLZGRkaSkZHhsvskJSUREhJifsTExLjs2vYItwjaO4RMBh0eNVphHf9pjFP8+YExoH/WOSGv0PKqonpJ1L4kVYNBYtb6Y/x27LLT51ptD7JyGx07v3T36vLyvIJ8jakPBkli49Er3Cvfy6/q8Tyh3AzAEl0/7te8R5O4h+zezxTzEpaXFzBx4kRycnLMj3PnzlX7PSNMbqMjlpclAWHw2CJ44msIjIKrJ4pLT4+3W3q6NJWJeTmzmVuSSldW9X4h+9/BS8zbdJKnl+4p85xE+ULk6I4Gu26jrVQJO6aXyW0M0ucQnfJvFqtmEyW7xilDFP/UTGG6biR+AcE0Dw+weT5AdLHllV2gdW0LNDfgNvEKDw9HoVBw+bL1t93ly5ddGoxXq9UEBwdbPaobk9tY6bI4bR6Ef+2Erv8HSLDrU5jfC05trvDUij5Mtgwvrd6AsqL1eQssLa/SH8pLOTd4efV+Dp53LEnTE7iU47x7bqLITveg0gF6e6kJtvY22lttDFYreFC+g+95iQ5X16OXZCzQPcQD2pnsltoAcEer8HL3Rob4+Zg3Z1/J9Y6yTfZwm3ipVCq6detGSkqK+ZjBYCAlJYVevXq5a1ouwdy/0VG30RZ+ofDIPPi/742lp6+nw7JHYM2/jUXi7FCR22jLUrpZZECpcEy8bG0Psrzmiyv38d3e8zz88VaHrucJ2Oo87ShWSaoW70PpP8Mff9kualg25mU7+TSC6/z76lt8ovqQMHI5r2zKYO2bzNQ9iQYV/dtHsXB4N6Y+3L7c+cpkMou4V+VF2xNwq9uYmJjIZ599xhdffMHRo0d5/vnnKSgoICEhAYARI0YwceJE83itVsu+ffvYt28fWq2WCxcusG/fPvM+S0+hJMveBYmALe8zZuf3GG38fe8y40bv4+tsDq/YbSx7TFOkx6eiOizFSJJkFaTO1+roM2szk344CMDfl2tfuRx77+jNIj0/7nOsBJFKafv9LbvaWDpVQuJR+e9sUL9K57zfKZIUzDM8xvMB73NAamEe5a9WcH/7KHMp8vIw53rVYNxLpze4vHilW8VryJAhzJ49mylTptClSxf27dvHunXrzEH89PR0Ll0q6bZy8eJFunbtSteuXbl06RKzZ8+ma9euPPPMM+56CTYxBexPZuaTmecC01wdBANmw1O/GDP18y7C10Pg+zFQmG01tGK3sexHsVDrhHhhLYAPJP9BenYhK3amO3S+J1LZXTJv/nzEqkO11Ttb6n22l9irtek2Gn+O5ipLfN5jrmoBobICMgJaM1D7Nh8Y/smlAuvrqe2Ioy1MlldNJapKksTUNYcZ+PFWTlxxLHbrCG4vkzhu3DjGjRtn87nNmzdb/R4bG+sVAeLOMSFEBfuSkXuToZ/tYMUzPWkQ7IKWcLG3w3N/wuZ3YPs8OLDK2HL9wdmAcSWqMgF7o3g5/gm2vMaF697telREef/dSgu25djSWmUvN6500FwhB4UMhipSmKRcQZDsBhrJh2TdY4R2T+To+hMoDWXLjKsc/PIBC/GqIcvr099PsXxnOjKZ8Qu9ZQPX9Hms9auN7sBfpWTlmNuIDvHlxJV8nli4w3Xfcip/uP9tGLUBItpAQSZ8M5JPfJIJJ8eBVImyxwq1OoctL6h924MqKv5n6jxdFez9XUrHvHxy01GvGEySz+cEyW6QamjFg9p3mK8fSKC/UXRspaqoLSqmVkRNbhFas/8iM9ceA+CNAe3o1951i3FCvKqJ2PAAVo3pRaNQP05lFfDEwu1cdKWV0rg7PPs73PUqyJU8qNjFBvWrDJJvLddcsGUBFGj1jn9zS+UnunqjrJUnXeeuFTp8HUuvoPRqoz3BN4mXHAMJirXErLwPxdk/uCGpeLNoOP/UTuWkZKyMa0pStfXndcbyiq4hy2vX6WxeWb0fgITbY3n6DtfWsRPiVY00CfNn5ZjbiKnvx5mrhQxZuJ3zTnwYKkSphnsnw+hNHDLEUk+WT7LqE1jxOOTYbgBiU7w0jltexpI43ihR9inP8Cr9UqetOUz6Vdt/Q0vrav+5HGIn/I/b3kkhX6Oz685rdHpayC6wWvUmU32+RK67gdT0Dvpp32Wx/gEMFh/R4OIkVVvYWxCwRaQpUbUaLa+TmfmMXrYHrd5Av/aRTB7QzuX3EOJVzcTU92fVmF40DfPnXPYNhny6g3PZLhQwgOhODNK+yXtFj6ORlPD3r8YWVqlLy3z6bBkABRqd46kSUtWz9D0NZ+L1S7edYfAntqtO/JBW8oWx9UQWYLRuFv1xyqblpUDPM/zEL6pJdJf/RZ7kR/Y978LINaRLZUuqB/vZD1E7E7A3rUhWV2PkzDwNTy3ZRc6NIrrEhJI8pGuFNcoqgxCvGqBhqB+rxvSieXgAF67fYMin2zmTVeDSe+hQ8ol+EAO070DjHqDJhf++aMwNu3bGPM7WgkfS2mMUaByrKSZJtW97UEUxr9JctbPkP2u97e7ahVp9mZhXG1k6P6im8JrPStSyIjbpO3O/5j0KO41AJi+JXwWoSn4OcpHlZWqMXKjVl1ntrCo3tHqeWbaHc9k3aFLfn0Uju+Oncjwe5wxCvGqIqBBfVo65jRYRAVzMucmQhds5lem6ZWMTJ6TG8PR66PeOsfT06S3wSS/YsQAMBpuWV3aBljN2XKHSSEjlWl7epmt6g1TtK9hZ+RpzuMAHHS8pv+W/qtfpJD9NjuRPovY5EorGc4mwMgmqlq3LTDEvW6iVjgtEkK+P2VXOcaH1pTdIvLAyjf3nrhPq78PShB7mtKHqQIhXDdIg2JeVY3pxS2Qgl3M1DFm4w6V5LyZSjmdBr38ZO3o3vaO49PRrsOQBlNeqntBrT7vsiVr61ULWHrzkcWkueoPEfXM2M+2/R5w+1xnX+fu9F1i+M51OspP8V/U6Lyq/x0emZ52+B/GaWXxvuAuT81pavCzdQdPeRls4Y3kp5DKCioXQVeIlSRJv/XyEDUcuo1LKWTSiO80jXJMSYQ8hXjVMRJCar0ffRpuoIDLzNDyxcDt/uSArvW10yZ7NZ79MZfXuc3y0z8Cdl1/im6hEUAXCuR00+7YfzynWoKBym3LLcxvtpQPcNWsTzy/fy/rDrqsW4gouXr/hsMVZmitOJB+r0TJB+TU/qKbQRn6OLCmYsdoXeK7oP2Ri3Z6sdGjIJEo+CpmVFVYaZ8QLINTfGPfKueGarPfPt55m6bYzALz/eBe62+gZ6WqEeLmBsECjgLWLDiYrX8sTC3eYGydUFtP/+Xr+PugMEuO/O8CcDX9x7rqGV89058yQ36DFfcj1Gib4rOQH1RRay5zPipewv+xf+vjO4k42Jkz9BGsDtyWlVDwI6C47xi+qiTyn/C8KmcSP+t701bzHL4bbsLVUUDr+ZnIHA9TKcrsK3RLpnJVjinu5wvJae/ASM345CsCkB9swoFN0la/pCEK83ES9ABUrRvekY6MQsgu0DP1sB38Wr1BVBpNsvD+kC0/2bAJAu+hg2hVbZKv+luD/viP9ztnkSP50kp82ujCK7/DBuQYg9lwmncFg5RoO/9y6H0FtS7EoD39uMlX5BatVb9FCfonLUijPaF/mP0XjuIb9yialLS+1j/EjGqBS2m24sWJ0T9pEOVctJbS4Ldr1wqqJV+rZa/xn1T4kCUb0asroO5tX6XrOIMTLjYT6q/jqmZ50iQnlemERwxbt5OmluyvlRppEQyGX8c7gjux+PZ7/vXAH4+5tCcAPey+glyD7ln8Sr5nFr/puqGR6XvL5jjWq1+koO+Xgfey7jaUtL22pqq6epl3VNZ/e8kOsV71GgnI9cpnESt3d9NXMYqOhW4Xn2ot5+asUdjsLRVZi61mwCyyv01kFPPPFbjQ6A/FtGzD14fZOr9xWBbfvbazrhPj58OWoOGatP86Knen8duwKm49f4Z/dYnip7y3mfWjOEhFkXOW5r20DQvx8yMi9ybaTWQSqlWRSjzFFiTyk38F0n6W0lZ/jR9UbLNQ/RLLuMTSUV5nA/mqjrZiX5YfD0wL2N3WuLcYXRCGTlMsZqtwEwHkpnAlFo9lq6OjwNUqLlymW5V8cYJfLZBgkCbVSzvN3tyC7QEuLSgTGQ4vFa8b/jvLl9rM0Cw+geUQAzcIDaRYeQIuIACKC1HbFKLtAS8KSXVwrLKJT4xA+HFo9uVzlIcTLAwjy9eHNRzrwVO9YZq0/ztpDGazac46f9l/gmTua82yf5uXm+FhSujaVWqlgYOeGfLnjLN+lnmdE71jzyJ8Nvdimac80ny8YqNjO88r/cr98D+OLxpAqtbZ7D3sLbVqdgdyb1i5o5+m/mn/2LOmyn5dVGe6V72WGz2KiZcZ9kF/o+vKe7gkK8HPqOrJSvlD3pvX588RVmtb3B4rbohkkokJ8+U/8LZWe7/3to1iz7yJ5Gh2nsgo4lVVAyjHrMQEqBc0iAmheLGjNi3+ODvVlzLI9nLlaSON6fiwa2R1/Vc1LiRAvD6J5RCDz/68bqWezeeeXY6SevcbHm07w9a50XoxvxdC4JmW28azec47GoSUfEFtflI/e2ogvd5xl/eHLDO/V1Oq5bIJ5oejf/Fffi7d9FtNCfolvVG/yhf5+3tMN4QbWll95buOnpdqGlcbTYl4bjjhfs740oeQxxedLHlUYiy+eNkTyWtEYdkltK3U9k+X1+cjuHMvIY+zdLbinTQNaRBhLO5tqfVXGVbSkzy0R7J96Pxm5NzldLF6nMws4lZXP6awCzmUXUqDVc+hCLocu2F5MCvZVsjShBw2CXFAxpRII8fJAujWtz7fP9WL94cu8t+4Yp7IKmPLTYZb8eYbx/VrTv0MUMpmMnaeuMv7bAwDllhlp19AYzL1RpCf3hu3g/AZDd3Zq2jBZuZzHlVtIUK7nPvleXtONYbuhpDqnsaa7bRFaubv8/gDXqhgcrgwGg8QTC3cQHqSiU+NQZq49xpej4rizVUSVr/2AfCdv+iwhQpaLXpKxSP8g7+v+wU3KJma2bBDoUE6f6bvnvraR3NfWuEWoS0yo+XmTZxblghJLcrmMhqF+NAz14/aW4VbPaXUG0rMLOJVZYBQ3079ZBWTla/D1kbNwRHdaNgiq8jwqixAvD0Umk9G/QxT3tW3Ayt3n+GDjX5zOKuD55Xu5tUkokx5sa7U6afpg2Io6qJUKAtVK8jW6cuvq5xLIeN2z/Gy4jXd8PqeJPJOvVTNYobuXJN2T5OFfppKqJaXLu5Tmfwcu8f7jBnMcR2+QkFFxh+iq8NeVPHadMbpyvxw05pkN/3wXJ2Y8UOlrRnCd6T5LeVBhXE09bmjM+KIx7Jda2j0nKtjXMfGq4K0wvVeVjYU6ikopp2WDIJvilHuzCMkAIf6OhTKqC7Ha6OH4KOQMv60pm1+9hxfubYmfj4K96df5x4LtfPbHaYevY9qMa29fniW/GzrTT/Muy3R9AXhS+Rvr1eO5W55WnOdVqZcCwPXipEi9QaJ/8u88On+bywP5kiTxvwOXOHu1wG6Pycrtz5QYJN/Kr+rxPKjYRZGk4APdYB7WzihXuIByE0wtqaievikoXlW3sSoE+/q4XbhAiJfXEKhWknh/aza/ejdD42KQy4xuYBns/N83i5eDHY0K8GOKLoEhmjc4Y4ikoSybpapZjM56F19dFToDFWvG2asF/H0ln33nrldYQNFZ1h3K4F8r9tJn1mb703DyllFcZbHPLJJVn1BPls8hQyyPaN/ifd0/0VLxB9nRzckVWV6mmJcr3EZvR4iXlxEZ7EvSo51Y/5+7eOzWxtzeMszqeXvf3GFOWF6W7JTa0l87k4W6AeglGbcXbOS1kyPoL99V8cnlsHrPefPPrtgeZYllJv8LK9Nsjsl3sIoGSDyh+I1f1eO5V7EPjaTkvaIhDNK+yREp1uE5+ZbavmOvhE1FpW0UZrex+jY8ewtCvLyUVpFBzHm8M4tG9HCo52KJ5eX8XrabqHlHN4x/aKdxQdmEYP01FqiSmVdcetqpaxUZWL37HAssViYHf7LN6Tk5ir04kyOlrBvLrvCVzzvM9FlEsOwGew0teVCbxCf6R9A5GS4ubXnZs/wqSvIc0asp97VpQMdGoU7dvzYiAvZejp9KQYdGIew7dx2w73bUd0EvyTSpFWP83+cfBV8zXPc9AxS76C0/wrSiEfxkuB1Hyvr9Z1Uae9OvWx1zdU0pR5K8y3MbZRgYqfiV8cpV+Ms03JBUzNY9zhJ9f6vKps7gVyrmJSHho5BZNa11hHH3tqrU/WsjwvKqBfRsXvEO/rAqWF6WHL6iYXrBozyifZsjhqbUk+XzgeoTFvnMJpKKG1WUFi53YS9g31x2kdWqN5nmswx/mYYdBqPb/Ln+wUoLF5QN2E94oC1fjupJo1DnklgFJQjxqgVYboa1V/zNlEh4yUV1yw9LsQzUvsWs4tLT8Yo0NqhfZYhiE5XNpT9xJY+ktUer1Jz0ZpGe/cVWaHmU3uKkQM9zijWsVU2kh/wv8iVfXi96mqHa1zkrVb3jTWnxGnVHM25rHsafE+4luJw6XQL7iHetFhAeqGb7xHs5eaXAbrJqbHiAy++rQ8k8/SB+NXRnls+ndJGf5F2fz3hIvp2Jumc4LzVw6nr3v/87BglOZxawcET3Ss1p9LI97DlbcekdS8OrtSydWT6f0kluTD3Zou/ExKJnuEi4nbOdxzIuWdraUvso4KZzlT0EwvKqNUSH+HFHK/sftuYRrhcvE39LjXlUO523ioZxU/LhToWxqsJIxXpkOB7PMhlD+89fr/Rc/vjbsbJCo5ftwQcdLyq+syrJ/LL2OUYWveZS4QLo1SKMuY93JjxQzbxht1o992ScsYRRXA0U8KtNCMurjhDs60N4oLrcDPuqYEDO5/oBpBhu5V2fz+gpP8Z0ny8YoNjBa0VjOC05XqCuJrZAqq7sZ43qU9rKjVuaftV34/Wip8tUNq0qiX1v4ZbIIDo0CqFDoxAGd21UZkXx3/e25Nam9bi1SahL713bEZZXHcKW9fXYrY1p7kKX8owUzRPayUwuSiBf8iVOfpy1qgmMUfy33NLT09YcrvQ9U89m8/oPB8lxYO+kGi2vKb/mR9UbtJWf46oUxDjtvxlTlFhl4QpQKdg+8V6WP9PTfKxXizD6dyiJmdlKhVAq5PS5JcLhyiECI0K86hC2SgU/2DGK316526X3kZDzlb4v/TTv8ru+I76yIib5fM13qqncIrO9edtU/7wyPDZ/O8t3ptP5zV/LHddNdpxfVBN5vrgk80/63vTVzOJnQy+c695om/Uv3UV0iHGT81uDOjCyV1O6N3WtJScoQYhXHWLMnS2sfr+1SSh33WKsrnBHcVWBitrGr7CwKiriAhGMKJrAq0VjyJX86SI/xc+qSbyg+L7c0tOu9hr9iksyf6N601ySebQ2kReLxpFdTklmR0mdHE/Ky31oXM/ffGz4bU2Z/kiHGq0sWtcQMa86RJMwf/ZMjuf9DX9xT+sGxLcr6cr86fBupJ69Rq8WYXSYuh6NzoC/SkGh1trV69UirPRlK0DGN/q72aLvzAyfxfRVpJLo8y0PKHbxatEYDklla55n5mlYe/ASm49nMmNwB5QVCGp59JYfYqbyM5rIMwFYpbubGbonycV1bbnCAtWEVWN/QoFtZJKn1eatZnJzcwkJCSEnJ4fg4Kp/69ZGruZruHD9Bq2jgjh7tZCIQDV/nMji/naR+Poo6J/8O8cyKrMfUeJh+Xam+yylviwfnSTnU/1DfKh71G7p6Zfib+HF+PKzymMn/K/MsSAKmahcwZPK3wBjSeaJRc/wh6FTJeZtnxWje9K7hWtXJus6jn5GhXgJnOaGVs8vBy/x8jf7K3V+GDlM8/mChxU7ADhhaMj4ojHslcqWNW4bHczaF++0e63cm0V0mmYd67pHnsY7Pp+bSzIv0/Xl3UqUZC6PAR2jy6Q8CFyDo59REfMSOI2fSsH97SMrHmiHq4Tw76IXeFb7ElekUFrKL/KtajpvKL/ED+sdAOV9t+4+k20lXCHkM8fnE5aoZhEty+aMIZIhmjeYoktwqXABzHzM8aYagupBWF6CSnMp5wYnruRz5mohn2w6UamtR8Hk84byK/6p/B2As4YGTNCNNpeeVshlfDWqp81Ym6W72F++i7d8lhAhy0Evyfhc/yBz7ZRkdoZ3H+vItpNXSbi9GQcv5LDj5FUGdmlIv/ZV3zIksI1wG+0gxKv6sBV7cpQ+8v2847OIRjJjl+3luvtI0g0lH+MK3i8v3Gmuxf/plpNEBKlJXL2fcHKY7rOEAcUlmf82NGJ80RjSpKpXX/j533fQoVFIla8jcA6vchvnzZtHbGwsvr6+9OzZk127yi90980339CmTRt8fX3p2LEjv/zySw3NVFAeuybdB0BsmD9pb/Qt8/x/x91h99wtxaWnv9TFAzBMmcKv6vHcLd8HwIMf/sHNIj1p6ddIWnuMxNX7eES+lQ3qVxmg2IVOkvOhbhADtO9USrjua9OAv2c8wIkZD3Bm5gDOzBwghMvDcbvltWrVKkaMGMGCBQvo2bMnycnJfPPNNxw/fpwGDcpu7N22bRt33XUXSUlJPPTQQ6xYsYJ3332XvXv30qFDhwrvJyyv6iW7QEuAWoFaqcBgkDiVVcD2U1e5v10kkcG+nMsu5M73NpV7jdvkR3hXuZCm8isAfKu/i7eK/o/n+nenVYNAXl/2KzN8PideYaySetjQlPFFz3LYwcqmi5/qzoXrN+nXPtJtbbsE9vEat7Fnz5706NGDjz/+GACDwUBMTAz//ve/mTBhQpnxQ4YMoaCggJ9//tl87LbbbqNLly4sWLCgwvsJ8XI/+85dZ+vfmdzXNpK20cHo9AZavr7WaowfN3lZ+Q1PK9Yhl0lkSiFMLkogVFbA68qvCJbdQCMp+VD3KJ/qH7Jb2bRJfX/G3duSns3q0zSs+janC1yHo59RtyaparVaUlNTmThxovmYXC4nPj6e7du32zxn+/btJCYmWh3r168fP/74o83xGo0GjaZkM3Juru0GmoKao0tMqFUvQqVCzpmZA7iaryHvpo67Z2/mBr68rRvOL/qevOezkJbyi3yqSjafk2ZoyatFYzghNba69qg7mvFa/zbm9mqC2otbxSsrKwu9Xk9kpPWye2RkJMeOHbN5TkZGhs3xGRkZNscnJSUxffp010xYUK2YMtXPzBxARs5NDl/MYem2cAb8Hcu/lT/wnOK/6FAwS/c4S/QPUC/Al1fvaMboO5sLsaqD1PrtQRMnTrSy1HJzc4mJiXHjjASOEBXiS1SIr7lrNAxmd9o+AgL8eeOWW3jDrbMTeAJuFa/w8HAUCgWXL1+2On758mWiomzn0URFRTk1Xq1Wo1aLfWe1gR5du7h7CgIPwq22tkqlolu3bqSkpJiPGQwGUlJS6NWrl81zevXqZTUeYMOGDXbHCwSC2onb3cbExERGjhxJ9+7diYuLIzk5mYKCAhISEgAYMWIEjRo1IikpCYAXX3yRPn36MGfOHAYMGMDKlSvZs2cPCxcudOfLEAgENYzbxWvIkCFkZmYyZcoUMjIy6NKlC+vWrTMH5dPT05HLSwzE3r17s2LFCiZPnsykSZNo1aoVP/74o0M5XgKBoPbg9jyvmkbkeQkEno1XbQ8SCAQCZxHiJRAIvBIhXgKBwCtxe8C+pjGF+MQ2IYHAMzF9NisKx9c58crLM9ZeF1n2AoFnk5eXR0iI/bJEdW610WAwcPHiRYKCgryuLZVpa9O5c+fq5EqpeP114/VLkkReXh4NGza0SpMqTZ2zvORyOY0bN654oAcTHBxcq//zVoR4/bX/9ZdncZkQAXuBQOCVCPESCAReiRAvL0KtVjN16tQ6WyVDvP66/fpLU+cC9gKBoHYgLC+BQOCVCPESCAReiRAvgUDglQjxEggEXokQLw8mOzubYcOGERwcTGhoKKNGjSI/P7/ccxYuXMjdd99NcHAwMpmM69ev18xkXURd757uzOs/fPgwjz32GLGxschkMpKTk2tuoh6AEC8PZtiwYRw+fJgNGzbw888/8/vvvzNmzJhyzyksLKR///5MmjSphmbpOlatWkViYiJTp05l7969dO7cmX79+nHlyhWb47dt28bQoUMZNWoUaWlpDBo0iEGDBnHo0KEanrlrcPb1FxYW0rx5c2bOnGm3AU2tRhJ4JEeOHJEAaffu3eZja9eulWQymXThwoUKz9+0aZMESNeuXavGWbqWuLg46V//+pf5d71eLzVs2FBKSkqyOf7xxx+XBgwYYHWsZ8+e0rPPPlut86wunH39ljRt2lR6//33q3F2noewvDyU7du3ExoaSvfu3c3H4uPjkcvl7Ny5040zqx5M3dPj4+PNxxzpnm45Hozd0+2N92Qq8/rrOkK8PJSMjAwaNGhgdUypVFK/fn273cG9mfK6p9t7vc52T/dkKvP66zpCvGqYCRMmIJPJyn0cO3bM3dMUCDyeOlcSx928/PLLPPXUU+WOad68OVFRUWUCtTqdjuzs7FoZnK2J7umeTGVef11HWF41TEREBG3atCn3oVKp6NWrF9evXyc1NdV87m+//YbBYKBnz55ufAXVQ13vnl6Z11/ncfeKgcA+/fv3l7p27Srt3LlT2rp1q9SqVStp6NCh5ufPnz8vtW7dWtq5c6f52KVLl6S0tDTps88+kwDp999/l9LS0qSrV6+64yU4xcqVKyW1Wi0tXbpUOnLkiDRmzBgpNDRUysjIkCRJkoYPHy5NmDDBPP7PP/+UlEqlNHv2bOno0aPS1KlTJR8fH+ngwYPueglVwtnXr9FopLS0NCktLU2Kjo6WXnnlFSktLU36+++/3fUSahQhXh7M1atXpaFDh0qBgYFScHCwlJCQIOXl5ZmfP336tARImzZtMh+bOnWqBJR5LFmypOZfQCX46KOPpCZNmkgqlUqKi4uTduzYYX6uT58+0siRI63Gr169WrrlllsklUoltW/fXvrf//5XwzN2Lc68ftPfv/SjT58+NT9xNyBK4ggEAq9ExLwEAoFXIsRLIBB4JUK8BAKBVyLESyAQeCVCvAQCgVcixEsgEHglQrwEAoFXIsRLIBB4JUK8BB7BU089ZbPCRv/+/d09NYGHIqpKCDyG/v37s2TJEqtj9rpDFxUV4ePjY3VMq9WiUqmcvm9lzxO4F2F5CTwGtVpNVFSU1aNevXoAyGQy5s+fz8CBAwkICGDGjBlMmzaNLl26sGjRIpo1a4avry8A6enpPPLIIwQGBhIcHMzjjz9uVWrG3nkC70KIl8BrmDZtGoMHD+bgwYM8/fTTAJw4cYLvvvuO77//nn379mEwGHjkkUfIzs5my5YtbNiwgVOnTjFkyBCra5U+T+B9CLdR4DH8/PPPBAYGWh2bNGmSuRPSk08+SUJCgtXzWq2WZcuWERERARjreR08eJDTp08TExMDwLJly2jfvj27d++mR48eNs8TeB9CvAQewz333MP8+fOtjtWvX9/8s2UzEhNNmza1EqCjR48SExNjFi6Adu3aERoaytGjR83iVfo8gfchxEvgMQQEBNCyZctyn3fkmKP3Eng3IuYlqFW0bduWc+fOce7cOfOxI0eOcP36ddq1a+fGmQlcjbC8BB6DRqMp0+ZLqVQSHh7u8DXi4+Pp2LEjw4YNIzk5GZ1Ox9ixY+nTp49Nt1PgvQjLS+AxrFu3jujoaKvHHXfc4dQ1ZDIZP/30E/Xq1eOuu+4iPj6e5s2bs2rVqmqatcBdiDLQAoHAKxGWl0Ag8EqEeAkEAq9EiJdAIPBKhHgJBAKvRIiXQCDwSoR4CQQCr0SIl0Ag8EqEeAkEAq9EiJdAIPBKhHgJBAKvRIiXQCDwSoR4CQQCr+T/ARwfSgCRAeX8AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.018603, + "end_time": "2024-03-24T05:07:27.403129", + "exception": false, + "start_time": "2024-03-24T05:07:27.384526", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4509.109971, + "end_time": "2024-03-24T05:07:30.143404", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/tvae/3/mlu-eval.ipynb", + "output_path": "eval/contraceptive/tvae/3/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/contraceptive/tvae/3", + "path_prefix": "../../../../", + "random_seed": 3, + "single_model": "tvae" + }, + "start_time": "2024-03-24T03:52:21.033433", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/contraceptive/tvae/model.pt b/contraceptive/tvae/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..3adebe1bc149c5700e1ffcc70cafa8c45e3a9f47 --- /dev/null +++ b/contraceptive/tvae/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68604d18cab4b7967c1faeb7515500d8cbff2ae913c0a1ce7c2d30acd8a323e3 +size 30477550 diff --git a/contraceptive/tvae/params.json b/contraceptive/tvae/params.json new file mode 100644 index 0000000000000000000000000000000000000000..2596ab81f5fa7c676910b93bbbf57de356b03481 --- /dev/null +++ b/contraceptive/tvae/params.json @@ -0,0 +1 @@ +{"gradient_penalty_mode": "ALL", "loss_balancer_beta": 0.7999999999999999, "loss_balancer_r": 0.95, "tf_pma_low_exp_2": 3, "grad_loss_fn": "mae", "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.7, "bias_weight_decay": 0.05, "head_activation": "relu6", "tf_activation": "tanh", "dataset_size_exp_2": 11, "batch_size_exp_2": 2, "epochs": 100, "lr_mul": 0.07, "n_warmup_steps": 200, "Optim": "amsgradw", "fixed_role_model": "tvae", "mse_mag_target": 0.5, "g_loss_mul": 0.1, "d_model_exp_2": 8, "attn_activation": "leakyhardtanh", "tf_d_inner_exp_2": 9, "tf_n_layers_enc": 3, "tf_n_head_exp_2": 6, "tf_activation_final": "leakyhardtanh", "tf_num_inds_exp_2": 7, "ada_d_hid_exp_2": 9, "ada_n_layers": 9, "ada_activation": "softsign", "ada_activation_final": "leakyhardsigmoid", "head_d_hid_exp_2": 9, "head_n_layers": 9, "head_n_head_exp_2": 6, "head_activation_final": "leakyhardsigmoid", "mse_mag": true, "mse_mag_multiply": true, "single_model": true, "models": ["tvae"], "max_seconds": 3600} \ No newline at end of file diff --git a/insurance/lct_gan/eval.csv b/insurance/lct_gan/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..8605b0cdfb44acd67e923a40d8f46f2380c1ce96 --- /dev/null +++ b/insurance/lct_gan/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +lct_gan,0.0014914689685398724,0.010001089336377238,0.0007886611743015237,3.719151735305786,0.015319732017815113,0.531665027141571,0.047635164111852646,8.764879453337926e-07,1.2892181873321533,0.018356479704380035,0.5159444808959961,0.028083112090826035,0.1561143845319748,0.0019985174294561148,5.0083699226379395 diff --git a/insurance/lct_gan/history.csv b/insurance/lct_gan/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..639219db8bf6273d8354d8ca5176b8dbb7b8d3b2 --- /dev/null +++ b/insurance/lct_gan/history.csv @@ -0,0 +1,19 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.04117060134704742,0.8109188975505845,0.012500455402668946,0.4644619934923119,0.0,0.0,0.0,0.0,0.041821396528846685,900,113,145.03757858276367,1.2835183945377315,0.16115286509195964,0.14025535803716793,0.017303582032004165,2.386988934791206,0.0012482430997963294,0.0,0.0,0.0,0.0,0.0,0.017303582032004165,450,57,47.186697244644165,0.8278367937656871,0.10485932721032036,0.04331476881838681 +1,0.00788458850596928,1.4652959933179,0.0002524864735020014,0.07177837291939392,0.0,0.0,0.0,0.0,0.007992250232620993,900,113,145.16301083564758,1.2846284144747573,0.16129223426183065,0.08565499742342307,0.005107812441161109,0.07783917244306761,0.00024728326528121317,0.0,0.0,0.0,0.0,0.0,0.005107812441161109,450,57,47.019214153289795,0.8248984939173648,0.10448714256286622,0.08467314502616462 +2,0.00557574656041753,0.7293474927610862,0.00022519498775718153,0.041499399857388604,0.0,0.0,0.0,0.0,0.005658741251487906,900,113,145.352201461792,1.286302667803469,0.16150244606865777,0.0888019731532024,0.005094906620474325,0.33239709677336415,1.4148092069338326e-05,0.0,0.0,0.0,0.0,0.0,0.005094906620474325,450,57,47.298946142196655,0.8298060726701167,0.10510876920488145,0.0653855954051802 +3,0.004928552754507917,0.590203044300799,7.935110965993373e-05,0.03116383927563826,0.0,0.0,0.0,0.0,0.005009777373375578,900,113,145.0207061767578,1.283369081210246,0.16113411797417534,0.09011044628522565,0.005199750801224986,0.648273554320065,2.5179529602438558e-05,0.0,0.0,0.0,0.0,0.0,0.005199750801224986,450,57,47.28320384025574,0.8295298919343111,0.10507378631167942,0.051663709136559384 +4,0.004318670859793201,0.4876711248132525,4.46604487823629e-05,0.029432983928256565,0.0,0.0,0.0,0.0,0.004396132711424596,900,113,145.67448663711548,1.289154749001022,0.1618605407079061,0.08900898528507853,0.0041498291804115675,0.44782343388953705,3.687838003341986e-05,0.0,0.0,0.0,0.0,0.0,0.0041498291804115675,450,57,47.23726558685303,0.8287239576640882,0.10497170130411784,0.057800088480131274 +5,0.003727001822941626,0.4257845432669564,5.2791223793633036e-05,0.027504234943124983,0.0,0.0,0.0,0.0,0.0037818589322139613,900,113,145.0672550201416,1.2837810178773592,0.16118583891126845,0.09003757454652701,0.003250846434198643,0.4149662161665363,8.010250678769003e-05,0.0,0.0,0.0,0.0,0.0,0.003250846434198643,450,57,47.08042764663696,0.8259724148532801,0.10462317254808214,0.06960962812228356 +6,0.003335649493189218,0.29143936353892297,0.00010225355728581073,0.03192889101389382,0.0,0.0,0.0,0.0,0.0033789488727537296,900,113,144.45740246772766,1.278384092634758,0.16050822496414185,0.08855825027994878,0.003431849268866548,0.27368448856224714,3.499706230363269e-05,0.0,0.0,0.0,0.0,0.0,0.003431849268866548,450,57,47.77770447731018,0.8382053417071962,0.10617267661624484,0.06641790176856152 +7,0.002982409707296433,0.37030739115790007,3.9599444266533356e-05,0.03103297157213092,0.0,0.0,0.0,0.0,0.0030181357321432895,900,113,144.7265179157257,1.280765645271909,0.16080724212858413,0.0891251541443367,0.003124318533429889,0.24963311337495198,7.335967463523267e-05,0.0,0.0,0.0,0.0,0.0,0.003124318533429889,450,57,47.88541555404663,0.8400950097201163,0.10641203456454806,0.07890400658933479 +8,0.0020741428555467994,0.2619519646582818,1.8381270814843714e-05,0.031121474682456917,0.0,0.0,0.0,0.0,0.002098895235814982,900,113,145.16141033172607,1.2846142507232396,0.16129045592414007,0.09325064248116934,0.004463403613384192,0.8222081985106039,7.022714163104686e-05,0.0,0.0,0.0,0.0,0.0,0.004463403613384192,450,57,47.134621143341064,0.826923177953352,0.10474360254075792,0.05030834952598078 +9,0.0023060996746709053,0.2621581708226265,4.253982956825255e-05,0.029572723129143316,0.0,0.0,0.0,0.0,0.002331820259375187,900,113,143.75192093849182,1.2721408932609897,0.15972435659832424,0.09426238097712002,0.0023851372594233707,0.25001330026493196,9.203486099863416e-05,0.0,0.0,0.0,0.0,0.0,0.0023851372594233707,450,57,46.45239043235779,0.8149542181115401,0.10322753429412841,0.06987214349046872 +10,0.0013159852624004189,0.218279704756602,7.74645728111526e-06,0.024696054148177306,0.0,0.0,0.0,0.0,0.0013293167201401147,900,113,143.5770845413208,1.2705936685072636,0.1595300939348009,0.09105278838392908,0.00223359671033298,0.03436504584376627,1.4256753953578257e-05,0.0,0.0,0.0,0.0,0.0,0.00223359671033298,450,57,46.45383548736572,0.8149795699537846,0.10323074552747938,0.07930388350627925 +11,0.002150901930160924,0.22495316012830485,3.4149523725070684e-05,0.03553776060955392,0.0,0.0,0.0,0.0,0.0021734700896518513,900,113,143.75677585601807,1.2721838571329032,0.15972975095113118,0.09556485827913326,0.003228269326912899,0.28536299721485003,1.6670719938160433e-05,0.0,0.0,0.0,0.0,0.0,0.003228269326912899,450,57,46.40282607078552,0.814084667908518,0.10311739126841227,0.07249125089136917 +12,0.0019467951555270703,0.23753629187743283,3.624726925012959e-05,0.03207059481077724,0.0,0.0,0.0,0.0,0.0019664278747707916,900,113,143.81293630599976,1.2726808522654847,0.15979215145111084,0.09223815666890778,0.003769443849644934,0.12684818327569333,0.0005272575768817463,0.0,0.0,0.0,0.0,0.0,0.003769443849644934,450,57,46.357746601104736,0.8132938000193813,0.10301721466912164,0.08174848222386158 +13,0.0011665212795681631,0.09113499079847809,2.4942857778784166e-05,0.02439475072444313,0.0,0.0,0.0,0.0,0.001178674958355259,900,113,143.67677998542786,1.2714759290745827,0.1596408666504754,0.09811776785789865,0.0026310520773727654,0.5994252953760104,0.0001863106027298025,0.0,0.0,0.0,0.0,0.0,0.0026310520773727654,450,57,46.223793745040894,0.8109437499129981,0.10271954165564642,0.06012566037590436 +14,0.0007886418905885269,0.1318949167653181,1.084472590072184e-06,0.019767768517550494,0.0,0.0,0.0,0.0,0.0007973033490513141,900,113,142.2467658519745,1.258820936743137,0.15805196205774943,0.09370011994532779,0.0024774786025712576,0.23988299155125448,0.0003575192506448812,0.0,0.0,0.0,0.0,0.0,0.0024774786025712576,450,57,45.4305682182312,0.7970275126005474,0.100956818262736,0.07729252947396353 +15,0.0007920145899212609,0.07880277095880331,1.3125071642120135e-06,0.022073325576881568,0.0,0.0,0.0,0.0,0.0007999837701855641,900,113,142.16578197479248,1.2581042652636503,0.15796197997199166,0.09601840465865304,0.0018119772487261798,0.28852804994411196,0.0001659401577897141,0.0,0.0,0.0,0.0,0.0,0.0018119772487261798,450,57,45.49597787857056,0.7981750505012378,0.10110217306349012,0.07632629704465599 +16,0.0005196595300286491,0.0565358007824824,5.911478473423242e-07,0.015494050164189603,0.0,0.0,0.0,0.0,0.0005250095642016579,900,113,143.4526925086975,1.26949285405927,0.15939188056521947,0.09898689962857593,0.0018605626394532413,0.42602644269920564,0.00010228292684815276,0.0,0.0,0.0,0.0,0.0,0.0018605626394532413,450,57,46.52276873588562,0.8161889251909757,0.10338393052419027,0.07228483269889757 +17,0.0003834019511123188,0.05582865150848945,1.0770429865198301e-07,0.013134720898750755,0.0,0.0,0.0,0.0,0.00038761303258878695,900,113,144.13887667655945,1.275565280323535,0.1601543074183994,0.09740801119303281,0.0022972054127603767,0.26383435410354306,0.0004230380179690739,0.0,0.0,0.0,0.0,0.0,0.0022972054127603767,450,57,47.14963889122009,0.8271866472143876,0.10477697531382243,0.0769950772080113 diff --git a/insurance/lct_gan/mlu-eval.ipynb b/insurance/lct_gan/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..869659f2955b13fe3be4c9f53359fae10fe035f6 --- /dev/null +++ b/insurance/lct_gan/mlu-eval.ipynb @@ -0,0 +1,2441 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:27.386652Z", + "iopub.status.busy": "2024-03-26T15:11:27.385797Z", + "iopub.status.idle": "2024-03-26T15:11:27.419554Z", + "shell.execute_reply": "2024-03-26T15:11:27.418692Z" + }, + "papermill": { + "duration": 0.048996, + "end_time": "2024-03-26T15:11:27.421555", + "exception": false, + "start_time": "2024-03-26T15:11:27.372559", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:27.446341Z", + "iopub.status.busy": "2024-03-26T15:11:27.445996Z", + "iopub.status.idle": "2024-03-26T15:11:27.452563Z", + "shell.execute_reply": "2024-03-26T15:11:27.451772Z" + }, + "papermill": { + "duration": 0.021004, + "end_time": "2024-03-26T15:11:27.454322", + "exception": false, + "start_time": "2024-03-26T15:11:27.433318", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:27.477463Z", + "iopub.status.busy": "2024-03-26T15:11:27.477188Z", + "iopub.status.idle": "2024-03-26T15:11:27.481080Z", + "shell.execute_reply": "2024-03-26T15:11:27.480283Z" + }, + "papermill": { + "duration": 0.017728, + "end_time": "2024-03-26T15:11:27.483003", + "exception": false, + "start_time": "2024-03-26T15:11:27.465275", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:27.506237Z", + "iopub.status.busy": "2024-03-26T15:11:27.505987Z", + "iopub.status.idle": "2024-03-26T15:11:27.509940Z", + "shell.execute_reply": "2024-03-26T15:11:27.509134Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017873, + "end_time": "2024-03-26T15:11:27.511897", + "exception": false, + "start_time": "2024-03-26T15:11:27.494024", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:27.535719Z", + "iopub.status.busy": "2024-03-26T15:11:27.535395Z", + "iopub.status.idle": "2024-03-26T15:11:27.541003Z", + "shell.execute_reply": "2024-03-26T15:11:27.540149Z" + }, + "papermill": { + "duration": 0.019808, + "end_time": "2024-03-26T15:11:27.542897", + "exception": false, + "start_time": "2024-03-26T15:11:27.523089", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "39e26bbe", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:27.567995Z", + "iopub.status.busy": "2024-03-26T15:11:27.567704Z", + "iopub.status.idle": "2024-03-26T15:11:27.572458Z", + "shell.execute_reply": "2024-03-26T15:11:27.571649Z" + }, + "papermill": { + "duration": 0.019417, + "end_time": "2024-03-26T15:11:27.574274", + "exception": false, + "start_time": "2024-03-26T15:11:27.554857", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"lct_gan\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 3\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/lct_gan/3\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011037, + "end_time": "2024-03-26T15:11:27.596218", + "exception": false, + "start_time": "2024-03-26T15:11:27.585181", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:27.619868Z", + "iopub.status.busy": "2024-03-26T15:11:27.619570Z", + "iopub.status.idle": "2024-03-26T15:11:27.628448Z", + "shell.execute_reply": "2024-03-26T15:11:27.627690Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023014, + "end_time": "2024-03-26T15:11:27.630351", + "exception": false, + "start_time": "2024-03-26T15:11:27.607337", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/lct_gan/3\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:27.653930Z", + "iopub.status.busy": "2024-03-26T15:11:27.653657Z", + "iopub.status.idle": "2024-03-26T15:11:29.608562Z", + "shell.execute_reply": "2024-03-26T15:11:29.607578Z" + }, + "papermill": { + "duration": 1.969006, + "end_time": "2024-03-26T15:11:29.610577", + "exception": false, + "start_time": "2024-03-26T15:11:27.641571", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:29.636448Z", + "iopub.status.busy": "2024-03-26T15:11:29.636020Z", + "iopub.status.idle": "2024-03-26T15:11:29.648363Z", + "shell.execute_reply": "2024-03-26T15:11:29.647447Z" + }, + "papermill": { + "duration": 0.027471, + "end_time": "2024-03-26T15:11:29.650335", + "exception": false, + "start_time": "2024-03-26T15:11:29.622864", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:29.674921Z", + "iopub.status.busy": "2024-03-26T15:11:29.674227Z", + "iopub.status.idle": "2024-03-26T15:11:29.681025Z", + "shell.execute_reply": "2024-03-26T15:11:29.680255Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.020903, + "end_time": "2024-03-26T15:11:29.682921", + "exception": false, + "start_time": "2024-03-26T15:11:29.662018", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:29.706777Z", + "iopub.status.busy": "2024-03-26T15:11:29.706506Z", + "iopub.status.idle": "2024-03-26T15:11:29.800272Z", + "shell.execute_reply": "2024-03-26T15:11:29.799447Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.108394, + "end_time": "2024-03-26T15:11:29.802588", + "exception": false, + "start_time": "2024-03-26T15:11:29.694194", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:29.829803Z", + "iopub.status.busy": "2024-03-26T15:11:29.829512Z", + "iopub.status.idle": "2024-03-26T15:11:34.481606Z", + "shell.execute_reply": "2024-03-26T15:11:34.480800Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.668801, + "end_time": "2024-03-26T15:11:34.483976", + "exception": false, + "start_time": "2024-03-26T15:11:29.815175", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-26 15:11:32.116367: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-26 15:11:32.116434: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-26 15:11:32.118072: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:34.508965Z", + "iopub.status.busy": "2024-03-26T15:11:34.508378Z", + "iopub.status.idle": "2024-03-26T15:11:34.515802Z", + "shell.execute_reply": "2024-03-26T15:11:34.515104Z" + }, + "papermill": { + "duration": 0.02189, + "end_time": "2024-03-26T15:11:34.517618", + "exception": false, + "start_time": "2024-03-26T15:11:34.495728", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:34.543039Z", + "iopub.status.busy": "2024-03-26T15:11:34.542763Z", + "iopub.status.idle": "2024-03-26T15:11:43.009224Z", + "shell.execute_reply": "2024-03-26T15:11:43.008182Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.481876, + "end_time": "2024-03-26T15:11:43.011682", + "exception": false, + "start_time": "2024-03-26T15:11:34.529806", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-26T15:11:43.039872Z", + "iopub.status.busy": "2024-03-26T15:11:43.039463Z", + "iopub.status.idle": "2024-03-26T15:11:43.046117Z", + "shell.execute_reply": "2024-03-26T15:11:43.045279Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.023643, + "end_time": "2024-03-26T15:11:43.048107", + "exception": false, + "start_time": "2024-03-26T15:11:43.024464", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 36,\n", + " 'realtabformer': (19, 551, Embedding(551, 800), True),\n", + " 'lct_gan': 29,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:43.073075Z", + "iopub.status.busy": "2024-03-26T15:11:43.072825Z", + "iopub.status.idle": "2024-03-26T15:11:43.077485Z", + "shell.execute_reply": "2024-03-26T15:11:43.076731Z" + }, + "papermill": { + "duration": 0.019439, + "end_time": "2024-03-26T15:11:43.079481", + "exception": false, + "start_time": "2024-03-26T15:11:43.060042", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:43.104939Z", + "iopub.status.busy": "2024-03-26T15:11:43.104696Z", + "iopub.status.idle": "2024-03-26T15:11:43.168541Z", + "shell.execute_reply": "2024-03-26T15:11:43.167600Z" + }, + "papermill": { + "duration": 0.078936, + "end_time": "2024-03-26T15:11:43.170495", + "exception": false, + "start_time": "2024-03-26T15:11:43.091559", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/aug_test/insurance 0\n", + "Caching in ../../../../insurance/_cache_bs_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/bs_test/insurance 0\n", + "Caching in ../../../../insurance/_cache_synth_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/synthetics/insurance 600\n", + "600\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:43.197320Z", + "iopub.status.busy": "2024-03-26T15:11:43.197043Z", + "iopub.status.idle": "2024-03-26T15:11:43.517226Z", + "shell.execute_reply": "2024-03-26T15:11:43.516318Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.335995, + "end_time": "2024-03-26T15:11:43.519210", + "exception": false, + "start_time": "2024-03-26T15:11:43.183215", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'loss_balancer_beta': 0.79,\n", + " 'loss_balancer_r': 0.95,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'lct_gan',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.PReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['lct_gan'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "# params = {\n", + "# **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "# }\n", + "params = getattr(PARAMS, dataset_name).BEST_DICT[gp][gp_multiply][single_model]\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).DEFAULTS,\n", + " **params,\n", + "}\n", + "if isinstance(params, (list, tuple)):\n", + " params = params[param_index]\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:43.545198Z", + "iopub.status.busy": "2024-03-26T15:11:43.544912Z", + "iopub.status.idle": "2024-03-26T15:11:43.646358Z", + "shell.execute_reply": "2024-03-26T15:11:43.645446Z" + }, + "papermill": { + "duration": 0.116761, + "end_time": "2024-03-26T15:11:43.648456", + "exception": false, + "start_time": "2024-03-26T15:11:43.531695", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_train/lct_gan/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/insurance [400, 0]\n", + "Caching in ../../../../insurance/_cache_aug_val/lct_gan/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/insurance [0, 200]\n", + "Caching in ../../../../insurance/_cache_bs_train/lct_gan/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/insurance [100, 0]\n", + "Caching in ../../../../insurance/_cache_bs_val/lct_gan/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/insurance [0, 50]\n", + "Caching in ../../../../insurance/_cache_synth/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/insurance [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-26T15:11:43.675708Z", + "iopub.status.busy": "2024-03-26T15:11:43.675364Z", + "iopub.status.idle": "2024-03-26T15:11:44.094325Z", + "shell.execute_reply": "2024-03-26T15:11:44.093448Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.434814, + "end_time": "2024-03-26T15:11:44.096306", + "exception": false, + "start_time": "2024-03-26T15:11:43.661492", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['lct_gan'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:44.124295Z", + "iopub.status.busy": "2024-03-26T15:11:44.123994Z", + "iopub.status.idle": "2024-03-26T15:11:44.128047Z", + "shell.execute_reply": "2024-03-26T15:11:44.127274Z" + }, + "papermill": { + "duration": 0.020133, + "end_time": "2024-03-26T15:11:44.129857", + "exception": false, + "start_time": "2024-03-26T15:11:44.109724", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:44.155930Z", + "iopub.status.busy": "2024-03-26T15:11:44.155651Z", + "iopub.status.idle": "2024-03-26T15:11:44.162412Z", + "shell.execute_reply": "2024-03-26T15:11:44.161630Z" + }, + "papermill": { + "duration": 0.022141, + "end_time": "2024-03-26T15:11:44.164306", + "exception": false, + "start_time": "2024-03-26T15:11:44.142165", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "9631369" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:44.190208Z", + "iopub.status.busy": "2024-03-26T15:11:44.189948Z", + "iopub.status.idle": "2024-03-26T15:11:44.277547Z", + "shell.execute_reply": "2024-03-26T15:11:44.276663Z" + }, + "papermill": { + "duration": 0.102872, + "end_time": "2024-03-26T15:11:44.279505", + "exception": false, + "start_time": "2024-03-26T15:11:44.176633", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 29] --\n", + "├─Adapter: 1-1 [2, 1071, 29] --\n", + "│ └─Sequential: 2-1 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 30,720\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 29] (recursive)\n", + "│ └─Sequential: 2-2 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-31 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-37 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-40 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-42 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-43 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-45 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-46 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-48 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-49 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-18 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-51 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-52 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─PReLU: 4-38 [2, 128] 1\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-40 [2, 128] 1\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-42 [2, 128] 1\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-44 [2, 128] 1\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-46 [2, 128] 1\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-48 [2, 128] 1\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-50 [2, 128] 1\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-52 [2, 128] 1\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 9,631,369\n", + "Trainable params: 9,631,369\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 37.10\n", + "========================================================================================================================\n", + "Input size (MB): 0.31\n", + "Forward/backward pass size (MB): 307.09\n", + "Params size (MB): 38.53\n", + "Estimated Total Size (MB): 345.93\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:11:44.310299Z", + "iopub.status.busy": "2024-03-26T15:11:44.309495Z", + "iopub.status.idle": "2024-03-26T16:12:56.494119Z", + "shell.execute_reply": "2024-03-26T16:12:56.493098Z" + }, + "papermill": { + "duration": 3672.218682, + "end_time": "2024-03-26T16:12:56.512737", + "exception": false, + "start_time": "2024-03-26T15:11:44.294055", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 600]\n", + "Creating model of type \n", + "[*] Embedding False True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.04117060134704742, 'avg_role_model_std_loss': 0.8109188975505845, 'avg_role_model_mean_pred_loss': 0.012500455402668946, 'avg_role_model_g_mag_loss': 0.4644619934923119, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.041821396528846685, 'n_size': 900, 'n_batch': 113, 'duration': 145.03757858276367, 'duration_batch': 1.2835183945377315, 'duration_size': 0.16115286509195964, 'avg_pred_std': 0.14025535803716793}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.017303582032004165, 'avg_role_model_std_loss': 2.386988934791206, 'avg_role_model_mean_pred_loss': 0.0012482430997963294, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.017303582032004165, 'n_size': 450, 'n_batch': 57, 'duration': 47.186697244644165, 'duration_batch': 0.8278367937656871, 'duration_size': 0.10485932721032036, 'avg_pred_std': 0.04331476881838681}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00788458850596928, 'avg_role_model_std_loss': 1.4652959933179, 'avg_role_model_mean_pred_loss': 0.0002524864735020014, 'avg_role_model_g_mag_loss': 0.07177837291939392, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007992250232620993, 'n_size': 900, 'n_batch': 113, 'duration': 145.16301083564758, 'duration_batch': 1.2846284144747573, 'duration_size': 0.16129223426183065, 'avg_pred_std': 0.08565499742342307}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005107812441161109, 'avg_role_model_std_loss': 0.07783917244306761, 'avg_role_model_mean_pred_loss': 0.00024728326528121317, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005107812441161109, 'n_size': 450, 'n_batch': 57, 'duration': 47.019214153289795, 'duration_batch': 0.8248984939173648, 'duration_size': 0.10448714256286622, 'avg_pred_std': 0.08467314502616462}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00557574656041753, 'avg_role_model_std_loss': 0.7293474927610862, 'avg_role_model_mean_pred_loss': 0.00022519498775718153, 'avg_role_model_g_mag_loss': 0.041499399857388604, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005658741251487906, 'n_size': 900, 'n_batch': 113, 'duration': 145.352201461792, 'duration_batch': 1.286302667803469, 'duration_size': 0.16150244606865777, 'avg_pred_std': 0.0888019731532024}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005094906620474325, 'avg_role_model_std_loss': 0.33239709677336415, 'avg_role_model_mean_pred_loss': 1.4148092069338326e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005094906620474325, 'n_size': 450, 'n_batch': 57, 'duration': 47.298946142196655, 'duration_batch': 0.8298060726701167, 'duration_size': 0.10510876920488145, 'avg_pred_std': 0.0653855954051802}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004928552754507917, 'avg_role_model_std_loss': 0.590203044300799, 'avg_role_model_mean_pred_loss': 7.935110965993373e-05, 'avg_role_model_g_mag_loss': 0.03116383927563826, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005009777373375578, 'n_size': 900, 'n_batch': 113, 'duration': 145.0207061767578, 'duration_batch': 1.283369081210246, 'duration_size': 0.16113411797417534, 'avg_pred_std': 0.09011044628522565}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.005199750801224986, 'avg_role_model_std_loss': 0.648273554320065, 'avg_role_model_mean_pred_loss': 2.5179529602438558e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005199750801224986, 'n_size': 450, 'n_batch': 57, 'duration': 47.28320384025574, 'duration_batch': 0.8295298919343111, 'duration_size': 0.10507378631167942, 'avg_pred_std': 0.051663709136559384}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004318670859793201, 'avg_role_model_std_loss': 0.4876711248132525, 'avg_role_model_mean_pred_loss': 4.46604487823629e-05, 'avg_role_model_g_mag_loss': 0.029432983928256565, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004396132711424596, 'n_size': 900, 'n_batch': 113, 'duration': 145.67448663711548, 'duration_batch': 1.289154749001022, 'duration_size': 0.1618605407079061, 'avg_pred_std': 0.08900898528507853}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0041498291804115675, 'avg_role_model_std_loss': 0.44782343388953705, 'avg_role_model_mean_pred_loss': 3.687838003341986e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0041498291804115675, 'n_size': 450, 'n_batch': 57, 'duration': 47.23726558685303, 'duration_batch': 0.8287239576640882, 'duration_size': 0.10497170130411784, 'avg_pred_std': 0.057800088480131274}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003727001822941626, 'avg_role_model_std_loss': 0.4257845432669564, 'avg_role_model_mean_pred_loss': 5.2791223793633036e-05, 'avg_role_model_g_mag_loss': 0.027504234943124983, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0037818589322139613, 'n_size': 900, 'n_batch': 113, 'duration': 145.0672550201416, 'duration_batch': 1.2837810178773592, 'duration_size': 0.16118583891126845, 'avg_pred_std': 0.09003757454652701}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003250846434198643, 'avg_role_model_std_loss': 0.4149662161665363, 'avg_role_model_mean_pred_loss': 8.010250678769003e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003250846434198643, 'n_size': 450, 'n_batch': 57, 'duration': 47.08042764663696, 'duration_batch': 0.8259724148532801, 'duration_size': 0.10462317254808214, 'avg_pred_std': 0.06960962812228356}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003335649493189218, 'avg_role_model_std_loss': 0.29143936353892297, 'avg_role_model_mean_pred_loss': 0.00010225355728581073, 'avg_role_model_g_mag_loss': 0.03192889101389382, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0033789488727537296, 'n_size': 900, 'n_batch': 113, 'duration': 144.45740246772766, 'duration_batch': 1.278384092634758, 'duration_size': 0.16050822496414185, 'avg_pred_std': 0.08855825027994878}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003431849268866548, 'avg_role_model_std_loss': 0.27368448856224714, 'avg_role_model_mean_pred_loss': 3.499706230363269e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003431849268866548, 'n_size': 450, 'n_batch': 57, 'duration': 47.77770447731018, 'duration_batch': 0.8382053417071962, 'duration_size': 0.10617267661624484, 'avg_pred_std': 0.06641790176856152}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002982409707296433, 'avg_role_model_std_loss': 0.37030739115790007, 'avg_role_model_mean_pred_loss': 3.9599444266533356e-05, 'avg_role_model_g_mag_loss': 0.03103297157213092, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0030181357321432895, 'n_size': 900, 'n_batch': 113, 'duration': 144.7265179157257, 'duration_batch': 1.280765645271909, 'duration_size': 0.16080724212858413, 'avg_pred_std': 0.0891251541443367}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003124318533429889, 'avg_role_model_std_loss': 0.24963311337495198, 'avg_role_model_mean_pred_loss': 7.335967463523267e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003124318533429889, 'n_size': 450, 'n_batch': 57, 'duration': 47.88541555404663, 'duration_batch': 0.8400950097201163, 'duration_size': 0.10641203456454806, 'avg_pred_std': 0.07890400658933479}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0020741428555467994, 'avg_role_model_std_loss': 0.2619519646582818, 'avg_role_model_mean_pred_loss': 1.8381270814843714e-05, 'avg_role_model_g_mag_loss': 0.031121474682456917, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002098895235814982, 'n_size': 900, 'n_batch': 113, 'duration': 145.16141033172607, 'duration_batch': 1.2846142507232396, 'duration_size': 0.16129045592414007, 'avg_pred_std': 0.09325064248116934}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004463403613384192, 'avg_role_model_std_loss': 0.8222081985106039, 'avg_role_model_mean_pred_loss': 7.022714163104686e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004463403613384192, 'n_size': 450, 'n_batch': 57, 'duration': 47.134621143341064, 'duration_batch': 0.826923177953352, 'duration_size': 0.10474360254075792, 'avg_pred_std': 0.05030834952598078}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0023060996746709053, 'avg_role_model_std_loss': 0.2621581708226265, 'avg_role_model_mean_pred_loss': 4.253982956825255e-05, 'avg_role_model_g_mag_loss': 0.029572723129143316, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002331820259375187, 'n_size': 900, 'n_batch': 113, 'duration': 143.75192093849182, 'duration_batch': 1.2721408932609897, 'duration_size': 0.15972435659832424, 'avg_pred_std': 0.09426238097712002}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023851372594233707, 'avg_role_model_std_loss': 0.25001330026493196, 'avg_role_model_mean_pred_loss': 9.203486099863416e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023851372594233707, 'n_size': 450, 'n_batch': 57, 'duration': 46.45239043235779, 'duration_batch': 0.8149542181115401, 'duration_size': 0.10322753429412841, 'avg_pred_std': 0.06987214349046872}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013159852624004189, 'avg_role_model_std_loss': 0.218279704756602, 'avg_role_model_mean_pred_loss': 7.74645728111526e-06, 'avg_role_model_g_mag_loss': 0.024696054148177306, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013293167201401147, 'n_size': 900, 'n_batch': 113, 'duration': 143.5770845413208, 'duration_batch': 1.2705936685072636, 'duration_size': 0.1595300939348009, 'avg_pred_std': 0.09105278838392908}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00223359671033298, 'avg_role_model_std_loss': 0.03436504584376627, 'avg_role_model_mean_pred_loss': 1.4256753953578257e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00223359671033298, 'n_size': 450, 'n_batch': 57, 'duration': 46.45383548736572, 'duration_batch': 0.8149795699537846, 'duration_size': 0.10323074552747938, 'avg_pred_std': 0.07930388350627925}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002150901930160924, 'avg_role_model_std_loss': 0.22495316012830485, 'avg_role_model_mean_pred_loss': 3.4149523725070684e-05, 'avg_role_model_g_mag_loss': 0.03553776060955392, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0021734700896518513, 'n_size': 900, 'n_batch': 113, 'duration': 143.75677585601807, 'duration_batch': 1.2721838571329032, 'duration_size': 0.15972975095113118, 'avg_pred_std': 0.09556485827913326}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003228269326912899, 'avg_role_model_std_loss': 0.28536299721485003, 'avg_role_model_mean_pred_loss': 1.6670719938160433e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003228269326912899, 'n_size': 450, 'n_batch': 57, 'duration': 46.40282607078552, 'duration_batch': 0.814084667908518, 'duration_size': 0.10311739126841227, 'avg_pred_std': 0.07249125089136917}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0019467951555270703, 'avg_role_model_std_loss': 0.23753629187743283, 'avg_role_model_mean_pred_loss': 3.624726925012959e-05, 'avg_role_model_g_mag_loss': 0.03207059481077724, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0019664278747707916, 'n_size': 900, 'n_batch': 113, 'duration': 143.81293630599976, 'duration_batch': 1.2726808522654847, 'duration_size': 0.15979215145111084, 'avg_pred_std': 0.09223815666890778}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003769443849644934, 'avg_role_model_std_loss': 0.12684818327569333, 'avg_role_model_mean_pred_loss': 0.0005272575768817463, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003769443849644934, 'n_size': 450, 'n_batch': 57, 'duration': 46.357746601104736, 'duration_batch': 0.8132938000193813, 'duration_size': 0.10301721466912164, 'avg_pred_std': 0.08174848222386158}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0011665212795681631, 'avg_role_model_std_loss': 0.09113499079847809, 'avg_role_model_mean_pred_loss': 2.4942857778784166e-05, 'avg_role_model_g_mag_loss': 0.02439475072444313, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001178674958355259, 'n_size': 900, 'n_batch': 113, 'duration': 143.67677998542786, 'duration_batch': 1.2714759290745827, 'duration_size': 0.1596408666504754, 'avg_pred_std': 0.09811776785789865}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0026310520773727654, 'avg_role_model_std_loss': 0.5994252953760104, 'avg_role_model_mean_pred_loss': 0.0001863106027298025, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0026310520773727654, 'n_size': 450, 'n_batch': 57, 'duration': 46.223793745040894, 'duration_batch': 0.8109437499129981, 'duration_size': 0.10271954165564642, 'avg_pred_std': 0.06012566037590436}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007886418905885269, 'avg_role_model_std_loss': 0.1318949167653181, 'avg_role_model_mean_pred_loss': 1.084472590072184e-06, 'avg_role_model_g_mag_loss': 0.019767768517550494, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007973033490513141, 'n_size': 900, 'n_batch': 113, 'duration': 142.2467658519745, 'duration_batch': 1.258820936743137, 'duration_size': 0.15805196205774943, 'avg_pred_std': 0.09370011994532779}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0024774786025712576, 'avg_role_model_std_loss': 0.23988299155125448, 'avg_role_model_mean_pred_loss': 0.0003575192506448812, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0024774786025712576, 'n_size': 450, 'n_batch': 57, 'duration': 45.4305682182312, 'duration_batch': 0.7970275126005474, 'duration_size': 0.100956818262736, 'avg_pred_std': 0.07729252947396353}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007920145899212609, 'avg_role_model_std_loss': 0.07880277095880331, 'avg_role_model_mean_pred_loss': 1.3125071642120135e-06, 'avg_role_model_g_mag_loss': 0.022073325576881568, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007999837701855641, 'n_size': 900, 'n_batch': 113, 'duration': 142.16578197479248, 'duration_batch': 1.2581042652636503, 'duration_size': 0.15796197997199166, 'avg_pred_std': 0.09601840465865304}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0018119772487261798, 'avg_role_model_std_loss': 0.28852804994411196, 'avg_role_model_mean_pred_loss': 0.0001659401577897141, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0018119772487261798, 'n_size': 450, 'n_batch': 57, 'duration': 45.49597787857056, 'duration_batch': 0.7981750505012378, 'duration_size': 0.10110217306349012, 'avg_pred_std': 0.07632629704465599}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0005196595300286491, 'avg_role_model_std_loss': 0.0565358007824824, 'avg_role_model_mean_pred_loss': 5.911478473423242e-07, 'avg_role_model_g_mag_loss': 0.015494050164189603, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005250095642016579, 'n_size': 900, 'n_batch': 113, 'duration': 143.4526925086975, 'duration_batch': 1.26949285405927, 'duration_size': 0.15939188056521947, 'avg_pred_std': 0.09898689962857593}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0018605626394532413, 'avg_role_model_std_loss': 0.42602644269920564, 'avg_role_model_mean_pred_loss': 0.00010228292684815276, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0018605626394532413, 'n_size': 450, 'n_batch': 57, 'duration': 46.52276873588562, 'duration_batch': 0.8161889251909757, 'duration_size': 0.10338393052419027, 'avg_pred_std': 0.07228483269889757}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0003834019511123188, 'avg_role_model_std_loss': 0.05582865150848945, 'avg_role_model_mean_pred_loss': 1.0770429865198301e-07, 'avg_role_model_g_mag_loss': 0.013134720898750755, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00038761303258878695, 'n_size': 900, 'n_batch': 113, 'duration': 144.13887667655945, 'duration_batch': 1.275565280323535, 'duration_size': 0.1601543074183994, 'avg_pred_std': 0.09740801119303281}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022972054127603767, 'avg_role_model_std_loss': 0.26383435410354306, 'avg_role_model_mean_pred_loss': 0.0004230380179690739, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022972054127603767, 'n_size': 450, 'n_batch': 57, 'duration': 47.14963889122009, 'duration_batch': 0.8271866472143876, 'duration_size': 0.10477697531382243, 'avg_pred_std': 0.0769950772080113}\n", + "Epoch 18\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00036346292850794273, 'avg_role_model_std_loss': 0.047643846057415055, 'avg_role_model_mean_pred_loss': 2.4050790815127115e-07, 'avg_role_model_g_mag_loss': 0.013326084169869622, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00036746932837154923, 'n_size': 900, 'n_batch': 113, 'duration': 144.99755716323853, 'duration_batch': 1.2831642226835267, 'duration_size': 0.16110839684804282, 'avg_pred_std': 0.09879991837439284}\n", + "Time out: 3602.547290802002/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'lct_gan', 'n_size': 600, 'n_batch': 75, 'role_model_metrics': {'avg_loss': 0.0007886611748836003, 'avg_g_mag_loss': 0.008810463138737153, 'avg_g_cos_loss': 0.0036491415455626943, 'pred_duration': 1.261091709136963, 'grad_duration': 3.734239101409912, 'total_duration': 4.995330810546875, 'pred_std': 0.1561143845319748, 'std_loss': 0.0019985174294561148, 'mean_pred_loss': 8.764879453337926e-07, 'pred_rmse': 0.028083112090826035, 'pred_mae': 0.018356479704380035, 'pred_mape': 0.5159444212913513, 'grad_rmse': 0.047635164111852646, 'grad_mae': 0.015319733880460262, 'grad_mape': 0.531665027141571}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0007886611748836003, 'avg_g_mag_loss': 0.008810463138737153, 'avg_g_cos_loss': 0.0036491415455626943, 'avg_pred_duration': 1.261091709136963, 'avg_grad_duration': 3.734239101409912, 'avg_total_duration': 4.995330810546875, 'avg_pred_std': 0.1561143845319748, 'avg_std_loss': 0.0019985174294561148, 'avg_mean_pred_loss': 8.764879453337926e-07}, 'min_metrics': {'avg_loss': 0.0007886611748836003, 'avg_g_mag_loss': 0.008810463138737153, 'avg_g_cos_loss': 0.0036491415455626943, 'pred_duration': 1.261091709136963, 'grad_duration': 3.734239101409912, 'total_duration': 4.995330810546875, 'pred_std': 0.1561143845319748, 'std_loss': 0.0019985174294561148, 'mean_pred_loss': 8.764879453337926e-07, 'pred_rmse': 0.028083112090826035, 'pred_mae': 0.018356479704380035, 'pred_mape': 0.5159444212913513, 'grad_rmse': 0.047635164111852646, 'grad_mae': 0.015319733880460262, 'grad_mape': 0.531665027141571}, 'model_metrics': {'lct_gan': {'avg_loss': 0.0007886611748836003, 'avg_g_mag_loss': 0.008810463138737153, 'avg_g_cos_loss': 0.0036491415455626943, 'pred_duration': 1.261091709136963, 'grad_duration': 3.734239101409912, 'total_duration': 4.995330810546875, 'pred_std': 0.1561143845319748, 'std_loss': 0.0019985174294561148, 'mean_pred_loss': 8.764879453337926e-07, 'pred_rmse': 0.028083112090826035, 'pred_mae': 0.018356479704380035, 'pred_mape': 0.5159444212913513, 'grad_rmse': 0.047635164111852646, 'grad_mae': 0.015319733880460262, 'grad_mape': 0.531665027141571}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T16:12:56.549761Z", + "iopub.status.busy": "2024-03-26T16:12:56.549467Z", + "iopub.status.idle": "2024-03-26T16:12:56.553324Z", + "shell.execute_reply": "2024-03-26T16:12:56.552597Z" + }, + "papermill": { + "duration": 0.024866, + "end_time": "2024-03-26T16:12:56.555175", + "exception": false, + "start_time": "2024-03-26T16:12:56.530309", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T16:12:56.588832Z", + "iopub.status.busy": "2024-03-26T16:12:56.588556Z", + "iopub.status.idle": "2024-03-26T16:12:56.667436Z", + "shell.execute_reply": "2024-03-26T16:12:56.666643Z" + }, + "papermill": { + "duration": 0.098493, + "end_time": "2024-03-26T16:12:56.669788", + "exception": false, + "start_time": "2024-03-26T16:12:56.571295", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T16:12:56.706930Z", + "iopub.status.busy": "2024-03-26T16:12:56.706615Z", + "iopub.status.idle": "2024-03-26T16:12:56.972561Z", + "shell.execute_reply": "2024-03-26T16:12:56.971642Z" + }, + "papermill": { + "duration": 0.287381, + "end_time": "2024-03-26T16:12:56.974641", + "exception": false, + "start_time": "2024-03-26T16:12:56.687260", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAESCAYAAABU2qhcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAzkUlEQVR4nO3de3xTVb738U/aJun9QqE3KC33IpeCYEvxgmilOIjWQUEODyAPwuijjp56QRyhzsw5dhxxRIUjB2dGnDMiiCOOgxwUKowjFLlDsYCAQIGSXoCm91uynj/Spg2kpSltE5rf+/XKq8neKztrl+bL2muvvbZGKaUQQggX4OHsCgghRAMJJCGEy5BAEkK4DAkkIYTLkEASQrgMCSQhhMuQQBJCuAwvZ1egPZjNZvLy8ggICECj0Ti7OkKIKyilKC0tJSoqCg+P5ttBXSKQ8vLyiI6OdnY1hBDXcPbsWXr16tXs+i4RSAEBAYBlZwMDA51cGyHElUpKSoiOjrZ+V5vTJQKp4TAtMDBQAkkIF3atLhXp1BZCuAwJJCGEy5BAEkK4jC7RhySuj8lkora21tnVEDcwrVaLp6fndW9HAsmNKaUwGAwUFxc7uyqiCwgODiYiIuK6xgJKILmxhjAKCwvD19dXBpWKNlFKUVFRQUFBAQCRkZFt3pZbBVKdycyBs8UUV9RyV1wYHh7u+wU0mUzWMAoNDXV2dcQNzsfHB4CCggLCwsLafPjmVp3aJqV4aEUWj/1lD6XVdc6ujlM19Bn5+vo6uSaiq2j4W7qe/ki3CiS9lye+OktyF1fUOLk2rkEO00R7aY+/JbcKJIBgHy0AxRVyVkkIV+N+geSrA+CytJCEcDluGEjSQhLXR6PR8Pnnnzu7Gu3q1VdfZcSIEc6uhvsFUkh9C0n6kMSNbNWqVQQHB7fb9p5//nkyMzPbbXtt1aZAWr58ObGxsXh7e5OYmMiuXbtaLL9u3Tri4uLw9vZm2LBhbNy4sdmyjz/+OBqNhqVLl7alatcUVN9CuiwtJOEGampa9x+vv7+/Swz/cDiQ1q5dS1paGunp6ezbt4/4+HhSUlKsg6KutGPHDqZPn87cuXPZv38/qamppKamcvjw4avKrl+/np07dxIVFeX4nrRSSH0gGSslkK6klKKipq7TH47ePHnTpk3cdtttBAcHExoayn333cfJkycBGDt2LAsWLLApX1hYiFar5dtvvwXgwoULTJo0CR8fH/r06cPq1auJjY1t83+C2dnZ3HXXXfj4+BAaGsr8+fMpKyuzrt+2bRsJCQn4+fkRHBzMrbfeypkzZwA4ePAg48ePJyAggMDAQEaNGsWePXta/Lxt27YxZ84cjEYjGo0GjUbDq6++CkBsbCy//e1vmTVrFoGBgcyfPx+ABQsWMHDgQHx9fenbty+LFi2yOT1/5SHbo48+SmpqKkuWLCEyMpLQ0FCefPLJDr/EyOGBkX/4wx+YN28ec+bMAWDFihV8+eWX/PnPf+all166qvzbb7/NxIkTeeGFFwD47W9/y+bNm1m2bBkrVqywljt//jxPP/00X331FZMmTWrr/lxTsI90ajenstbETYu/6vTPzflNCr661v8plpeXk5aWxvDhwykrK2Px4sU8+OCDHDhwgBkzZvD73/+e3/3ud9bT0GvXriUqKorbb78dgFmzZlFUVMS2bdvQarWkpaU1+x9qa+qSkpJCUlISu3fvpqCggMcee4ynnnqKVatWUVdXR2pqKvPmzePjjz+mpqaGXbt2Wes2Y8YMRo4cyXvvvYenpycHDhxAq9W2+Jljx45l6dKlLF68mGPHjgGWFk6DJUuWsHjxYtLT063LAgICWLVqFVFRUWRnZzNv3jwCAgJ48cUXm/2crVu3EhkZydatWzlx4gTTpk1jxIgRzJs3r02/q9ZwKJBqamrYu3cvCxcutC7z8PAgOTmZrKwsu+/JysoiLS3NZllKSopNp6DZbGbmzJm88MILDBky5Jr1qK6uprq62vq6pKSk1fsgndo3vilTpti8/vOf/0yPHj3Iyclh6tSpPPvss3z33XfWAFq9ejXTp09Ho9Fw9OhRtmzZwu7duxk9ejQAf/zjHxkwYECb6rJ69Wqqqqr4y1/+gp+fHwDLli1j8uTJvP7662i1WoxGI/fddx/9+vUDYPDgwdb35+bm8sILLxAXFwfQqnrodDqCgoLQaDRERERctf6uu+7iueees1n2yiuvWJ/Hxsby/PPPs2bNmhYDKSQkhGXLluHp6UlcXByTJk0iMzPTdQKpqKgIk8lEeHi4zfLw8HCOHj1q9z0Gg8FueYPBYH39+uuv4+XlxS9/+ctW1SMjI4Nf//rXjlTdSjq1m+ej9STnNylO+VxHHD9+nMWLF/P9999TVFSE2WwGLF/uoUOHMmHCBD766CNuv/12Tp06RVZWFv/93/8NwLFjx/Dy8uLmm2+2bq9///6EhIS0qe5HjhwhPj7eGkYAt956K2azmWPHjnHHHXfw6KOPkpKSwj333ENycjJTp061Xu+VlpbGY489xv/8z/+QnJzMww8/bA2utmoI2qbWrl3LO++8w8mTJykrK6Ouru6as6sOGTLE5hKQyMhIsrOzr6tu1+L0s2x79+7l7bffZtWqVa0e6blw4UKMRqP1cfbs2VZ/XrB0ajdLo9Hgq/Pq9IejI3wnT57MpUuXeP/99/n+++/5/vvvgcYO3BkzZvDpp59SW1vL6tWrGTZsGMOGDWv331drffDBB2RlZTF27FjWrl3LwIED2blzJ2Dpu/nhhx+YNGkS33zzDTfddBPr16+/rs9rGo5gOUqZMWMGP/vZz9iwYQP79+/nV7/61TU7vK88dNRoNNbw7ygOBVL37t3x9PQkPz/fZnl+fr7dpiNAREREi+X/9a9/UVBQQO/evfHy8sLLy4szZ87w3HPPERsba3eber3eOn+2o/NoB0sL6YZ28eJFjh07xiuvvMLdd9/N4MGDuXz5sk2ZBx54gKqqKjZt2sTq1auZMWOGdd2gQYOoq6tj//791mUnTpy4ahutNXjwYA4ePEh5ebl12fbt2/Hw8GDQoEHWZSNHjmThwoXs2LGDoUOHsnr1auu6gQMH8u///u98/fXX/PznP+eDDz645ufqdDpMJlOr6rhjxw5iYmL41a9+xejRoxkwYIC1U93VOBRIOp2OUaNG2YxXMJvNZGZmkpSUZPc9SUlJV41v2Lx5s7X8zJkzOXToEAcOHLA+oqKieOGFF/jqq/bvYG1oIZVU1VFn6ti0F+0vJCSE0NBQVq5cyYkTJ/jmm2+u6qP08/MjNTWVRYsWceTIEaZPn25dFxcXR3JyMvPnz2fXrl3s37+f+fPn4+Pj06ZrsWbMmIG3tzezZ8/m8OHDbN26laeffpqZM2cSHh7OqVOnWLhwIVlZWZw5c4avv/6a48ePM3jwYCorK3nqqafYtm0bZ86cYfv27ezevdumj6k5sbGxlJWVkZmZSVFRERUVFc2WHTBgALm5uaxZs4aTJ0/yzjvvXHcrrMMoB61Zs0bp9Xq1atUqlZOTo+bPn6+Cg4OVwWBQSik1c+ZM9dJLL1nLb9++XXl5eaklS5aoI0eOqPT0dKXValV2dnaznxETE6PeeuutVtfJaDQqQBmNxmuWra0zqZgFG1TMgg3qYll1qz+jq6msrFQ5OTmqsrLS2VVx2ObNm9XgwYOVXq9Xw4cPV9u2bVOAWr9+vbXMxo0bFaDuuOOOq96fl5en7r33XqXX61VMTIxavXq1CgsLUytWrGjV51/5WYcOHVLjx49X3t7eqlu3bmrevHmqtLRUKaWUwWBQqampKjIyUul0OhUTE6MWL16sTCaTqq6uVo888oiKjo5WOp1ORUVFqaeeeqrV/yaPP/64Cg0NVYBKT09XSjX/3XnhhRdUaGio8vf3V9OmTVNvvfWWCgoKsq5PT09X8fHx1tezZ89WDzzwgM02nnnmGTVu3Lhm69PS31Rrv6MOB5JSSr377ruqd+/eSqfTqYSEBLVz507runHjxqnZs2fblP/kk0/UwIEDlU6nU0OGDFFffvlli9vvyEBSSqmhizepmAUb1ImC0lZ/RldzIwdSezt79qwC1JYtW5xdlRtaewSSRikHR6W5oJKSEoKCgjAaja3qT7r9999w9lIlf3tiLKNi2nZ25UZXVVXFqVOn6NOnD97e3s6uTqf65ptvKCsrY9iwYVy4cIEXX3yR8+fP8+OPP15zDJBoXkt/U639jjr9LJszyKl/91ZbW8vLL7/MkCFDePDBB+nRo4d1kORHH32Ev7+/3Udrxsi1l3vvvbfZerz22mudVo/O5lZT2DYIkjmR3FpKSgopKfbHW91///0kJibaXdeZrac//vGPVFZW2l3XrVu3TqtHZ3PLQAqROZFEMwICAq55//nO0LNnT2dXwSnc8pBNLh8RwjW5aSDV9yFVSgtJCFfinoHkI5ePCOGK3DKQQvzq50SSQBLCpbhlIMmcSEK4JvcMJOnUFtehK07y7yrcMpBkYKS40bX3JP9gmRpXo9FQXFzcrtt1hFsGUkMLqbzGRE2dXPEvhKtwy0AK9NbSMNOEnPpvQimoKe/8h0zy326T/FdXV/P888/Ts2dP/Pz8SExMZNu2bdb3njlzhsmTJxMSEoKfnx9Dhgxh48aNnD59mvHjxwOWKV40Gg2PPvpom34f18MtR2p7eGgI8tFSXFFLcUUtYQHudXFps2or4LWOu+NLs17OA53ftcvVk0n+m5/k/6mnniInJ4c1a9YQFRXF+vXrmThxItnZ2QwYMIAnn3ySmpoavv32W/z8/MjJycHf35/o6Gj+9re/MWXKFI4dO0ZgYCA+Pj5t+p1cD7cMJLD0IzUEkrixyCT/9if5z83N5YMPPiA3N9d6K7Hnn3+eTZs28cEHH/Daa6+Rm5vLlClTrFP69u3b1/r+hmvkwsLC2r1/qrXcNpCCrIMj5ZDNSutraa0443MdIJP825ednY3JZGLgwIE2y6urq603gfzlL3/JE088wddff01ycjJTpkxh+PDhbfq8juCWfUjQ5IaR0kJqpNFYDp06+yGT/LfLJP9lZWV4enqyd+9emymhjxw5wttvvw3AY489xk8//cTMmTPJzs5m9OjRvPvuu+22r9fLjQNJBkfeiGSSfwt7k/yPHDkSk8lEQUEB/fv3t3k0PbSLjo7m8ccf57PPPuO5557j/ffft24TaPXNAzqC2wZSkNwO6YYkk/xb2Jvkf+DAgcyYMYNZs2bx2WefcerUKXbt2kVGRgZffvklAM8++yxfffUVp06dYt++fWzdutX6eTExMWg0GjZs2EBhYaHNmcJO0zGz63YuR+fUVkqpt7f8qGIWbFAv/e1gB9bMdd3Ic2rLJP8W9ib5r6mpUYsXL1axsbFKq9WqyMhI9eCDD6pDhw4ppZR66qmnVL9+/ZRer1c9evRQM2fOVEVFRdZt/uY3v1ERERFKo9FcNTf+tcic2vUcnVMb4C9Zp1n89x+YOCSCFTNHdXANXY87z6l9pXPnzhEdHc2WLVu4++67nV2dG1Z7zKnttmfZZE4k92Vvkv/Y2FjuuOMOZ1fN7bltH1KwzKvttmSSf9flti2kxgtsJZDcjUzy77rcNpCCfWVgpLiaTPLvXO57yFYfSNV1ZqpqnTfuwtm6wDkN4SLa42/JbQPJX++Fl4dl3Ik7tpIaDj8qKiqcXBPRVTT8LV3Poa3bHrJpNBqCfbUUldVwubyWyKDOv7LZmTw9PQkODrZe5e7r69umgYFCKKWoqKigoKCA4OBgPD0927wttw0ksJz6LyqrcdtT/w2XE7R16g0hmgoODra5RKUt3DuQ3PzUv0ajITIykrCwMGpr3fN3INqHVqu9rpZRA/cOJDn1D1gO39rjj0mI6+W2ndogp/6FcDVuHUjWOZEq3buFJISrcOtAajhku1wuLSQhXIGbB5LMiSSEK3HrQGq4ns3opqf9hXA1bh1IwT7SQhLClbh3IMlpfyFcipsHUsPAyBq5yFQIF+DWgdTQh1RnVpTXuO8V/0K4CrcOJG+tBzovy69ATv0L4XxtCqTly5cTGxuLt7c3iYmJ7Nq1q8Xy69atIy4uDm9vb4YNG8bGjRtt1r/66qvExcXh5+dHSEgIycnJ1pv/dSSNRmMdHCn9SEI4n8OBtHbtWtLS0khPT2ffvn3Ex8eTkpLS7BXjO3bsYPr06cydO5f9+/eTmppKamoqhw8ftpYZOHAgy5YtIzs7m++++47Y2FgmTJhAYWFh2/eslUJksn8hXIdDN15SSiUkJKgnn3zS+tpkMqmoqCiVkZFht/zUqVPVpEmTbJYlJiaqX/ziF81+RsM9nLZs2dKqOrXlvmzW+q3YoWIWbFB/P3De4fcKIVqntd9Rh1pINTU17N27l+TkZOsyDw8PkpOTycrKsvuerKwsm/JgmWS9ufI1NTWsXLmSoKAg4uPj7Zaprq6mpKTE5tFW1sGRcoGtEE7nUCAVFRVhMpkIDw+3WR4eHo7BYLD7HoPB0KryGzZswN/fH29vb9566y02b95M9+7d7W4zIyODoKAg6yM6OtqR3bAhl48I4Tpc5izb+PHjOXDgADt27GDixIlMnTq12X6phQsXYjQarY+zZ8+2+XNlcKQQrsOhQOrevTuenp7k5+fbLM/Pz2926sqIiIhWlffz86N///6MGTOGP/3pT3h5efGnP/3J7jb1ej2BgYE2j7ZqOjhSCOFcDgWSTqdj1KhRZGZmWpeZzWYyMzNJSkqy+56kpCSb8gCbN29utnzT7VZXVztSvTaxnvaXOZGEcDqHp7BNS0tj9uzZjB49moSEBJYuXUp5eTlz5swBYNasWfTs2ZOMjAwAnnnmGcaNG8ebb77JpEmTWLNmDXv27GHlypUAlJeX85//+Z/cf//9REZGUlRUxPLlyzl//jwPP/xwO+6qfdY5kaSFJITTORxI06ZNo7CwkMWLF2MwGBgxYgSbNm2ydlzn5ubi4dHY8Bo7diyrV6/mlVde4eWXX2bAgAF8/vnnDB06FLDM53z06FE+/PBDioqKCA0N5ZZbbuFf//pXp9xL3d0n+hfClWiUuvGvKi0pKSEoKAij0ehwf9KP+aVMeOtbQny17F88oYNqKIR7a+131GXOsjlLQwvJWFmL2XzDZ7MQNzS3D6Sg+k5ts4LSqjon10YI9+b2gaT38sRXZ7knmXRsC+Fcbh9I0PQCW+nYFsKZJJCAIB+5YaQQrkACCQjxk9HaQrgCCSTkejYhXIUEEnI7JCFchQQSMieSEK5CAgmZE0kIVyGBRJM+JDntL4RTSSDR9AJbOWQTwpkkkGg87S/jkIRwLgkk5LS/EK5CAonGQ7bSqjrqTGYn10YI9yWBROOlI2CZhkQI4RwSSICXpwcB3pbJM+XUvxDOI4FUzzo4Um6pLYTTSCDVsw6OLJcWkhDOIoFUTwZHCuF8Ekj1QuSGkUI4nQRSvWCZpE0Ip5NAqieDI4VwPgmkesG+csNIIZxNAqle40T/csgmhLNIINULktP+QjidBFK9xoGREkhCOIsEUj05yyaE80kg1WtoIVXUmKiuMzm5NkK4JwmkegHeXnhoLM+NcqZNCKeQQKrn4aFpcgdbCSQhnEECqQnrqX/pRxLCKSSQmgiS2yEJ4VQSSE3InEhCOJcEUhNyS20hnEsCqYmGC2xlLJIQziGB1ETDnEhy2l8I55BAasI6ja20kIRwCgmkJmROJCGcSwKpCZkTSQjnalMgLV++nNjYWLy9vUlMTGTXrl0tll+3bh1xcXF4e3szbNgwNm7caF1XW1vLggULGDZsGH5+fkRFRTFr1izy8vLaUrXrInMiCeFcDgfS2rVrSUtLIz09nX379hEfH09KSgoFBQV2y+/YsYPp06czd+5c9u/fT2pqKqmpqRw+fBiAiooK9u3bx6JFi9i3bx+fffYZx44d4/7777++PWuDppeOKKU6/fOFcHca5eA3LzExkVtuuYVly5YBYDabiY6O5umnn+all166qvy0adMoLy9nw4YN1mVjxoxhxIgRrFixwu5n7N69m4SEBM6cOUPv3r2vWl9dXU11dbX1dUlJCdHR0RiNRgIDAx3ZHRtl1XUMTf8KgJzfpOCr82rztoQQjUpKSggKCrrmd9ShFlJNTQ179+4lOTm5cQMeHiQnJ5OVlWX3PVlZWTblAVJSUpotD2A0GtFoNAQHB9tdn5GRQVBQkPURHR3tyG40y0/niVf9Jf/SjyRE53MokIqKijCZTISHh9ssDw8Px2Aw2H2PwWBwqHxVVRULFixg+vTpzSbpwoULMRqN1sfZs2cd2Y1maTQaGRwphBO51DFJbW0tU6dORSnFe++912w5vV6PXq/vkDqE+GopKquWwZFCOIFDgdS9e3c8PT3Jz8+3WZ6fn09ERITd90RERLSqfEMYnTlzhm+++ea6+oKuR7Bc8S+E0zh0yKbT6Rg1ahSZmZnWZWazmczMTJKSkuy+JykpyaY8wObNm23KN4TR8ePH2bJlC6GhoY5Uq10Fy6l/IZzG4UO2tLQ0Zs+ezejRo0lISGDp0qWUl5czZ84cAGbNmkXPnj3JyMgA4JlnnmHcuHG8+eabTJo0iTVr1rBnzx5WrlwJWMLooYceYt++fWzYsAGTyWTtX+rWrRs6na699rVVGq74l05tITqfw4E0bdo0CgsLWbx4MQaDgREjRrBp0yZrx3Vubi4eHo0Nr7Fjx7J69WpeeeUVXn75ZQYMGMDnn3/O0KFDATh//jxffPEFACNGjLD5rK1bt3LnnXe2cdfaJsRPZo0UwlkcHofkilo7xqE1lm89wRtfHeOhUb1Y8nB8O9VQCPfWIeOQ3IHMqy2E80ggXSFELrAVwmkkkK4QJHMiCeE0EkhXaJzoX1pIQnQ29wskpaDM/swEYDsnUhfo7xfihuJegVRWAG/0h7eGQJ39Q7KGFlKdWVFWXdeZtRPC7blXIPn1AHMtmGqg8IjdIt5aT/Rell+LdGwL0bncK5A0GogcYXmet7/ZYiFyxb8QTuFegQQQNdLyM+9As0Vkbm0hnMMNA2mE5WcLLSS5HZIQzuGGgVTfQsr/Aeqq7RaRU/9COIf7BVJwDHgHWzq3C3LsF2loIZVLIAnRmdwvkDSaa/YjyZxIQjiH+wUSXLMfSeZEEsI53DSQ6ltIFw7YXS2n/YVwDvcMpIaxSPk5dju2g+S0vxBO4Z6BFNwbfLpZOrbzf7hqtcyJJIRzuGcgaTQt9iNZ50SS0/5CdCr3DCRosR+p4ZDNWFmLySxX/AvRWdw3kFq4pi3Yx3LIphSUVkkrSYjO4r6B1NBCKjgCtVU2q3ReHvjpPAG5YaQQncl9AymoF/iGgrnObsd2sHRsC9Hp3DeQmo7YvmDnsE1O/QvR6dw3kKDFfiQZHClE53PvQLJe03bwqlUyOFKIzufmgTTC8rMgB2orbVY13p9NWkhCdBb3DqTAnpZ5tpXpqo5t62htGRwpRKdx70BqYY7tIJ+GWSMlkIToLO4dSNDs3EhyPZsQnU8CqZlr2uS0vxCdTwKpoYVUeBRqKqyLg+W0vxCdTgIpIBL8wuo7tg9bFze0kIzSQhKi00ggNTPHdkMfUml1HbUmsxMqJoT7kUACu/1IDWfZQG6HJERnkUACu3MjeXpoCPT2AuRMmxCdRQIJGsciFR6FmnLr4hC/hlP/0kISojNIIAEERoJ/BCgzGJp0bMvgSCE6lQRSAzv9SHLqX4jOJYHUwE4/kpz6F6JztSmQli9fTmxsLN7e3iQmJrJr164Wy69bt464uDi8vb0ZNmwYGzdutFn/2WefMWHCBEJDQ9FoNBw4cKAt1bo+dq5pkzmRhOhcDgfS2rVrSUtLIz09nX379hEfH09KSgoFBQV2y+/YsYPp06czd+5c9u/fT2pqKqmpqRw+3NhXU15ezm233cbrr7/e9j25Xg2HbEU/QnUZ0HjqX674F6KTKAclJCSoJ5980vraZDKpqKgolZGRYbf81KlT1aRJk2yWJSYmql/84hdXlT116pQC1P79+x2qk9FoVIAyGo0Ove8qSwYplR6o1OkdSimlPvjuJxWzYIN64q97rm+7Qri51n5HHWoh1dTUsHfvXpKTk63LPDw8SE5OJisry+57srKybMoDpKSkNFu+NaqrqykpKbF5tIsr+pHktL8QncuhQCoqKsJkMhEeHm6zPDw8HIPBYPc9BoPBofKtkZGRQVBQkPURHR3d5m3ZuKIfSeZEEqJz3ZBn2RYuXIjRaLQ+zp492z4bvuKatoZObaN0agvRKbwcKdy9e3c8PT3Jz8+3WZ6fn09ERITd90RERDhUvjX0ej16vb7N72+WTcd2qfW0v7SQhOgcDrWQdDodo0aNIjMz07rMbDaTmZlJUlKS3fckJSXZlAfYvHlzs+Wdyj/MMs82CgzZ1oGRlbUmqmpNzq2bEG7AoRYSQFpaGrNnz2b06NEkJCSwdOlSysvLmTNnDgCzZs2iZ8+eZGRkAPDMM88wbtw43nzzTSZNmsSaNWvYs2cPK1eutG7z0qVL5ObmkpeXB8CxY8cAS+vqelpSbRI5AkrOQ95+AqKT8NCAWVmu+PfWenZuXYRwMw73IU2bNo0lS5awePFiRowYwYEDB9i0aZO14zo3N5cLFy5Yy48dO5bVq1ezcuVK4uPj+fTTT/n8888ZOnSotcwXX3zByJEjmTRpEgCPPPIII0eOZMWKFde7f45r0o/k4aGRy0eE6EQapZRydiWuV0lJCUFBQRiNRgIDA69vY8c3w0cPQegAeHoPd725jZ8Ky1kzfwxj+oa2T4WFcDOt/Y7ekGfZOlTDqf+LJ6CqxHrFv8yJJETHk0C6kn8PCOyFpWP7UJPbIcmZNiE6mgSSPdapSA4QVH/q31BS5bz6COEmJJDsaTI30sDwAAD+a9tJtuTkN/8eIcR1k0Cyp8k1bXNujWXCTeHU1Jl5/K97+cfBPOfWTYguTALJnsj6QLp4An1dGctn3MwDI6KoMyueWbOfT/a006UqQggbEkj2+IVCUG/L8wuH0Hp68IepI5ieEI1ZwYufHmLV9lPOraMQXZAEUnOi4i0/66/89/TQ8NqDw5h7Wx8AXv1HDsu3nnBW7YTokiSQmmNnjm2NRsMrkwbzy7sHAPDGV8d446ujdIGxpUK4BAmk5tiZYxssoZR2z0AW3hsHwPKtJ/n1P3IwmyWUhLheEkjNaWghXfoJKouvWv2Lcf347QNDAFi14zQvfXYIk4SSENdFAqk5vt0guKFj+6DdIjOTYlnycDweGvhkzzmeWbOfWpO5EyspRNcigdQSO/1IV3poVC+W/dvNaD01bDh0gSf+ulfmThKijSSQWtJMP9KVfjYskpWzRqP38mDLkQLmfribipq6jq+fEF2MBFJLrphjuyXjB4Wxak4CfjpPtp+4yD1/+JaX12ezMfsCl8tlpgAhWkPmQ2pJxSX4vWXcEQtOg0/INd+yP/cy/3fVbpt5uDUaGBIVyK39u3Nb/+6MjumGj05mnxTuo7XfUYensHUrvt0gJBYun4Yj/4Comy3povEANHaeaxjpr2H7L/pz8HQ+h05f4MfcfIouX8bnQjUFF6rZ9F012z1q6BesoV+wB9H+ilBdHR6eWhj2MMTeZtmWEG5IAulaIkdYAumLp1v9Fl8gqf4BgM5OodL6R1P7PqSqWxy6sU/gMXwq6Hwdr68QNzA5ZLuWn7bBhn+HmgpQZkCBUlc8V/XPzY3rPHWWQNH61v/0A50vSutLmVnHhUpPzpTASaOZ4lot0ZpCHvT8Dl9NNQDlHgGcjP45PmN/Qb8BN+HhIa0mceNq7XdUAsnJTGbFD3lGtp+4SPbJM8Se+YxH2ERvj0LLeqXhn5rR7I+cRshNdzOmX3fiIgIkoMQNRQLpBlVrMnP47CUu7PmC3if+h6FV+6zrjpqj+dA0gW+044nvG8mYvqGM6Rt6YwZUlRG2vw05f4ekp2D0HGfXSHQgCaQuotaQQ/G25QT/+Clas2Ua3WLlx1rTnXxkSiZXhRPsqyUhthtj+oaS1C+UQeEuHFB1NbDnz/DP16HyUuPyMf8PJvwHeMjZx65IAqmrqSyGAx+hdq1Ec/m0dfE+NZBP625ngymREvwBCPbVktinm7UF5RIBpRT8sB4yf205SQCWW031HQe7/2h5PXAiTPkj6AOcVk3RMSSQuiqzCY5/Dbveh5+21neug0mjZa/3GFaVJ/F1zVDqmpxADfHVktgnlNGxIQyKCGBAWADhgXo0nTW84PR38PUiyKs//PQLg/ELYeQs8PSCw5/B509AXRWED4V/WwtBvTqnbterygjn91pOegy4B7z0zq6RS5JAcgclFyB7HRxcAwU/WBfXenfjSOg9/K3udtZd6E5FzdUX/Abovegf7s+AMH8GhAVYn0cF+bRfa6rgCGx5FX7cZHmt9YNbn4GkJ0Hvb1v23B74eDqUF4B/OEz/GHqOap96tBezGYp+hHO74NxuOLsbCo8C9V8h/3AY8wSM/r/gHeTUqroaCSR3Y8i2BNOhTyxf6nqq+yDyYh5gi9c4sop8OF5QyumLFXiaawiijBBNGSGUEawpJURTRphnObF+1fTUV9LDqxptYA+8e/QhKLIfutA+lhkQ/MNaHrxZkgdbX4MDH1lacBpPS6f1uAWW9zanOBdWP2IJVy8feHAFDEltv9+RoyouWVo/53bD2V2W59UlV5cLjgFTDZTW30JeH2jZ3zH/DwIiOrfOLkoCyV2Z6iyHcgfXwNENlsMgADTQYxDUlKMqLqGpLW/zR9RqdJT7RGEOjEbbvQ++YX3w7BYDQdHw41eQtRzqKi2FB0+Gu9Oh+4DWbbyqBP4213JYCnD3YrgtrXNGrytlaakd+Cuc3g4Xj19dRutrabn1Gg29Eiw//cMsnfWHP7WcOSw8ainrqYPh0yytwtbufxclgSQsX+6cv1vC6cx3V6/XeFiuz/PpBj4hmH1CKPMM5KLJH0OtD+cqtdSVFOBbcZ4IVUBPTRGRXMRT04o/megxcM9voHei4/U21cHXv4LvV1hex/8bTH4bvOwNeW8H1aWWluWeDyA/23Zdt34QndAYQGE3Wfq9mmM2w/Gv4LulcHZn/UINxE2CW5+F6Fs6Zh9cnASSsFWcC4U/gk+wJYR8u4E+CDyuPeGDUoqishpOXyznTH4xhXmnqCr4CXNxLvryc0SqAnppiojWFHBRBfJO3c8xRt/DlNG9+NmwSAK8tW2r86734X8XgDJBzK0w7a+WereXCwctQxAOrYOGFqOXNwz5Odz0APS6xXIHmrbK3WlpMR3b2Lgs5lZLMA24x62uWZRAEp3CbFbkGSs5XVTBycIyMo8W8N3xQhpm8/XWenDv0EgeGtWLpL6hjneYn9gC6+ZY+m669YV/++T6Dn9qKuCHzyxBdH5v4/LuA2HUHIh/pH1DD6DgKOx4Fw6tBXP9LBBhQ6D3GEvn91WPYNvXHdUy7EQSSMJpDMYq1u8/z6d7z3KysLGvqmewDz+/uSdTbu5FbHe/1m+w4AisngrFuZj1QZjvSscrpJel87jpF1fn13yro+CI5ZDs4BqoNlqWeWjhpvstZ8Vibu34FovxPOz8L9i7CmrKWv8+Lx/L/gX1sozb6neX5fDRFYLKVGc5cXGNukggCadTSnHgbDGf7j3HFwfzKK1qnEXzltgQHhplOaTz8vCgoLSKwtJqy6OsmoKSJs9Lq6grKeC16gxu9rDT0dxA4wneV4SUPhDKC+Hs943lQmJh1KMw4v+Afw+bTdSZzOw9c5lakyLUX0eov45uvjq8PNtxLsPKy/DD51BqsIxjau7REJz2aP0sU9X0G28JqO4DOz5Qywog/4fGR8EPltbfA8tg+NQW3yqBJFxKVa2JzTn5fLr3HP9qckin0dRPltAKemr4pddnDNOcIlBTTiAVhHpVEqAq8FC1Lb9Z4wmD7rW0hvqOt+k7M5sVu09f4h+H8vjfbAMX7czwGeyrJdRPR6ifvjGo/PR099fRzU9HrxBfhvUMwrM9R8SbTZYO96piy0j9ghw4+Y1lBoryQtuyAVGWYOo3HvreCX7d2/65tZWWM4X5OfXhc9jy2Vd+ZoPb0iA5vcVNSiAJl5VfYjmkW7en8ZDOW+tBWIA3PQL09PDXExZo+dkjoOG5ZV2In5adP13irzvPkHkkvz7YFD39YEZ8EA/eFECkvqZJS6PYknhxkyAwyloHpRQHzxn5x8E8vjx0AUNJlXVdNz8d3f11XCyr4XJFDa29u1V3fz0ThoQzcUgESf1C0bZnq6ops9kSEj9thZNb4cwOMFXblokYDn3usBzGmmotfVemOst4qYbn5tr6dXWNZYrPwqWT1isAbGks/XjhQ2wfwbHXPDkigSRcnlIKQ0kV/nov/PVeDl/KcsFYyce7zrJmVy4FpZYvpEZjmd98RmJv7hwUZtNiUUpx1FDKhkN5/OPgBXIvVVjXBXh7MXFIBJPjoxjbL9R6iGYyK4orarhYXsPFshoullfX/6zhYpnl+aXyGo4YSmwOSQO9vUgeHM7EoRHcMbAH3toOvGi4thJysyytp5Pbrh660BY+3eoDZyiE32R53iPOEnBtIIEk3EatyUzmkXz+ujOX704UWZf3DPZhekI0tw/owT9/LOSLg3mcKGjsTPbRenLPTeFMjo/ijoHd0Xu1PTRq6sxk/XSRTYcNbM4xUFTWeNjno/VkfFwPUoZEcFdcWNuHQbRWWYHlsO7cbktLx0NrGTvloQVPrZ3XXo3L/XtYQsg/vF37pCSQhFv6qbCMj3flsm7vOYorru5X0nl6cOegHkyOj+LuwWH46tp/FmeTWbH3zGX+9/AFvjpsIM/YeDio8/Tg1v6hTBwawaiYEHp380Pn1fVv/iOBJNxaVa2JjdkX+OvOMxzOKyGpbyiT46OYMCScwI5uoTShlCL7vJFNhw1sOmzgpyLbS3Y8PTTEdPOlbw8/+vXwtzzCLM+DfV3gtH47kUASwsUopTheUMamwwa+OVrA8fxSymuav8txqJ+Ofj38rWEV1mTKmOa+tk0XazTQw19PZLAPkUHeHduPdQ0SSEK4OKUU+SXVnCws46fCMk4WlnOysIyTBWU2h3ntpZufjohAb6KCvYkM8iEiqPF5VJAP4UH66+pHa0mHBtLy5ct54403MBgMxMfH8+6775KQkNBs+XXr1rFo0SJOnz7NgAEDeP311/nZz35mXa+UIj09nffff5/i4mJuvfVW3nvvPQYMaN0lAhJIoqspr67jVFF9QBWWc7KgjEvlNVf1M1/1msYFdWYzBaXVXCiuorK2+ZZYU4HeXgR4a/HXexHg7YW/t1f9c63ldcPyJsv69vAjMsinxe122I0i165dS1paGitWrCAxMZGlS5eSkpLCsWPHCAu7eq6bHTt2MH36dDIyMrjvvvtYvXo1qamp7Nu3j6FDhwLw+9//nnfeeYcPP/yQPn36sGjRIlJSUsjJycHb29vRKgpxw/PTezG0ZxBDe17/RG9KKYyVtVwwVnHBWElecRUGYxV5xkouFFuWXTBWUV1npqSqjpImwxdaY8HEOJ64s9911xPa0EJKTEzklltuYdmyZQCYzWaio6N5+umneemll64qP23aNMrLy9mwYYN12ZgxYxgxYgQrVqxAKUVUVBTPPfcczz//PABGo5Hw8HBWrVrFI488cs06SQtJiOujlOJyRS2XK2ooraqjrKqOsupaSqzP6yitqq3/Wdf4s6qOJ+7sR+rIni1uv0NaSDU1Nezdu5eFCxdal3l4eJCcnExWVpbd92RlZZGWlmazLCUlhc8//xyAU6dOYTAYSE5Otq4PCgoiMTGRrKwsu4FUXV1NdXXjyNSSEjuz+AkhWk2j0dDNz3IZjDM5NACiqKgIk8lEeHi4zfLw8HAMBoPd9xgMhhbLN/x0ZJsZGRkEBQVZH9HR0Y7shhDCRd2QI7IWLlyI0Wi0Ps6ePevsKgkh2oFDgdS9e3c8PT3Jz8+3WZ6fn09EhP3JzCMiIlos3/DTkW3q9XoCAwNtHkKIG59DgaTT6Rg1ahSZmZnWZWazmczMTJKSkuy+JykpyaY8wObNm63l+/TpQ0REhE2ZkpISvv/++2a3KYToopSD1qxZo/R6vVq1apXKyclR8+fPV8HBwcpgMCillJo5c6Z66aWXrOW3b9+uvLy81JIlS9SRI0dUenq60mq1Kjs721rmd7/7nQoODlZ///vf1aFDh9QDDzyg+vTpoyorK1tVJ6PRqABlNBod3R0hRCdo7XfU4UBSSql3331X9e7dW+l0OpWQkKB27txpXTdu3Dg1e/Zsm/KffPKJGjhwoNLpdGrIkCHqyy+/tFlvNpvVokWLVHh4uNLr9eruu+9Wx44da3V9JJCEcG2t/Y7KpSNCiA7XYSO1XVFDpsp4JCFcU8N381rtny4RSKWlpQAyHkkIF1daWkpQUPOXw3SJQzaz2UxeXh4BAQHXnAa1pKSE6Ohozp4961aHd7Lf7rXf4Fr7rpSitLSUqKgoPFqYf7tLtJA8PDzo1auXQ+9x1/FLst/ux1X2vaWWUYMbcqS2EKJrkkASQrgMtwskvV5Peno6er3e2VXpVLLf7rXfcGPue5fo1BZCdA1u10ISQrguCSQhhMuQQBJCuAwJJCGEy5BAEkK4DLcLpOXLlxMbG4u3tzeJiYns2rXL2VXqUK+++ioajcbmERcX5+xqtbtvv/2WyZMnExUVhUajsd5EooFSisWLFxMZGYmPjw/JyckcP37cOZVtR9fa70cfffSqf/+JEyc6p7Kt4FaB1HBPufT0dPbt20d8fDwpKSkUFBQ4u2odasiQIVy4cMH6+O6775xdpXZXXl5OfHw8y5cvt7u+4d5/K1as4Pvvv8fPz4+UlBSqqtr/DrGd6Vr7DTBx4kSbf/+PP/64E2vooA6dlcnFJCQkqCeffNL62mQyqaioKJWRkeHEWnWs9PR0FR8f7+xqdCpArV+/3vrabDariIgI9cYbb1iXFRcXK71erz7++GMn1LBjXLnfSik1e/Zs9cADDzilPm3hNi2khnvKNb3/27XuKddVHD9+nKioKPr27cuMGTPIzc11dpU61bXu/dfVbdu2jbCwMAYNGsQTTzzBxYsXnV2lZrlNILXlnnJdQWJiIqtWrWLTpk289957nDp1ittvv906h5Q7aMu9/7qKiRMn8pe//IXMzExef/11/vnPf3LvvfdiMpmcXTW7usT0I6J59957r/X58OHDSUxMJCYmhk8++YS5c+c6sWaiMzS98/OwYcMYPnw4/fr1Y9u2bdx9991OrJl9btNCass95bqi4OBgBg4cyIkTJ5xdlU7Tlnv/dVV9+/ale/fuLvvv7zaB1JZ7ynVFZWVlnDx5ksjISGdXpdPIvf8anTt3josXL7rsv79bHbKlpaUxe/ZsRo8eTUJCAkuXLqW8vJw5c+Y4u2od5vnnn2fy5MnExMSQl5dHeno6np6eTJ8+3dlVa1dlZWU2/+ufOnWKAwcO0K1bN3r37s2zzz7Lf/zHfzBgwAD69OnDokWLiIqKIjU11XmVbgct7Xe3bt349a9/zZQpU4iIiODkyZO8+OKL9O/fn5SUFCfWugXOPs3X2Vq6p1xXNG3aNBUZGal0Op3q2bOnmjZtmjpx4oSzq9Xutm7dqoCrHg33CLzee/+5qpb2u6KiQk2YMEH16NFDabVaFRMTo+bNm2e9qasrkvmQhBAuw236kIQQrk8CSQjhMiSQhBAuQwJJCOEyJJCEEC5DAkkI4TIkkIQQLkMCSQjhMiSQhBAuQwJJCOEyJJCEEC7j/wPOvNkGgqR1ygAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T16:12:57.011228Z", + "iopub.status.busy": "2024-03-26T16:12:57.010431Z", + "iopub.status.idle": "2024-03-26T16:14:07.425494Z", + "shell.execute_reply": "2024-03-26T16:14:07.424662Z" + }, + "papermill": { + "duration": 70.436218, + "end_time": "2024-03-26T16:14:07.428007", + "exception": false, + "start_time": "2024-03-26T16:12:56.991789", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T16:14:07.466107Z", + "iopub.status.busy": "2024-03-26T16:14:07.465789Z", + "iopub.status.idle": "2024-03-26T16:14:07.486809Z", + "shell.execute_reply": "2024-03-26T16:14:07.485894Z" + }, + "papermill": { + "duration": 0.042464, + "end_time": "2024-03-26T16:14:07.488829", + "exception": false, + "start_time": "2024-03-26T16:14:07.446365", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
lct_gan0.0014910.0100010.0007893.7191520.015320.5316650.0476358.764879e-071.2892180.0183560.5159440.0280830.1561140.0019995.00837
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "lct_gan 0.001491 0.010001 0.000789 3.719152 0.01532 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "lct_gan 0.531665 0.047635 8.764879e-07 1.289218 0.018356 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "lct_gan 0.515944 0.028083 0.156114 0.001999 5.00837 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T16:14:07.524962Z", + "iopub.status.busy": "2024-03-26T16:14:07.524666Z", + "iopub.status.idle": "2024-03-26T16:14:07.979077Z", + "shell.execute_reply": "2024-03-26T16:14:07.978106Z" + }, + "papermill": { + "duration": 0.47451, + "end_time": "2024-03-26T16:14:07.981174", + "exception": false, + "start_time": "2024-03-26T16:14:07.506664", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T16:14:08.020165Z", + "iopub.status.busy": "2024-03-26T16:14:08.019765Z", + "iopub.status.idle": "2024-03-26T16:15:22.391668Z", + "shell.execute_reply": "2024-03-26T16:15:22.390855Z" + }, + "papermill": { + "duration": 74.394834, + "end_time": "2024-03-26T16:15:22.394155", + "exception": false, + "start_time": "2024-03-26T16:14:07.999321", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/lct_gan/all inf False\n", + "Caching in ../../../../insurance/_cache_bs_test/lct_gan/all inf False\n", + "Caching in ../../../../insurance/_cache_synth_test/lct_gan/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T16:15:22.433392Z", + "iopub.status.busy": "2024-03-26T16:15:22.433072Z", + "iopub.status.idle": "2024-03-26T16:15:22.452838Z", + "shell.execute_reply": "2024-03-26T16:15:22.452120Z" + }, + "papermill": { + "duration": 0.041998, + "end_time": "2024-03-26T16:15:22.454820", + "exception": false, + "start_time": "2024-03-26T16:15:22.412822", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T16:15:22.490669Z", + "iopub.status.busy": "2024-03-26T16:15:22.490377Z", + "iopub.status.idle": "2024-03-26T16:15:22.495611Z", + "shell.execute_reply": "2024-03-26T16:15:22.494812Z" + }, + "papermill": { + "duration": 0.025648, + "end_time": "2024-03-26T16:15:22.497503", + "exception": false, + "start_time": "2024-03-26T16:15:22.471855", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'lct_gan': 0.037697755648017243}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T16:15:22.534825Z", + "iopub.status.busy": "2024-03-26T16:15:22.534103Z", + "iopub.status.idle": "2024-03-26T16:15:22.948433Z", + "shell.execute_reply": "2024-03-26T16:15:22.947449Z" + }, + "papermill": { + "duration": 0.435297, + "end_time": "2024-03-26T16:15:22.950481", + "exception": false, + "start_time": "2024-03-26T16:15:22.515184", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+uUlEQVR4nO3deXxU5b348c+ZOTOTSTIzSUjIAgkJq8omshVRAaUuWJXa2lZpBXGvVi3lvpR7f8WltVjrgl0ut+2tRG/de1HbenEpilaruLIIyiZLWEIC2ZPZz/P7Y5IhQzJZJstJwvf9es2LzJlnznxPSL55lvM8j6aUUgghhIksZgcghBCSiIQQppNEJIQwnSQiIYTpJBEJIUwniUgIYTpJREII00kiEkKYThKREMJ0kohEm4qLi9E0jb1795odihjAJBGJbvf000+zcuVKs8MQ/YgkItHtJBGJzpJEJIQwnSQi0Wlr165l1qxZuFwu3G43U6dO5emnnwZg9uzZvPLKK+zbtw9N09A0jcLCwg6f2zAM7rnnHvLy8khOTmbOnDls27aNwsJCFi1aFC1XUVHB0qVLGT9+PKmpqbjdbi666CI2bdoUc77169ejaRrPP/88999/P0OHDiUpKYnzzjuPXbt2dce3Q3QD3ewARP9SXFzM4sWLGTt2LMuWLSMtLY3PPvuMV199lauuuor/+I//oLq6mgMHDvDoo48CkJqa2uHzL1u2jAcffJBLLrmECy64gE2bNnHBBRfg8/liyn311Ve89NJLXHHFFRQVFXHkyBF+//vfM2vWLLZt20ZeXl5M+QceeACLxcLSpUuprq7mwQcfZMGCBWzYsKHr3xTRdUqINqxevVoBas+ePaqqqkq5XC41ffp05fV6Y8oZhhH9+uKLL1bDhg3r9GeVlpYqXdfV/PnzY47fc889ClALFy6MHvP5fCocDseU27Nnj3I4HOq+++6LHnvrrbcUoE499VTl9/ujxx977DEFqC1btnQ6TtH9pGkmOuyNN96gtraWu+66i6SkpJjXNE3r8vnXrVtHKBTihz/8YczxH/3oRy3KOhwOLJbIj284HObYsWOkpqYyZswYPv300xblr7nmGux2e/T52WefDURqVsJ8kohEh+3evRuAcePG9cj59+3bB8DIkSNjjmdkZJCenh5zzDAMHn30UUaNGoXD4SAzM5OsrCw2b95MdXV1i3MXFBTEPG86X2VlZXdegkiQJCLRL/3iF79gyZIlnHPOOfz5z3/mtdde44033mDs2LEYhtGivNVqbfU8SlZK7hOks1p02IgRIwD4/PPPW9Ramku0mTZs2DAAdu3aRVFRUfT4sWPHWtRc/vKXvzBnzhz+9Kc/xRyvqqoiMzMzoc8X5pEakeiw888/H5fLxYoVK1qMYjWvWaSkpLTaPGrPeeedh67rrFq1Kub4b3/72xZlrVZri9rMCy+8wMGDBzv9ucJ8UiMSHeZ2u3n00Ue57rrrmDp1KldddRXp6els2rSJhoYGnnjiCQAmT57Mc889x5IlS5g6dSqpqalccskl7Z4/Ozub22+/nYcffphLL72UCy+8kE2bNrF27VoyMzNjalrf+MY3uO+++7jmmms488wz2bJlC0899RTDhw/vsesXPcjkUTvRxzUfvm/y17/+VZ155pnK6XQqt9utpk2bpp555pno63V1deqqq65SaWlpCujUUH4oFFI//elPVU5OjnI6nercc89VX3zxhRo0aJC66aabouV8Pp/6yU9+onJzc5XT6VQzZ85U77//vpo1a5aaNWtWtFzT8P0LL7wQ8zl79uxRgFq9enVnvyWiB2hKSW+d6NuqqqpIT0/n5z//Of/xH/9hdjiiB0gfkehTvF5vi2NNE2hnz57du8GIXiN9RKJXlJeXEw6H475ut9vJyMjgueeeo7i4mHnz5pGamsq7777LM888w/nnn8/MmTN7MWLRmyQRiV4xderU6A2LrZk1axbr169nwoQJ6LrOgw8+SE1NTbQD++c//3kvRit6m/QRiV7x3nvvtdrsapKens7kyZN7MSLRl0giEkKYTjqrhRCm69d9RIZhcOjQIVwuV7fM/hZCdC+lFLW1teTl5UVXS4hX0FQHDhxQCxYsUBkZGSopKUmNGzdOffTRRx16b0lJiQLkIQ959PFHSUlJm7/LptaIKisrmTlzJnPmzGHt2rVkZWWxc+fOFks+xONyuQAoKSnB7Xb3ZKhCiATU1NSQn58f/V2Nx9RE9Mtf/pL8/HxWr14dPdZ81nV7mppjbrdbEpEQfVh7XSemdlb/9a9/ZcqUKVxxxRUMHjyYSZMm8cc//jFueb/fT01NTcxDCNH/mZqIvvrqK1atWsWoUaN47bXXuPnmm7ntttuis7hPtGLFCjweT/SRn5/fyxELIXqCqfcR2e12pkyZwr/+9a/osdtuu42PPvqI999/v0V5v9+P3++PPm9qf1ZXV0vTTIg+qKamBo/H0+7vqKl9RLm5uZx22mkxx0499VT+93//t9XyDocDh8PRqc9QShEKhdqc5yRaZ7Va0XVdbo0QPc7URDRz5ky2b98ec2zHjh3RJUO7KhAIcPjwYRoaGrrlfCej5ORkcnNzY3bAEKK7mZqIfvzjH3PmmWfyi1/8gu985zt8+OGH/OEPf+APf/hDl89tGAZ79uzBarWSl5eH3W6Xv+ydoJQiEAhQXl7Onj17GDVqVNs3pAnRBaYmoqlTp/Liiy+ybNky7rvvPoqKili5ciULFizo8rkDgQCGYZCfn09ycnI3RHvycTqd2Gw29u3bRyAQaLGXmWjpWJ2fN78s47Q8N2PzPGaH02+YPsXjG9/4Bt/4xjd67PzyV7xr5PvXOe/uOsqBSi8HKr2MHJyKQ299GyMRS37KhOgmhqHYf+x4f2R5rb+N0qI502tEQgwUx+oDhIzI3TA3zx5Bkk1qQx0lNaKTXGFhYXRNaNE1lQ0BAIYn1ZFUf8jkaPoXqREJ0U1qfUHsoTpOPfwShJww7luQNdrssPoFqRENAIFAwOwQBJCWbGdy0iGcuuKro3Vs/LTl7ADRupMyEQVCRtxHKGx0uGywA2UTMXv2bG699VZuvfVWPB4PmZmZ/PSnP41usVxYWMjPfvYzrr76atxuNzfccAMA7777LmeffTZOp5P8/Hxuu+026uvro+ctKyvjkksuwel0UlRUxFNPPZVQfKJ1I7JS+Vqmj8EuB0dq/NSUftViW2zRupOyafa7t3bFfa0oM4X5k4ZEn//hnd0Ew63/MA1Nd3LFlOMTbx9/bw/eQOxUkh9/PbGq+RNPPMG1117Lhx9+yMcff8wNN9xAQUEB119/PQAPPfQQy5cv5+677wZg9+7dXHjhhfz85z/n8ccfp7y8PJrMmpZZWbRoEYcOHeKtt97CZrNx2223UVZWllB8Io66MmzWyN93W6gen9eLU+5ja9dJmYj6g/z8fB599FE0TWPMmDFs2bKFRx99NJqIzj33XH7yk59Ey1933XUsWLCAO+64A4BRo0bx61//mlmzZrFq1Sr279/P2rVr+fDDD5k6dSoAf/rTnzj11FN7/doGqgafnyRfNRZNw2bVCIYV3ppynMndM2VpIDspE9Etc0bGfc1ywiyQG84ZEbfsiTNGFs/s+KJu7fna174WMyVlxowZPPzww9HJu1OmTIkpv2nTJjZv3hzT3FJKRae67NixA13XY7bsOeWUU0hLS+u2mE9mSimeWL+F0w8e44yiTIIpuVBzCH/NMciRRNSekzIR2fWOd431VNmuSklJiXleV1fHjTfeyG233daibEFBATt27Oit0E5K/pCBPVgLgC05Da3ehqo5hK9BFu/riJMyEfUHGzZsiHn+wQcfMGrUKKzW1m+SO+OMM9i2bRsjR7Ze2zvllFMIhUJ88skn0abZ9u3bqaqq6ta4T1beQBhHqA6rRcPiTEN3WggCwYZqs0PrF07KUbP+YP/+/SxZsoTt27fzzDPP8Jvf/Ibbb789bvk777yTf/3rX9x6661s3LiRnTt38vLLL3PrrbcCMGbMGC688EJuvPFGNmzYwCeffMJ1112H0+nsrUsa0BqCYRzhOmxWDRxuLEmRxeLDvjqTI+sfJBH1UVdffTVer5dp06Zxyy23cPvtt0eH6VszYcIE3n77bXbs2MHZZ5/NpEmTWL58OXl5edEyq1evJi8vj1mzZnH55Zdzww03MHjw4N64nAHPGwhjC3uxWSxgT2FsYR5TC9M5JUOWnukIaZr1UTabjZUrV7Jq1aoWr+3du7fV90ydOpXXX3897jlzcnL4+9//HnPsBz/4QZfiFBG+YBib4UW3amBPwZHsBosFAvXtv1lIjUiI7uANhtHDPnSrBWxOcDTu4+WvNTewfkJqREJ0g/RkO8mpCpemgy2ZYwErVeV16NZ6hhlhsMhM/LZIIuqD1q9fb3YIopNGDk6FLBsEksCegi9op6zWT5LNwrCQD+wp7Z/kJCZNMyG6g1IQ9Ea+tjlJsumELXZCYQVBn7mx9QOSiIToBn5fHYbROMnZlkySzUrI4iBsKFRQdpFpjyQiIbrByx/tYsOeCioDGlisOHQLIYsDBQR8kojaI4lIiG6g/JFkY3FE+oJ0qwXDGtn1JOCXRNQeSURCdIdgbCICwBZZ/iPsl3uJ2iOJSIhuYDR2SFvtx6fMaLZIjSjk95oSU38iw/dCdFEobKCFIlsH6fbjm1CedVo+tpISrPZwvLeKRpKIhOiiYFhhUUEAdNvxROR0pkSmeYRlTfH2mNo0u+eee9A0LeZxyimnmBmSEJ0WCBlYjSBWDSw2x/EXrPbIvyHZaLE9pteIxo4dyz/+8Y/oc13vwZCUgnCw584fj9XWcjnHOJ588kl+/OMfc+jQIRyO4z/U8+fPx+Vy8T//8z89FaVIkMUChR4rHmUH6/H/s92VQSivw22tIcvE+PoD0xORruvk5OT0zoeFg/DPh3vns5o7+yeg2ztU9IorruC2227jr3/9K1dccQUQ2X3jlVdeaXNmvTCPK8nG5KEpYHeBfjwRldYpLLV+NFe9JKJ2mD5qtnPnTvLy8hg+fDgLFixg//79ccv6/X5qampiHgON0+nkqquuiu68AfDnP/+ZgoICZs+ebV5gom1N/UBWW/SQ1RHpLzKC0jRrj6k1ounTp1NcXMyYMWM4fPgw9957L2effTaff/45LperRfkVK1Zw7733Jv6BVlukdtLbmv1wdsT111/P1KlTOXjwIEOGDKG4uJhFixbFLKYv+o6wodCCvshf9WZNM92eRBBQ0kfULlMT0UUXXRT9esKECUyfPp1hw4bx/PPPc+2117Yov2zZMpYsWRJ9XlNTQ35+fotycWlah5tIZpo0aRITJ07kySef5Pzzz2fr1q288sorZocl4vjicA1Hth5gqL2OMROP/3wdT0S+SP+k/CGJy/Q+oubS0tIYPXo0u3a1vgGiw+GI6cAdyK677jpWrlzJwYMHmTt3bucSruhV/pCBVQUiW1E1qxHZ7El4ASMcBiPU6ZrxycT0PqLm6urq2L17N7m5uWaHYrqrrrqKAwcO8Mc//pHFixebHY5oQzDcOHxv0WI6q232JEAjbCgZwm+HqYlo6dKlvP322+zdu5d//etffPOb38RqtXLllVeaGVaf4PF4+Na3vkVqairz5883OxzRhkAwjNUIYNW04/cOAXbdSthiiyQiuamxTaY2zQ4cOMCVV17JsWPHyMrK4qyzzuKDDz4gK0sGOwEOHjzIggULTprmaH8VDAaxoVrUiIakO8kZkYMerIWQLI7WFlMT0bPPPmvmx/dZlZWVrF+/nvXr1/Of//mfZocj2hFumvBqia0R2awWbElOCNdL06wdfaqzWkRMmjSJyspKfvnLXzJmzBizwxHtCDUmIs3maDky1lRDkqZZmyQR9UHx9i0TfVN2soWUZBtJjthdcwMhg93lARw1dRSFfMjgfXySiITooslDkqHcDcmpLV7bXRFkUIOffL8PGbyPr08N3/cEpZTZIfRr8v3rgKZm1wk3y9qsGmEt8rc+FJKmWVsGbCKy2SJ/fxoaZL3grmj6/jV9P0VLTaszNr+ZEYgsbdOYnMJBSURtGbBNM6vVSlpaGmVlZQAkJyfLXK1OUErR0NBAWVkZaWlpWK2yU2k8f/lwN4XHjjEubQTJJ7x2PBHJqFlbBmwiAqLLizQlI9F5aWlpvbdMSz8UNhQqFCCswKontXhdaxw1C8vwfZsGdCLSNI3c3FwGDx5MMGjCgmj9nM1mk5pQO5qmd0DsetVNLI3zy8IBaZq1ZUAnoiZWq1V+oUSPaD7h1WpreQd809KxhtSI2nRSJCIhekq8Ca9NZo7Jw6bSsDmlf7ItkoiE6ILIwvlNE15bjiy6kp2gW0FJ10BbBuzwvRC9IRAysKrGGpG1lcnJTXPPzNi0oR+RGpEQXeCwWchO1nCFbK02zfZWBdGO1eMK2ckwIb7+QhKREF2Q63GSm5cMDSkxM++blNSE0Kt95OoNkojaIE0zIboq3Dgi1kqNSG8cNVOhQGTdatEqSURCdIFSzZaBbaVGpNsixwzDiKxbLVolTTMhuuDdneU4dh8h15NEfis1IqvNTggwVONysbKAfqukRiREFwQD/sia1NDqqJld1zE0XdatbockIiG6IByIzLy3WHWwtLx7X7dqhDUbhgLC0jSLRxKREF3QtEysVbe3uoGizWrBsEiNqD3SRyREF0RrRLaWE14BhqQ5yR6Rjc17VBJRGyQRCdEFTesMWVqZ8AqQZLNCcjIErXJ3dRukaSZEF0S3ErK3sfdcdJqH1IjikRqREF2QnazhTtKx2Z2tvu4Phdl/1E9SVQP5kojikkQkRBfMLHRDyBNpfrUibCi+KPeRVe9laDggWwrF0WeaZg888ACapnHHHXeYHYoQHdfG9A4A3WLB0CI3MYZklca4+kQi+uijj/j973/PhAkTzA5FiM5pam61Mr0DGrcUsjQuFysL6MdleiKqq6tjwYIF/PGPfyQ9Pd3scITosBpfkL9/tpdP91fGrRFpmoZmlUTUHtMT0S233MLFF1/M3Llz2y3r9/upqamJeQhhlkDIQAUDkXlkcWpEAJbGLYVCsm51XKZ2Vj/77LN8+umnfPTRRx0qv2LFCu69994ejkqIjgmGIwvnR5aJjZ+ImmpLhuz2GpdpNaKSkhJuv/12nnrqKZKSWr8r9UTLli2juro6+igpKenhKIWIr2m9akuchfObWPRI08yQpllcptWIPvnkE8rKyjjjjDOix8LhMO+88w6//e1v8fv9LbYAcjgcOBxt3DgmRC8KhAx0I4hubbtGdNboPBzKg8Mug/fxmJaIzjvvPLZs2RJz7JprruGUU07hzjvvlH3IRJ8XaGqatVMjynCngF0HZIpHPKYlIpfLxbhx42KOpaSkMGjQoBbHheiLIk2zIBYtzg4eTZoWQ5O5ZnHJndVCJMhpt6I7DJx2K+jxm2b7qoJYKxtwGXY8vRhff9KnEtH69evNDkGIDjslKxlyXZEnbdSIdlUESK70kq9LIorH9PuIhOi3mk9ibaOzWm8aNQsFZSePOCQRCZGopnlmVh0s8X+VLI0d2cowpJ8oDklEQiRo3eclfLKvkrKGtss17W0WVgoMSUStkUQkRIL8Ph+BsIFq665qwKZbIgvoG8jiaHFIIhIiQU2rM8ZbJraJbrVgaHrj3mZSI2qNJCIhEtS0cL41zsL5TWxWLbKThySiuPrU8L0Q/YlqnE1vbadGVDgohSGFg3H4j0nTLA5JREIkyGhMRLq97RpRikOH1GQwqqRGFIc0zYRIgGEojGCkdmNtJxE1Fmp8oySi1iSUiL766qvujkOIfiWsFIOTIdlubbdG5A2E2VMVoLTGJ02zOBJKRCNHjmTOnDn8+c9/xufzdXdMQvR5NquFWSM8TByaFr1PKJ6GQIjPD3s5UNkgTbM4EkpEn376KRMmTGDJkiXk5ORw44038uGHH3Z3bEL0bU1Lv7Y1857G4XuLTthQUiOKI6FEdPrpp/PYY49x6NAhHn/8cQ4fPsxZZ53FuHHjeOSRRygvL+/uOIXoe5qSShsz76FxJw/NhqFASY2oVV3qrNZ1ncsvv5wXXniBX/7yl+zatYulS5eSn5/P1VdfzeHDh7srTiH6lJKKBv75xQG+LK1pt0Zka7yhESAky8W2qkuJ6OOPP+aHP/whubm5PPLIIyxdupTdu3fzxhtvcOjQIS677LLuilOIPsUfChMM+AgZqt0akW45vreZrFvduoTuI3rkkUdYvXo127dvZ968eTz55JPMmzcPS+MM5KKiIoqLiyksLOzOWIXoM/yNqzNabe2szkjs3mahYBBZdb2lhBLRqlWrWLx4MYsWLSI3N7fVMoMHD+ZPf/pTl4IToq8KhtXxrYTaWK+6iSW6pZDUiFqTUCJ64403KCgoiNaAmiilKCkpoaCgALvdzsKFC7slSCH6mkAwjNUIYbXY297TrNHMMbmkWN04LUYvRNf/JNRHNGLECI4ePdrieEVFBUVFRV0OSoi+LhjwA6rdHTya5Ga4cSfZ0An1fHD9UEKJSMVZ7rKurq7DmyUK0Z+Fgl6ASKvA0oGGhezk0aZONc2WLFkCRDrfli9fTnJycvS1cDjMhg0bOP3007s1QCH6ohSrgc1uxeZIAq39jRP3VwWx1/hwa16S2y198ulUIvrss8+ASI1oy5Yt2O3H28Z2u52JEyeydOnS7o1QiD5oytAUOJIGSe4Olf+81Ev60XqKdIckolZ0KhG99dZbQGRH1sceewy3u2P/CUIMONHpHe13VANYbJFycmd16xIaNVu9enV3xyFE/xKd3tGxu4Ks0UQUBiMMFtlSvbkOJ6LLL7+c4uJi3G43l19+eZtl16xZ0+XAhOjLXt+8n+xDVRQlD6Uj7QJr493XkeViA2Bx9myA/UyHR808Hg9aY6ecx+Np89FRq1atYsKECbjdbtxuNzNmzGDt2rWdvwohell9fT0NgTDK2rFRYl23oTQLhiHrVremwzWi5s2x7mqaDR06lAceeIBRo0ahlOKJJ57gsssu47PPPmPs2LHd8hlC9ITjy8R2rGlms1oIy04ecSV0H5HX66Wh4fiucvv27WPlypW8/vrrnTrPJZdcwrx58xg1ahSjR4/m/vvvJzU1lQ8++CCRsIToFWFDRTurbR1ZJpbjS4GEZW+zViXUWX3ZZZdx+eWXc9NNN1FVVcW0adOw2+0cPXqURx55hJtvvrnT5wyHw7zwwgvU19czY8aMVsv4/X78/uNzdWpqahIJX4guCYQMdBVJJrq9Y309IwenEsrPJDlUJetWtyLhFRrPPvtsAP7yl7+Qk5PDvn37ePLJJ/n1r3/dqXNt2bKF1NRUHA4HN910Ey+++CKnnXZaq2VXrFgR0xeVn5+fSPhCdEmgaea9BpZ29jRrkpZsJ9OTSrJdl6ZZKxJKRA0NDbhcLgBef/11Lr/8ciwWC1/72tfYt29fp841ZswYNm7cyIYNG7j55ptZuHAh27Zta7XssmXLqK6ujj5KSkoSCV+ILvGHw1iNQOM8s47dRwQ0m+YhTbMTJbx4/ksvvURJSQmvvfYa559/PgBlZWWdvsnRbrczcuRIJk+ezIoVK5g4cSKPPfZYq2UdDkd0hK3pIUSvU+Cxh0myWdtdi6hJnT/EgZoQx+r9kohakVAiWr58OUuXLqWwsJDp06dH+3Ref/11Jk2a1KWADMOI6QcSoq8Z7E7izGEuxuZ5OnxDY2V9gI2HGjhQ6YWwzMA/UUKd1d/+9rc566yzOHz4MBMnToweP++88/jmN7/Z4fMsW7aMiy66iIKCAmpra3n66adZv349r732WiJhCdF7OnlnddO61bKTR+sS3nI6JyeHnJycmGPTpk3r1DnKysqii+x7PB4mTJjAa6+9xte//vVEwxKid3RwK6EmenQnD0lErUkoEdXX1/PAAw+wbt06ysrKMIzYVec6uhOsLCUr+qPNJZWovWUMSrExtIOd1TZLZG8zw1AyfN+KhBLRddddx9tvv80PfvADcnNzo1M/hDgZ1DU0oAVCuJ066B28oVHXCGs6YQUqFEB+Y2IllIjWrl3LK6+8wsyZM7s7HiH6vFDAhw2wWqwdW50R0C0WDK1pJ48Ath6Mrz9KaNQsPT2djIyM7o5FiH4hsl41WGyODq3OCI1TPJr2NgtJH9GJEkpEP/vZz1i+fHnMfDMhThbhoA9oTEQdpGkaM0fnMCbHhVXJ8P2JEmqaPfzww+zevZvs7GwKCwux2WIrmp9++mm3BCdEXxTyNy6cb+vcmkKFg9OhzA5KOqtPlFAimj9/fjeHIUT/YTTWiKydqBE1viHyrwzft5BQIrr77ru7Ow4h+o1kaxi71YLewSVAmhysDWOv9eO2BWTb6RMk1EcEUFVVxX//93+zbNkyKioqgEiT7ODBg90WnBB90bkj05g8LJ10d2qn3rdhXw27yuuol77VFhKqEW3evJm5c+fi8XjYu3cv119/PRkZGaxZs4b9+/fz5JNPdnecQvQdoUjTrKP3EDXRGm9+lFGzlhKqES1ZsoRFixaxc+fOmJ1d582bxzvvvNNtwQnRJzUlkg5uJdSkaScPIxSAOLsln6wSSkQfffQRN954Y4vjQ4YMobS0tMtBCdFX1fqCvLWthG2Hazo84bVJNBEpZHG0EySUiBwOR6vLtO7YsYOsrKwuByVEX+UNhmmor8cbCHU+ETVOkDVkBn4LCSWiSy+9lPvuu49gMJLVNU1j//793HnnnXzrW9/q1gCF6Ev8QQPd8KNbLZ3uI7LplsgC+komvp4ooUT08MMPU1dXR1ZWFl6vl1mzZjFy5EhcLhf3339/d8coRJ/hD4XRo8vEdjIRNa5JJFsKtZTQqJnH4+GNN97gvffeY9OmTdTV1XHGGWcwd+7c7o5PiD7FFzSwGn50W+cT0ZgcF2pIBi5VJ02zE3Q6ERmGQXFxMWvWrGHv3r1omkZRURE5OTkopWRJEDGg+UNhdOVHt2jQySke2e4kSHdBnU8S0Qk61TRTSnHppZdy3XXXcfDgQcaPH8/YsWPZt28fixYt6tQysUL0R/5AKLKVkMXS6c5qACxN0zxk4mtznaoRFRcX884777Bu3TrmzJkT89qbb77J/PnzefLJJ7n66qu7NUgh+gqbCmCxWrBZO980q/UFaag3SPIG8UiNKEanakTPPPMM//7v/94iCQGce+653HXXXTz11FPdFpwQfc3UoclMHpbO0EwPWKydeu+hKh8fH6jnQGWDNM1O0KlEtHnzZi688MK4r1900UVs2rSpy0EJ0Wc1Te/oZP8QRBZHMyx65IZGQ5pmzXUqEVVUVJCdnR339ezsbCorK7sclBB9VjCyFlFnm2Vw4vC91Iia61QiCofD6Hr8biWr1UooJJleDFxvbi1h66FqasOdv/OlaUsh2duspU59N5VSLFq0CIej9dEC2aFVDHSV1TU4fSGUtfM1It1iaba3mfzBbq5TiWjhwoXtlpERMzFQGYbCCETWErIndb6PyG61HO8jkhpRjE4lotWrV/dUHEL0ef6QgaUxgdiSUjr9/uhur4ZChWVvs+YSXqGxO6xYsYKpU6ficrkYPHgw8+fPZ/v27WaGJERcDYFQZMKrRcNq63zTzKFbmDJiMCOyUlGyOFoMUxPR22+/zS233MIHH3zAG2+8QTAY5Pzzz6e+vt7MsIRolTcYRjf8kZsZE0hEutXC6NwMslwOLLKlUIyEJr12l1dffTXmeXFxMYMHD+aTTz7hnHPOMSkqIVrnC4bRDV/jEiCd7yMCjq/qKH1EMUxNRCeqrq4GiLuLrN/vjxmZa21xNiF6ilKQaglit1jAnpzQOUrrw9gbArgcftl2uhlTm2bNGYbBHXfcwcyZMxk3blyrZVasWIHH44k+8vPzezlKcTIble1iZoGT0dkusCWWiN7cUcH20lq8PrnVpbk+k4huueUWPv/8c5599tm4ZZYtW0Z1dXX0UVJS0osRipOeYTSb4pFYImralNEISSJqrk80zW699Vb+/ve/88477zB06NC45RwOR9ybKYXocSHv8d03EphrBmDVmxKR9BE1Z2qNSCnFrbfeyosvvsibb75JUVGRmeEI0ab1n+9n66FqqoLWTs+8b6LbI4koHA6DEe7O8Po1U2tEt9xyC08//TQvv/wyLpcruhWRx+PB6UxwVEKIHlJVXYnDFyIngekdTZq2FIrON7PIzzmYXCNatWoV1dXVzJ49m9zc3OjjueeeMzMsIVoV8kXub9OTOrfVdHN2mw2lWSKJSPqJokytESnZ7VL0E0opQv5IIrI5Oz+9o4ldtxDSHDID/wR9orNaiL7OFzSwhCJrETmcroTPM2pwKlpuBh7qpEbUjCQiITqgzh/CFvZhs2pYHYkN3QMMTU+GTA/Uyk4ezfWZ+4iE6Mvq/SFshg+b1QK2xJtmADRuPR29J0lIjUiIjggrhcvqx4El4XuIIDJfrc6v4fAFcUnTLEoSkRAdMCIrlRGFKah6FzgSHzU7WOVl695aikINjJemWZQ0zYToKH8tGhrYE++stlsthDV7ZLlYqRFFSSISoiNCgeOJows1IrtuIWyxy31EJ5BEJEQHrP1sV2T3jpB2fE2hBNitkUQUMhSEJRE1kUQkRAdUHDtGjS8E9lTQEl9t2q5bjq9bLTWiKElEQrQjGDYI+2oBSErxdOlcNquFkMVOZCMPGb5vIolIiHbU+kLYw/VYLRp6F+6qhsZtpxubdqGgJKImMnwvRDtqvEHs4QYcugXN0bVEpGkaZwzPZrCRgm7I8H0TSURCtKPGF8QRrsOhW8Dh7vL5JgzLhrIkUMFuiG5gkKaZEO2o8YZwhGpx6FZI6lofEXB81E06q6MkEQnRDqtFw009STYLONO6fL6KgIVqb5BAMAhh2d8MpGkmRLtmDEuF/ckoVLfUiNbvqiL3cA0js1LJCvvBKr+GUiMSoj2+yH57mi0Z9K5v3mC36YQ1G2GZ5hEliUiI9nirIv92R/8Q4NCthCyNqzTKUiCAJCIh2vRVeR0vf7CVr8rruqV/CMChWwhbHITCBgS93XLO/k4SkRBtKK/1Q8OxSDPK2fpW6J2VZLMSsjoi882kRgRIIhKiTWW1fpzBKlIcOqRkdss5k2wWQpbGRCR3VwOSiIRo05FqL8nBClLsOqRkdcs5m/qIQmEjsnuskOF7IeLxBsL46muwGgFSklK7rWmW5XJgDMkiq/KA1IgaSSISIo6DVV6Sg5U4bVb0lEHddr9PRoqdjIJsCDqkRtRImmZCxFFS2UBqoAyPU4fU7O49ud64bbWMmgEmJ6J33nmHSy65hLy8PDRN46WXXjIzHDGANQRC7DtWT2V9x2e8ZyTbydcr8Tjt4B7SbbEopagIWKn2BjECkojA5KZZfX09EydOZPHixVx++eVmhiIGKKUUn5VU8e7Oo5EbCIERg1M5/7RskmzWNt87cagH9vkhaAdPdyYiWLOlgrFlNUz21JP4wrMDh6mJ6KKLLuKiiy4yMwQxwG09VMPb28sBcDtt1PqC7C6r49k6P9+ekk+qo41fgYZjkaaTpXubZhaLhtUR2Rst5PdKIqKfdVb7/X78/uNzc2pqakyMRvQHo7JT2XusnqxUB9OKMiiv8/PXjYeobAjy902H+O7UfLQT1qBWSrHlYDWnBLdHkkRaAVjarj11ltUe2bY6FGiIVJG6sA72QNCvOqtXrFiBx+OJPvLz880OSfRxDt3KxeNzmVaUgaZpDHYlccXkfAal2jlndFaLJASwu7yOdV+UseHDDSilYNDIbo/L1lgjCocNmfhKP0tEy5Yto7q6OvooKSkxOyTRR4XCRvRrTdOOJxyl8AQO8/2hx8jz7QZ/HQBGY/+RPxTmnR1HcQYrKdQr0SxWyBrd7fHZ7Q4MTW+c5iEd1v2qaeZwOHA4ur4Mgxj4Xtp4CKsFZo0eTEZKYy9MzWHY/grUlR//C6xpVKSO5tX6UYwaNoQdR2qp9gYZ791MbloSDBoBXVynujVJtmZ3Vwe94Ezv9s/oT/pVIhKiI0qrfZRUNGDRNHRrY03o2G74/H/BCINuh7Rh4K+F2lJKd3zEMN8G9pWNpzr1VIYGDzIjpRTdosOwmT0SY5LNQtDqJGTUQ6ChRz6jPzE1EdXV1bFr167o8z179rBx40YyMjIoKCgwMTLRn20+UAXAmJxU3Ek2qCqBrWsiSShzFIyZB42dxVQfZNTOdZSV7CLLvw1Hw5fkuJNw6DrkTwN3bo/EODwrlXBuFp5gAIL1PfIZ/Ympiejjjz9mzpw50edLliwBYOHChRQXF5sUlejPAiGDnWWRfp9xQzxQVwZbXoisDT1oBIz9ZuwImGcItsk/YMiwnbD/fag9HFmFcehUKDizx+IsykyBIdlQWgYBSUSmJqLZs2dHRiWE6CY7y2oJhAzSkm0Mcfhh43ORUSnPkJZJqImmRTqks0aDYUSe98Zwuj0l8q80zaSPSAwsWw9F7i0bn6WjbX4uMiqWkgnjrwCrrf0TWHpnIDkUNqgN2dB9QVyBul75zL6sXw3fC9GWqoYAByu96MrPuGOvgrcyss70xO+BzWl2eDFqfCH+9kUNX5TWStMMSURiAEm263z9lEGcb7xHku9opEN64vd6ZPi9q5LtVoJWJ2FDEfZLjUiaZmLAsFtgXPV6cFRGhugnfBeSu2cxs+7m0C0YeqSPKOiro3snkPQ/UiMSA8fuN6F8R2SS6rhvgyvH7Iji0jQN3RlJRCFfQ+TWgpOYJCIxIGzesI7DW98hZBhw6jcgfZjZIbXLnpSCoVkJho3IzZUnMUlEot/zl27Hu/VV9h5roCbvLBh8qtkhdUhqko2ANYVAyAD/yb2ShCQi0b81VFDx8V8wlMKXNY70MWebHVGHuZJ0Anoq/pABvpM7EUlntei/wkHY+iJHq2qpdeQwaPw30HrpPqDuMDwzlcDQPNIa6qRGZHYAQiRs1z+orzhEZdDGV1nncdqQ/jWDvTAzhdEFeZH5cL5qs8MxlSQi0T+Vfg6HNlJWG2DXoDkMzc3Gae+Hg+AOd+Tfk7xpJolI9D8NFbDjVQyl2GwdR7VzKOPyPGZH1WlKKSqMJCobAiipEQnRj4RDsPVFCAcJuPKxFM3E47RRkJFsdmSdZih4fksNX5bWEqyvjKxdfZKSzmrRv3z1VmRpD5uTpAnzucThImwoLJb+t/i81aJhSx2E0iz4/H7svmpwppkdlimkRiT6j/IdcODjyNenXhKdQ2bth0moSUaqA5/uxhsIR7YvOklJIhL9Q/1R+PJvka/zp7LNn0VVQ8d3be2r0pPteG3p+ILhSN/XSUoSkej7gl7Y8hcIBSAtn+rcs/jHF0co/tfeTm0h3RelJ9vx6ml4g1IjEqLvCvpg07PH1xYa+03e31NF2FDkpyeTntK/90nNSLHjtTUmovpys8MxjXRWi77LVxPZeaO2NLKw2fgrKPNb+bI0cs/NzJGZJgfYdYNS7dTZs/AHDUI1h9ENo9dWiexLJBGJvunYbtj+f5GlXm1OmHgl4eRMXv9wP0rB6GwXOZ4ks6PssmS7zhljhjPCMgjNCEF9WZ9evqSnSCISfUvNYdj3HhzdGXmekgnjv43hSOONbUcor/XjtFuZPSbL3Di70fQRmVBXBBV7oOagJCIhTKEUVO6Fkg2Ej35FMGwQBoy8KbhOORd0O2XVPr44XINF07hgbA4pjgH2o+vOiySiqv0wZLLZ0fS6Afa/KfoVpeDYbsJfvc3Rw/spq/VTFwhTnjySQ+7TUUczuVmPdEbneJIYne1iVHZqZE+wAUQpRak+hEBFPUMtu7Ea4da3PRrAJBEJc9SVw+51VJRsZ++xerxhC2UpYzg8aDyGw4NutZCkx3bazhufg9Yb+42Z4JU9MLpWw51UT3rl3shmkCcRSUQDhFKqf/yShgKw95+RO6SVQW3AYE/yeGqyzmDi8Dy+nu0iNU6zq19cXwI0TWPEYBcVh4sor91F+uGNkojM8Lvf/Y5f/epXlJaWMnHiRH7zm98wbdo0s8Pqe5SKdGaWfYmq3Et9zVHKq72UeSE/v4C8IYXgHkKVbTD/+KqBXE9S48PZ+hIZSkUWF7PoPT9krBQc3Yna+TphbzW6xQKZo8idci75R2FSQRo268k3bN1k3BAP//vVWHIOb8N7+EucheWQOnA65NtjeiJ67rnnWLJkCf/1X//F9OnTWblyJRdccAHbt29n8ODBZodnPqUi+7GXfQHlXxKor+JonZ/yWj8NgeM7P4SP7QWjLPJ1vZ/MY1Bpz+aw7sLQrLhtBqmWIE58DPdopOsBCDTgDQSpagigO5zoqYOwu7JwpOWgu7MhZXBkW+TGmkhFfYADlQ34gga6VSNJt5KRYic9xYZDj9OnYYSh4ivUvn9RXbqXkkovIbuLiXO+jSVrFHZgWt/bdqzXZbkc5OQO5VhVIV+VHeDUL/6GZdL3I9sinQQ0ZfLm89OnT2fq1Kn89re/BcAwDPLz8/nRj37EXXfd1eZ7a2pq8Hg8VFdX43a7eyPcnqVUZDqDvxbqjqBqDqId2w3+WkKGwfbSWqoCGhXOYVQ4i/A7MhiW5WHMIAt5ei32+iNQcwB/9RGq6gPU+UPU+kKRu3abGZPtIqPxjuSjdX52lrXc4E9vnEg6cuhg0rOGQpKbvVVhPjxQj9J0FBpKs6CwoDQLyUk2phRlkp+RCiE/9XXV1B49iFG5H299LdXeIN6wRqlrHGUZU7h8ahHZ7v5/H1B3qmoI8Px7X3DqgefJsIcZXliEc/QcSCsAW//8XnX0d9TUGlEgEOCTTz5h2bJl0WMWi4W5c+fy/vvvtyjv9/vx+/3R5zU1HVzVrmIP7F7XYr2XD/dUgDIAUICGin7tSdI5JafpG6f4eF8FYUOBUjSdRWv82pWkMzb3eNlP91cRDB3/5VeNZQGSHVbG5x3/D9lYUokvGInBooJYVDgaptNmZdwQD1htWAePZHvIRQm55GSkMinXzehsF0m2ljURR8hPds0hsmsOQbCBQCBAVUDDjx2/JQlb5iBwpYE9BX+9onrfUfzeWsJ1R6H+KHZ/BcnBCpzBakK++sjQOjDIF2RiyIdu0TCUIhhWeAMhAuFIwG7lhsOR/eVra3zsORrZSjlocVKeMo6KjImMys/lwsKMgTf83g3Sku2cP2kkb4XmMbJ0LYV1ZZE7y4F9NWHK6g2URUdpJzZhNU7PT8PR2Lm/t6KBI9W+mNebfzlhqAdn489NSUUDh2LKxhqX5yHFESl7pMZPtttx/MXx34Gk7qkAmPrTcPToUcLhMNnZ2THHs7Oz+fLLL1uUX7FiBffee2/nPyjkj4zSnMCoO0a86qAybOA9nkyMhioMI7a0iv6rg//4f7by1WKEjThlrRC0HT8e9KIaE1G48RGyOGiwpWOk5DBu/NcgvRDNqjN9iJe5SXpkjeO26A7IKIo8ADsQr5E7JAmGDDq+uqFSCn/IoN4fQoWDuI1q8B2FQD2ukI8xIX+kuaXCkSSuDAKhMHVeXyS5aAZY7YSSoN5hx3Dl4cwaxunpKRRkJJ/U/UAdUZiZwjdnTWXL3qEk23fB0R3grSLk9xLy+uO/sT4ITc3junqMuvjJhboANCYiVdeAUett47x+COqNZX1gaVYzU0acN3Vev/qztGzZMpYsWRJ9XlNTQ35+fvtvTMuP7IEO0f4O0Mgv9MUeavbEoVsh9Xj2HzbKj6Ea/7Y0lon8o0V+uVKalT01gGosG01dWuQNumaBZHv0HEXjAjRWKNAsOsqeAhYdTdMif+Ga1RyGpDnbv9Yu0jSNJJu1sablAFKBIW2+xw6cuLHz0MaH6Ly0ZDtnnzYMGAYjz4OQn8yqYzi9vsjgQpNmNXyrywHWyM9UekMAmz/+zrHNy6b5glh98cvqLjs0/vFICYTA3ixl2Lvvfi5TE1FmZiZWq5UjR47EHD9y5Ag5OS1vc3c4HDgcjhbH22VPidYOmuvMpg/ZnVgSObMTna9pA+vePNETdAeezDw6+iPoSafDZd2Nj47oyTEFU+vJdrudyZMns27duugxwzBYt24dM2bMMDEyIURvMr1ptmTJEhYuXMiUKVOYNm0aK1eupL6+nmuuucbs0IQQvcT0RPTd736X8vJyli9fTmlpKaeffjqvvvpqiw5sIcTAZfp9RF0x4O4jEmKA6ejvqIylCiFMJ4lICGE6SURCCNOZ3lndFU3dWx2e6iGE6FVNv5vtdUX360RUW1sL0LG7q4UQpqmtrcXjiX+bZb8eNTMMg0OHDuFyufrNollN01JKSkoG1EifXFf/0lvXpZSitraWvLw8LG2sedWva0QWi4WhQ/vnjCa32z2gfrCbyHX1L71xXW3VhJpIZ7UQwnSSiIQQppNE1MscDgd33313YqsI9GFyXf1LX7uuft1ZLYQYGKRGJIQwnSQiIYTpJBEJIUwniUgIYTpJRD3s/vvv58wzzyQ5OZm0tLQOvUcpxfLly8nNzcXpdDJ37lx27tzZs4F2UkVFBQsWLMDtdpOWlsa1115LXV3L/dGamz17NpqmxTxuuummXoo4vt/97ncUFhaSlJTE9OnT+fDDD9ss/8ILL3DKKaeQlJTE+PHj+b//+79eirRzOnNdxcXFLf5vkpJ6cS81JXrU8uXL1SOPPKKWLFmiPB5Ph97zwAMPKI/Ho1566SW1adMmdemll6qioiLl9Xp7NthOuPDCC9XEiRPVBx98oP75z3+qkSNHqiuvvLLN98yaNUtdf/316vDhw9FHdXV1L0XcumeffVbZ7Xb1+OOPq61bt6rrr79epaWlqSNHjrRa/r333lNWq1U9+OCDatu2ber//b//p2w2m9qyZUsvR962zl7X6tWrldvtjvm/KS0t7bV4JRH1ktWrV3coERmGoXJyctSvfvWr6LGqqirlcDjUM88804MRdty2bdsUoD766KPosbVr1ypN09TBgwfjvm/WrFnq9ttv74UIO27atGnqlltuiT4Ph8MqLy9PrVixotXy3/nOd9TFF18cc2z69Onqxhtv7NE4O6uz19XRn8+eIk2zPmbPnj2UlpYyd+7c6DGPx8P06dNb3f3WDO+//z5paWlMmTIlemzu3LlYLBY2bNjQ5nufeuopMjMzGTduHMuWLaOhoaGnw42raafh5t/rtnYahsi1Ny8PcMEFF/SZ/xtI7LoA6urqGDZsGPn5+Vx22WVs3bq1N8IF+vmk14GotLQUoNXdb5teM1tpaSmDB8fuHavrOhkZGW3GeNVVVzFs2DDy8vLYvHkzd955J9u3b2fNmjU9HXKrOrvTMESuvS//30Bi1zVmzBgef/xxJkyYQHV1NQ899BBnnnkmW7du7ZWJ5VIjSsBdd93VomPvxEe8//C+rKev64YbbuCCCy5g/PjxLFiwgCeffJIXX3yR3bt3d+NViETMmDGDq6++mtNPP51Zs2axZs0asrKy+P3vf98rny81ogT85Cc/YdGiRW2WGT58eELnbtrh9siRI+Tm5kaPHzlyhNNPPz2hc3ZUR68rJyeHsrKymOOhUIiKiopWd+iNZ/r06QDs2rWLESNGdDrerursTsMQ+f/pTHkzJHJdJ7LZbEyaNIldu3b1RIgtSCJKQFZWFllZWT1y7qKiInJycli3bl008dTU1LBhwwZuvvnmHvnMJh29rhkzZlBVVcUnn3zC5MmTAXjzzTcxDCOaXDpi48aNADEJtzc132l4/vz5wPGdhm+99dZW3zNjxgzWrVvHHXfcET32xhtv9KmdiRO5rhOFw2G2bNnCvHnzejDSZkzrJj9J7Nu3T3322Wfq3nvvVampqeqzzz5Tn332maqtrY2WGTNmjFqzZk30+QMPPKDS0tLUyy+/rDZv3qwuu+yyPjl8P2nSJLVhwwb17rvvqlGjRsUM3x84cECNGTNGbdiwQSml1K5du9R9992nPv74Y7Vnzx718ssvq+HDh6tzzjnHrEtQSkWGuR0OhyouLlbbtm1TN9xwg0pLS4sOXf/gBz9Qd911V7T8e++9p3RdVw899JD64osv1N13391nh+87c1333nuveu2119Tu3bvVJ598or73ve+ppKQktXXr1l6JVxJRD1u4cKECWjzeeuutaBlArV69OvrcMAz105/+VGVnZyuHw6HOO+88tX379t4Pvg3Hjh1TV155pUpNTVVut1tdc801Mcl1z549Mde5f/9+dc4556iMjAzlcDjUyJEj1b/927+Zfh+RUkr95je/UQUFBcput6tp06apDz74IPrarFmz1MKFC2PKP//882r06NHKbrersWPHqldeeaWXI+6YzlzXHXfcES2bnZ2t5s2bpz799NNei1WWARFCmE5GzYQQppNEJIQwnSQiIYTpJBEJIUwniUgIYTpJREII00kiEkKYThKREMJ0kohEv1JcXByz5O4999wTMxl40aJF0flVov+QRCRatWjRorhrSt9yyy1omhYzU78nEkBhYSErV66MOfbd736XHTt2xH3PY489RnFxcfT57NmzYyaoir5JEpGIKz8/n2effRav1xs95vP5ePrppykoKDAlJqfT2WJRtuY8Hk+HNykQfYckIhHXGWecQX5+fswKimvWrKGgoIBJkyZ16dyt1VTmz58frWXNnj2bffv28eMf/zi6KBu0bJqdqHnNbNGiRbz99ts89thj0XPs2bOHkSNH8tBDD8W8b+PGjWia1mvr74hYkohEmxYvXszq1aujzx9//HGuueaaHv/cNWvWMHToUO677z4OHz7M4cOHO32Oxx57jBkzZnD99ddHz1FQUNDimgBWr17NOeecw8iRI7vrEkQnSCISbfr+97/Pu+++y759+9i3bx/vvfce3//+93v8czMyMrBarbhcLnJychJaAdHj8WC320lOTo6ew2q1smjRIrZv3x7d5ysYDPL000+zePHi7r4M0UGyQqNoU1ZWFhdffDHFxcUopbj44ovJzMw0O6wuycvL4+KLL+bxxx9n2rRp/O1vf8Pv93PFFVeYHdpJS2pEol2LFy+muLiYJ554ottqDRaLhROXwgoGg91y7o647rrroh3xq1ev5rvf/S7Jycm99vkiliQi0a4LL7yQQCBAMBjkggsu6JZzZmVlxfT7hMNhPv/885gydrudcDjcpc+Jd4558+aRkpLCqlWrePXVV6VZZjJpmol2Wa1Wvvjii+jX8VRXV0cXxG8yaNAg8vPzW5Q999xzWbJkCa+88gojRozgkUceoaqqKqZMYWEh77zzDt/73vdwOBwJNQkLCwvZsGEDe/fuJTU1lYyMDCwWS7SvaNmyZYwaNapPLX5/MpIakegQt9uN2+1us8z69euZNGlSzOPee+9ttezixYtZuHAhV199NbNmzWL48OHMmTMnpsx9993H3r17GTFiRMK7pixduhSr1cppp51GVlYW+/fvj7527bXXEggEemUUULRN1qwWJ61//vOfnHfeeZSUlLTYFVX0LklE4qTj9/spLy9n4cKF5OTk8NRTT5kd0klPmmbipPPMM88wbNgwqqqqePDBB80ORyA1IiFEHyA1IiGE6SQRCSFMJ4lICGE6SURCCNNJIhJCmE4SkRDCdJKIhBCmk0QkhDDd/wf31p/0gYRQ7QAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T16:15:22.988556Z", + "iopub.status.busy": "2024-03-26T16:15:22.987881Z", + "iopub.status.idle": "2024-03-26T16:15:23.313508Z", + "shell.execute_reply": "2024-03-26T16:15:23.312571Z" + }, + "papermill": { + "duration": 0.346653, + "end_time": "2024-03-26T16:15:23.315453", + "exception": false, + "start_time": "2024-03-26T16:15:22.968800", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEnCAYAAAANc04FAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA81UlEQVR4nO3deXxU1d348c+9d7askwRCFkhI2ILsKEIRFVFaKmq1tu6tIC5d8Gktj63y61Na7MuirVpp64uuEH2qtVpxaa1SHlRQVEQURBCQQCBAQgLZk9nv+f0xmUkCWSfLZPm+X695JZm5c+c7Wb4559xzvkdTSimEECKK9GgHIIQQkoiEEFEniUgIEXWSiIQQUSeJSAgRdZKIhBBRJ4lICBF1koiEEFEniUgIEXWWaAfQFaZpcuLECRISEtA0LdrhCCHOoJSipqaGzMxMdL2Ndo+KsmPHjqlbbrlFpaSkKIfDoSZNmqS2b9/eoecWFRUpQG5yk1sfvxUVFbX5txzVFlFFRQVz5sxh3rx5vPbaa6SmpvL555+TnJzcoecnJCQAUFRURGJiYk+GKoSIQHV1NVlZWeG/1dZENRE9/PDDZGVlsW7duvB9ubm5HX5+qDuWmJgoiUiIPqy9oZOoDla/8sorzJgxg+uuu45hw4Yxffp0/vSnP7V6vMfjobq6utlNCNH/RTURHTp0iDVr1jB27Fg2bNjAd77zHb73ve/x5JNPtnj8qlWrcDqd4VtWVlYvRyyE6AmaUtGrR2Sz2ZgxYwbvvvtu+L7vfe97bN++nffee++s4z0eDx6PJ/x1qP9ZVVUlXTMh+qDq6mqcTme7f6NRHSPKyMhgwoQJze4755xzeOGFF1o83m63Y7fbO/UaSin8fj+BQCDiOEXXWa1WDMOIdhiij4pqIpozZw779+9vdt+BAwcYOXJkt5zf6/VSXFxMfX19t5xPRE7TNEaMGEF8fHy0QxF9UFQT0Q9+8AMuuOACfvGLX3D99dfzwQcf8Mc//pE//vGPXT63aZocPnwYwzDIzMzEZrPJpMcoUUpRVlbGsWPHGDt2rLSMxFmimojOP/98XnzxRZYvX84DDzxAbm4ujz/+OLfcckuXz+31ejFNk6ysLGJjY7shWtEVqampFBYW4vP5BnQiOl3r4Y19pUzITGRipjPa4fQbUV/iceWVV3LllVf22PnbnFYues1gaY2+c/AUxypcHKtwMWZYPHbLwE263Un+SoXoJqapOHq6cTyyrMbTxtGiqai3iIQYKE7XefGbwdkw37lkNA6rtIY6SlpEfdAll1zCPffcE+0wRCdV1HsBGOWoxVF3IsrR9C+SiPqxt956C03TqKysjHYoAqhx+7D5azmn+CX4+K9QdiDaIfUbkoiE6CZJsTbOc5wgxqI4dKqWnR+dvTpAtGxQJiKv32z15g+YHT7W14Fju8rj8XDfffeRlZWF3W5nzJgx/OUvf6GwsJB58+YBkJycjKZpLF68uN3z1dTUcMsttxAXF0dGRga//vWvz+oK/u///i8zZswgISGB9PR0br75ZkpLS8OPh1pimzZtYsaMGcTGxnLBBRecNTl1sBmdGs8XhroZlmDnZLWH6pJDRHEFVb8yKAern3jzYKuP5Q6N45rpw8Nf/3FLAb5Ay79MI5JjuG5G48LbtVsP4/I2X0rygy+O61Kst956K++99x6/+c1vmDp1KocPH+bUqVNkZWXxwgsv8LWvfY39+/eTmJhITExMu+dbtmwZW7du5ZVXXiEtLY0VK1bw0UcfMW3atPAxPp+Pn//85+Tl5VFaWsqyZctYvHgx//73v5ud68c//jGPPvooqampfPvb32bJkiVs3bq1S++336stxWoE/79b/XW4XS5iZB5buwZlIuovDhw4wHPPPcfGjRuZP38+AKNGjQo/npKSAsCwYcNISkpq93w1NTU8+eSTPPPMM1x22WUArFu3jszMzGbHLVmyJPz5qFGj+M1vfsP5559PbW1tsyUaDz74IHPnzgXg/vvv54orrsDtduNwOCJ7w/1cvduDw12FrmlYDQ1fQOGqLiMmtnuWLA1kgzIRLZ03ptXH9DPm3d118ehWjz1zjt6SOR0v6tYRO3fuxDCM8B97Vx06dAifz8fMmTPD9zmdTvLy8podt2PHDn72s5+xa9cuKioqMM1gF/Po0aPNFilPmTIl/HlGRgYApaWlZGdnd0u8/YlSiiff2s2046c5N3covrgMqD6Bp/o0pEsias+gTEQ2S8eHxnrq2I7oSFeru9XV1bFgwQIWLFjA008/TWpqKkePHmXBggV4vd5mx1qt1vDnoZnToaQ12Hj8JjZfDQDW2CS0Oiuq+gTueine1xGDcrC6v5g8eTKmabJ58+YWH7fZbAAdLnEyatQorFYr27dvD99XVVXFgQONl5n37dvH6dOneeihh7jooosYP358s4Fq0TKXN4DdX4uha+gxSVhigjWaffVVUY6sf5BE1Ifl5OSwaNEilixZwksvvcThw4d56623eO655wAYOXIkmqbxr3/9i7KyMmpra9s8X0JCAosWLeKHP/whb775Jnv27OH2229H1/VwiyY7OxubzcZvf/tbDh06xCuvvMLPf/7zHn+v/V29L4A9UIvV0MCeiO4IJqKAu+2fiQiSRNTHrVmzhq9//et897vfZfz48dx5553U1dUBMHz4cFauXMn9999PWload999d7vne+yxx5g9ezZXXnkl8+fPZ86cOZxzzjnhAebU1FTy8/N5/vnnmTBhAg899BCPPPJIj77HgcDlDWANuLDqOtjimJiTyfk5yYxPGRyLfbsqqqViu6qtMpRut5vDhw+Tm5s7aK/idERdXR3Dhw/n0Ucf5fbbb++x1xnoP49Pj1dxZMtfGc0xxl/0NYgdAruehbihMPPOaIcXNf2iVKzofR9//DH79u1j5syZVFVV8cADDwBw9dVXRzmy/s3lC2AJuLHYdLDGgL1hHy9PTXQD6yckEQ0gZ15eP9PevXsBeOSRR9i/fz82m43zzjuPt99+m6FDh/ZWmANScqyN2HhFgmYBayynvQaVZbVYjDpGmgHQZSV+WyQRDSCZmZns3Lmzzcezs7PZsWNH7wU1SIwZFg+pVvA6wBaH22ejtMaDw6oz0u8GW1y0Q+zTJBENIBaLhTFjWp+sKXqQUuBzBT+3xuDAQkC34Q/4wCeJqD1y1UyIbuBx1zZO5rTG4rAa+HU7AVOhfLKLTHskEQnRDV7efpBth8up8GqgG9gtOn7djgK8bklE7ZFEJEQ3UJ5gstHtwS6YxdAxjeA0Ba9HElF7JBEJ0R18zRMRANZg+Y+Apy4aEfUrkoiE6Aamzw2AYWtcqKxZgy0iv8cVlZj6E7lqJkQX+QMmmj+4dZDF1jhr/MIJWViLijBsHVuUPJhJi6gP6mu7ePS1ePoaX0ChKx8AFmtjIoqJicOi62gBb2tPFQ2imoh+9rOfoWlas9v48eOjGdKAcWbtINFzvH4Tw/RhaKBb7Y0PGMEyLfhlo8X2RL1FNHHiRIqLi8O3d955p+deTCnwe3v/1ol1xYsXL2bz5s2sXr06nJwLCgq4/fbbyc3NJSYmhry8PFavXn3W86655hoefPBBMjMzw1UX3333XaZNm4bD4WDGjBm89NJLaJrWbAb2p59+yuWXX058fDxpaWl885vf5NSpU63GU1hY2OUfxUCi65DjNEiJs4HRmIgKKnwUlNVSVinF0doT9TEii8VCenp677xYwAdvP9o7r9XURf8NFluHDl29ejUHDhxg0qRJ4QWpycnJjBgxgueff54hQ4bw7rvvctddd5GRkcH1118ffu6mTZtITExk48aNQHDl81VXXcXChQt55plnOHLkyFldrMrKSi699FLuuOMOfv3rX+Nyubjvvvu4/vrreeONN1qMJzU1tRu+KQNHgsPKeSPiwJYAlsZEVFKr0Gs8aAl1yHesbVFPRJ9//jmZmZk4HA5mz57NqlWrWq157PF48Hgam7nV1QPvP43T6cRmsxEbG9ssQa9cuTL8eW5uLu+99x7PPfdcs0QUFxfHn//853Dlxt///vdomsaf/vQnHA4HEyZM4Pjx49x5Z2NZit/97ndMnz6dX/ziF+H71q5dS1ZWFgcOHGDcuHEtxiPOEBoHMhrL5xp2BwowfdI1a09UE9GsWbPIz88nLy+P4uJiVq5cyUUXXcSnn35KQkLCWcevWrWq2R9kpxnWYOuktzX55YzUE088wdq1azl69Cgulwuv19tsCyAIlpYNJSGA/fv3M2XKlGb1f5oWzgfYtWsXb775ZrPdOUIKCgoYN65r2yENBgFTofncwXGOJl0zi82BD1AyRtSuqCaiyy+/PPz5lClTmDVrFiNHjuS5555rsUjX8uXLWbZsWfjr6upqsrKyzjquVZrW4S5SX/Lss89y77338uijjzJ79mwSEhL41a9+xbZt25odFxfX+YWVtbW1XHXVVTz88MNnPRbamUO07bPiak7uOcYIWy15Uxt/vxoTkTs4Tnjmti8iLOpds6aSkpIYN24cBw+2vAGi3W7Hbre3+NhAYrPZmhXE37p1KxdccAHf/e53w/cVFBS0e568vDz++te/4vF4wt+3poXzAc4991xeeOEFcnJysFha/nU4Mx7RnMdvYihvcCuqJi0iq82BCzADATD93dIyHqiiftWsqdraWgoKCgb9f+KcnBy2bdtGYWEhp06dYuzYsXz44Yds2LCBAwcO8JOf/OSshNKSm2++GdM0ueuuu/jss8/YsGFDuP50qFj+0qVLKS8v56abbmL79u0UFBSwYcMGbrvttnDyOTOewbplUGt8gYbL97rWbLDaanMAGgFTySX8dkQ1Ed17771s3ryZwsJC3n33Xb761a9iGAY33XRTNMOKunvvvRfDMJgwYQKpqaksWLCAa6+9lhtuuIFZs2Zx+vTpZq2j1iQmJvLPf/6TnTt3Mm3aNH784x+zYsUKgPC4UWZmJlu3biUQCPClL32JyZMnc88995CUlISu6y3Gc/To0Z578/2Q1xfAML0YmtY4dwiwWQwCujWYiGRSY5uiWjz/xhtvZMuWLZw+fZrU1FQuvPBCHnzwQUaPbn131aakeH7nPf3009x2221UVVX16gaOA/nnsenT48Ru+zVZyTGM+Mr/hFtFvoBJYOvvsPhqsJx/GyRmtnOmgadfFM9/9tlno/nyg8JTTz3FqFGjGD58OLt27QrPEYrGLrIDVSC04FVv3iKyGjpWRwwE6qRr1o4+NVgtul9JSQkrVqygpKSEjIwMrrvuOh588MFohzWg+BsSkWa1n31lLDRmJF2zNkkiGuB+9KMf8aMf/SjaYQxoabE6cbFWHPbmrUyv36SgzIu9upZcvxu5eN86SURCdNF5w2OhLBFiW5gUWu5jSL2HLI8buXjfuj51+b4n9OONbAeUAf1zCHW7zpgsazU0Alrwf73fL12ztgzYRGS1Bv//1NdLveC+IFSWxDAG3kaDoeqMTSczQnCultaQnAI+SURtGbBdM8MwSEpKorS0FIDY2NjwJD7Ru0zTpKysjNjY2FZnb/dn//iggJzTp5mUNJrYMx5rTERy1awtA++3oonQavFQMhLRo+s62dnZA+6fQcBUKL+XgALDcvb8KK3hqllALt+3aUAnIk3TyMjIYNiwYfh8vmiHM6jZbLbwTO2BJLS8A5rXqw7RG9aXBaRiZpsGdCIKMQxjQI5NiOhruuDVsJ69IDtUOtaUFlGbBkUiEqKntLbgNWROXiZWlYQ1ZmB1SbubJCIhuiBYOD+04PXsmUIJsTFgMUDJ0EBbBl6nXYhe5PWbGKqhRWS0UCsrtPYsIImoLdIiEqIL7FadtFiNBL+1xa5ZYaUP7XQdCX4bKVGIr7+QRCREF2Q4Y8jIjIX6uGYr70OKqv1YqtxkWOolEbVBumZCdFWg4YpYCy0iS8NVM9XJ/e0GG0lEQnSBUk3KwLbQIrJYg/eZphmsWy1aJF0zIbrgnc/LsBecJMPpIKuFFpFhteEHTNVQLlYK6LdIWkRCdIHP6wnWpIYWr5rZLBZMzSJ1q9shiUiILgh4gyvvdcMC+tmz9y2GRkCzYiogIF2z1kgiEqILQmViDYutxQ0UrYaOqUuLqD0yRiREF4RbRNaWdyYZnhRD2ug0rK5TkojaIIlIiC4I1RnSW1jwCuCwGhAbCz5DZle3QbpmQnRBeCshWxtboYeXeUiLqDXSIhKiC9JiNRIdFqy2lveJ8/gDHD3lwVFZT5YkolZJIhKiC+bkJILfGex+tSBgKj4rc5Na52JEwCtbCrWiz3TNHnroITRN45577ol2KEJ0XBvLOwAsuo6pBScx+qVKY6v6RCLavn07f/jDH5gyZUq0QxGic0LdrRaWd0DDlkJ6Q7lYKaDfqqgnotraWm655Rb+9Kc/kZycHO1whOiwarePf31cyEdHK1ptEWmahmZIImpP1BPR0qVLueKKK5g/f367x3o8Hqqrq5vdhIgWr99E+bzBdWSttIgA9IYthfxSt7pVUR2sfvbZZ/noo4/Yvn17h45ftWoVK1eu7OGohOgYXyBYOD9YJrb1RBRqLZmy22urotYiKioq4vvf/z5PP/00DkfLs1LPtHz5cqqqqsK3oqKiHo5SiNaF6lXrrRTOD9Etwa6ZKV2zVkWtRbRjxw5KS0s599xzw/cFAgG2bNnC7373Ozwez1lbANntduz2NiaOCdGLvH4Ti+nDYrTdIrpwXCZ25cRuk4v3rYlaIrrsssvYvXt3s/tuu+02xo8fz3333Sf7kIk+zxvqmrXTIkpJjAObBZAlHq2JWiJKSEhg0qRJze6Li4tjyJAhZ90vRF8U7Jr50LVWdvAICRVDk7VmrZKZ1UJEKMZmYLGbxNgMsLTeNTtS6cOoqCfBtOHsxfj6kz6ViN56661ohyBEh41PjYWMhOAXbbSIDpZ7ia1wkWWRRNSaqM8jEqLfarqItY3BakvoqpnfJzt5tEISkRCRCq0zMyygt/6npDcMZCvTlHGiVkgiEiJCmz4tYseRCkrr2z4utLdZQCkwJRG1RBKREBHyuN14AyaqrVnVgNWiBwvom0hxtFZIIhIiQqHqjK2ViQ2xGDqmZmnY20xaRC2RRCREhEKF841WCueHWA0tuJOHJKJW9anL90L0J6phNb3RTosoZ0gcw3OGYfeclq5ZKyQRCREhsyERWWxtt4ji7BaIjwWzUlpErZCumRARME2F6Qu2box2ElHDQQ1PlETUEklEQkQgoBTDYiHWZrTbInJ5Axyu9FJS7ZauWSsiSkSHDh3q7jiE6Feshs7c0U6mjkgKzxNqTb3Xz6fFLo5V1EvXrBURJaIxY8Ywb948/vrXv+J2u7s7JiH6h1Dp17ZW3tNw+V63EDCVtIhaEVEi+uijj5gyZQrLli0jPT2db33rW3zwwQfdHZsQfVsoqbSx8h4advLQrJgKlLSIWhRRIpo2bRqrV6/mxIkTrF27luLiYi688EImTZrEY489RllZWXfHKUSfUlRez9ufHWNfSXW7LSJrw4RGAL+Ui21RlwarLRYL1157Lc8//zwPP/wwBw8e5N577yUrK4tbb72V4uLi7opTiD7F4w/g87rxm6rdFpFFb9zbTOpWt6xLiejDDz/ku9/9LhkZGTz22GPce++9FBQUsHHjRk6cOMHVV1/dXXEK0ad4GqozGno71RlpvreZ3ydds5ZENKHxscceY926dezfv5+FCxfy1FNPsXDhQvSGUgi5ubnk5+eTk5PTnbEK0Wf4AqpxK6E26lWH6OEthaRF1JKIEtGaNWtYsmQJixcvJiMjo8Vjhg0bxl/+8pcuBSdEX+X1BTBMP4Zua3tPswZz8jKIMxKJ0c1eiK7/iSgRbdy4kezs7HALKEQpRVFREdnZ2dhsNhYtWtQtQQrR1/i8HnRUuzt4hGSkJILDCvh7Prh+KKIxotGjR3Pq1Kmz7i8vLyc3N7fLQQnR1/l9LoDgP2O9A//PZSePNkXUIlKt1N2tra3t8K6tQvRncYaJ1WZgtTtAa3/jxKOVPmzVbhI1F7G9EF9/06lEtGzZMiB4FWDFihXExjZ+SwOBANu2bWPatGndGqAQfdGMEXFwMgkciR06/tMSF8mn6si12CURtaBTiejjjz8Ggi2i3bt3Y7M1DtLZbDamTp3Kvffe270RCtEXhZd3tD9QDaBbg8fJzOqWdSoRvfnmm0Bwa+jVq1eTmNix/wZCDDjh5R3tD1QDGOFEFAAzALpsqd5URIPV69atkyQkBrX/fHKUXccqqfa1Pz4EYDTMvg6Wi5WFr2fqcIvo2muvJT8/n8TERK699to2j12/fn2HzrlmzRrWrFlDYWEhABMnTmTFihVcfvnlHQ1LiKioq6uj3htAGR27OGOxWFGajmk21K22xvRwhP1LhxOR0+lEa7g64HR2z8a5I0aM4KGHHmLs2LEopXjyySe5+uqr+fjjj5k4cWK3vIYQPaGxTGzHumZWQycgO3m0qsOJaN26dS1+3hVXXXVVs68ffPBB1qxZw/vvvy+JSPRZAVOFB6utHSkTS7AUiFezEjD90jVrQUTziFwuF0qp8OX7I0eO8OKLLzJhwgS+9KUvRRRIIBDg+eefp66ujtmzZ7d4jMfjweNpXKtTXV0d0WsJ0RVev4lFBZOJxdaxLtaYYfH4s4YS66+UutUtiGiw+uqrr+app54CoLKykpkzZ/Loo49y9dVXs2bNmk6da/fu3cTHx2O32/n2t78dTmgtWbVqFU6nM3zLysqKJHwhusQbWnmvgd7OnmYhSbE2hjrjibVZpGvWgogrNF500UUA/OMf/yA9PZ0jR47w1FNP8Zvf/KZT58rLy2Pnzp1s27aN73znOyxatIi9e/e2eOzy5cupqqoK34qKiiIJX4gu8QQCGKa3YZ1Zx+YRAU2WeUjX7EwRdc3q6+tJSEgA4D//+Q/XXnstuq7zhS98gSNHjnTqXDabjTFjxgBw3nnnsX37dlavXs0f/vCHs4612+3Y7R0bHBSixyhw2gI4MNqtRRRS6/FTWe0nps7DEElEZ4m4eP5LL71EUVERGzZsCI8LlZaWdnl+kWmazcaBhOhrhiU6uGBkAhMznR2e0FhR52XniXqOVbggICvwzxRRi2jFihXcfPPN/OAHP+Cyyy4LDy7/5z//Yfr06R0+z/Lly7n88svJzs6mpqaGZ555hrfeeosNGzZEEpYQvaeTM6tDdatlJ4+WRZSIvv71r3PhhRdSXFzM1KlTw/dfdtllfPWrX+3weUpLS8O1rZ1OJ1OmTGHDhg188YtfjCQsIXpPB7cSCrGEd/KQRNSSiBIRQHp6Ounp6c3umzlzZqfOIRUcRX/0SVEFqrCUIXFWRnRwsNqqB/c2M00ll+9bEFEiqqur46GHHmLTpk2UlpZims3LX8pOsGIgq62vR/P6SYyxgKWDExotGgHNQkCB8nvp2Aq1wSOiRHTHHXewefNmvvnNb5KRkRFe+iHEYOD3urEChm50rDojYNF1TC20k4cXaw/G1x9FlIhee+01Xn31VebMmdPd8QjR5/m8HqyAbrV3qDojNOz2GtrbzC9jRGeKKBElJyeTkpLS3bEI0S8EfG6gIRF1kKZpzBmXTmJhAoaSy/dnimge0c9//nNWrFhBfX19d8cjRJ/n9zQUzu9kKY+cYcmkxNqwKBmsPlNELaJHH32UgoIC0tLSyMnJwWpt3uP96KOPuiU4Ifois6FFZHSiRdTwhOBHuXx/logS0TXXXNPNYQjRf8QaAWyGjqWDJUBCjtcEsNV4SLR6kYVKzUWUiH760592dxxC9BuXjkkCkiExvlPP23akmoyyWvIccZKIzhDRGBEEy3/8+c9/Zvny5ZSXlwPBLtnx48e7LTgh+iR/sGvW0TlEIVrD5Ee5ana2iFpEn3zyCfPnz8fpdFJYWMidd95JSkoK69ev5+jRo+FaRUIMSKFE0sGthEJCO3mYfi8o1eFL/4NBRC2iZcuWsXjxYj7//PNmO7suXLiQLVu2dFtwQvQ1NW4fb+4tYm9xdYcXvIaEE5FCiqOdIaJEtH37dr71rW+ddf/w4cMpKSnpclBC9FUuX4D6ujpcXn/nE1HDAllTVuCfJaJEZLfbW6wXfeDAAVJTU7sclBB9lcdnYjE9WAy902NEVotOQLMG9zaTha/NRJSIvvKVr/DAAw/g8wW/mZqmcfToUe677z6+9rWvdWuAQvQlHn8AS7hMbCcTUUNNItlS6GwRJaJHH32U2tpaUlNTcblczJ07lzFjxpCQkMCDDz7Y3TEK0We4fSaG6cESQSLKS09g3PAUhsbbpWt2hoiumjmdTjZu3MjWrVvZtWsXtbW1nHvuucyfP7+74xOiT/H4A1hUQyLq5BKPtEQHJCdArVsS0Rk6nYhM0yQ/P5/169dTWFiIpmnk5uaSnp6OUkpKgogBzeP1B7cS0h2dHqwGQA8t85CFr011qmumlOIrX/kKd9xxB8ePH2fy5MlMnDiRI0eOsHjx4k6ViRWiP7IqLzZDx2p0vmtW4/Zxss6kyuWTFtEZOtUiys/PZ8uWLWzatIl58+Y1e+yNN97gmmuu4amnnuLWW2/t1iCF6CvOHxELI5ODC1h1o1PPPVHp5uCxOnJUPU5JRM10qkX0t7/9jf/3//7fWUkI4NJLL+X+++/n6aef7rbghOhzQss7Ojk+BMHiaKZuCU5oNKVr1lSnEtEnn3zCl7/85VYfv/zyy9m1a1eXgxKiz/IFaxF1tlsGZ16+lxZRU51KROXl5aSlpbX6eFpaGhUVFV0OSoi+6o09Rew5UUVNoPMXnENbCsneZmfr1HczEAhgsbT+FMMw8PulySkGroqqamLcfpTR+RaRRdeb7G0mfydNdSoRKaVYvHhxq/vPy1bRYiAzTYXpDZZHtjk6P0ZkM/TGMSJpETXTqUS0aNGido+RK2ZioPL4TfSGBGJ1xHX6+eHdXk2FCsjeZk11KhGtW7euW1981apVrF+/nn379hETE8MFF1zAww8/TF5eXre+jhDdod7rDy541TUMa+e7ZnaLzozRw0g+Ei+bLJ4h4gqN3WHz5s0sXbqU999/n40bN+Lz+fjSl75EXV1dNMMSokUuXwCL6QlOZowgEVkMnXEZKaQm2NFlS6FmIlpr1l1ef/31Zl/n5+czbNgwduzYwcUXXxylqIRomdsXwGK6G0qAdH6MCGis6ihjRM1ENRGdqaqqCqDVzRs9Hk+zAfGWaiIJ0VOUgnjdh03XwRYb0TlK6gLY6r0k2D2y7XQTUe2aNWWaJvfccw9z5sxh0qRJLR6zatUqnE5n+JaVldXLUYrBbGxaAnOyYxiXlgDWyBLRGwfK2V9Sg8stV5ib6jOJaOnSpXz66ac8++yzrR6zfPlyqqqqwreioqJejFAMeqbZZIlHZIkotCmj6ZdE1FSf6Jrdfffd/Otf/2LLli2MGDGi1ePsdnurc5iE6HF+V7B/BhGtNQMwLKFEJGNETUW1RaSU4u677+bFF1/kjTfeIDc3N5rhCNGmtz49yp4TVVT6jE6vvA+x2IKJKBAIgBnozvD6tai2iJYuXcozzzzDyy+/TEJCQngHEKfTSUxMhFclhOghlVUV2N1+0iNY3hES2lIovN5Ml99ziHKLaM2aNVRVVXHJJZeQkZERvv3973+PZlhCtMjvDs5vszg6t9V0UzarFaXpwUQk40RhUW0RqVB/W4g+TimF3xNMRNaYzi/vCLFZdPyaXVbgn6FPDFYL0de5fSa6P1iLyB6TEPF5xg6LR8tIwUmttIiakEQkRAfUevxYA26shoZhj+zSPcCI5FgY6oQa2cmjqT4zj0iIvqzO48dqurEaOlgj75oB0LD1dHhOkpAWkRAdEVCKBMODHT3iOUQQXK9W69Gwu30kSNcsTBKREB0wOjWe0TlxqLoEsEd+1ex4pYs9hTXk+uuZLF2zMOmaCdFRnho0NLBFPlhtM3QCmi1YLlZaRGGSiIToCL+3MXF0oUVks+gEdJvMIzqDJCIhOuC1jw8Gd+/wa401hSJgM4KJyG8qCEgiCpFEJEQHlJ8+TbXbD7Z40CIv8mqz6I11q6VFFCaJSIh2+AImAXcNAI44Z5fOZTV0/LqN4EYecvk+RBKREO2ocfuxBeowdA1LF2ZVQ8O20w1dO79PElGIXL4Xoh3VLh+2QD12i45m71oi0jSNc0elMcyMw2LK5fsQSURCtKPa7cMeqMVu0cGe2OXzTRmZBqUOUL5uiG5gkK6ZEO2odvmx+2uwWwxwdG2MCGi86iaD1WGSiIRoh6FrJFKHw6pDTFKXz1fu1aly+fD6fBCQ/c1AumZCtGv2yHg4GotCdUuL6K2DlWQUVzMmNZ7UgAcM+TOUFpEQ7XEH99vTrLFg6frmDTarhYBmJSDLPMIkEQnRHldl8GN3jA8BdouBX2+o0iilQABJREK06VBZLS+/v4dDZbXdMj4EYLfoBHQ7/oAJPle3nLO/k0QkRBvKajxQfzrYjYppeSv0znJYDfyGPbjeTFpEgCQiIdpUWuMhxldJnN0CcUO75ZwOq45fb0hEMrsakEQkRJtOVrmI9ZUTZ7NAXGq3nDM0RuQPmMHdY4VcvheiNS5vAHddNYbpJc4R321ds9QEO+bwVFIrjkmLqIEkIiFacbzSRayvghirgSVuSLfN90mJs5GSnQY+u7SIGkjXTIhWFFXUE+8txRljgfi07j25pWHbarlqBkiLSAwS9V4/ZTUeEh1WkuM6VmExJdaGw1KB02KDxOHdFotSigqvgeHykeB1SWuAKLeItmzZwlVXXUVmZiaapvHSSy9FMxwxACml+OhoBX9++zDrPzpO/ruFvLLrBG5foN3nTh3hZPZQDylxNnB2ZyKC9bvL2VtcHd7GerCLaiKqq6tj6tSpPPHEE9EMQwxge05Us3l/GQFTkRhjRdOgoLSWZz84Sq2nnQWn9aeDXSe9e7tmuq5h2IN7o/k90jWDKHfNLr/8ci6//PIOH+/xePB4GtfmVFdX90RYYgAZmxZP4ek6UuPtzMxNoazWwys7T1BR7+Nfu05ww/lZaGfUoFZKsft4FeN9+7EBJGWDbnRrXIYtuG2131sfbCJ1oQ72QNCvuqerVq3C6XSGb1lZWdEOSfRxdovBFZMzmJmbgqZpDEtwcN15WQyJt3HxuNSzkhBAQVktmz4rZdsH21BKwZAx3R6XtaFFFAiYsvCVfpaIli9fTlVVVfhWVFQU7ZBEH+UPmOHPNU1rTDhK4fQW840Rp8l0F4CnFgDTVAB4/AG2HDhFjK+CHEsFmm5A6rhuj89ms2NqloZlHtI961dXzex2O3Z718swiIHvpZ0nMHSYO25YcLAZoLoY9r8KtWWN/4E1jfL4cbxeN5axI4dz4GQNVS4fk12fkJHkgCGjoYt1qlvisDaZXe1zQUxyt79Gf9KvEpEQHVFS5aaovB5d07AYDS2h0wXw6QtgBsBig6SR4KmBmhJKDmxnpHsbR0onUxV/DiN8x5kdV4JFt8DIOT0So8Oq4zNi8Jt14K3vkdfoTyQRiQHnk2OVAOSlx5PosEJlEexZH0xCQ8dC3kJoGCym6jhjP99EadFBUj17sdfvIz3Rgd1igayZkJjRIzGOSo0nkJGK0+cFn1zCj2oiqq2t5eDBg+GvDx8+zM6dO0lJSSE7OzuKkYn+yus3+bw0OO4zabgTakth9/PB2tBDRsPErza/AuYcjvW8bzJ85Odw9D2oKQ5WYRxxPmRf0GNx5g6Ng+FpUFIKXklEUU1EH374IfPmzQt/vWzZMgAWLVpEfn5+lKIS/dnnpTV4/SZJsVaG2z2w8+/Bq1LO4WcnoRBNCw5Ip44D0wx+3RuX021xwY/SNYtuIrrkkkuCl0eF6CZ7TgTnlk1OtaB98vfgVbG4oTD5OjCs7Z9A750Lyf6ASY3fisXtI8Fb2yuv2Zf1q8v3QrSlst7L8QoXFuVh0unXwVURrDM99UawxkQ7vGaq3X7++Vk1n5XUSNcMSURiAIm1Wfji+CF8ydyKw30qOCA99cYeufzeVbE2A58RQ8BUBDzSIpKrZmLAsOkwqeotsFcEL9FPuQFiu6eYWXezW3RMS3CMyOeupXsXkPQ/0iISA0fBG1B2ILhIddLXISE92hG1StM0LDHBROR31wenFgxikojEgPDJtk0U79mC3zThnCsheWS0Q2qXzRGHqRn4AmZwcuUgJolI9Huekv249rxO4el6qjMvhGHnRDukDol3WPEacXj9JngGdyUJSUSif6svp/zDf2AqhTt1Esl5F0U7og5LcFjwWuLx+E1wD+5EJIPVov8K+GDPi5yqrKHGns6QyVei9dI8oO4wamg83hGZJNXXSoso2gEIEbGD/0dd+QkqfFYOpV7GhOH9awV7ztA4xmVnBtfDuauiHU5USSIS/VPJp3BiJ6U1Xg4OmceIjDRibP3wIrg9MfhxkHfNJBGJ/qe+HA68jqkUnxiTqIoZwaRMZ7Sj6jSlFOWmg4p6L0paREL0IwE/7HkRAj68CVnouXNwxljJTomNdmSdZip4bnc1+0pq8NVVBGtXD1IyWC36l0NvBkt7WGNwTLmGq+wJBEyFrve/4vOGrmGNH4LSdNweDzZ3FcQkRTusqJAWkeg/yg7AsQ+Dn59zVXgNmdEPk1BISrwdtyURlzcQ3L5okJJEJPqHulOw75/Bz7POZ68nlcp6b3Rj6gbJsTZc1uTgho/15dEOJ2okEYm+z+eC3f8AvxeSsqjKuJD/++wk+e8WUlHXv5NRcqwNlyUJl09aREL0XT437Hq2sbbQxK/y3uFKAqYiKzm2w/vY91UpcTZc1oZEVFcW7XCiRgarRd/lrg7uvFFTEixsNvk6Sj0G+0qCc27mjBka5QC7bki8jVpbKh6fib+6GItp9lqVyL5EEpHom04XwP5/B0u9WmNg6k0EYofynw+OohSMS0sg3emIdpRdFmuzcG7eKEbrQ9BMP9SV9unyJT1FEpHoW6qL4chWOPV58Ou4oTD565j2JDbuPUlZjYcYm8EleanRjbMbzRo9FGpzofwwVB+XRCREVCgFFYVQtI3AqUP4AiYBwMycQcL4S8Fio7TKzWfF1eiaxoKJ6cTZB9ivbmJmMBFVHoXh50U7ml43wH6aol9RCk4XEDi0mVPFRymt8VDrDVAWO4YTidNQp4byHUtwMDrd6WBcWgJj0+KDe4INIEopSizD8ZbXMUIvwDADLW97NIBJIhLRUVsGBZsoL9pP4ek6XAGd0rg8iodMxrQ7sRg6DkvzQduFk9PRemO/sSh49TCMq9FIdNSRXFEY3AxyEJFENEAopfrHH6nfC4VvB2dIK5Mar8nh2MlUp57L1FGZfDEtgfhWul394v1FQNM0Rg9LoLw4l7KagyQX75REJPowpYKDmaX7UBWF1FWfoqzKRakLsrKyyRyeA4nDqbQO4/8O1ZPhdDTcYloukaFUsLiYbun5S8ZKwanPUZ//h4CrCouuw9CxZMy4lKxTMD07Casx+C5bh0wa7uSFQxNJL96Lq3gfMTllED9wBuTb0ycS0RNPPMGvfvUrSkpKmDp1Kr/97W+ZOXNmtMPqG5QK7sde+hmU7cNbV8mpWg9lNR7qvY07PwROF4JZGvy8zsPQ01BhS6PYkoCpGSRaTeJ1HzG4GeXUSLZ4wVuPy+ujst6LxR6DJX4ItoRU7EnpWBLTIG5YcFvkhpZIeZ2XYxX1uH0mFkPDYTFIibORHGfFbmllTMMMQPkh1JF3qSoppKjChd+WwNR5X0dPHYsNmNn3th3rdakJdtIzRnC6ModDpcc457N/ok//RnBbpEEg6ono73//O8uWLeP3v/89s2bN4vHHH2fBggXs37+fYcOGRTu83qVUcDmDpwZqT6Kqj6OdLgBPDX7TZH9JDZVejfKYkZQn5uKxpzAy1UneEJ00Sw3UnYTqY8QHTnJOkpdazzFqXP7grN0GXkBpCdAwI7nO46fwdD1QD5wGDgBgaVhIOmbEMJJTR4AjkerKAPuO1aE0CwoNpekodJSmE+uwMiN3KFkp8eD3UFdbRc2p45gVR3HV1VDl8uEKaJQkTKI0ZQaZ9izSeve72+ddkpfKc6cuIvHYc3x24ACjvOuIGTcPkrLB2v/nTLVFU1HefH7WrFmcf/75/O53vwPANE2ysrL4r//6L+6///5mx3o8HjweT/jr6upqsrKyqKqqIjExsfUXKT8MBZvOqvfyweFyUCYACtBQ4c+dDgvj00PnVHx4pJyAqUApQmfRGj5PcFiYmNF47EdHK/H5G//4VcOxALF2g8mZjbHuLKrA7QvGoCsfugqEw4yxGkwa7gTDihoyhhePJVBEBukp8ZyTkci4tAQc1hZaIn4PVJ8I3nz1eL1eKr0aHmx4dAdpQ4eQkJAEtjiO1yl2HzmFx1VDoPYU1J3C5ikn1ldOjK+KMcPiSI23A1Dj9nGiyo1F1zCVwhdQuLx+vIFgwBMyEnHGBPeXL6l2c/hUcCtlnx5DWdxYylOmMjYrg/NzUgbe5fduUniqjje372RMyWucm2knzhb8Ph2pDlBaZ6J0C0o7swurMS0rCXvD4H5heT0nq9zNHm/66ZQRTmIafm+Kyus50ezY5iZlOomzB489We0hLdHe+ODk68HRxt8dwb9Rp9PZ7t9oVH8bvF4vO3bsYPny5eH7dF1n/vz5vPfee2cdv2rVKlauXNn5F/J7gldpzmDWnqa1LKxMK7gak4lZX4lpNj9ahT9awNP4w1buGsyA2cqxBvisjff7XKiGRBRouPl1O/XWZMy4dCZN/gIk56AZFmYNdzHfYQnWOG6LxQ4pucEbYANaa1sOd8DwIY3VDZVSePwmdR4/KuAj0awC9ynw1pHgd5Pn9wS7WyoQTOLKxOsPUOtyB5OLZoJhw++AOrsNMyGTmNSRTEuOIzsldlCPA3VEztA4vjr3fHYXjiDWdhBOHQBXJX6PC7/L0/oT63wQ6h7X1mHWtp5cqPVCQyJStfWYNa42zusBn6XhWDfoTVpmymzlSZ0X1RbRiRMnGD58OO+++y6zZ88O3/+jH/2IzZs3s23btmbHR9wi8tYFi2lBeLwDNI5Xupvf1eQLu8VgSHxj9j9Z48FUDf9bGo4JftCwGjopcY3HnqrzohqODX9zteATLJpOcqwtfI7Kei8NDQo03YKyxYFuQdM07BZdWg4C/B6qKk9T53IHLy6ENPnTHZZgx2IEf6eq6r3UelrfOTY1wY6t4dhqt48ad+vHDk2wYW/451Hj9ZNga/L76BwBRtv/FPtFi6iz7HY7dru9/QPPZIsLtw6a6symD2mdKIk8tBODr0kDa26e6AkWO86hmXT0V9CZTIePTWy4dURPXlOIajt56NChGIbByZMnm91/8uRJ0tMH33obIQarqCYim83Geeedx6ZNm8L3mabJpk2bmnXVhBADW9S7ZsuWLWPRokXMmDGDmTNn8vjjj1NXV8dtt90W7dCEEL0k6onohhtuoKysjBUrVlBSUsK0adN4/fXXSUuTWSZCDBZRn0fUFR0dkRdCREdH/0ZlUocQIuokEQkhoi7qY0RdEepVVldXRzkSIURLQn+b7Y0A9etEVFNTA0BWVlaUIxFCtKWmpgans/Vplv16sNo0TU6cOEFCQkK/KZoVWpZSVFQ0oAbY5X31L731vpRS1NTUkJmZid5Gzat+3SLSdZ0RI0ZEO4yIJCYmDqhf7BB5X/1Lb7yvtlpCITJYLYSIOklEQoiok0TUy+x2Oz/96U8jqyLQh8n76l/62vvq14PVQoiBQVpEQoiok0QkhIg6SURCiKiTRCSEiDpJRD3swQcf5IILLiA2NpakpKQOPUcpxYoVK8jIyCAmJob58+fz+eef92ygnVReXs4tt9xCYmIiSUlJ3H777dTW1rb5nEsuuQRN05rdvv3tb/dSxK174oknyMnJweFwMGvWLD744IM2j3/++ecZP348DoeDyZMn8+9//7uXIu2czryv/Pz8s342Dkcv7qWmRI9asWKFeuyxx9SyZcuU0+ns0HMeeugh5XQ61UsvvaR27dqlvvKVr6jc3Fzlcrl6NthO+PKXv6ymTp2q3n//ffX222+rMWPGqJtuuqnN58ydO1fdeeedqri4OHyrqqrqpYhb9uyzzyqbzabWrl2r9uzZo+68806VlJSkTp482eLxW7duVYZhqF/+8pdq79696n/+53+U1WpVu3fv7uXI29bZ97Vu3TqVmJjY7GdTUlLSa/FKIuol69at61AiMk1Tpaenq1/96lfh+yorK5Xdbld/+9vfejDCjtu7d68C1Pbt28P3vfbaa0rTNHX8+PFWnzd37lz1/e9/vxci7LiZM2eqpUuXhr8OBAIqMzNTrVq1qsXjr7/+enXFFVc0u2/WrFnqW9/6Vo/G2VmdfV8d/f3sKdI162MOHz5MSUkJ8+fPD9/ndDqZNWtWi5tORsN7771HUlISM2bMCN83f/58dF0/ay+6Mz399NMMHTqUSZMmsXz5curr63s63FaFNvhs+r1ua4NPCL73pscDLFiwoM/8bCCy9wVQW1vLyJEjycrK4uqrr2bPnj29ES7Qzxe9DkQlJSUAZ9XsTktLCz8WbSUlJQwb1nzvWIvFQkpKSpsx3nzzzYwcOZLMzEw++eQT7rvvPvbv38/69et7OuQWnTp1ikAg0OL3et++fS0+p6SkpE//bCCy95WXl8fatWuZMmUKVVVVPPLII1xwwQXs2bOnVxaWS4soAvfff/9ZA3tn3lr7gfdlPf2+7rrrLhYsWMDkyZO55ZZbeOqpp3jxxRcpKCjoxnchIjF79mxuvfVWpk2bxty5c1m/fj2pqan84Q9/6JXXlxZRBP77v/+bxYsXt3nMqFGjIjp3aGPJkydPkpGREb7/5MmTTJs2LaJzdlRH31d6ejqlpaXN7vf7/ZSXl3dqY8xZs2YBcPDgQUaPHt3peLsqkg0+09PT+/yGoN2xcanVamX69OkcPHiwJ0I8iySiCKSmppKamtoj587NzSU9PZ1NmzaFE091dTXbtm3jO9/5To+8ZkhH39fs2bOprKxkx44dnHfeeQC88cYbmKYZTi4dsXPnToBmCbc3Nd3g85prrgEaN/i8++67W3zO7Nmz2bRpE/fcc0/4vo0bN/apDUEjeV9nCgQC7N69m4ULF/ZgpE1EbZh8kDhy5Ij6+OOP1cqVK1V8fLz6+OOP1ccff6xqamrCx+Tl5an169eHv37ooYdUUlKSevnll9Unn3yirr766j55+X769Olq27Zt6p133lFjx45tdvn+2LFjKi8vT23btk0ppdTBgwfVAw88oD788EN1+PBh9fLLL6tRo0apiy++OFpvQSkVvMxtt9tVfn6+2rt3r7rrrrtUUlJS+NL1N7/5TXX//feHj9+6dauyWCzqkUceUZ999pn66U9/2mcv33fmfa1cuVJt2LBBFRQUqB07dqgbb7xRORwOtWfPnl6JVxJRD1u0aJECzrq9+eab4WMAtW7duvDXpmmqn/zkJyotLU3Z7XZ12WWXqf379/d+8G04ffq0uummm1R8fLxKTExUt912W7Pkevjw4Wbv8+jRo+riiy9WKSkpym63qzFjxqgf/vCHUZ9HpJRSv/3tb1V2dray2Wxq5syZ6v333w8/NnfuXLVo0aJmxz/33HNq3LhxymazqYkTJ6pXX321lyPumM68r3vuuSd8bFpamlq4cKH66KOPei1WKQMihIg6uWomhIg6SURCiKiTRCSEiDpJREKIqJNEJISIOklEQoiok0QkhIg6SURCiKiTRCT6lfz8/GYld3/2s581Wwy8ePHi8Poq0X9IIhItWrx4cas1pZcuXYqmac1W6vdEAsjJyeHxxx9vdt8NN9zAgQMHWn3O6tWryc/PD399ySWXNFugKvomSUSiVVlZWTz77LO4XK7wfW63m2eeeYbs7OyoxBQTE3NWUbamnE5nhzcpEH2HJCLRqnPPPZesrKxmFRTXr19PdnY206dP79K5W2qpXHPNNeFW1iWXXMKRI0f4wQ9+EC7KBmd3zc7UtGW2ePFiNm/ezOrVq8PnOHz4MGPGjOGRRx5p9rydO3eiaVqv1d8RzUkiEm1asmQJ69atC3+9du1abrvtth5/3fXr1zNixAgeeOABiouLKS4u7vQ5Vq9ezezZs7nzzjvD58jOzj7rPQGsW7eOiy++mDFjxnTXWxCdIIlItOkb3/gG77zzDkeOHOHIkSNs3bqVb3zjGz3+uikpKRiGQUJCAunp6RFVQHQ6ndhsNmJjY8PnMAyDxYsXs3///vA+Xz6fj2eeeYYlS5Z099sQHSQVGkWbUlNTueKKK8jPz0cpxRVXXMHQoUOjHVaXZGZmcsUVV7B27VpmzpzJP//5TzweD9ddd120Qxu0pEUk2rVkyRLy8/N58sknu63VoOs6Z5bC8vl83XLujrjjjjvCA/Hr1q3jhhtuIDY2ttdeXzQniUi068tf/jJerxefz8eCBQu65ZypqanNxn0CgQCffvpps2NsNhuBQKBLr9PaORYuXEhcXBxr1qzh9ddfl25ZlEnXTLTLMAw+++yz8OetqaqqChfEDxkyZAhZWVlnHXvppZeybNkyXn31VUaPHs1jjz1GZWVls2NycnLYsmULN954I3a7PaIuYU5ODtu2baOwsJD4+HhSUlLQdT08VrR8+XLGjh3bp4rfD0bSIhIdkpiYSGJiYpvHvPXWW0yfPr3ZbeXKlS0eu2TJEhYtWsStt97K3LlzGTVqFPPmzWt2zAMPPEBhYSGjR4+OeNeUe++9F8MwmDBhAqmpqRw9ejT82O23347X6+2Vq4CibVKzWgxab7/9NpdddhlFRUVn7YoqepckIjHoeDweysrKWLRoEenp6Tz99NPRDmnQk66ZGHT+9re/MXLkSCorK/nlL38Z7XAE0iISQvQB0iISQkSdJCIhRNRJIhJCRJ0kIiFE1EkiEkJEnSQiIUTUSSISQkSdJCIhRNT9f6m6ifvzrjFqAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T16:15:23.355932Z", + "iopub.status.busy": "2024-03-26T16:15:23.355325Z", + "iopub.status.idle": "2024-03-26T16:15:23.576390Z", + "shell.execute_reply": "2024-03-26T16:15:23.575455Z" + }, + "papermill": { + "duration": 0.243939, + "end_time": "2024-03-26T16:15:23.578338", + "exception": false, + "start_time": "2024-03-26T16:15:23.334399", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAErCAYAAAChCiRcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA3jElEQVR4nO3dd1gU97oH8C8sdalSBFF0QRQwwRJUDh4RC4gSvSgajRIrRxMTTBRLNOdE4/EmxAh2o09i1BhrooTkEERR6lFERJNjQVSE2MACujRdlt3f/cOzc1lpuwpsmffzPDwyM7+ZfYdx3p3yKwaMMQZCCNFzhpoOgBBC2gMlO0IIL1CyI4TwAiU7QggvULIjhPACJTtCCC9QsiOE8AIlO0IIL1CyI4TwAiU7Qggv6Fyy27p1K0QiEczMzODn54ezZ882Wfbbb79FQEAAOnTogA4dOiAoKKjZ8oQQ/aVTye7QoUOIjo7GypUrcf78efTp0wchISF48OBBo+XT09MxZcoUpKWlITs7G66urhg5ciTu3r3bzpETQjTNQJc6AvDz88OAAQOwZcsWAIBcLoerqyvmz5+PZcuWtbi+TCZDhw4dsGXLFkyfPr2twyWEaBEjTQegqtraWuTl5WH58uXcPENDQwQFBSE7O1ulbdTU1EAqlcLOzq7JMhKJBBKJhJuWy+UoLy+Hvb09DAwMXn4HCCGtjjGGyspKuLi4wNCw+RtVnUl2jx49gkwmg5OTk9J8JycnXL16VaVtfPzxx3BxcUFQUFCTZWJiYrBq1apXipUQ0r5u376NLl26NFtGZ5Ldq/ryyy9x8OBBpKenw8zMrMlyy5cvR3R0NDctFovRtWtXFBUVwcrKqj1C1QipVIq0tDQMGzYMxsbGmg6HvCK+HM/Kykq4ubmpdG7qTLJzcHCAQCDA/fv3lebfv38fzs7Oza4bGxuLL7/8EidOnEDv3r2bLWtqagpTU9MG8+3s7GBtba1+4DpCKpVCKBTC3t5er08OvuDL8VTsmyqPmHTmbayJiQl8fX1x8uRJbp5cLsfJkyfh7+/f5HpfffUVVq9ejeTkZPTv3789QiWEaCGdubIDgOjoaMyYMQP9+/fHwIEDsWHDBlRXV2PWrFkAgOnTp6Nz586IiYkBAKxZswYrVqzA/v37IRKJUFpaCgCwtLSEpaWlxvaDENL+dCrZTZ48GQ8fPsSKFStQWlqKvn37Ijk5mXtpcevWLaU3Mtu2bUNtbS0mTpyotJ2VK1fis88+a8/QCSEaplPJDgCioqIQFRXV6LL09HSl6eLi4rYPiBCiE3TmmR0hhLwKSnaEEF7QudtYQkhDNTU1SpXrq55KcPpiITo4nIOluXJVKi8vLwiFwvYOUeMo2fGUqicHX08MXXP16lX4+vo2mP9VI2Xz8vLwxhtvtH1QWoaSHU+penLw9cTQNV5eXsjLy+OmC0qeIPqni1j3lg88O9k2KMtHlOx4StWTg68nhq4RCoVKX0qGf5bBNOspvF/vg77d7DUYmfagZMdTdHLovqJH1aiW1DW6rPBhNfevkVHTp7mFqRHcHCzaJD5tQ8mOJ5o7MQA6OXRN0aNqDItNb7HcosMXWyyTtngoL44pJTseUPXEAOjk0BWKL64Nk/vCo2PDpo/VTyVITM/GmKH+sDBv2LEFANx4UIUFh35v9ktQn1Cy44GWTgyATg5d5dHREq93tmkwXyqVotQReKNbB73u9UQdlOx4pKkTA6CTQ9dIZM9gaHYXRRUFMDRr+AVWV1eHe3X3kF+e3+RjiaKKKhia3YVE9gxA4/8v9AklOx5o6cQA6OTQNfeq/4SF22Z80sJgeV8nf93scgs34F51X/jCqdly+oCSHQ+oemIAdHLoCheLbqgumo+Nk/uieyOPJurq6nDq36fw18F/bfLLq/BBFT469DtchnVr63C1AiU7HmjpxADo5NA1pgIzyJ91hpu1J3rZN/7MrsioCN523k0+lpA/E0P+7CFMBU0PU6BPKNnxQEsnBkAnB9F/1OsJIYQXKNkRQniBkh0hhBco2RFCeIGSHSGEFyjZEUJ4gZIdIYQXKNkRQniBkh0hhBco2RFCeIGSHSGEFyjZEUJ4gZIdIYQXKNkRQniBkh0hhBco2RFCeIGSHSGEF3Qu2W3duhUikQhmZmbw8/PD2bPND6zw008/wcvLC2ZmZvDx8UFSUlI7RUoI0SY6lewOHTqE6OhorFy5EufPn0efPn0QEhKCBw8eNFr+9OnTmDJlCiIjI3HhwgWMGzcO48aNw6VLl9o5ckKIpulUslu3bh3mzJmDWbNmoVevXti+fTuEQiF27tzZaPmNGzdi1KhRWLJkCby9vbF69Wq88cYb2LJlSztHTgjRNJ0ZcKe2thZ5eXlYvnw5N8/Q0BBBQUHIzs5udJ3s7GxER0crzQsJCUFCQkKTnyORSCCRSLjpiooKAM8HpJFKpa+wB5pT+fT5/vxxqxx1dXWNlql+JsG5h4DDzYewMDNttMyNh9UAno9Epqt/C33R0jHly/FUJ26dSXaPHj2CTCaDk5PyeKVOTk64evVqo+uUlpY2Wr60tLTJz4mJicGqVasazD9+/DiEQuFLRK552fcNAAjw91+utFDSCD/cuNDi9nKz/40/zVslNPKSVDum+n88a2pqVC6rM8muvSxfvlzparCiogKurq4YOXIkrK2tNRjZy/tLdS188h/A3dEC5saCRstcKxVj6c/5+Gq8N3o6Nz7cIgBYmAogsrdoq1CJilo6pnw5noo7L1XoTLJzcHCAQCDA/fv3lebfv38fzs7Oja7j7OysVnkAMDU1halpw8t+Y2PjJsdT1XZOtsaI8HdTqWxPZxv07WbfxhGRV6XqMdX346nOOakzLyhMTEzg6+uLkydPcvPkcjlOnjwJf3//Rtfx9/dXKg8AKSkpTZYnhOgvnbmyA4Do6GjMmDED/fv3x8CBA7FhwwZUV1dj1qxZAIDp06ejc+fOiImJAQB89NFHCAwMRFxcHN58800cPHgQ586dwzfffKPJ3SCEaIBOJbvJkyfj4cOHWLFiBUpLS9G3b18kJydzLyFu3boFQ8P/v1gdNGgQ9u/fj3/84x/45JNP0KNHDyQkJOD111/X1C4QQjREp5IdAERFRSEqKqrRZenp6Q3mvfXWW3jrrbfaOCpCiLbTmWd2hBDyKijZEUJ4gZIdIYQXKNkRQniBkh0hhBco2RFCeIGSHSGEFyjZEUJ4gZIdIYQXKNkRQniBkh0hhBco2RFCeIGSHSGEFyjZEUJ4gZIdIYQXKNkRQniBkh0hhBco2RFCeIGSHSGEFyjZEUJ4gZIdIYQXKNkRQniBkh0hhBco2RFCeIGSHSGEFyjZEUJ4gZIdIYQXKNkRQniBkh0hhBco2RFCeIGSHSGEFyjZEUJ4QWeSXXl5OSIiImBtbQ1bW1tERkaiqqqq2fLz58+Hp6cnzM3N0bVrV3z44YcQi8XtGDUh7a+8vBwTRw7GrY1TMHHkYJSXl2s6JK2gM8kuIiICly9fRkpKChITE5GZmYm5c+c2Wf7evXu4d+8eYmNjcenSJezevRvJycmIjIxsx6h1Q21tLfZ9tw3lKdux77ttqK2t1XRI5CU5OzvD3t4ehdeugj2rROG1q7C3t4ezs7OmQ9M8pgOuXLnCALDc3Fxu3tGjR5mBgQG7e/euytv58ccfmYmJCZNKpSqvIxaLGQAmFovVillXLFmyhAkEAgaA+xEIBGzJkiWaDo2oycnJSek4vvjj5OSk6RBbnTrnp5G6yfHmzZtwd3dvpVSrmuzsbNja2qJ///7cvKCgIBgaGiInJwfjx49XaTtisRjW1tYwMmp6tyUSCSQSCTddUVEBAJBKpZBKpS+5B9qnpqYGUVFR2Lt3b4NlMpkMa9euRUlJCbZs2QKhUKiBCIk67ty5g/v37zdb5v79+0hOToa/v7/eHFN1zkm1k52HhwcCAwMRGRmJiRMnwszMTN1NqK20tBQdO3ZUmmdkZAQ7OzuUlpaqtI1Hjx5h9erVzd76AkBMTAxWrVrVYP7x48f15j8IABQUFDSa6Orbu3cvfHx84Onp2U5RkZc1Z84clcqNHj0acXFx6N69extH1D5qampULqt2sjt//jx27dqF6OhoREVFYfLkyYiMjMTAgQPV3RSWLVuGNWvWNFsmPz9f7e2+qKKiAm+++SZ69eqFzz77rNmyy5cvR3R0tNK6rq6uGDlyJKytrV85Fm1x4cIFlcrV1NQgNDS0jaMhr6r+izcTExOl5671p42NjREZGak3X9yKOy9VGDDG2Mt8SF1dHX799VfuwX/Pnj0xe/ZsTJs2DY6Ojipt4+HDhygrK2u2jLu7O/bu3YtFixbh8ePHSp9vZmaGn376qdnb2MrKSoSEhEAoFCIxMVHtK9GKigrY2Nhwt8D6olevXtwXibm5OZ4+fcotqz/t7e2NK1euaCRGojozMzPu8cuYMWPw8ccf486dO+jSpQvWrFmDxMREAICpqSmePXumyVBblVrn56s+IHz27Blbt24dMzU1ZQYGBszU1JRNmzaN3bt371U3zVG8oDh37hw379ixYy2+oBCLxewvf/kLCwwMZNXV1S/12fr6gkIoFHIPrs3NzZUeZNefFgqFmg6VqMDHx4c7ZuXl5SwlJYVFR0ezlJQUVl5ezi3z8fHRdKitSp3z86WTXW5uLps3bx7r0KED69KlC/v73//Obt68yTIzM9mIESPYgAEDXnbTjRo1ahTr168fy8nJYf/+979Zjx492JQpU7jld+7cYZ6eniwnJ4cx9vyP4Ofnx3x8fNiNGzdYSUkJ91NXV6fy5+prsrOwsOBOgNDQUJaVlcUOHDjAsrKyWGhoKLfMwsJC06ESFUyePLnZN7GKn8mTJ2s61FbVpm9j161bh127dqGgoAChoaHYs2cPQkNDYWj4vMqem5sbdu/eDZFIpO6mm7Vv3z5ERUVhxIgRMDQ0xIQJE7Bp0yZuuVQqRUFBAffA8vz588jJyQHw/KVKfUVFRa0en65xdnZGYWEhAODcuXO4ePEizMzMcPHiRZw7d06pHNF+qr5w0JcXEy9F3Uzq4eHBvvjii2ZvUyUSCdu9e7e6m9ZK+nplx9crAX117NgxlY7nsWPHNB1qq2rTK7uUlBR07dqVu5KrlzRx+/ZtdO3aFSYmJpgxY8ZLJV/SPuhKgPCN2s3FunfvjkePHjWYX15eDjc3t1YJirS94cOHt2o5olkZGRmtWk4fqZ3sWBM1VaqqqtqlgjFpHUOHDuWqCL143BTTHTt2xNChQ9s7NPISrl+/zv1uamqqtKz+8a1fjm9Uvo1VVLQ1MDDAihUrlColymQy5OTkoG/fvq0eIGkbAoEA27dvx4QJE2BgYKC0TPGIYtu2bRAIBJoIj6hJUUlcIBCgY8eOuH37NrfM0dER9+7dg0wmU7kyuT5SOdkp/kiMMVy8eBEmJibcMhMTE/Tp0weLFy9u/QhJmwkPD8eRI0cQHR2NP//8k5vfsWNHxMXFITw8XIPREXUoujuTyWRKiQ6A0nRz3aLpO5WTXVpaGgBg1qxZ2Lhxo161JuCz8PBwhIWFIS0tDUePHsXo0aMxbNgwuqLTMS4uLiq1E3dxcWmHaLST2m9jd+3a1RZxEA0SCAQIDAxEdXU1AgMDKdHpoPHjx+P8+fMqleMrlZJdeHg4du/eDWtr6xZvbeLj41slMEKI6lRtv8znds4qJTsbGxvuIbaNjU2bBkQIUV/9Z66tUU4fqZTs6t+60m0sIdqnfu2I5nqx0ZeunV6G2s/siP6RyWTIyMhAZmYmLCws6AWFDvL19cWJEycAAMOGDUNISAiuX7+OHj164NixY0hKSuLK8ZVKya5fv34N6mI1RZWHpER7xMfHY9GiRSguLgbwvKMHkUhEVU90TP0+JJOSkrjkBkDp3FW1r0l9pFKyGzduXBuHQTQhPj4eEydOxJtvvomFCxdyVwIpKSmYOHEiDh8+TAlPRzTXO039Vk987sXmpXsq5gt97alYJpPBw8MDDg4OePjwodKD627dusHR0RFlZWW4fv063dLqgPT0dAwbNgzA8xYwcrmcW1Z/Oi0tTa+aAKpzflKya4G+Jrv6J8ebb74JNzc3XLt2DT179kRRURF+++03APp3cuir2tpaWFhYwMLCAra2tkpfXiKRCI8fP0Z1dTWqq6uVWj/pOnXOT5VuY+3s7HDt2jU4ODigQ4cOzT6/o9HHdcPdu3cBPO9sNTk5GTKZDMDzUdQEAgHc3NxQVFTElSPa7fTp06irq4NYLEZAQACio6OVHksoxqA4ffo0b7+8VEp269evh5WVFfe7qi8riPZ6+PAhgOe9Nr9IJpNx8xXliHYrKSkB8Hz4y08++YRLbsDzxxJ79+7FO++8w5XjI5WSXf2OOGfOnNlWsZB21KFDh1YtRzSrU6dOAICjR482uBq/c+cO93ZWUY6P1O7PTiAQ4MGDBw3ml5WV0YNsHXLq1Cnud0dHR2zbtg07d+7Etm3blKon1C9HtFdAQACsra2xb9++Bn1OMsawf/9+WFtbIyAgQEMRap7alYqbep8hkUj06sGnvlMkMWNjYwiFQsybN49bJhKJ8OTJE0ilUkp2OkImk6GyshJAw3NUMV1ZWQmZTMbbixKVk51iJC8DAwPs2LEDlpaW3DKZTIbMzEx4eXm1foSkTShODKlU2qC9pKKCcf1yRLtt3ryZS2pmZmYNmovV1NSAMYbNmzdj0aJFmgpTo1ROduvXrwfw/Fti+/btSt8OJiYmEIlE2L59e+tHSNpEz549uU4dDQwMlK4G6tfL6tmzp0biI+rJysoCAAwcOBD//ve/kZGRwfVPGBgYiEGDBuHcuXPIysqiZNcSxdu5YcOGIT4+nh5c67j58+fj5MmTAAAHBwdERESgpqYGQqEQ+/bt497Czp8/X5NhEhVVV1cDAAYPHgxjY2Ol/gmNjY0xePBgnDt3jivHR2o/s1P0WEx0240bN7jfHz58iA0bNrRYjmiv/v3748SJE9i1axe++OILZGVlcR07BAQE4Pvvv+fK8ZXayW727NnNLt+5c+dLB0PaT/3ncq1RjmhWUFAQvvzySzx+/BhCoZB7DLFu3TqlxxJBQUGaDFOj1K568vjxY6WfBw8eIDU1FfHx8Xjy5EkbhEjagmLw63nz5qFz585Ky7p06YL33ntPqRzRbkOHDuWaS9VvF1t/2tramretJ4BWahsrl8sxb948dO/eHUuXLm2NuLSGvraNVbSlNDExgUQi4ZqLAc/rUpqamqK2tlbv2lLqK5lMBnt7e4jF4ibL2NjY6F19WHXOT7Wv7BrdiKEhoqOjuTe2RPuZmJigX79+qKmpgVwuR0REBOLi4hAREQG5XI6amhr069ePEp2OSE9PbzbRAYBYLEZ6enr7BKSFWq2n4sLCQtTV1bXW5kgbq62txYULFyAUCiGRSLBv3z7s27cPwP9f2V24cAG1tbWU8HRAcnIy93tVVRWys7O5qif+/v5cvdjk5GSMGDFCU2FqlNrJLjo6WmmaMYaSkhL89ttvSm1oiXb7+uuvUVdXh23btmH69OnYvHkzUlNTMXz4cMyfPx+7d+/Gu+++i6+//hoLFizQdLikBb/88guA5y8gLCwsGlQ9GT58OFJTU/HLL79g7dq1Go5WM9ROdhcuXFCaNjQ0hKOjI+Li4lp8U0u0R2FhIQBgzJgxMDExwYcffggPDw+EhobC2NgYY8aMUSpHtJtUKgXwvI16Yy8oFF2vKcrxEdWz4ynFW9bExET87W9/a7Bc0UUQvY3VDT4+PiguLsaFCxcQFhaGpUuX4unTpzhz5gy++uor/P7771w53mI6oqysjE2dOpVZWVkxGxsbNnv2bFZZWanSunK5nI0aNYoBYD///LNanysWixkAJhaLXyJq7SWRSJiRkRFzcnJiUqmU1dbWsoSEBFZbW8ukUilzcnJiRkZGTCKRaDpUooLKykoGgAFg5ubm3O8vTqt6zugKdc7PVnkb2x4iIiJw+fJlrtfVzMxMzJ07V6V1N2zYQB2OvsDExAQLFy7E/fv30aVLF+zYsQPl5eXYsWMHunTpgvv372PhwoX0ckJHWFpaYsCAAQCg1AlA/ekBAwYodeDBO+2QfF/ZlStXGACWm5vLzTt69CgzMDBgd+/ebXbdCxcusM6dO7OSkhK6smvEkiVLmJGRkdKVgJGREVuyZImmQyMvYcCAAUrHUvEzYMAATYfWJtQ5P3VikOzs7GzY2toqtesLCgqCoaEhcnJyMH78+EbXq6mpwdSpU7F161aVh5CTSCSQSCTcdEVFBYDnD3b18eHu559/jpUrV2Lr1q3IyMhAYGAgPvjgA5iYmOjl/uq7U6dOoaqqCtOnT8fFixfh4+ODPXv2wNLSUi+Ppzr71GrJ7s6dO/jnP/+Jb775prU2ySktLUXHjh2V5hkZGcHOzg6lpaVNrrdw4UIMGjQIYWFhKn9WTEwMVq1a1WD+8ePHIRQKVQ9ax3h6esLT0xMAuJHlie6q/9IpMzNTg5G0rZqaGpXLtlqyKysrw3fffadWslu2bBnWrFnTbJn8/PyXiufXX39Fampqg6oyLVm+fLlSXcKKigq4urpi5MiRetVc7EVSqRQpKSkIDg6GsbGxpsMhr4gvx1Nx56UKjd7GLlq0qMUBfNzd3eHs7Nxg3Iu6ujqUl5c3eXuampqKwsJC2NraKs2fMGECAgICmmw2Y2pqClNT0wbzjY2N9fo/jQJf9pMv9P14qrNvGk12jo6OSoO7NMXf3x9PnjxBXl4efH19ATxPZnK5HH5+fo2us2zZsgb1x3x8fLB+/XqMHTv21YMnhOgUnXhB4e3tjVGjRmHOnDnYvn07pFIpoqKi8Pbbb8PFxQXA80GfR4wYgT179mDgwIFwdnZu9Kqva9eucHNza+9dIIRomMrJLjw8vNnlbd2X3b59+xAVFYURI0bA0NAQEyZM4AYBAp4/oygoKFDrgSUhhD9UTnY2NjYtLp8+fforB9QUOzs77N+/v8nlIpGoyWEeFVpaTgjRXyonu127drVlHIQQ0qZ0prkYIYS8CpWv7FTtvokG3CGEaCOVk93u3bvRrVs39OvXj559EUJ0jsrJbt68eThw4ACKioowa9YsvPPOO7Czs2vL2AghpNWo/Mxu69atKCkpwdKlS/Gvf/0Lrq6umDRpEo4dO0ZXeoQQrafWCwpTU1NMmTIFKSkpuHLlCl577TW8//77EIlEqKqqaqsYCSHklb3021hDQ0MYGBiAMaY05ighhGgjtZKdRCLBgQMHEBwcjJ49e+LixYvYsmULbt26xe8eUAkhWk/lFxTvv/8+Dh48CFdXV8yePRsHDhyAg4NDW8ZGCCGtRuVkt337dnTt2hXu7u7IyMhARkZGo+Xi4+NbLThCCGktKie76dOn06A1hBCdpValYkII0VXUNpYQwguU7AghvEDJjhA9I5PJkJGRgczMTGRkZFA92P+iZEeIHomPj4eHhweCg4Oxbt06BAcHw8PDg2pJgJIdIXojPj4eEydOhI+PD7KysnDgwAFkZWXBx8cHEydO5H3Co2RH6LZHD8hkMixatAhjxoxBQkIC/Pz8YG5uDj8/PyQkJGDMmDFYvHgxr48tJTueo9se/ZCVlYXi4mJ88sknMDRUPq0NDQ2xfPlyFBUVISsrS0MRap5ODKVI2obitic0NBRjxozBtWvX0LNnTxQVFWHixIk4fPhwi6PKEe1QUlICAHj99dcbXa6YryjHR5TseEpx2+Pu7o7k5GTu9ub48eMQCARwd3fH4sWLERYWBoFAoOFoSUs6deoEALh06RL+8pe/NFh+6dIlpXJ8RLexPKW47SksLISDgwO2b9+OXbt2Yfv27XBwcEBhYSHvb3t0SUBAAEQiEb744gvI5XKlZXK5HDExMXBzc0NAQICGItQ8SnY8dfv2bQBAx44dcefOHcyePRsdOnTA7NmzcefOHXTs2FGpHNFuAoEAcXFxSExMxLhx43DmzBk8ffoUZ86cwbhx45CYmIjY2FheX6VTsuOpnJwcAM9HjTMyUn6aYWRkhJkzZyqVI9ovPDwchw8fxsWLFzFkyBBMmTIFQ4YMwaVLl+j5K+iZHW8pxg3Jy8tr9LbnwoULSuWIbggPD0dYWBjS0tJw9OhRjB49GsOGDeP1FZ0CXdnxVI8ePQAAKSkpjd72pKSkKJUjukMgECAwMBBDhgxBYGAgJbr/MmD01d2siooK2NjYQCwWw9raWtPhtJra2lpYWFjAwsICtra2+PPPP7llIpEIjx8/RnV1Naqrq2FiYqLBSMnLkEqlSEpKQmhoKIyNjTUdTptR5/yk21ieMjExwcKFC7F27VqYmZlhwYIFqKmpgVAoxIEDByAWi7FkyRJKdERvULLjsa+++goAsH79emzYsIGbb2RkhCVLlnDLCdEH9MyO57766itUV1cjNjYWoaGhiI2NRXV1NSU6onfoyo7AxMQEH374ITw8PPT+GQ/hL525sisvL0dERASsra1ha2uLyMhIVFVVtbhednY2hg8fDgsLC1hbW2PIkCF4+vRpO0RMCNEmOpPsIiIicPnyZaSkpCAxMRGZmZmYO3dus+tkZ2dj1KhRGDlyJM6ePYvc3FxERUU16BWCEKL/dOI2Nj8/H8nJycjNzUX//v0BAJs3b+aeMbm4uDS63sKFC/Hhhx9i2bJl3DxPT892iZkQol10ItllZ2fD1taWS3QAEBQUBENDQ+Tk5GD8+PEN1nnw4AFycnIQERGBQYMGobCwEF5eXvj8888xePDgJj9LIpFAIpFw0xUVFQCe11uSSqWtuFfaRbFv+ryPfMKX46nO/ulEsistLeUapisYGRnBzs4OpaWlja5z8+ZNAMBnn32G2NhY9O3bF3v27MGIESNw6dKlJlsGxMTEYNWqVQ3mHz9+HEKh8BX3RPspWk4Q/aDvx7OmpkblshpNdsuWLcOaNWuaLZOfn/9S21a093z33Xcxa9YsAEC/fv1w8uRJ7Ny5EzExMY2ut3z5ckRHR3PTFRUVcHV1xciRI/WqBcWLpFIpUlJSEBwcTG9j9QBfjqfizksVGk12ixYt4nrXaIq7uzucnZ3x4MEDpfl1dXUoLy+Hs7Nzo+spOins1auX0nxvb2/cunWryc8zNTWFqalpg/nGxsZ6/Z9GgS/7qc9kMhlOnz6NzMxMWFhY6HVHAOr8X9VosnN0dISjo2OL5fz9/fHkyRPk5eXB19cXAJCamgq5XA4/P79G1xGJRHBxcUFBQYHS/GvXrmH06NGvHjwhWig+Ph6LFi1CcXExAGDdunUQiUSIi4vjfRdPOlEHw9vbG6NGjcKcOXNw9uxZnDp1ClFRUXj77be5N7F3796Fl5cXzp49CwAwMDDAkiVLsGnTJhw+fBg3btzAp59+iqtXryIyMlKTu0NIm6ChFFvAdERZWRmbMmUKs7S0ZNbW1mzWrFmssrKSW15UVMQAsLS0NKX1YmJiWJcuXZhQKGT+/v4sKytLrc8Vi8UMABOLxa2xG1qrtraWJSQksNraWk2HQl5CXV0dE4lEbOzYsUwmkykdT5lMxsaOHcvc3NxYXV2dpkNtVeqcnzrxNhYA7OzssH///iaXi0SiRjuaXLZsmVI9O0L0kWJMkQMHDsDQ0FBpfFjFUIqDBg1CVlYWhg4dqrlANUgnbmMJIc2joRRbRsmOED1QfyhFmUyGjIwMZGZmIiMjAzKZjIZSBPVU3CJ97an4RXzp2VZfyWQyeHh4wMHBAQ8ePFCqXtW1a1d07NgRZWVluH79ul5VQ1Hn/KQrO0L0gEAgwFtvvYVz587hzp07Ssvu3LmDc+fOYeLEiXqV6NRFyY6gtrYWmzZtwjfffINNmzahtrZW0yERNclkMuzevRsAGlSKNzMzAwB8//33Si8u+IaSHc8tXboUFhYWWLx4MZKSkrB48WJYWFhg6dKlmg6NqCE9PR0PHz6Et7d3g4r6Dg4O8PLywoMHD5Cenq6ZALUAJTseW7p0KdauXQt7e3ssXLgQc+fOxcKFC2Fvb4+1a9dSwtMhiiSWn5+PPn36KFUq7tOnD65evapUjo90pp4daV21tbVYv349bGxsYGZmhvXr13PLunXrBhsbG6xfvx7/+7//SyOM6QBFxxf+/v5ISEiATCZDWVkZ/Pz8kJCQgL/+9a84c+ZMgwHR+YSu7Hjq66+/Rl1dHcRiMXr37q10JdC7d2+IxWLU1dXh66+/1nSoRAV2dnYAgOrq6kaXK+YryvERJTueun79OgAgODgYCQkJ8PPzg7m5OXclEBwcrFSOaDdF7z//+c9/EBYWhjNnzuDp06c4c+YMwsLCcPHiRaVyfES3sTxlYGAAAPD19W20eVG/fv2QkpLClSParXPnzgCeH9eTJ08iMTGRWyYUCmFgYADGGFeOj+jKjqcUXWPt3LkTdXV1Ssvq6uq4agxNdaFFtEtAQABEIhF8fX3h5OSktMzJyQm+vr5wc3NDQECAhiLUPEp2POXq6grg+VgdXbp0wY4dO1BeXo4dO3agS5cuXGepinJEuwkEAsTFxSEvLw+vv/46Nm7ciKioKGzcuBGvvfYa8vLyEBsby+tKxdRcrAX62lxM0bxIIBCguLhY6TbWyMgI3bp1g1wu17vmRfruxc47AcDNzQ2xsbF62XmnOucnPbPjKcWVwMSJExEaGgo3Nzdcu3YNPXv2RFFREZKSknD48GFKdDomPDwcY8aMwebNm5Gamorhw4dj/vz5VH0I0J3OOzVF3zvvPHLkCBOJRAwA9+Pm5saOHDmi6dDIS2jseIpEIr09nuqcn/TMjufCw8Nx48YNpKSkIDo6GikpKbh+/bpe3vLoO+qWvXn0zK4F+vrM7kXUxZNuUzyD9fHx4VpQKI6nQCDAuHHjcOnSJb17BktdPBHCM4pu2T/55BMYGiqf1opu2YuKipCVlaWhCDWPkh0heoC6ZW8ZJTtC9ED9btkbQ92yU7IjRC8oWlB88cUXDXo2kcvliImJoRYUmg6AEPLqFPUmExMTMW7cOKWOAMaNG4fExETet6CgSsWE6Inw8HAcPnwYixYtwpAhQ7j5bm5uOHz4MO+rE1GyI0SPhIeHIywsDGlpaTh69ChGjx6NYcOG8fqKToGSHSF6RiAQIDAwENXV1QgMDKRE91/0zI40OqgyIfqGkh3PxcfHw8PDA8HBwVi3bh2Cg4Ph4eHB+6ZFRP9QsuMxRVvK+/fvK82/f/8+taUkeoeSHU/JZDLMmzcPjDG82DxaMW/evHl0S0v0Br2g4Kn09HSuN+Lhw4eje/fuKCgogKenJwoLC5GUlMQNqjxixAgNR0vIq6Nkx1OpqakAnjcfOn78ODcOxfHjx2FkZIROnTqhpKQEqamplOyIXtCZ29jy8nJERETA2toatra2iIyMRFVVVbPrlJaWYtq0aXB2doaFhQXeeOMNHDlypJ0i1m63bt0C8LxheGPNixQNxhXlCNF1OpPsIiIicPnyZaSkpCAxMRGZmZmYO3dus+tMnz4dBQUF+PXXX3Hx4kWEh4dj0qRJuHDhQjtFrb1cXFy430eNGqXU2eOoUaMaLUd0A1UlapxOJLv8/HwkJydjx44d8PPzw+DBg7F582YcPHgQ9+7da3K906dPY/78+Rg4cCDc3d3xj3/8A7a2tsjLy2vH6LWTWCzmfjc0NOReUjDGlPpDq1+OaL/4+Hh0795dqSpR9+7d6c06dOSZXXZ2NmxtbdG/f39uXlBQEAwNDZGTk4Px48c3ut6gQYNw6NAhvPnmm7C1tcWPP/6IZ8+eYejQoU1+lkQigUQi4aYrKioAPO/JVyqVts4OaYH6XxIvDqpsbm6uVE6f9luf/fzzz5g8ebLS8QOeD5c5YcIEHDp0qMlzRVep839TJ5JdaWkpOnbsqDTPyMgIdnZ2KC0tbXK9H3/8EZMnT4a9vT2MjIwgFArx888/w8PDo8l1YmJisGrVqgbzjx8/DqFQ+PI7oWUUSRxAg0Gy69/2VFRUICkpqd3iIi9HJpNhzpw5ABoeT8X03LlzYWRkpFfNx2pqalQuq9Fkt2zZMqxZs6bZMvn5+S+9/U8//RRPnjzBiRMn4ODggISEBEyaNIkbhKQxy5cvR3R0NDddUVEBV1dXjBw5Uq/GoDA2NkZGRgYsLS3RoUMH3L59m1vm7OyMsrIyVFdXY+nSpQgODtZgpEQVqampSnch9SmmxWIxLCwsMHz48HaPr63U/9JuiUaT3aJFizBz5sxmy7i7u8PZ2ZmrE6ZQV1eH8vJyODs7N7peYWEhtmzZgkuXLuG1114DAPTp0wdZWVnYunUrtm/f3uh6pqamMDU1bTDf2NhYrwaiCQkJgbW1NSoqKiAUCrFgwQLU1NRAKBRi//79qK6uhrW1NUJCQvTqSkBfZWZmcr+bmJhgwYIFcHNzQ1FRETZs2IDa2lquXEhIiKbCbHXqnJMaTXaOjo5wdHRssZy/vz+ePHmCvLw8+Pr6Anj+TSaXy+Hn59foOorL2xcHHxEIBA2qWvCRQCDArl27MGHCBDx8+BAbNmzglhkYGAAAdu3aRYlOR9y8eRPA8+NaWVkJAwMDJCUlITIyEqtXr4a5uTnkcjlXjo904m2st7c3Ro0ahTlz5uDs2bM4deoUoqKi8Pbbb3NVI+7evQsvLy+cPXsWAODl5QUPDw+8++67OHv2LAoLCxEXF4eUlBSMGzdOg3ujPcLDw3HkyBG4uroqze/atSuOHDnC+84edcm1a9cAAE5OTjAyUr6GMTIy4u6AFOX4SCeSHQDs27cPXl5eGDFiBEJDQzF48GB888033HKpVIqCggLuis7Y2BhJSUlwdHTE2LFj0bt3b+zZswfff/89QkNDNbUbWic8PBw3b95UGiS7sLCQEp2OUVyN37t3D2FhYUrdsoeFhXFv3xXl+Egn3sYCgJ2dHfbv39/kcpFI1KBBe48ePajFhAqos0fdN2DAAK7+6ItVierXIhgwYEC7x6YtdObKjhDStHXr1jW5rP5FQHPl9B0lO0L0gLm5OcLCwgAAT58+VVqmmA4LC2tQ4ZhPKNkRoicSEhK4hPeisLAwJCQktG9AWoaSHSF6JCEhATU1NXjvvffQt29fvPfee6ipqeF9ogN06AUFIUQ15ubm2LRpE5KSkhAaGqpXleFfBV3ZEUJ4gZIdIYQXKNkRQniBntm1QFFHSZ3eFXSRVCpFTU0NKioq6BmPHuDL8VScly82KGgMJbsWVFZWAkCD9qOEEO1RWVkJGxubZssYMFVSIo/J5XLcu3cPVlZWet2uUNFv3+3bt/Wq3z6+4svxZIyhsrISLi4uDXo4ehFd2bXA0NAQXbp00XQY7cba2lqvTw6+4cPxbOmKToFeUBBCeIGSHSGEFyjZEQDPu6NfuXJlo13SE91Dx7MhekFBCOEFurIjhPACJTtCCC9QsiOE8AIlOx0ydOhQLFiwQNNhEKKTKNnpofT0dBgYGODJkyeaDoWoQdu+zLQtnldFyY4QPVJbW6vpELQWJTsdJZFI8PHHH8PV1RWmpqbw8PDAd999h+LiYgwbNgwA0KFDBxgYGGDmzJktbq+yshIRERGwsLBAp06dsH79+gbf7D/88AP69+8PKysrODs7Y+rUqXjw4AG3XHFFefLkSfTv3x9CoRCDBg1CQUFBa+++3pk5cyYyMjKwceNGGBgYwMDAAIWFhYiMjISbmxvMzc3h6emJjRs3Nlhv3Lhx+Pzzz+Hi4gJPT08AwOnTp9G3b1+YmZmhf//+SEhIgIGBAX7//Xdu3UuXLmH06NGwtLSEk5MTpk2bhkePHjUZT3FxcXv9OdoGIzojMDCQffTRR4wxxiZNmsRcXV1ZfHw8KywsZCdOnGAHDx5kdXV17MiRIwwAKygoYCUlJezJkyctbvtvf/sb69atGztx4gS7ePEiGz9+PLOysuI+jzHGvvvuO5aUlMQKCwtZdnY28/f3Z6NHj+aWp6WlMQDMz8+Ppaens8uXL7OAgAA2aNCg1v5T6J0nT54wf39/NmfOHFZSUsJKSkrYs2fP2IoVK1hubi67efMm27t3LxMKhezQoUPcejNmzGCWlpZs2rRp7NKlS+zSpUtMLBYzOzs79s4777DLly+zpKQk1rNnTwaAXbhwgTHG2OPHj5mjoyNbvnw5y8/PZ+fPn2fBwcFs2LBhTcZTV1eniT9Nq6Fkp0MUya6goIABYCkpKY2WUySdx48fq7TdiooKZmxszH766Sdu3pMnT5hQKFRKdi/Kzc1lAFhlZaXS5544cYIr89tvvzEA7OnTpyrFwmf1v8ya8sEHH7AJEyZw0zNmzGBOTk5MIpFw87Zt28bs7e2V/ubffvutUrJbvXo1GzlypNK2b9++zX1JqhqPLqHbWB30+++/QyAQIDAwsFW2d/PmTUilUgwcOJCbZ2Njw90SKeTl5WHs2LHo2rUrrKysuM+/deuWUrnevXtzv3fq1AkAlG53ieq2bt0KX19fODo6wtLSEt98802Dv7ePjw9MTEy46YKCAvTu3RtmZmbcvPrHFgD++OMPpKWlwdLSkvvx8vICABQWFrbhHmkOdfGkgzQx0HF1dTVCQkIQEhKCffv2wdHREbdu3UJISEiDh+L1e8ZV9AEol8vbNV59cPDgQSxevBhxcXHw9/eHlZUV1q5di5ycHKVyFhYWam+7qqoKY8eOxZo1axosU3xB6RtKdjrIx8cHcrkcGRkZCAoKarBc8S0vk8lU2p67uzuMjY2Rm5uLrl27AgDEYjGuXbuGIUOGAACuXr2KsrIyfPnll1yvzefOnWuN3SH/ZWJionTMTp06hUGDBuH999/n5qly1eXp6Ym9e/dCIpFwHQHk5uYqlXnjjTdw5MgRiEQiGBk1ngZejEfX0W2sDhKJRJgxYwZmz56NhIQEFBUVIT09HT/++CMAoFu3bjAwMEBiYiIePnyIqqqqZrdnZWWFGTNmYMmSJUhLS8Ply5cRGRkJQ0ND7sqsa9euMDExwebNm3Hz5k38+uuvWL16dZvvK5+IRCLk5OSguLgYjx49Qo8ePXDu3DkcO3YM165dw6efftogaTVm6tSpkMvlmDt3LvLz83Hs2DHExsYC+P8r7Q8++ADl5eWYMmUKcnNzUVhYiGPHjmHWrFlcgnsxHl2/Oqdkp6O2bduGiRMn4v3334eXlxfmzJmD6upqAEDnzp2xatUqLFu2DE5OToiKimpxe+vWrYO/vz/GjBmDoKAg/PWvf4W3tzf33MfR0RG7d+/GTz/9hF69euHLL7/kTiDSOhYvXgyBQIBevXrB0dERISEhCA8Px+TJk+Hn54eysjKlq7ymWFtb41//+hd+//139O3bF3//+9+xYsUKAOCOp4uLC06dOgWZTIaRI0fCx8cHCxYsgK2tLde9+YvxvPisUNdQF0+kUdXV1ejcuTPi4uIQGRmp6XDIK9q3bx9mzZoFsViskWe+2oCe2REAwIULF3D16lUMHDgQYrEY//znPwEAYWFhGo6MvIw9e/bA3d0dnTt3xh9//IGPP/4YkyZN4m2iAyjZ8cKtW7fQq1evJpdfuXIFABAbG4uCggKYmJjA19cXWVlZcHBwaK8wSSsqLS3FihUrUFpaik6dOuGtt97C559/rumwNIpuY3mgrq6u2aY+zb2RI0RfULIjhPACvY0lhPACJTtCCC9QsiOE8AIlO0IIL1CyI4TwAiU7ojVmzpzJ9YprbGwMJycnBAcHY+fOnWq1y9y9ezdsbW3bLtAmKHoNJtqJkh3RKqNGjUJJSQmKi4tx9OhRDBs2DB999BHGjBmDuro6TYdHdJkmew4lpL4ZM2awsLCwBvNPnjzJALBvv/2WMcZYXFwce/3115lQKGRdunRh8+bNa9Bbcv2flStXMsYY27NnD/P19WWWlpbMycmJTZkyhd2/f5/7nPLycjZ16lTm4ODAzMzMmIeHB9u5cye3/NatW+ytt95iNjY2rEOHDux//ud/WFFREWOMsZUrVzb43LS0tDb5O5GXQ1d2ROsNHz4cffr0QXx8PADA0NAQmzZtwuXLl/H9998jNTUVS5cuBQAMGjQIGzZsgLW1NUpKSlBSUoLFixcDAKRSKVavXo0//vgDCQkJKC4uVhqM6NNPP8WVK1dw9OhR5OfnY9u2bVxzOalUipCQEFhZWSErKwunTp2CpaUlRo0ahdraWixevBiTJk3irkxLSkowaNCg9v1DkeZpOtsSotDUlR1jjE2ePJl5e3s3uuynn35i9vb23PSuXbuYjY1Ni5/34hgaY8eOZbNmzWq07A8//MA8PT2ZXC7n5kkkEmZubs6OHTvWYvxE8+jKjugExhjX8eSJEycwYsQIdO7cGVZWVpg2bRrKyspQU1PT7DZaGkNj3rx5OHjwIPr27YulS5fi9OnT3Lp//PEHbty4ASsrK27MBjs7Ozx79kxvx2zQN5TsiE7Iz8+Hm5sbiouLMWbMGPTu3RtHjhxBXl4etm7dCqD5AaIVY2hYW1tj3759yM3Nxc8//6y03ujRo/Hnn39i4cKFuHfvHkaMGMHdAldVVcHX1xe///670s+1a9cwderUNt570hqoqwui9VJTU3Hx4kUsXLgQeXl5kMvliIuL43rUVXRHr9DY2AmqjqHh6OiIGTNmYMaMGQgICMCSJUsQGxuLN954A4cOHULHjh1hbW3daJz6NmaDvqErO6JVJBIJSktLcffuXZw/fx5ffPEFwsLCMGbMGEyfPh0eHh6QSqXcWBg//PADtm/frrQNkUiEqqoqnDx5Eo8ePUJNTY1KY2isWLECv/zyC27cuIHLly8jMTER3t7eAICIiAg4ODggLCwMWVlZ3LgfH374Ie7cucN97n/+8x8UFBTg0aNHkEql7fNHI6rR9ENDQhRmzJjBVdswMjJijo6OLCgoiO3cuZPJZDKu3Lp161inTp2Yubk5CwkJYXv27GkwKPh7773H7O3tlaqe7N+/n4lEImZqasr8/f3Zr7/+2mDgaG9vb2Zubs7s7OxYWFgYu3nzJrfNkpISNn36dObg4MBMTU2Zu7s7mzNnDhOLxYwxxh48eMCCg4OZpaUlVT3RQtSfHSGEF+g2lhDCC5TsCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbxAyY4Qwgv/B7L8PO5Jo9oLAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T16:15:23.618757Z", + "iopub.status.busy": "2024-03-26T16:15:23.618455Z", + "iopub.status.idle": "2024-03-26T16:15:23.890191Z", + "shell.execute_reply": "2024-03-26T16:15:23.889351Z" + }, + "papermill": { + "duration": 0.294565, + "end_time": "2024-03-26T16:15:23.892255", + "exception": false, + "start_time": "2024-03-26T16:15:23.597690", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEpCAYAAADCh6TFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABbJ0lEQVR4nO2dd3xTdffH3zdp0kUXFFrAQhnKkKVACy5Ei6g4cDwCIqMiDkR9rAtQQRQtKiCPiiDIElFA5VF/gij2ARRB0DJlj0LLaOmge2Td3x9pbpM2bZM0bTq+79crL5qb773329B8cs75nu85kizLMgKBQNDAUHl6AgKBQOAKQrwEAkGDRIiXQCBokAjxEggEDRIhXgKBoEEixEsgEDRIhHgJBIIGiRAvgUDQIBHiJRAIGiRCvAQCQYOkXojXggULiIyMxMfHh+joaHbv3l3p2JtvvhlJkio8hg0bVoczFggEnsbL0xNYu3YtcXFxLFq0iOjoaObPn8/QoUM5duwYrVq1qjB+/fr16HQ65XlmZia9e/fmX//6l0P3M5lMXLhwgYCAACRJctvvIRAI3IMsy+Tl5dGmTRtUqirsK9nDREVFyU8//bTy3Gg0ym3atJHj4+MdOv+DDz6QAwIC5Pz8fIfGp6SkyIB4iId41PNHSkpKlZ9lj1peOp2OxMREpk6dqhxTqVTExMSwc+dOh66xdOlSRo4cib+/v93XS0pKKCkpUZ7LpUU0UlJSCAwMrMHsBQJBbZCbm0tERAQBAQFVjvOoeGVkZGA0GgkLC7M5HhYWxtGjR6s9f/fu3fzzzz8sXbq00jHx8fHMnDmzwvHAwEAhXgJBPaa6sE69CNi7ytKlS+nZsydRUVGVjpk6dSo5OTnKIyUlpQ5nKBAIaguPWl6hoaGo1WrS0tJsjqelpREeHl7luQUFBaxZs4Y333yzynHe3t54e3vXeK4CgaB+4VHLS6vV0rdvXxISEpRjJpOJhIQEBg4cWOW5X3/9NSUlJTzyyCO1PU2BQFAP8XiqRFxcHOPGjaNfv35ERUUxf/58CgoKiI2NBWDs2LG0bduW+Ph4m/OWLl3K8OHDadGihSemLagEWZYxGAwYjUZPT0VQT1Gr1Xh5edU4Vcnj4jVixAjS09OZPn06qamp9OnTh02bNilB/OTk5Aq5HseOHWP79u388ssvnpiyoBJ0Oh0XL16ksLDQ01MR1HP8/Pxo3bo1Wq3W5WtIsty0GnDk5uYSFBRETk6OWG10IyaTiRMnTqBWq2nZsiVarVYkAQsqIMsyOp2O9PR0jEYjV155ZQXjxNHPqMctr/rKgXPZvP79IQJ9vFg1IdrT06n36HQ6TCYTERER+Pn5eXo6gnqMr68vGo2Gs2fPotPp8PHxcek6QrwqwdtLzf6UbAJ8vJBlWVgRDlLldg6BoBR3/J2Iv7RKaN/CbD3kFRvIKtBVM1ogENQ1QrwqwUejpk2Q2Zw9k1ng4dkIBILyCPGqgshQ837JpAyxetaYufnmm/n3v//t6Wk0GrZu3YokSWRnZ9fqfYR4VYFFvM5kCMtLUHcfSoFjCPGqgo4Wy0u4jYImhnXNvPqKEK8qiGxRKl7pQrxcQZZlCnUGjzxcTV8sKSnhlVdeISIiAm9vbzp37szSpUs5c+YMgwcPBiAkJARJkhg/fny118vLy2P06NH4+/vTunVrPvjggwpu6qpVq+jXrx8BAQGEh4fz8MMPc+nSJeV1i8WXkJBAv3798PPz47rrruPYsWMO/16zZs2iVatWBAQE8NhjjzFlyhT69OmjvD5+/HiGDx/O22+/TZs2bejSpYtDcwPYuHEjV111Fb6+vgwePJgzZ844PK+aIFIlqkBxGzMLRLqECxTpjXSf/rNH7n34zaH4aZ3/8x47diw7d+7kww8/pHfv3iQlJZGRkUFERATffvstDzzwAMeOHSMwMBBfX99qrxcXF8cff/zBDz/8QFhYGNOnT2fPnj02wqHX63nrrbfo0qULly5dIi4ujvHjx7Nx40aba7366qvMnTuXli1b8uSTT/Loo4/yxx9/VDuH1atX8/bbb/PJJ59w/fXXs2bNGubOnUuHDh1sxiUkJBAYGMjmzZsdnltKSgr3338/Tz/9NI8//jh///03L7zwQrVzcgdCvKqgXXM/VBIU6oyk55XQKtC1ZDpBw+D48eOsW7eOzZs3ExMTA0DHjh2V15s3bw5Aq1atCA4OrvZ6eXl5rFy5ki+//JJbb70VgOXLl9OmTRubcY8++qjyc8eOHfnwww/p378/+fn5NGvWTHnt7bffZtCgQQBMmTKFYcOGUVxcXG2S50cffcSECROU/cLTp0/nl19+IT8/32acv78/n332mc2WnermtnDhQjp16sTcuXMB6NKlCwcPHuTdd9+t9v2pKUK8qkDrpaJtiC8pWUUkZRQI8XISX42aw28O9di9nWXfvn2o1WpFIGrK6dOn0ev1NvXmgoKCFJfMQmJiIm+88Qb79+/n8uXLmEwmwLyvt3v37sq4Xr16KT+3bt0agEuXLtGuXbsq53Hs2DEmTZpkcywqKor//e9/Nsd69uxZYa9hdXM7cuQI0dG2O1CqqwjjLoR4VUNkC39Ssoo4k1lAdEdRwcIZJElyyXXzFI64ge6moKCAoUOHMnToUFavXk3Lli1JTk5m6NChFYLmGo1G+dkSwrCIiTsoX0rdmbl5AhGwr4YOIterydCzZ09MJhPbtm2z+7rFKnG03E/Hjh3RaDT89ddfyrGcnByOHz+uPD969CiZmZnMnj2bG2+8ka5du1YIiNeULl262MwBqPDcHo7MrVu3bhVaFf755581n7QDCPGqBsuKo8j1avxERkYybtw4Hn30Ub777juSkpLYunUr69atA6B9+/ZIksSPP/5Ienp6hZhReQICAhg3bhwvvfQSW7Zs4dChQ0yYMAGVSqVYTu3atUOr1fLRRx9x+vRpfvjhB9566y23/l7PPPMMS5cuZeXKlZw4cYJZs2Zx4MCBahegHJnbk08+yYkTJ3jppZc4duwYX375JStWrHDr/CtDiFc1dLBacRQ0fhYuXMiDDz7IpEmT6Nq1KxMnTqSgwPx/37ZtW2bOnMmUKVMICwtj8uTJ1V5v3rx5DBw4kLvuuouYmBiuv/56unXrpgTZW7ZsyYoVK/j666/p3r07s2fPZs6cOW79nUaPHs3UqVN58cUXufbaa0lKSmL8+PHVBvodmVu7du349ttv+e677+jduzeLFi3inXfecev8K0PU86qGpIwCBs/Zio9GxeGZt6NSiXQJexQXF5OUlESHDh1cLnHSFCgoKKBt27bMnTuXCRMmeGweQ4YMITw8nFWrVnnk/lX9vYh6Xm7iihBf1CqJYr2JtLxiWgfVfVBX0HDZu3cvR48eJSoqipycHKVhzL333ltncygsLGTRokUMHToUtVrNV199xa+//mqTz9UQEW5jNWjUKiJCzIKVJOJeAiuSk5Np1qxZpY/k5GQA5syZQ+/evYmJiaGgoIDff/+d0NBQt83j6quvrnQOq1evRpIkNm7cyE033UTfvn35v//7P7799lsll62hIiwvB+gQ6s+ZzELOZBRyXSdPz0ZQX2jTpg379u2r8vV27dqRmJhYq/PYuHEjer3e7mthYWH4+vry66+/1uocPIEQLweIDPWHY+kkZVS9uiRoWnh5edG5c2dPT4P27dt7egoeweNu44IFC4iMjMTHx4fo6OgKOSPlyc7O5umnn6Z169Z4e3tz1VVXVdgD5m5ErpdAUP/wqOW1du1a4uLiWLRoEdHR0cyfP5+hQ4dy7NgxWrVqVWG8TqdjyJAhtGrVim+++Ya2bdty9uxZh/aZ1QQl10ukSwgE9QaPite8efOYOHGismF00aJFbNiwgWXLljFlypQK45ctW0ZWVhY7duxQtkpERkbW+jwtlldyZiFGk4xapEsIBB7HY26jTqcjMTHRZsVDpVIRExPDzp077Z7zww8/MHDgQJ5++mnCwsLo0aMH77zzTpXbNUpKSsjNzbV5OEubYF+0ahU6o4kL2UVOn+9O0nKLuWXuVj77/bRH5yEQeBqPiVdGRgZGo1HpjG0hLCyM1NRUu+ecPn2ab775BqPRyMaNG3n99deZO3cus2bNqvQ+8fHxBAUFKY+IiAin56pWSUQ0N6dLeNp13HEqg9PpBXy/74JH5yEQeBqPB+ydwWQy0apVKxYvXkzfvn0ZMWIEr776KosWLar0nKlTp5KTk6M8UlJSXLp3h3pSzz4z37ybP7fY/tK4oO6JjIxk/vz5ynNJkvjuu+88Np+mgsdiXqGhoajVatLS0myOp6WlER4ebvec1q1bo9FoUKvLajV169aN1NRUdDpdhVpEAN7e3nh7e9d4vkpJaA+vOKbnlwCQWyTEq75y8eJFQkJCPD2NRo/HLC+tVkvfvn1JSEhQjplMJhISEiotZnb99ddz8uRJmxpGx48fp3Xr1naFy51E1pMN2mWWl+t12gW1S3h4uFu+MGuCLMsYDAaPzqG28ajbGBcXx5IlS1i5ciVHjhzhqaeeoqCgQFl9HDt2LFOnTlXGP/XUU2RlZfHcc89x/PhxNmzYwDvvvMPTTz9d63OtP26j2fIymmQKdI7VlfIYsgy6As88nBR2RxplOIq123jmzBkkSWL9+vUMHjwYPz8/evfuXWFRavv27dx44434+voSERHBs88+q1SzAMebdPz000/07dsXb29vtm/f7vTcGxIeTZUYMWIE6enpTJ8+ndTUVPr06cOmTZuUIH5ycjIqVZm+RkRE8PPPP/P888/Tq1cv2rZty3PPPccrr7xS63NV0iWyCjEYTXipPaP7GfllFSxzi/Q0867HmyT0hfBOm+rH1QbTLoDWv/pxpTjSKKMmvPrqq8yZM4crr7ySV199lVGjRnHy5Em8vLw4deoUt99+O7NmzWLZsmWkp6czefJkJk+ezPLlywHHm3RMmTKFOXPm0LFjx0bvunr8L9/yn2SPrVu3Vjg2cODAOqvUaE14oA/eXipKDCbOXS5S3Mi6xmJ5gTlo3wZR5aKmONoooya8+OKLDBs2DICZM2dy9dVXc/LkSbp27Up8fDyjR49WrLwrr7ySDz/8kEGDBrFw4UJ8fHwcbtLx5ptvMmTIELfNuz7jcfFqKKhUEpEt/DmWlkdSZoFHxEuWZTIKrC2veh7T0PiZLSBP3dtBHG2UURMqa57RtWtX9u/fz4EDB1i9erUyRpZlTCYTSUlJdOvWzeEmHf369XPbnOs7QrycIDLUj2Npeea4l/v+rh0mr8SAzlC2WJFT31ccJckp160xU1XzjPz8fJ544gmeffbZCue1a9fOqUYY5ZtoNGaEeDlBpIeD9pn5tn+oIl3CPVg3yrC0EbM0yrjppptq/f7XXnsthw8frrRCxcGDB5VGGJYk67///rvW51XfaVBJqp6mgyXXK9MzuV4ZVvEuEImq7sKRRhm1ySuvvMKOHTuYPHky+/bt48SJE3z//fdKLLgumnQ0RIR4OYHnLa9y4lXfY14NiOoaZdQmvXr1Ytu2bRw/fpwbb7yRa665hunTpysLBnXRpKMhItxGJ7CkS5y7XIjOYELrVbfan1HObaz3Ma8GREBAgE3AvKCggJkzZ/L4449Xe+6ZM2dsnlsnD0dGRlZIJg4ODq5wrH///vzyyy+V3mPUqFGMGjWq0vvcfPPNTS5pWYiXE7QK8MZPq6ZQZyTlciGdWjar/iQ3ItzG2qM+NMoQOIdwG51AkiTae7AJrSVg38LfvBVKBOzdi71GGUeOHKmyyYbAcwjLy0k6hPpx5GKuRzoJZRaUlM7Bn8wCnXAb3cg111xjt1FGUVFRlU02BJ5DiJeTeLKDdkae2fLq2NKfv89eJrdYBOxrG19f33rRZENQEeE2OklZaRwPiFep5dWxNNZWH93GphY0FriGO/5OhHg5SVl1ibrP9bLEvDqWzqE+BewtGeSFhaLDkqB6LH8n1jsPnEW4jU5iyfW6kFNEsd6Ij0ZdzRnuQWcwKTGuji3Nc8grNtSbhiBqtZrg4GClTIufn1+dJHgKGhayLFNYWMilS5cIDg62KSzqLEK8nKSFv5YAby/ySgwkZxVyVVhAndw3q3RDtlolcUVI2abj/GIDQX6uf3u5E0sFXOs6UwKBPYKDgyutmOwoQrycRJIkIkP9OXg+h6SMgjoTL0uOVwt/LT4aNb4aNUV6I7nF+nojXpIk0bp1a1q1alVp+3mBoHwpd1cR4uUCFvGqy1wvRbyamcsLB/p6UaQ3klOkx/l+SLWLWq12yx+nQFAVImDvAh1amN22ukyXsATrQ5uZE1QDfczWVn1ccRQI6gIhXi5gCdrXZbqExfIKVSyvUvGqRyuOAkFdIsTLBSI9kC6RWWC7NSjIIl6isoSgiSLEywUsdb1Sc4spqqMOPhViXj7mcKXYIiRoqtQL8VqwYAGRkZH4+PgQHR3N7t27Kx27YsUKJEmyedRFzSVrQvy1iuVTV3GvjPIxL+E2Cpo4HhevtWvXEhcXx4wZM9izZw+9e/dm6NChVeYKBQYGcvHiReVx9uzZOpyxmbru45hZPuYlAvaCJo7HxWvevHlMnDiR2NhYunfvzqJFi/Dz82PZsmWVniNJEuHh4crD0uexLrGI1+k6Ey+L5WUWLyXmJTZnC5ooHhUvnU5HYmIiMTExyjGVSkVMTEyFjsLW5Ofn0759eyIiIrj33ns5dOhQpWNLSkrIzc21ebiDyDqs6yXLslIOp4XiNoqYl6Bp41HxysjIwGg0VrCcwsLCSE1NtXtOly5dWLZsGd9//z1ffPEFJpOJ6667jnPnztkdHx8fT1BQkPKwdF+pKZGhdZfrlVtkQG8078Jv7i/yvAQCqAduo7MMHDiQsWPH0qdPHwYNGsT69etp2bIln376qd3xU6dOJScnR3mkpKS4ZR4dlFyv2k+XsJTCCfDxUjaCB4mAvaCJU6PtQfn5+UrjTAuBgYEOnx8aGoparSYtLc3meFpamsObNjUaDddccw0nT560+7q3tzfe3t4Oz8lRLLleGfkl5BXrCfCpvf2FGXm2wXqwWm0UeV6CJorTlldSUhLDhg3D39+foKAgQkJCCAkJITg4mJCQEKeupdVq6du3LwkJCcoxk8lEQkICAwcOdOgaRqORgwcPKi3U64pAH42SMHq2lvs4lk9QtdwfRMxL0HRx2vJ65JFHkGWZZcuWERYWVuOaTXFxcYwbN45+/foRFRXF/PnzKSgoIDY2FoCxY8fStm1b4uPjAXjzzTcZMGAAnTt3Jjs7m/fff5+zZ8/y2GOP1WgerhBZWks+KaOAHm2Dau0+5bcGQVnAvkhv9EgbNoHA0zgtXvv37ycxMZEuXbq4ZQIjRowgPT2d6dOnk5qaSp8+fdi0aZMSxE9OTkalKvtgXr58mYkTJ5KamkpISAh9+/Zlx44ddO/e3S3zcYbIFv4knr1c6yuOlgRVy0ojYOOm5hXrlcx7gaCp4LR49e/fn5SUFLeJF8DkyZOV1ubl2bp1q83zDz74gA8++MBt964JHUpXHJNqecUxs9zWIDAXJbQURcwpEuIlaHo4LV6fffYZTz75JOfPn6dHjx4ValD36tXLbZOr70TWUZa9xW1saWV5gTlon1diEImqgiaJ0+KVnp7OqVOnlJgUmDPeZVlGkiSMxrrZqFwfKGuDVssBe8VttLWuAko3Z4tcL0FTxGnxevTRR7nmmmv46quv3BKwb8hYsuyzCnTkFNZeOWbLamNoOfESuV6CpozT4nX27Fl++OEH0YgT8Pf2olWAN5fySkjKLKCPX3Ct3MeS59XCjtsIIl1C0DRxen39lltuYf/+/bUxlwZJbce9ivVG8krMMa1Qf1vLq2yLkIh5CZoeTlted999N88//zwHDx6kZ8+eFQL299xzj9sm1xDo0MKf3UlZtVYS2tLyTKOWlNwuC5bnwm0UNEWcFq8nn3wSMCeLlqepBezByvKqpXSJspZn3hXii2WloIV4Ceo3vx5O49CFXJ4e3AkvtXsSqp0Wr/J7GZs6llyv2nIbM+0kqFoQW4QEDYG03GJe+mY/lwv1BPp6EXt9B7dc1ykJ1Ov1eHl58c8//7jl5o0B605Csiy7/fr2tgZZCBQFCQX1HJNJJm7dPi4X6rm6TSAPR7dz27WdEi+NRkO7du2anGtYFe2bm8Urt9jA5UL3W0D2tgZZEG6joL6z+PfT/HEyE1+Nmg9HXYO3l/uaETvtfL766qtMmzaNrKwst02iIeOrVdM6yNwApDaC9uVr11sTKJJUBfWY/SnZzPn5GAAz7u5Op5bN3Hp9p2NeH3/8MSdPnqRNmza0b98ef39/m9f37Nnjtsk1FCJb+HMxp5gzGQX0be9cWaDqKHMb7cS8RJKqoJ6SklXIpNV7MJhk7ugRzoj+7qlgbI3T4jV8+HC3T6Kh06GlPztPZ9bKimNZLa8qYl5FBmV7lkDgac5mFvDwkl2czy4isoUfs+/vVSt/m06L14wZM9w+iYaOpQltbbiNSr/GgIriZYl56YwmSgwmpUS0QOApkjIKGLX4T1Jzi+kY6s+XEwfU2rY5l8tAJyYmcuTIEQCuvvpqrrnmGrdNqqFhveLobsryvCq6jf5aNSoJTLI5XUKIl8CTnLyUz8NL/uRSXgmdWzXjy4nRtAqovYbQTovXpUuXGDlyJFu3biU4OBiA7OxsBg8ezJo1a2jZsqW751jvsc71cqf7ZjLJSoa9vYC9JEkE+mrILtSTW6QnLLBuO4cLBBaOp+Xx8JJdZOSX0CUsgNUTo+3+zboTp1cbn3nmGfLy8jh06BBZWVlkZWXxzz//kJuby7PPPlsbc6z3RDT3QyVBgc5Ieqml5A5yivQYTbYtz8qj7G8UQXuBhzhyMZdRi/8kI7+Ebq0D+erxAbUuXOCC5bVp0yZ+/fVXunXrphzr3r07CxYs4LbbbnPr5BoK3l5q2gT7cu5yEWcyCt1mKltcxiBfTaU16oNEFyGBBzl0IYdHPtvF5UI9PdoG8sWEaIL97H/RuhunLS+TyVRhMzaYE1ib8tahDrVQXaKqBFULonO2wFMcOJfNw0vMwtX7iiBWTxhQZ8IFLpbEee6557hw4YJy7Pz58zz//PPceuutbp1cQ8JSmNCd9ewzCypPULUg3EaBJ9ibfJnRn+0ip0jPte2CWfVYdK2tKlaG0+L18ccfk5ubS2RkJJ06daJTp0506NCB3NxcPvroo9qYY4OgNup6lTWbrfzbTGwREtQ1f5/JYszS3eQVG4iKbM7nE6KVL9G6xGnxioiIYM+ePWzYsIF///vf/Pvf/2bjxo3s2bOHK664wqVJLFiwgMjISHx8fIiOjmb37t0OnbdmzRokSaoXibNKJyE3ildVCaoWRDVVQV2y63QmY5ftJr/EwICOzVnxaH+aebuccVUjXLqrJEkMGTKEIUOG1HgCa9euJS4ujkWLFhEdHc38+fMZOnQox44do1WrVpWed+bMGV588UVuvPHGGs/BHVjcxrOZhW5Ll6iqooSFsv2NImAvqF12nMxgwsq/KdIbuaFzKEvG9sNX67ncQpfEKyEhgYSEBC5dulQhSL9s2TKnrjVv3jwmTpyodCNatGgRGzZsYNmyZUyZMsXuOUajkdGjRzNz5kx+//13srOzXfk13EpEcz/UKokivZG03BLCg2q+4uhYwF7EvAS1z+8n0nls5d+UGEwMuqoln47p6/GkaKfdxpkzZ3LbbbeRkJBARkYGly9ftnk4g06nIzExkZiYmLIJqVTExMSwc+fOSs978803adWqFRMmTKj2HiUlJeTm5to8agONWkVEiC/gPtexqooSFkQHIUFts+XYJSaUCtctXVvVC+ECFyyvRYsWsWLFCsaMGVPjm2dkZGA0GgkLC7M5HhYWxtGjR+2es337dpYuXcq+ffscukd8fDwzZ86s6VQdIjLUnzOZhSRlFDCwU4saX0/Z11iV5SWqqQpqkV8PpzFp9R50RhO3dQ/j44evrTTnsK5xehY6nY7rrruuNuZSLXl5eYwZM4YlS5YQGhrq0DlTp04lJydHeaSkpNTa/CxxL3dVl7BYXuWbzVqjNOEQMS+Bm9n0TypPfpGIzmjizp7hLBhdf4QLXLC8HnvsMb788ktef/31Gt88NDQUtVpNWlqazfG0tDTCw8MrjD916hRnzpzh7rvvVo5ZYm5eXl4cO3aMTp062Zzj7e2Nt3ftb1WAskRVd7iNRTojBTpzxVpHLC/hNgrcyYYDF3l2zV6MJpm7e7fhg4d6u61xhrtwWryKi4tZvHgxv/76K7169aqQbT9v3jyHr6XVaunbty8JCQlKuoPJZCIhIYHJkydXGN+1a1cOHjxoc+y1114jLy+P//znP0REuL/gmTO4M9fLstKo9VJVuRRtnedlMsmoVKKml6BmfL/vPHHr9mM0ydx3TVvef7BXvRMucEG8Dhw4QJ8+fQAqNOJwJT0gLi6OcePG0a9fP6Kiopg/fz4FBQXK6uPYsWNp27Yt8fHx+Pj40KNHD5vzLZUtyh/3BJa6XmezCmssJJYcr1B/bZXvq2W10SRDgc5AgAeSBQWNh/V7zvHi1/sxyfBg3yt494FeqOvpF6LT4rVlyxa3TmDEiBGkp6czffp0UlNT6dOnD5s2bVKC+MnJyahU9U/17dEm2AeNWkJnMHEhp4grQvxcvpay0minCKE13l4qtGoVOqOJ3GIhXgLXWfdXCq+sP4Asw6ioCN4e3rNeW/KeSY0tx+TJk+26iQBbt26t8twVK1a4f0Iu4qVWEdHcj9PpBZzJKKyReFVVhNAaS02vjPwScov0tA32dfmegqbL6l1nefW/Zk/qkQHtePOeHvVauMCF1UZB1XRw0wbtsgTV6hcbRGUJQU34fOcZRbhir4/krXvrv3BBPbG8GhPuCtpn5ldeQbU8yoqjEC+BkyzdnsRbPx4G4PGbOjL1jq4NppGLEC834y7xqqrlWXlE52yBK3y67RTxP5mTwSfd3ImXhnZpMMIFLriNv/32GwZDxQ+JwWDgt99+c8ukGjIdQ93jNjpSy8uCKIsjcJYFW04qwvXsrVfWvnBdOgo7PnbrJZ22vAYPHszFixcrVHzIyclh8ODBGI1Gt02uIWKxvJIzCzEYTS7nx2TkVb8p24KlsoSIeQkc4T+/nuCDX48DEDfkKp699craveGlI7DybihIB+9m0He8Wy7r9CersnIvmZmZFbpnN0VaB/rg7aXCYJI5n13k8nUslldVtbwsiMoSAkeQZZm5vxxThOvl27vUvnClHYYVd5mFK7wndLvHbZd22PK6//77AfPS/Pjx42223BiNRg4cOOCxPY/1CZVKon0LP46n5ZOUUUD7Fs4LutG65VmAI5aXaMIhqBpZlpm96SifbjsNwKt3dmPiTR1r96Zph8wWV2EmhPeCsd+DX3O3Xd5h8QoKCgLMb0JAQAC+vmX5RFqtlgEDBjBx4kS3TawhE9nCn+Np+eagfRfnz79cqKO04xnNHWhoECSqqQqqQJZlZm04wtLtSQDMuLs7sdd3qN2bph6ElfdAURa07gNj/utW4QInxGv58uUAREZG8uKLLwoXsQqUTkKZhS6db0mTCPHTOBQzUypLCLdRUA5Zlpn5f4dZseMMAG8N78GYAe1r96YXD8Dn90DRZWhzLYxZD74hbr+N0wH7GTNmuH0SjY3IGlaXcKQIoTUiz0tgD5NJ5vXv/2H1rmQkCd65ryejotrV7k0v7IPP74XibGjbFx5ZD77BtXIrpwP2aWlpjBkzhjZt2uDl5YVarbZ5CGpe1ytdqePlWA88i9uYJ/K8BKWYTDLT/ntQEa73HuhVB8K110q4+pldxVoSLnDB8ho/fjzJycm8/vrrtG7dukEltdUVFrfx3OUi9EYTGifTJTKd2BoEooOQwBajSeblbw7w7Z5zqCSY+1Bv7rvGtc5eDnN+D6waDsU5cEUUPPIt+ATW6i2dFq/t27fz+++/K2VxBBUJC/TGV6OmSG8kJauQji2bOXW+JU2ipcNuo/m/Mb/EUKPcMkHDx2A08cLX+/l+3wXUKokPRvThnt5tavem5xPh8/ugJAciomH0N7UuXOBi30ZZlmtjLo0GSTKnS4BrrqOSoFpNRQkLFssLzAImaJrojSaeW7uP7/ddwEsl8fGoa2pfuM79DZ8PLxWuAXVicVlwWrzmz5/PlClTOHPmTC1Mp/HQsaUlaO/8iqOyNaiaWl4WNGoVfqX980SuV9NEZzDxzJd72XDgIhq1xCejr+WOnq1r96Ypf8Gq+6AkF9pdB498A94BtXtPK5x2G0eMGEFhYSGdOnXCz8+vQhnorKwst02uIaME7V1YcVTK4ThoeYF5xbFQZxRxryZIicHI06v38uuRNLRqFQsfuZZbu4VVf2JNSNkNq+4HXR60vx4eXmfe+lOHOC1e8+fPr4VpND5qki6R4UDXoPIE+nqRmityvZoaxXojT32RyJZj6Wi9VCwe05ebu1Tead4tJP8JXzwAunyIvBEeXgvaus/7dFq8xo0bVxvzaHTUpJOQZbXR0YA9iFyvpkix3sjEz//m9xMZ+GhUfDa2Pzdc6VhLQJc5uxNWP+hx4QIXK6meOnWK1157jVGjRnHp0iUAfvrpJw4dOuTWyXmc7BQw6Fw61eI2XsgpoljveKWNghIDRaXjHc3zArFFqKlRqDPw6Iq/+P1EBn5aNcvHR9W+cJ35o8zi6jDI7Cp6SLjABfHatm0bPXv2ZNeuXaxfv578/HwA9u/f37iy7zNPwdLb4OtxYChx+vTQZlqaeXshy5CS5XjQ3mJ1+WjKgvCOICpLND7+b/8FNv2TWuF4QYmB8cv/YsepTPy1alY+GuWWDu1Vcma72eLSF0DHwaUWl+s9GtyB0+I1ZcoUZs2axebNm9FqyyyDW265hT///NOlSSxYsIDIyEh8fHyIjo5m9+7dlY5dv349/fr1Izg4GH9/f/r06cOqVatcum+V5KSYN5Ue2wjrxjotYJIkERlq/s91xnXMsCpC6EwCsCXXS6w2NmzOZBTw08GL5BTpeearvTz5RSLpeWV/e3nFesYt283upCwCvL34fEI0/SPdu+G5Akm/wep/gb4QOt0Co74CjecbvTgtXgcPHuS+++6rcLxVq1ZkZGQ4PYG1a9cSFxfHjBkz2LNnD71792bo0KGKO1qe5s2b8+qrr7Jz504OHDhAbGwssbGx/Pzzz07fu0o63mz+T/LygeObYO0joC926hKubBPKyHM+WA/C8mosPLtmL0+t3sP/jpZ1kd92PB0w/9+OXbabv89eJtDHi1WPRdO3vfs3PNtwehusfsgsXJ1jYGT9EC5wQbyCg4O5ePFiheN79+6lbdu2Tk9g3rx5TJw4kdjYWLp3786iRYvw8/Nj2bJldsfffPPN3HfffXTr1o1OnTrx3HPP0atXL7Zv3+70vaul0y1m89jLF078AmtHOyVgZUF7J9xGq2azzmAv5pVTpGfM0l2s+zvFqWsJPEORzsg/53MAOHAuRzm+5dglcgr1PPLZLvYmZxPkq+HLiQPoExFcuxM6tQW+fAgMRdB5CIxYDRqf2r2nEzgtXiNHjuSVV14hNTUVSZIwmUz88ccfvPjii4wdO9apa+l0OhITE4mJiSmbkEpFTEwMO3furPZ8WZZJSEjg2LFj3HTTTXbHlJSUkJuba/Nwio43lwnYyV9hzSjQO1Yh1ZVcL2crSliwt9r495ksfj+Rwec7zzh1LYFnOHwxR6njdjq97G/mt+PpPPzZnxw4l0OIn4avJg6gR9ug2p3Mqf/BVyPBUAxXDoWR9Uu4wAXxeuedd+jatSsRERHk5+fTvXt3brrpJq677jpee+01p66VkZGB0WhUumNbCAsLIzW1YqDSQk5ODs2aNUOr1TJs2DA++ugjhgwZYndsfHw8QUFByiMiIsKpOQLQcRCM/ho0fqX/qaNAV701pXQScsZtzHe8dr01ZTW9ymJextJPQmFJ0+4r0FA4aGVtnc7IV37OKzZw6EIuoc20rHl8IN3b1PL2m5O/wpelwnXV7TBiFXg592VaFzgtXlqtliVLlnDq1Cl+/PFHvvjiC44ePcqqVavqrCROQEAA+/bt46+//uLtt98mLi6u0s7aU6dOJScnR3mkpLjoQnW40bzhVOMPp7eYv5WqETCL23gxp5ginWMCkuGq5WWng5DlW1zsd2wYHDhfJl7nLtta9y0DvFnz+AC6hNfy9psTv8JXD4OxBLrcCQ99Xi+FC2rQt7Fdu3a0a1ez+kChoaGo1WrS0tJsjqelpREeHl7peSqVis6dOwPQp08fjhw5Qnx8PDfffHOFsd7e3jb19mtE5PXmjaerH4SkbeZ4QBVJeiF+GoJ8NeQU6TmbVUDX8Oq/MTNdtbx8Ksa8LBvoCx0UToFn+cdKvCy1D4ZeHUZEiB+PDGivWPK1xvHSuK5RB12Gwb9WgJdzf4d1iUPiFRcXx1tvvYW/vz9xcXFVjp03b57DN9dqtfTt25eEhASGDx8OgMlkIiEhgcmTJzt8HZPJREmJ87lYLtF+oFnAvngAzvwOX46oVMDM6RL+7E/JJindMfFy1fIKsrPaaLG8CnSGSrs+CeoHhToDJy/lVzje64pgnh7cufYncPxn84q6UQdd74IHl9dr4QIHxWvv3r3o9Xrl58pw5cMRFxfHuHHj6NevH1FRUcyfP5+CggJiY2MBGDt2LG3btiU+Ph4wx7D69etHp06dKCkpYePGjaxatYqFCxc6fW+XaTfAXN7WImCr/1XpxtQOLfzM4uVg3EtZbXQxYF+sN1FiMOLtpcZU+vUty+bjvk4kvQrqlsMXcpUvG2uaO7nq7BLHfoK1Y8CkN7cme3AZqDXVn+dhHBKvLVu22P3ZHYwYMYL09HSmT59Oamoqffr0YdOmTUoQPzk5GZWqLDRXUFDApEmTOHfuHL6+vnTt2pUvvviCESNGuHVe1dIu2lzm9ov74ewfZldy9NcVSoIoQXsHVhyL9Ual5VkrB8vhWAjw8UKSzEKVW2SgZUCZeIE57iXEq/5isboCvL3Is4pRhjjQPapGHN0A68aZhav7cHjgswYhXFCDmJc7mTx5cqVuYvlA/KxZs5g1a1YdzMoBIvrDmO/MNY2Sd5otsXJVJJVOQg7keiWXbiMK8PEi2M+5PyCVSqKZtxd5xQZyi/W0DPDGumZkoc4A1M/AqwAuF5o9m44t/dlvtepYq5bXkf+Dr8eDyQBX3w/3LwF1vZAEh3BoppaGs46wfv16lyfTILmiL4z9r1nAUnaZBcyqmqQl18sRt/Fsaau09i38XHLBA300ZvEqDdpbW14FIl2iXmNZaGnforx41ZIVdPgH+CbWLFw9HoD7Fjco4QIHUyWs86QCAwNJSEjg77//Vl5PTEwkISFBaUzb5Gjb19wN2CcYzu02u5LF5j9Ai9uYnldSbcrC2VKBc6XLNlhvETLfxzqGUqAT6RKeRJZlDl/IRWcw2X29TLxsNzvXitt46Lsyi6vnvxqkcIGD4rV8+XLlERYWxkMPPURSUhLr169n/fr1nD59mpEjRxIaWsslOeozba6xErDS8rhF5q0cFtO/uriXYnk1d223flBpoqrlgyDbWF5CvDxJwpFL3Pnh78zacNju6xZruYW/lgBv8/+jJJWtIruNQ/+Fbx4F2Qi9RsB9nzZI4QIXklSXLVvGiy++aJOQqlariYuLq3Q/YpOhTR8Y93/m7sDnE82toIouE+lgM46zWWVuoyuU3yJkG/MSbqMn2XLMXGigfPKpBcsXTpCfhuBSVzHI17GO6Q7zz7fwzYRS4RoJwxeCquEu4jj9zhgMBo4ePVrh+NGjRzGZ7JvETYrWvUoFrHlpE87hdA8xC0d1lldyqbi1a15Tt9FezEtYXp4k8exloGzLVnkU8fLVKK6iW13Gg9/At4+Zhav3wzD8kwYtXODCamNsbCwTJkzg1KlTREVFAbBr1y5mz56t5GY1ecJ7mgXs83vg4j6ezX2R/+OFKqtLGIwm5VvZVctLSVQtshPzEuLlMXKL9RxLywMcE69gRbzc5DIe+Br++zjIJujzCNzzYYMXLnBBvObMmUN4eDhz585VSuO0bt2al156iRdeeMHtE2ywhPeAcT/CyrtpVXCML7Vv8176e0Bvu8MvZBdjMMlovVSEB7q2e7/8FiEby0u4jR5jX3K24sIbKvFOLNZyoI+G4NIvIbekSexfC989aRaua8bA3R+Cyo2uqAdx+rdQqVS8/PLLnD9/nuzsbLKzszl//jwvv/xynW3MbjCEdYfxP2LwDeVq1VleTX8ZCjLtDj2bZXEZ/VCpXNvGU1ZZomLAvlCsNnoMi8sIlVteBqP5uNZLpVhcNXYb930F/33CLFzXjmtUwgUuNuCwEBgYSGBg3XTHbbC06kbJI9+TLgdxFWcxrrgLCipWnK3pSiNUDNjbuo1VW16n0vO5+6Pt/HKo8lJEAtewFi9DZeJVapGpVRK9S4sM9qpJscG9q+G7pwAZ+sbCXfMblXCBixn233zzDevWrSM5ORmdzra7zp49e9wyscaEf9sejNO8ySf6GbRKPwwr74axP0CzlsoYS3Z9OxfjXWAd83I+YB+3dh8Hz+fw+KpEzswe5vIcBLYYTTJ7k6u3vCzHvVQq7r/2Cm68siUtndwiprBnFfzwDCBDvwlw55xGJ1zgguX14YcfEhsbS1hYGHv37iUqKooWLVpw+vRp7rjjjtqYY6NAFdqFkbrXKPZpCZcOw8q7IL+sTr+SoFoTy6uKJNXqUiVEy7Ta4Vhqnk280eIelsdikalLQwYuC1fiSvhhMiBD/4kwbG6jFC5wQbw++eQTFi9ezEcffYRWq+Xll19m8+bNPPvss+Tk5FR/gSZKZKgfp+U2rLl6EQS0gfSjsOIuyDPXMlPcxhrUbFJiXvaSVKuJeZX/SOUU6vkw4YQiqgLXSDybBYC3l/mjZs/yMplkJaDv5WK8E4C/l8P/PWv+OeoJuPN9c6ZrI8Vp8UpOTua6664DwNfXl7w88xLwmDFj+Oqrr9w7u0aEZZvQvoIWMP5HCGwLGcdg5V3IuRcVt9EtMa9iPbIs1yjP64f955m3+TiLtp1yeT6CsnjXte3MXX7srTZax8HUahfF5u9l8OO/zT9HPwV3vNuohQtcEK/w8HCysszfJu3atVN6NSYlJdl80wts6aBs0C6EFp1KBewKyDiOcfkwmukyUElwRUjNY156o0yR3uhUwL48Ftczu1C4kzUhsTTeFd3R3FvRnuVlfcwly+uvz+DH580/D5gEt8c3euECF8Trlltu4YcffgDMCavPP/88Q4YMYcSIEXb7OQrMVKjr1byjWcCCIvC6fIo12rfoGViI1sv1+ISfVq3ETHKLDDaWl7OpEpbYTLFe5Ie5yqXcYlKyipAk6NfeLF72VhutrTG1s+K1ewlsKM2vHDgZhr7TJIQLXFhtXLx4sbIN6Omnn6ZFixbs2LGDe+65hyeeeMLtE2wsWErj5BTpuVygI8RfC807wPgNFCy+nY5FF/jU8DrkDISgK1y6hyRJBPp4cblQX+o6lr3mbJKq5QNVrBdbvlzF4jJ2CQtQ6rNVb3k58eW161P46WXzz9c9A0PeajLCBU5aXgaDgVmzZtm0JRs5ciQffvghzzzzDFpt/a557Ul8tWolc96mtldIe77stpAUU0vCjRdhxTDIdr1JrHW6hMnkWszr0IUc9BbLyyAsL1exiFff9iGKRWXf8io75rDh9efCMuG6/t9NTrjASfHy8vLivffew2AQ2dqu0KGSktD/FAYxQvc6uT5t4fKZUgFLdukegVads8unSpgqyTEqz+jPdmEUlleNscS7+kWGKLGsqiwvL5XkWBHKnQtg0xTzzzfEQcwbTU64wIWY16233sq2bdtqYy6Nnsrq2Z/NLOQCoSQO/gJCOkD2WbOAXT7r9D1sVhzLJUAUORi/yi7UK5ZXiYh5uUSx3qi0MuvbrnmZ5WWsfLXRoXjXjo/g52nmn298EW6d3iSFC1yIed1xxx1MmTKFgwcP0rdvX/z9bfOS7rnnHrdNrrHRIdS8kni6nHhZ0iRaRXSCrhvMCaxZp80CNv5HCIl0+B5luV6GCt1oCkoM+Hs79l9eFvMS4uUKB8+bXe/QZt5ENPclJctcMcSu5WUss7yq5I//wObp5p9vehkGT2uywgUuWF6TJk0iLS2NefPmMXr0aIYPH648XF1tXLBgAZGRkfj4+BAdHc3u3bsrHbtkyRJuvPFGQkJCCAkJISYmpsrx9QlL0N66KGFusV7pGNS+hT8EtYXxG6B5J8hJgeXDICvJ4XsEWbmN5VNXrIP2JQZjlaktympjJWWLBVVTFu8KRpIkJX+rqtXGKi2v7R+UCdegKXDLq01auMAF8TKZTJU+jEbnv6XXrl1LXFwcM2bMYM+ePfTu3ZuhQ4dy6dIlu+O3bt3KqFGj2LJlCzt37iQiIoLbbruN8+fPO33vusa6k5BFOJJLM+tDm2lpZrGKAtuYBazFlZB7zmyBZZ126B7Wm7NN5cWrNGifX2Lg+tlbiF3xl/JaeR3Ti1SJGnHusvn/9aowcys8h2JelVVN/X0u/PqG+eebp8Lgqe6dbAPF45ue5s2bx8SJE4mNjaV79+4sWrQIPz+/SktKr169mkmTJtGnTx+6du3KZ599pnTZru9ENPdDkszikZFvtrYs24Lalc+sD2xtdhlDr4Lc82YLLLP6bHfraqrlPyeW/Y1nMgrIyC9hx8nMSq0va7dRJB87jyW511LWxnq1sfz7WWXM67f3IeFN88+DX4Wbp9TSjBseDse8ioqKSEhI4K677gJg6tSplJSUKK+r1WreeustfHwcL6Sn0+lITExk6tSybxKVSkVMTAw7d+506BqFhYXo9XqaN29u9/WSkhKbeebm5jo8P3fjo1HTJsiX89lFnMksoGWAt1LHy27HoIBwc0HDz+8p3Qs5zPw8tPL274E+ZTGvkHLF7CyWl+VfndFEbpGBIDsVOy0fKJNstsK0Xk3bRXEWi3hZ8rus41kmGax3AVmvNtqw7T3Y8rb551teg5teqr0JN0ActrxWrlzJp59+qjz/+OOP2bFjB3v37mXv3r188cUXLFy40KmbZ2RkYDQale7YFsLCwmxyyarilVdeoU2bNsTExNh9PT4+3qZ1W0REhFNzdDcW1zGpNGifXJnlZSEgzFxSumU3yCvNA8s4Uen1A21iXravWTZnW1eYSM8vwR7Wq2Ii18t5sovMlrVFvKytqvL7G+1aXltnlwnXrdOFcNnBYfFavXo1jz/+uM2xL7/8ki1btrBlyxbef/991q1b5/YJVsXs2bNZs2YN//3vfyu1+KZOnUpOTo7ySElxPQHUHUSWrjha0iXOKL0aq9jT2KyVWcBadYf8VLOApR+zO9TGbSznNxaW7m+0rjCRnleZeJWdK+JeznO5wGJ52bqNUDHuZcmp81JJ5uDjlndga7z5xZg34EZRXt0eDovXyZMn6dmzp/Lcx8cHldVWhqioKA4ftt+TrjJCQ0NRq9WkpaXZHE9LSyM8PLzKc+fMmcPs2bP55Zdf6NWrV6XjvL29lYqv9aHya/kVx2SlS3Y1pXCatTQLWFgPyE8zl9O5VLGLk3WeV/mYl6XpbaHVJu2MSiwvvdXJJSJR1Wks9dEs9ehtLa9yMa/SLwq1hFm4tr1rfmHIW3DD87U/2QaKw+KVnZ1tEztKT08nMjJSeW4ymWxedwStVkvfvn1tgu2W4PvAgQMrPe+9997jrbfeYtOmTfTr18+pe3qaMrexkBKDkYu5xYCDHYP8Q80VWMN6QsElcz5Ymu0XhnUHofKrjZbN2Y5ZXlZuo7C8nEJnMClfFJaAvfWeRaOxvOUlAzKP6VfDb++ZD972Nlz/bJ3Mt6HisHhdccUV/PPPP5W+fuDAAa64wvkNxXFxcSxZsoSVK1dy5MgRnnrqKQoKCpQ2amPHjrUJ6L/77ru8/vrrLFu2jMjISFJTU0lNTSU/P9/pe3sCS5b92cwCUrIKkWXw16pp4WinGP8WMO4HCO8FBenmktJph5SXrZtwlHdPLHlejsW8rN1GYXk5g8XqkqQyN946nFXR8jLxstdaRpV8bT4wNB6um1wnc23IOCxed955J9OnT6e4uLjCa0VFRcycOZNhw5yvfT5ixAjmzJnD9OnT6dOnD/v27WPTpk1KED85OVlpsQawcOFCdDodDz74IK1bt1Yec+bMcfreniAixA+1SqJQZ+SvM+ZExnYt/B3b02bBrzmM/R5a94bCDLOApZq/WCxuoyxDXrFtLa7CcquNABmVWV4mEbB3lZzSYH2gj0ZxFyVJsp/rJcu03/sek7zMZaa4/V0YOKlO59tQcThVYtq0aaxbt44uXbowefJkrrrqKgCOHTvGxx9/jMFgYNq0aS5NYvLkyUyebP+bZuvWrTbPz5w549I96gtaLxVXhPhyNrOQraUt4F2qnmoRsM+Hw8V9pU09vsendS+0Xip0BhPZpRaAr0ZNkd5IfokTlpdJBOxdJavANk3CglolYTDJZV8MsgybX6f90SUALG72FI8PeLJO59qQcdjyCgsLY8eOHXTr1o0pU6Zw3333cd999zF16lS6d+/O9u3bK6Q8COxjCdr/cdLcw7F9qIvVU31DzALW5looyirt0L1fiXtdLs01alaa+6XEvKwtr1LxKr+JWy/cRpexvKehzWybaNhYXrIMv7xm3mgNvK4fz+ZmYl+wMziVYd+hQwc2bdpEeno6f/75J3/++Sfp6els2rSJjh071tYcGx2WoL0lqNu+uetNN/ANhrHfQdt+UHQZVt5DX425GkVOodl9CSgVL7sxLxGwdzsW8WpZTrxsKkv8PA12fgzAwT7TWWW8zfkqqk0cl7YHNW/enKioKKKioirNbBdUTmS5lUWHVhqrwicIxqyHK/pDcTZziqbTUzqtBI4DSvdMKhn2VquNmfk6TCYZCdsPjnAbXcfyhRAaYLsIY967KNP89+nw5yfmg3d9wOnIkebXG2mLstpCvFseILJce7NKs+udwScIHlkPEdE0k/NZrX2H9sXmPDCL21hgJ8/LYJKV2Jg1epsMe+E2OkNlbqNagje8VhJysHTf7t3/gX6PluV5CcvLKYR4eYAOVuKlUUu0CfZ1z4V9AuGRbznl24NAqZDPNe/QRzpJgLc5BmZxF8v3cLSXqGqdKiEKEjpHep7ZXbcRL1lmiukzxnv9gowE93wEfccDVextFFSJEC8P0DbYV/lDtaROuA3vAFZ3nscuU1cCpSJWaeO52mS2wArt7G0Es5tTPmAv3EbHKf/+KDEvS9drkwk2vMADpk2YZInkG96Fa8cq452qpCpQEOLlAbzUKsVVbFfTeJcdfPyDiNW9zJ+mbgRIRTx+9kWulY4rCwQW99GyKmnX8rLO8xKrjZUy68fD9HnzF05eKkuStnEbTSbY8Dz8vRQTEi/pnyC9879srqHsbXS14WwTRYiXh7DEvWrSIbsyAn01FOJDrO4ldhq7420q5HPtbHoYDmM0yYrlZVk4sLfiKDZmO8bvJzIo1pv48cAFAGRZVt7Plv4a+PE5SFwBSLzr8xzfmm6qmGGvWF7i4+gM4t3yEDddGYokwXWdQ91+bYtFVYQPsfqXOB/cn2ZSMSu171J86ncl5tWuNN/MXqKqXpTEAczxqONpeZUWZLyYY65NP//XE3zx51nySwyUGExImGjz+8uw53OQVHDfp2z1vlW5Zvl7gIh5OYsQLw8x/voOHHxjKEOvrrp6hitYtggBFOPNlr4fsd3UA3+pBN91I4jiCFCN5WWyTVKVZZnn1uzlpa/3U+RkA9uGzBs/HOK2D37jPwkVa6gV6gzkFpctfvwn4QQZ+TpUmJjnvQSv/atLhWsx9B5Rae9GEfNyDSFeHqSZg518nMWyOduCrPHj36op/GbsiUpfyHLtewxQHSaieeXiZSwXsE/PL+H7fRf4OvEc3yR6tiZaXbLqT3PC7/xfK4pXao7tPl9ZhozcQt7TLOY+aZtZuO5fAr3MMS5LTMtYrhihsLxcQ4hXI8TiNlpQSaDx9mOi/gUuhd2In1TCcs17XFWwB0Cpp18ZxXoT1p+3tFznSh81VlJzbcVLJRtps/UFHlT/hhEVPPAZ9HxQeb0sw76Sel5CvJxCiFcjxNptBFBJEn5aNSVo+fnqOWwx9sZX0tHrt4lcrzpY6RYhCyUGo00qhc5O49TGwocJJxizdBc6BxJzrS0vFSbeMH1M2+TvMcgqPgt7DXo8YDO+sg5CNpVUBQ4jxKsREmjH8rI0m71YCE/o49ih7ofKWMJSzRy6F/6FqYrPqrmDUNnzxpy0Om/zcX4/kcEvhyv2UCgsl9xrsbzUGJmn+YQ75d8wSmom658lpfVtFc6vPuYlPo7OIN6tRohlI7YFSZLw15qPpeeVoEPD7IBXMV15Oz6SniWauXQrqLxxb7HeZJPCWtIEtgvZ66848wfbqrWpOcWoMfKB5hOGq3dgQM1X7d5kkymqwtYgKNu7WOlqo8jzcgohXo0QjVqFv1atPFdJEv7e5ueWtAittw+qEavYSn+8JT0LVHO4WbXP7vXK9250xKVq6GjtNID96Z+LNs8vZRfwH80C7lHvRCereUX1AtvUA4CK+xrBEctLiJczCPFqpFi7jioJ/Eotr0ulwXY/by/w0jInaCqbjGYB+1Qzj8GqvRWuVWwo5zY2UvGyFmiNHfGyTovAqOeR829yl/pPdLKaSfp/s00VVVZRwq7lZX+10ZIrpnammq5AiFdjxTpob8/yslhmwQHNmKx/ho3GKLwlA59q5nGrKtHmWuW3B5U00qRV6wKMGi/7H43ZPx2lqKgYvnmUG3TbKZG9eFL/PL+a+gJ29jVaYc/y+uz302w8aI6vDejYwj2/SBNBiFcjxTrXS5JQYl6ZpR8uiyXWMsAbA148q5/Mj8ZotJKRhZr5DFH9rZxfIWDfSC0v61VUe24jwNJtxzjxyYNw5AdFuP5nuhYozfOqpBAhWOd5md/M7/edZ9YGc8LwlDu6csOV7t9t0ZgR4tVICfK1tbz8SlcbLV/6FksstJm5YJ4BL57TT+b/jAPQSkY+0fyHoaq/AHPfRutUicYqXtarqBo7wXMNBhZo/kOvvN+R1d48oY9ji+ka5fW8EoNipZYvRAhlq4kGo8zvJ9J58ev9AMReH8kTN4lKxM7icfFasGABkZGR+Pj4EB0dze7dla96HTp0iAceeIDIyEgkSWL+/Pl1N9EGRgW30SqAD7aWlwUjav6tf5ofjAPRSEY+1nzI7ard6IwmmxWy6sTrrR8P896mig1x6zvWllf58JMWPZ9o5nObOhEdGk7e8ilbTX1sWtZZFjL8tGrl/bXGEvPafy6bJ1clojfK3NWrNa8P6+5c9ygB4GHxWrt2LXFxccyYMYM9e/bQu3dvhg4dyqVLl+yOLywspGPHjsyePbvajtpNHeuAvWSV52XBImblYzNG1Dyvn8R3xusUAbtT9adN3Kuq1caLOUUs3Z7EJ1tPNbjYmHVncOsFQYtwDVHvoVjW8LrfaxwPMK8qlq+KC9Cimf0enJaY1/f7LlCgM3J95xbMfag3KrHK6BIeFa958+YxceJEYmNj6d69O4sWLcLPz49ly5bZHd+/f3/ef/99Ro4cibd3xZiCoIzyq40WN9GCxY20typmRE2cfhKb1DfjJZn4UPMxPse/V16vSpSsN21XUoih3mJtUVrm7o2ORZoPiFHvpVjW8Jj+RXbQiwVbTgIQHuRT4TrN/e3/bVpn0HdvHciiR/ri7aW2O1ZQPR4TL51OR2JiIjExMWWTUamIiYlh586dbrtPSUkJubm5No+mQKCPdcBequDGVGZ5WTChYq7vs/zXdBNekokO257jbtUOwNZCuZBdZGOJleun2qDQ2YiXDPpiFmk+4Bb1PopkLY/qX2K7qSclehOHL5r/jux1Oq+s+7lv6Xse0dyXFY/2J6DcNi6Bc3hMvDIyMjAajRV6PYaFhZGaWnFrhqvEx8cTFBSkPCIiItx27fpMoG/5mJeteFVleVlQe2l4QzWJrw03IclG5msWcI/qD3KL9MiyzIFz2Vw3+3/c8/F25RyTlWKZGph6WVuUsqEE1jzMYPV+Rbh2mHqUjisTuYej21W4TvNKxGvswEgeu6EDXz42gFYBFS02gXN4PGBf20ydOpWcnBzlkZLSNMq52AbsK7qNFssrxE9baWa3l1pCq9HwsuFxDoXdi1qS+UDzCbfqtzL/1xPc8/EfABxNzVPOsa6Y0NDEy2J5eaOjc8JEOJVAoexNrP5ldpquVsZZi5y9lIjKLK8Oof68dld3pRSRoGZ4TLxCQ0NRq9WkpaXZHE9LS3NrMN7b25vAwECbR1OgfKpE+YC9xY1Uq6RKLQW1SoWPRoWMirvO/osvDYNRSzLzNAs5t2WpzdhVO88A5S0vd/wmdUeJwYQPJXymmUNo2nYKZG/G617mT1N3m3HV1fSv7P0UuBePiZdWq6Vv374kJCQox0wmEwkJCQwcONBT02o0lE9S9SuXKmFtidmzHgA0Kgmf0oCyjIpXDRNYbbgVlSTzvuZTHlRvU8a+/v0hdp3OtEmpMDUA9SrUGbj7o+3M++UY+uIClmrmcKP6H4olH8brXmG33E0Ze0ePil+qkiRR3nAV4lU31E4pTweJi4tj3Lhx9OvXj6ioKObPn09BQQGxsbEAjB07lrZt2xIfHw+Yg/yHDx9Wfj5//jz79u2jWbNmdO7c2WO/R32kfJ5X+aqt1gH80ABvsN1zDJjdRh9NmcjJqHjNEIsJiTFev/Ke12IkZL423gyYy8l0btVMGd8Q3MZvE89x8HwOJ8+n8dsVn9JSfYh82Ye3g9/ir6K2NmOHdA/jp39s47ES8Oj1Hfhse5JyrLJUCYF78ah4jRgxgvT0dKZPn05qaip9+vRh06ZNShA/OTkZlVWNowsXLnDNNWUZzXPmzGHOnDkMGjSIrVu31vX06zXlA/YVVhsdsbzUKnw0tgIko+L1UgEb57WZ9zWLUWNijfEWdiVlsSspSxnbAAwvfjmchi/FLNe+T8uMI+TJvozTvUKx1A2wXZn21dhPayifX1pZqoTAvXhUvAAmT57M5MmT7b5WXpAiIyMr7eIisCXA2wtJMqcrqCTQeqnQqCVl87Gt5WXfUvBS2VpeZUjMMIzHhIpYr5+ZrfkMCZmvjLfajLL8XxXrjWw9ls4NV4bWWt1+V0k8cY4V2veIVh1Fp/ZnXOFL7JGvopOdXDaVSkLrpbJJqZAkKmTHVxawF7iXRr/a2FRRqSQCSoXC8uGyDtpbx8Aqs7wkSaqQRBnsZ7HoJGYaxrLUcAcA8ZqlPKLebDPWWCper333D09+kcgzX+5x/ReqBfJyslihfZdo1VFyZV9+vnYhe+SrAPtBebUk4V2u2oSERPm1WuE21g1CvBoxFtfRElC25HqZrbCy//rKElUz80vw0dj+idjWuZJ4y/AISwx3AjBLs5wx6l+UVy1u4zeJ5wDYcizd5d/F7ZTk4fXVQ0SpjpEr+zFGN5Vz/j2Ul3OK9BVOUasqijmAtXr5aFR29zUK3I8Qr0bMqKh2XNsumB5tg4Aya6v8Jm1ry+uloV2Uny/mFFdwGyXKx3gk3jaM5lPDMADe0qxgnPpnoB6vNhbnwhcP4Jv6F7myH4/oprJf7myTv5VfYqhwmiRRwfJCwsb2aiHiXXWGEK9GzNODO7N+0vWKu2jJqi9vGTS3cnNG9i/bgXApr6LlBVRwk0Ai3vAwCw13AzBTs5JH1T/Vz+1BxTnwxf2QsgudVwCjddM4IHcCqi9vrVZJeJd7PyQJm1QJkSZRdwjxakI0K11hLJ9tb20tlBc2n3JuUuWVWyTeNYxkgeEeAKZrVtFs76c1m7Cb+ed0Mlmf3gXn/gKfYNZ2X8BBuayOVnWlflR2YoDlLVEhXnWHcM6bEBZhKi9QLQO8mXZnV1SShK9WzUtDu/D+z8eYcEMHG6sMzC6SSpIqyeGSeN8wAhMqnvH6jubb3wBfNdCpdn4hB8kr1hMgF2BYMZzmqlMYvIPxGvcD+34HOKeMq87yUtkJ2EN5t1GIV10hxKsJYYl1lbe8AB6/qUxgnhrUiVu6tuKqsAB+2H/eZpw5NaCqu0jMNfwLEyqe81oPm1/nCfUoPjWaXcr9Kdl0bxNot8FFbTDrx8Os236QP9p8RB/VKbLkZvx+zafc27o3lwv/shlbXf0xlZ2YlyRJwvLyEMJtbEL4VxLzKo9KJdGtdSBqlUR4oK/NaxLYSQ4oj8QHhgf5QG/uGD1V8xWT1OZ6YPcu+INp6w+6NH9X+Hr7QVZr3yEg6yCZcgAP617jua1Gth1P53KhzmasI+Wtve0tYFg9L2+pCmoPIV5NCIt4lV9trIrW5YrtSZJkL2Jvl/8YH2Cu/kEAXtas5Wn1dwB8nXiuirPcSGEWq7Xv0FN1hhwpkId1r3JUNpeweWfDES4XlBOvajZcp+YW23UbrU0v4TbWHUK8mhAhfuYPVogTHzB7lUKdSYH4yHg/7+sfAuAlzTqeUa93+FxXKTEYeeSjn7j40W30UJ0hQw7kX8Wvckwuq73V3F/L5ULbXK7iatzG6zuH2nEby1leIlWizhAxrybEiP4R6I0mHuh7hcPnlM/z0qilCh2fq2OBcTgyEi9r1vKC5htUyMAwp67hDHuPnGJa+iu0Vp0lXQ7kYd1rnJBtf+dgPw25xbbiZV3CujyfPxpFaDNvO6uNIublKYTl1YRo7q/l2VuvpG2wb/WDK+HWbmHVD7LDJ8Z7idePAuB5zbecXDutdupEF2TQ6adRdFedJV0OYpQd4QLwUqsq3L64iphXQGlZ7fJ5XuWpbKuVwP0I8RI4RU36C35qvJu39Q8D0PnIAo58+Yp7BSw/HVbeTcvCk1ySgxmpe42TdoQLwE5bRrJLA/g3XdWy4vjSTFR7bqN17Myemy2oHYR4CZyiVWDNPpxLjHfxlv4RALqd+BQS3nSPgJUKF5cOk04II3WvcUpuW+nw7/ZdqHDsbGYhABNv7MDScf347unrlddUpb5hXnHFbUPns4uVn7X2AvqCWkG804JqiRtirrQwebB7Cj4uNd7JTP0Y85Pt8+DXN2omYPmXkFfeBelHIKA1T6hnclpu4/LlvL3U3NotjE4ty3oyWtIoTl7KrzD+Yk6Ry/cSuI4QL0G1TB7cmV/jBiki9nzMVXbHRTR3PJa23HgHM/TjzE/+mA+bX3dawIr1Rk4nnSL1wxik9KPIAW1g/Ab2FoY6dZ3yWFxD69pjuaVVJrq1DrAZ66WSaCHiXB5BrDYKqkWlkmzKOz8XcyWjB7Rj8+E0plolnDb305KS5bgVstI4FBMSb2lWwI6PzOJ126zqUvgBWPzbKZZs3MlX2rcJV13ggtycdREfsHLBKac00EejqlC7yxLfsi4yaPn9XxralRb+3gT7aegQ6o+XWsXbw3swe9NRJt7oejxQ4DyS3MRKk+bm5hIUFEROTk6T6SRUWxhNMp2mbVSeX9+5BX+czHT6Oo+oNzNLsxyApYY7iHpiET0jgqs8J2rKF3ylnUUn1UXOyy0YpXuNZNn5ldAW/loyrQLuV7cJ5NunrlNSRNJyi8kq0NGttfhbqSsc/YwKt1HgMmqVxDO3mONg/lq1y7WsvjAOYZp+AgATvH7i708fRzZVke2ee4E12rfopLrIOTmUkU4IVzNvLzY+e6Py3Ndqt8HXTw5kw7M32uS2hQX6COGqpwi3UVAjnhzUiSBfDUOvDkfrpeJUej5XtwnkwLkcm2a01fGl8VZMSMzWfEas18+c/+oZ2j78cUUXMuc88sq76KhKVYTrnNzK4ft8NXEA3duUiVG/9iH0jgimsMTAte1CHL6OwPPUC8trwYIFREZG4uPjQ3R0NLt3765y/Ndff03Xrl3x8fGhZ8+ebNy4scrxgtrD39uLx27sSERzP8ICfdjw7I2892BvfnzmBgZ0bO7UtdYYb+El/eOYZIm2J77g2LLHMRmtst5zzmFaMQwp6zQpppaM1L3ulHABinD9MPl6xg5szxv3XM2Ch69leWxUpZ3DBfUTj4vX2rVriYuLY8aMGezZs4fevXszdOhQLl26ZHf8jh07GDVqFBMmTGDv3r0MHz6c4cOH888//9TxzAVV4aVWsebxgSTF38m0O7s6fN7Xxpt52WAWsC4p60j6/EkwmSA7hdxFQ1FdTiLZ1LLU4qqYTFoV7z/YSxGoXlcE8+a9PQj2E9t5GioeD9hHR0fTv39/Pv74Y8DcNTsiIoJnnnmGKVOmVBg/YsQICgoK+PHHH5VjAwYMoE+fPixatKja+4mAvWc5eSmfL/48y4odZ6ocd7/qN+ZoPkUlyewKiKFD4T+0MqZy1tSKUbrXuED16RBXhTXjl+cHkV9iQKtWiQTSBoKjn1GPxrx0Oh2JiYlMnTpVOaZSqYiJiWHnzp12z9m5cydxcXE2x4YOHcp3331nd3xJSQklJSXK89zcXLvjBHVD51bNeOOeq5l+V3e2nUgn8cxlPt5yssK49aabMOlVzNUsJDrvVwDOmMIYpXuNi7SoML5jS3/+98LNyvML2UVKC7L61itS4B48+r+akZGB0WhUOmRbCAsL4+jRo3bPSU1NtTs+NTXV7vj4+HhmzpzpngkL3IZKJTG4SysGd2nFC7ddhSRJZOaXMHX9QX45nAbAd6YbMOkl5moWkSy34mHdq6RRFkd79PoOvHDbVfhq1KjKxava1GDzuaBh0Oi/kqZOnWpjqeXm5hIREVHFGYK6xpIM2qKZN4vH9rN5LafoNn4/9DBXtr+CnS2CKoiUoOniUfEKDQ1FrVaTlpZmczwtLY3w8HC754SHhzs13tvbG29vsX2joRLkq+GWfj2qHyhocng0gqnVaunbty8JCQnKMZPJREJCAgMHDrR7zsCBA23GA2zevLnS8QKBoHHicbcxLi6OcePG0a9fP6Kiopg/fz4FBQXExsYCMHbsWNq2bUt8fDwAzz33HIMGDWLu3LkMGzaMNWvW8Pfff7N48WJP/hoCgaCO8bh4jRgxgvT0dKZPn05qaip9+vRh06ZNSlA+OTkZlarMQLzuuuv48ssvee2115g2bRpXXnkl3333HT16CNdCIGhKeDzPq64ReV4CQf1GbMwWCASNGiFeAoGgQSLESyAQNEg8HrCvaywhPrFNSCCon1g+m9WF45uceOXlmWtMiSx7gaB+k5eXR1BQUKWvN7nVRpPJxIULFwgICLCpUV4fsWxlSklJadIro+J9MNNU3gdZlsnLy6NNmzY2aVLlaXKWl0ql4oorHG93Xx8IDAxs1H+sjiLeBzNN4X2oyuKyIAL2AoGgQSLESyAQNEiEeNVjvL29mTFjRpOviiHeBzPifbClyQXsBQJB40BYXgKBoEEixEsgEDRIhHgJBIIGiRAvgUDQIBHiVY/Iyspi9OjRBAYGEhwczIQJE8jPz6/ynMWLF3PzzTcTGBiIJElkZ2fXzWTdjOiabsaZ9+HQoUM88MADREZGIkkS8+fPr7uJ1gOEeNUjRo8ezaFDh9i8eTM//vgjv/32G48//niV5xQWFnL77bczbdq0Opql+xFd0804+z4UFhbSsWNHZs+eXWkDmkaNLKgXHD58WAbkv/76Szn2008/yZIkyefPn6/2/C1btsiAfPny5VqcZe0QFRUlP/3008pzo9Eot2nTRo6Pj7c7/qGHHpKHDRtmcyw6Olp+4oknanWetY2z74M17du3lz/44INanF39Q1he9YSdO3cSHBxMv35lfQtjYmJQqVTs2rXLgzOrXSxd02NiYpRjjnRNtx4P5q7plY1vCLjyPjR1hHjVE1JTU2nVqpXNMS8vL5o3b15pN/DGQFVd0yv7vZ3tmt4QcOV9aOoI8aplpkyZgiRJVT6OHj3q6WkKBA2OJlcSp6554YUXGD9+fJVjOnbsSHh4eIXArMFgICsrq1EHY+uia3pDwJX3oakjLK9apmXLlnTt2rXKh1arZeDAgWRnZ5OYmKic+7///Q+TyUR0dLQHf4PaRXRNN+PK+9Dk8fSKgaCM22+/Xb7mmmvkXbt2ydu3b5evvPJKedSoUcrr586dk7t06SLv2rVLOXbx4kV579698pIlS2RA/u233+S9e/fKmZmZnvgVXGLNmjWyt7e3vGLFCvnw4cPy448/LgcHB8upqamyLMvymDFj5ClTpijj//jjD9nLy0ueM2eOfOTIEXnGjBmyRqORDx486KlfwS04+z6UlJTIe/fulffu3Su3bt1afvHFF+W9e/fKJ06c8NSvUKcI8apHZGZmyqNGjZKbNWsmBwYGyrGxsXJeXp7yelJSkgzIW7ZsUY7NmDFDBio8li9fXve/QA346KOP5Hbt2slarVaOioqS//zzT+W1QYMGyePGjbMZv27dOvmqq66StVqtfPXVV8sbNmyo4xnXDs68D5a/h/KPQYMG1f3EPYAoiSMQCBokIuYlEAgaJEK8BAJBg0SIl0AgaJAI8RIIBA0SIV4CgaBBIsRLIBA0SIR4CQSCBokQL4FA0CAR4iUQCBokQrwE9YLx48fbLRd0++23e3pqgnqKKIkjqDfcfvvtLF++3OZYZa3t9Xo9Go3G5phOp0Or1Tp9X1fPE3gWYXkJ6g3e3t6Eh4fbPEJCQgCQJImFCxdyzz334O/vz9tvv80bb7xBnz59+Oyzz+jQoQM+Pj4AJCcnc++999KsWTMCAwN56KGHbOpkVXaeoGEhxEvQYHjjjTe47777OHjwII8++igAJ0+e5Ntvv2X9+vXs27cPk8nEvffeS1ZWFtu2bWPz5s2cPn2aESNG2Fyr/HmChodwGwX1hh9//JFmzZrZHJs2bZrS1u3hhx8mNjbW5nWdTsfnn39Oy5YtAXNRwoMHD5KUlERERAQAn3/+OVdffTV//fUX/fv3t3ueoOEhxEtQbxg8eDALFy60Oda8eXPlZ+vOShbat29vI0BHjhwhIiJCES6A7t27ExwczJEjRxTxKn+eoOEhxEtQb/D396dz585Vvu7IMUfvJWjYiJiXoFHRrVs3UlJSSElJUY4dPnyY7Oxsunfv7sGZCdyNsLwE9YaSkpIKPQq9vLwIDQ11+BoxMTH07NmT0aNHM3/+fAwGA5MmTWLQoEF23U5Bw0VYXoJ6w6ZNm2jdurXN44YbbnDqGpIk8f333xMSEsJNN91ETEwMHTt2ZO3atbU0a4GnEDXsBQJBg0RYXgKBoEEixEsgEDRIhHgJBIIGiRAvgUDQIBHiJRAIGiRCvAQCQYNEiJdAIGiQCPESCAQNEiFeAoGgQSLESyAQNEiEeAkEggbJ/wOOy+/1cAzUEwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019504, + "end_time": "2024-03-26T16:15:23.931535", + "exception": false, + "start_time": "2024-03-26T16:15:23.912031", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 3840.688081, + "end_time": "2024-03-26T16:15:26.675164", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/lct_gan/3/mlu-eval.ipynb", + "output_path": "eval/insurance/lct_gan/3/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/insurance/lct_gan/3", + "path_prefix": "../../../../", + "random_seed": 3, + "single_model": "lct_gan" + }, + "start_time": "2024-03-26T15:11:25.987083", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/insurance/lct_gan/model.pt b/insurance/lct_gan/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..2fa7bd71ffe1b3e7fe79493c417f278f0cc6ac37 --- /dev/null +++ b/insurance/lct_gan/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:098db33acce2a591bac340f287937c616657aa66331154377a1da52cd696363d +size 38581781 diff --git a/insurance/lct_gan/params.json b/insurance/lct_gan/params.json new file mode 100644 index 0000000000000000000000000000000000000000..008fbab11ed7ec7979d3862d582076ead0af535b --- /dev/null +++ b/insurance/lct_gan/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.7, "head_final_mul": "identity", "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "loss_balancer_beta": 0.79, "loss_balancer_r": 0.95, "dataset_size": 2048, "batch_size": 8, "epochs": 100, "n_warmup_steps": 100, "Optim": "diffgrad", "fixed_role_model": "lct_gan", "mse_mag": true, "mse_mag_target": 0.1, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "leakyrelu", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardsigmoid", "tf_activation_final": "leakyhardsigmoid", "tf_num_inds": 32, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "relu", "ada_activation_final": "softsign", "head_d_hid": 128, "head_n_layers": 9, "head_n_head": 64, "head_activation": "prelu", "head_activation_final": "softsign", "models": ["lct_gan"], "max_seconds": 3600} \ No newline at end of file diff --git a/insurance/realtabformer/eval.csv b/insurance/realtabformer/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..f9dfbe08dcd6b6c0c43114b8cedd1f03cc31b308 --- /dev/null +++ b/insurance/realtabformer/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +realtabformer,0.00010843533867349227,0.009143562115871948,0.0008848744287388399,0.6177115440368652,0.290086954832077,11.930258750915527,0.6675140261650085,3.047596464966773e-06,3.0843451023101807,0.01992633566260338,0.7107431292533875,0.02974683977663517,0.15960799157619476,0.0007001429912634194,3.702056646347046 diff --git a/insurance/realtabformer/history.csv b/insurance/realtabformer/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..565da66ad6e16086df9fd2e58b039468048cb5e9 --- /dev/null +++ b/insurance/realtabformer/history.csv @@ -0,0 +1,15 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.06546703377635114,0.2411056103362887,0.017943341929830138,15.073662688997057,0.0,0.0,0.0,0.0,0.06770005301468902,900,113,153.185537815094,1.3556242284521591,0.1702061531278822,0.19871398358218437,0.021130030669478907,0.19607622416553663,0.004915828111867692,0.0,0.0,0.0,0.0,0.0,0.021130030669478907,450,57,44.15434193611145,0.7746375778265167,0.09812075985802544,0.08530344322258443 +1,0.013551225868674617,0.425866672638608,0.0008923178030537808,3.086364604102241,0.0,0.0,0.0,0.0,0.014007968114585512,900,113,153.67027974128723,1.3599139800113915,0.17074475526809693,0.11838984252077288,0.00839744051038805,0.15208790417410342,0.0025543556639753243,0.0,0.0,0.0,0.0,0.0,0.00839744051038805,450,57,49.47366642951965,0.8679590601670114,0.10994148095448812,0.07164400266973596 +2,0.006413812188871412,0.4138342716928963,0.00027937432812316876,1.2699479508399962,0.0,0.0,0.0,0.0,0.006606090834312555,900,113,148.70240354537964,1.315950473852917,0.1652248928281996,0.10096772389272146,0.004124537513190363,0.06340608278020754,7.95078217205811e-05,0.0,0.0,0.0,0.0,0.0,0.004124537513190363,450,57,46.079535722732544,0.8084129074163604,0.10239896827273899,0.08735310192334286 +3,0.004092327245485245,0.39121937074457336,0.00015605472596730313,0.8052064700921376,0.0,0.0,0.0,0.0,0.004214407260684917,900,113,151.17974090576172,1.337873813325325,0.1679774898952908,0.09381394159147698,0.0034038320787820138,0.17477164815838414,3.0920931797607425e-05,0.0,0.0,0.0,0.0,0.0,0.0034038320787820138,450,57,44.10925340652466,0.7738465509916607,0.09802056312561035,0.06855134886542433 +4,0.0029089865246270264,0.3532221096504087,3.6831276020658194e-05,0.6396473573313819,0.0,0.0,0.0,0.0,0.003004638084805467,900,113,148.33039927482605,1.3126584006621775,0.16481155474980672,0.0951092214018634,0.002712128226024409,0.18766189036889222,0.00010364639543665517,0.0,0.0,0.0,0.0,0.0,0.002712128226024409,450,57,43.69719195365906,0.7666174026957729,0.09710487100813124,0.07099823879176065 +5,0.0025865390710532667,0.28755133922778336,5.002069715601034e-05,0.5206181814935472,0.0,0.0,0.0,0.0,0.0026650355748521785,900,113,149.53471994400024,1.3233161056991172,0.16614968882666695,0.09785628467138889,0.004123696400783956,0.10052104216206098,0.00014860794093957944,0.0,0.0,0.0,0.0,0.0,0.004123696400783956,450,57,45.542537689208984,0.7989918892843682,0.10120563930935329,0.07634249809933336 +6,0.0021268636879782812,0.31098353265446677,8.022330904053045e-06,0.46527117974228327,0.0,0.0,0.0,0.0,0.0021964997161800664,900,113,154.0214467048645,1.3630216522554381,0.17113494078318278,0.09568176698351724,0.002951748826243501,0.10900051805550315,5.319740680752188e-05,0.0,0.0,0.0,0.0,0.0,0.002951748826243501,450,57,45.68478465080261,0.801487450014081,0.10152174366845025,0.07844465254119744 +7,0.0016828670411551784,0.2405022908817118,1.4838149498900283e-05,0.41218649297952653,0.0,0.0,0.0,0.0,0.0017432555771018897,900,113,148.98203325271606,1.3184250730328855,0.16553559250301786,0.09567539716980099,0.0022270536331375802,0.2691560960501106,1.237651391853007e-05,0.0,0.0,0.0,0.0,0.0,0.0022270536331375802,450,57,44.04278039932251,0.772680357882851,0.0978728453318278,0.06789644183836092 +8,0.0012292804241102809,0.17017161006335066,4.924689298776446e-06,0.3241997027397156,0.0,0.0,0.0,0.0,0.0012760602385969832,900,113,148.87872958183289,1.3175108812551584,0.16542081064648098,0.09586640849577642,0.002435751861194149,0.36038146253469866,3.095573820521539e-05,0.0,0.0,0.0,0.0,0.0,0.002435751861194149,450,57,47.944602489471436,0.8411333770082708,0.1065435610877143,0.07143153931604486 +9,0.0011144688440577334,0.1618615282109625,8.490755512877918e-06,0.3029045389427079,0.0,0.0,0.0,0.0,0.0011581761015178119,900,113,153.11266350746155,1.354979323074881,0.17012518167495727,0.0970211002349326,0.002127645400275166,0.5217206840770483,1.6785784667897965e-05,0.0,0.0,0.0,0.0,0.0,0.002127645400275166,450,57,45.5362389087677,0.7988813843643456,0.10119164201948377,0.07289726209924802 +10,0.001183760426776846,0.12453773899971259,4.877423153579243e-06,0.3282844849758678,0.0,0.0,0.0,0.0,0.001230810873934792,900,113,151.34659552574158,1.3393504028826688,0.16816288391749065,0.09647954430068488,0.002083526839996921,0.7386151516803023,3.646810097101981e-06,0.0,0.0,0.0,0.0,0.0,0.002083526839996921,450,57,45.06990456581116,0.7907000801019501,0.10015534347958036,0.06614154230868607 +11,0.0009753633572189655,0.18075985179538961,2.4574399071989697e-06,0.3026174498928918,0.0,0.0,0.0,0.0,0.0010182713146140384,900,113,153.43047070503235,1.3577917761507288,0.17047830078336929,0.09725009125813974,0.0027126917298947876,0.44140643616923336,4.97356319702471e-05,0.0,0.0,0.0,0.0,0.0,0.0027126917298947876,450,57,46.485644578933716,0.8155376241918195,0.10330143239763048,0.060076543326048476 +12,0.0009962741199544527,0.1213041851699925,2.6763202339177273e-06,0.2919425904419687,0.0,0.0,0.0,0.0,0.0010380058418700678,900,113,150.9032688140869,1.3354271576467869,0.16767029868231878,0.09730801442885294,0.002843431096148884,0.4800476854505124,3.743590894896127e-05,0.0,0.0,0.0,0.0,0.0,0.002843431096148884,450,57,47.381882190704346,0.8312610910649885,0.10529307153489854,0.05945909781414166 +13,0.0009301258288420892,0.13480907791556043,4.767580377660616e-06,0.26487836490074795,0.0,0.0,0.0,0.0,0.0009680978746877776,900,113,152.54639387130737,1.3499680873567024,0.16949599319034153,0.09505416518819015,0.0015452833087571587,0.7066929681970184,1.9704139299798997e-05,0.0,0.0,0.0,0.0,0.0,0.0015452833087571587,450,57,45.19634485244751,0.7929183307446932,0.1004363218943278,0.06895738166119707 diff --git a/insurance/realtabformer/mlu-eval.ipynb b/insurance/realtabformer/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..9e1c684d864cc7af4607a4e214875a3193ee4f7f --- /dev/null +++ b/insurance/realtabformer/mlu-eval.ipynb @@ -0,0 +1,2380 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:28.970567Z", + "iopub.status.busy": "2024-03-26T10:27:28.970244Z", + "iopub.status.idle": "2024-03-26T10:27:29.002576Z", + "shell.execute_reply": "2024-03-26T10:27:29.001726Z" + }, + "papermill": { + "duration": 0.046868, + "end_time": "2024-03-26T10:27:29.004446", + "exception": false, + "start_time": "2024-03-26T10:27:28.957578", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:29.030111Z", + "iopub.status.busy": "2024-03-26T10:27:29.029784Z", + "iopub.status.idle": "2024-03-26T10:27:29.036163Z", + "shell.execute_reply": "2024-03-26T10:27:29.035356Z" + }, + "papermill": { + "duration": 0.021686, + "end_time": "2024-03-26T10:27:29.038051", + "exception": false, + "start_time": "2024-03-26T10:27:29.016365", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:29.061299Z", + "iopub.status.busy": "2024-03-26T10:27:29.061035Z", + "iopub.status.idle": "2024-03-26T10:27:29.064995Z", + "shell.execute_reply": "2024-03-26T10:27:29.064198Z" + }, + "papermill": { + "duration": 0.017664, + "end_time": "2024-03-26T10:27:29.066814", + "exception": false, + "start_time": "2024-03-26T10:27:29.049150", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:29.090252Z", + "iopub.status.busy": "2024-03-26T10:27:29.089982Z", + "iopub.status.idle": "2024-03-26T10:27:29.093679Z", + "shell.execute_reply": "2024-03-26T10:27:29.092858Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017585, + "end_time": "2024-03-26T10:27:29.095560", + "exception": false, + "start_time": "2024-03-26T10:27:29.077975", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:29.118593Z", + "iopub.status.busy": "2024-03-26T10:27:29.118322Z", + "iopub.status.idle": "2024-03-26T10:27:29.123628Z", + "shell.execute_reply": "2024-03-26T10:27:29.122788Z" + }, + "papermill": { + "duration": 0.019157, + "end_time": "2024-03-26T10:27:29.125713", + "exception": false, + "start_time": "2024-03-26T10:27:29.106556", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "c296c826", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:29.151783Z", + "iopub.status.busy": "2024-03-26T10:27:29.151046Z", + "iopub.status.idle": "2024-03-26T10:27:29.156204Z", + "shell.execute_reply": "2024-03-26T10:27:29.155397Z" + }, + "papermill": { + "duration": 0.020121, + "end_time": "2024-03-26T10:27:29.158086", + "exception": false, + "start_time": "2024-03-26T10:27:29.137965", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"realtabformer\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 2\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/realtabformer/2\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011484, + "end_time": "2024-03-26T10:27:29.180527", + "exception": false, + "start_time": "2024-03-26T10:27:29.169043", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:29.205158Z", + "iopub.status.busy": "2024-03-26T10:27:29.204916Z", + "iopub.status.idle": "2024-03-26T10:27:29.213862Z", + "shell.execute_reply": "2024-03-26T10:27:29.213156Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.02275, + "end_time": "2024-03-26T10:27:29.215725", + "exception": false, + "start_time": "2024-03-26T10:27:29.192975", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/realtabformer/2\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:29.239058Z", + "iopub.status.busy": "2024-03-26T10:27:29.238803Z", + "iopub.status.idle": "2024-03-26T10:27:31.237484Z", + "shell.execute_reply": "2024-03-26T10:27:31.236423Z" + }, + "papermill": { + "duration": 2.01271, + "end_time": "2024-03-26T10:27:31.239635", + "exception": false, + "start_time": "2024-03-26T10:27:29.226925", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:31.267367Z", + "iopub.status.busy": "2024-03-26T10:27:31.265675Z", + "iopub.status.idle": "2024-03-26T10:27:31.278502Z", + "shell.execute_reply": "2024-03-26T10:27:31.277826Z" + }, + "papermill": { + "duration": 0.028043, + "end_time": "2024-03-26T10:27:31.280357", + "exception": false, + "start_time": "2024-03-26T10:27:31.252314", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:31.304668Z", + "iopub.status.busy": "2024-03-26T10:27:31.303985Z", + "iopub.status.idle": "2024-03-26T10:27:31.311383Z", + "shell.execute_reply": "2024-03-26T10:27:31.310356Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021363, + "end_time": "2024-03-26T10:27:31.313340", + "exception": false, + "start_time": "2024-03-26T10:27:31.291977", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:31.337349Z", + "iopub.status.busy": "2024-03-26T10:27:31.337067Z", + "iopub.status.idle": "2024-03-26T10:27:31.427868Z", + "shell.execute_reply": "2024-03-26T10:27:31.426991Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.104998, + "end_time": "2024-03-26T10:27:31.429929", + "exception": false, + "start_time": "2024-03-26T10:27:31.324931", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:31.456316Z", + "iopub.status.busy": "2024-03-26T10:27:31.456033Z", + "iopub.status.idle": "2024-03-26T10:27:36.058454Z", + "shell.execute_reply": "2024-03-26T10:27:36.057694Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.617929, + "end_time": "2024-03-26T10:27:36.060923", + "exception": false, + "start_time": "2024-03-26T10:27:31.442994", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-26 10:27:33.638251: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-26 10:27:33.638308: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-26 10:27:33.639841: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:36.085770Z", + "iopub.status.busy": "2024-03-26T10:27:36.085187Z", + "iopub.status.idle": "2024-03-26T10:27:36.091876Z", + "shell.execute_reply": "2024-03-26T10:27:36.091187Z" + }, + "papermill": { + "duration": 0.021044, + "end_time": "2024-03-26T10:27:36.093742", + "exception": false, + "start_time": "2024-03-26T10:27:36.072698", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:36.120288Z", + "iopub.status.busy": "2024-03-26T10:27:36.119967Z", + "iopub.status.idle": "2024-03-26T10:27:45.134416Z", + "shell.execute_reply": "2024-03-26T10:27:45.133420Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 9.031105, + "end_time": "2024-03-26T10:27:45.136927", + "exception": false, + "start_time": "2024-03-26T10:27:36.105822", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-26T10:27:45.166700Z", + "iopub.status.busy": "2024-03-26T10:27:45.165898Z", + "iopub.status.idle": "2024-03-26T10:27:45.172799Z", + "shell.execute_reply": "2024-03-26T10:27:45.171962Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.024084, + "end_time": "2024-03-26T10:27:45.174609", + "exception": false, + "start_time": "2024-03-26T10:27:45.150525", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 36,\n", + " 'realtabformer': (19, 551, Embedding(551, 800), True),\n", + " 'lct_gan': 29,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:45.199812Z", + "iopub.status.busy": "2024-03-26T10:27:45.199535Z", + "iopub.status.idle": "2024-03-26T10:27:45.204048Z", + "shell.execute_reply": "2024-03-26T10:27:45.203222Z" + }, + "papermill": { + "duration": 0.019129, + "end_time": "2024-03-26T10:27:45.205947", + "exception": false, + "start_time": "2024-03-26T10:27:45.186818", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:45.230708Z", + "iopub.status.busy": "2024-03-26T10:27:45.230416Z", + "iopub.status.idle": "2024-03-26T10:27:45.294080Z", + "shell.execute_reply": "2024-03-26T10:27:45.293221Z" + }, + "papermill": { + "duration": 0.078268, + "end_time": "2024-03-26T10:27:45.295981", + "exception": false, + "start_time": "2024-03-26T10:27:45.217713", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/realtabformer/all inf False\n", + "../../../../ml-utility-loss/aug_test/insurance 0\n", + "Caching in ../../../../insurance/_cache_bs_test/realtabformer/all inf False\n", + "../../../../ml-utility-loss/bs_test/insurance 0\n", + "Caching in ../../../../insurance/_cache_synth_test/realtabformer/all inf False\n", + "../../../../ml-utility-loss/synthetics/insurance 600\n", + "600\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:45.322410Z", + "iopub.status.busy": "2024-03-26T10:27:45.322150Z", + "iopub.status.idle": "2024-03-26T10:27:45.648502Z", + "shell.execute_reply": "2024-03-26T10:27:45.647651Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.341991, + "end_time": "2024-03-26T10:27:45.650563", + "exception": false, + "start_time": "2024-03-26T10:27:45.308572", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.77,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'loss_balancer_beta': 0.75,\n", + " 'loss_balancer_r': 0.95,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'realtabformer',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': torch.nn.modules.activation.ReLU6,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.RReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['realtabformer'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "# params = {\n", + "# **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "# }\n", + "params = getattr(PARAMS, dataset_name).BEST_DICT[gp][gp_multiply][single_model]\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).DEFAULTS,\n", + " **params,\n", + "}\n", + "if isinstance(params, (list, tuple)):\n", + " params = params[param_index]\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:45.676533Z", + "iopub.status.busy": "2024-03-26T10:27:45.676248Z", + "iopub.status.idle": "2024-03-26T10:27:45.782685Z", + "shell.execute_reply": "2024-03-26T10:27:45.781738Z" + }, + "papermill": { + "duration": 0.121755, + "end_time": "2024-03-26T10:27:45.784854", + "exception": false, + "start_time": "2024-03-26T10:27:45.663099", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_train/realtabformer/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/insurance [400, 0]\n", + "Caching in ../../../../insurance/_cache_aug_val/realtabformer/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/insurance [0, 200]\n", + "Caching in ../../../../insurance/_cache_bs_train/realtabformer/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/insurance [100, 0]\n", + "Caching in ../../../../insurance/_cache_bs_val/realtabformer/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/insurance [0, 50]\n", + "Caching in ../../../../insurance/_cache_synth/realtabformer/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/insurance [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-26T10:27:45.815299Z", + "iopub.status.busy": "2024-03-26T10:27:45.814981Z", + "iopub.status.idle": "2024-03-26T10:27:46.290921Z", + "shell.execute_reply": "2024-03-26T10:27:46.289890Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.495672, + "end_time": "2024-03-26T10:27:46.294905", + "exception": false, + "start_time": "2024-03-26T10:27:45.799233", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding True True\n", + "['realtabformer'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:46.358247Z", + "iopub.status.busy": "2024-03-26T10:27:46.357815Z", + "iopub.status.idle": "2024-03-26T10:27:46.362748Z", + "shell.execute_reply": "2024-03-26T10:27:46.361668Z" + }, + "papermill": { + "duration": 0.04274, + "end_time": "2024-03-26T10:27:46.366019", + "exception": false, + "start_time": "2024-03-26T10:27:46.323279", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:46.426791Z", + "iopub.status.busy": "2024-03-26T10:27:46.425797Z", + "iopub.status.idle": "2024-03-26T10:27:46.434168Z", + "shell.execute_reply": "2024-03-26T10:27:46.433168Z" + }, + "papermill": { + "duration": 0.046946, + "end_time": "2024-03-26T10:27:46.437142", + "exception": false, + "start_time": "2024-03-26T10:27:46.390196", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "10420884" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:46.484952Z", + "iopub.status.busy": "2024-03-26T10:27:46.484384Z", + "iopub.status.idle": "2024-03-26T10:27:46.574362Z", + "shell.execute_reply": "2024-03-26T10:27:46.573512Z" + }, + "papermill": { + "duration": 0.107656, + "end_time": "2024-03-26T10:27:46.576226", + "exception": false, + "start_time": "2024-03-26T10:27:46.468570", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 15200] --\n", + "├─Adapter: 1-1 [2, 1071, 15200] --\n", + "│ └─Embedding: 2-1 [2, 1071, 19, 800] (440,800)\n", + "│ └─TensorInductionPoint: 2-2 [19, 1] 19\n", + "│ └─Sequential: 2-3 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 820,224\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 15200] (recursive)\n", + "│ └─Embedding: 2-4 [2, 267, 19, 800] (recursive)\n", + "│ └─TensorInductionPoint: 2-5 [19, 1] (recursive)\n", + "│ └─Sequential: 2-6 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-7 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─ReLU6: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─ReLU6: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─ReLU6: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ └─Encoder: 2-8 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─ReLU6: 6-31 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─ReLU6: 6-37 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-40 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-42 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─ReLU6: 6-43 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-45 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-46 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-48 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-49 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-18 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-51 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-52 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-9 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─RReLU: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─RReLU: 4-52 [2, 128] --\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 10,861,684\n", + "Trainable params: 10,420,884\n", + "Non-trainable params: 440,800\n", + "Total mult-adds (M): 42.02\n", + "========================================================================================================================\n", + "Input size (MB): 0.20\n", + "Forward/backward pass size (MB): 632.48\n", + "Params size (MB): 43.45\n", + "Estimated Total Size (MB): 676.13\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:27:46.606144Z", + "iopub.status.busy": "2024-03-26T10:27:46.605870Z", + "iopub.status.idle": "2024-03-26T11:15:05.472852Z", + "shell.execute_reply": "2024-03-26T11:15:05.471922Z" + }, + "papermill": { + "duration": 2838.900302, + "end_time": "2024-03-26T11:15:05.490822", + "exception": false, + "start_time": "2024-03-26T10:27:46.590520", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 600]\n", + "Creating model of type \n", + "[*] Embedding True True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.06546703377635114, 'avg_role_model_std_loss': 0.2411056103362887, 'avg_role_model_mean_pred_loss': 0.017943341929830138, 'avg_role_model_g_mag_loss': 15.073662688997057, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.06770005301468902, 'n_size': 900, 'n_batch': 113, 'duration': 153.185537815094, 'duration_batch': 1.3556242284521591, 'duration_size': 0.1702061531278822, 'avg_pred_std': 0.19871398358218437}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.021130030669478907, 'avg_role_model_std_loss': 0.19607622416553663, 'avg_role_model_mean_pred_loss': 0.004915828111867692, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.021130030669478907, 'n_size': 450, 'n_batch': 57, 'duration': 44.15434193611145, 'duration_batch': 0.7746375778265167, 'duration_size': 0.09812075985802544, 'avg_pred_std': 0.08530344322258443}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013551225868674617, 'avg_role_model_std_loss': 0.425866672638608, 'avg_role_model_mean_pred_loss': 0.0008923178030537808, 'avg_role_model_g_mag_loss': 3.086364604102241, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.014007968114585512, 'n_size': 900, 'n_batch': 113, 'duration': 153.67027974128723, 'duration_batch': 1.3599139800113915, 'duration_size': 0.17074475526809693, 'avg_pred_std': 0.11838984252077288}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00839744051038805, 'avg_role_model_std_loss': 0.15208790417410342, 'avg_role_model_mean_pred_loss': 0.0025543556639753243, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00839744051038805, 'n_size': 450, 'n_batch': 57, 'duration': 49.47366642951965, 'duration_batch': 0.8679590601670114, 'duration_size': 0.10994148095448812, 'avg_pred_std': 0.07164400266973596}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006413812188871412, 'avg_role_model_std_loss': 0.4138342716928963, 'avg_role_model_mean_pred_loss': 0.00027937432812316876, 'avg_role_model_g_mag_loss': 1.2699479508399962, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006606090834312555, 'n_size': 900, 'n_batch': 113, 'duration': 148.70240354537964, 'duration_batch': 1.315950473852917, 'duration_size': 0.1652248928281996, 'avg_pred_std': 0.10096772389272146}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004124537513190363, 'avg_role_model_std_loss': 0.06340608278020754, 'avg_role_model_mean_pred_loss': 7.95078217205811e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004124537513190363, 'n_size': 450, 'n_batch': 57, 'duration': 46.079535722732544, 'duration_batch': 0.8084129074163604, 'duration_size': 0.10239896827273899, 'avg_pred_std': 0.08735310192334286}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004092327245485245, 'avg_role_model_std_loss': 0.39121937074457336, 'avg_role_model_mean_pred_loss': 0.00015605472596730313, 'avg_role_model_g_mag_loss': 0.8052064700921376, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004214407260684917, 'n_size': 900, 'n_batch': 113, 'duration': 151.17974090576172, 'duration_batch': 1.337873813325325, 'duration_size': 0.1679774898952908, 'avg_pred_std': 0.09381394159147698}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0034038320787820138, 'avg_role_model_std_loss': 0.17477164815838414, 'avg_role_model_mean_pred_loss': 3.0920931797607425e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0034038320787820138, 'n_size': 450, 'n_batch': 57, 'duration': 44.10925340652466, 'duration_batch': 0.7738465509916607, 'duration_size': 0.09802056312561035, 'avg_pred_std': 0.06855134886542433}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0029089865246270264, 'avg_role_model_std_loss': 0.3532221096504087, 'avg_role_model_mean_pred_loss': 3.6831276020658194e-05, 'avg_role_model_g_mag_loss': 0.6396473573313819, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003004638084805467, 'n_size': 900, 'n_batch': 113, 'duration': 148.33039927482605, 'duration_batch': 1.3126584006621775, 'duration_size': 0.16481155474980672, 'avg_pred_std': 0.0951092214018634}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002712128226024409, 'avg_role_model_std_loss': 0.18766189036889222, 'avg_role_model_mean_pred_loss': 0.00010364639543665517, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002712128226024409, 'n_size': 450, 'n_batch': 57, 'duration': 43.69719195365906, 'duration_batch': 0.7666174026957729, 'duration_size': 0.09710487100813124, 'avg_pred_std': 0.07099823879176065}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0025865390710532667, 'avg_role_model_std_loss': 0.28755133922778336, 'avg_role_model_mean_pred_loss': 5.002069715601034e-05, 'avg_role_model_g_mag_loss': 0.5206181814935472, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0026650355748521785, 'n_size': 900, 'n_batch': 113, 'duration': 149.53471994400024, 'duration_batch': 1.3233161056991172, 'duration_size': 0.16614968882666695, 'avg_pred_std': 0.09785628467138889}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.004123696400783956, 'avg_role_model_std_loss': 0.10052104216206098, 'avg_role_model_mean_pred_loss': 0.00014860794093957944, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004123696400783956, 'n_size': 450, 'n_batch': 57, 'duration': 45.542537689208984, 'duration_batch': 0.7989918892843682, 'duration_size': 0.10120563930935329, 'avg_pred_std': 0.07634249809933336}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0021268636879782812, 'avg_role_model_std_loss': 0.31098353265446677, 'avg_role_model_mean_pred_loss': 8.022330904053045e-06, 'avg_role_model_g_mag_loss': 0.46527117974228327, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0021964997161800664, 'n_size': 900, 'n_batch': 113, 'duration': 154.0214467048645, 'duration_batch': 1.3630216522554381, 'duration_size': 0.17113494078318278, 'avg_pred_std': 0.09568176698351724}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002951748826243501, 'avg_role_model_std_loss': 0.10900051805550315, 'avg_role_model_mean_pred_loss': 5.319740680752188e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002951748826243501, 'n_size': 450, 'n_batch': 57, 'duration': 45.68478465080261, 'duration_batch': 0.801487450014081, 'duration_size': 0.10152174366845025, 'avg_pred_std': 0.07844465254119744}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0016828670411551784, 'avg_role_model_std_loss': 0.2405022908817118, 'avg_role_model_mean_pred_loss': 1.4838149498900283e-05, 'avg_role_model_g_mag_loss': 0.41218649297952653, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0017432555771018897, 'n_size': 900, 'n_batch': 113, 'duration': 148.98203325271606, 'duration_batch': 1.3184250730328855, 'duration_size': 0.16553559250301786, 'avg_pred_std': 0.09567539716980099}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022270536331375802, 'avg_role_model_std_loss': 0.2691560960501106, 'avg_role_model_mean_pred_loss': 1.237651391853007e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022270536331375802, 'n_size': 450, 'n_batch': 57, 'duration': 44.04278039932251, 'duration_batch': 0.772680357882851, 'duration_size': 0.0978728453318278, 'avg_pred_std': 0.06789644183836092}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0012292804241102809, 'avg_role_model_std_loss': 0.17017161006335066, 'avg_role_model_mean_pred_loss': 4.924689298776446e-06, 'avg_role_model_g_mag_loss': 0.3241997027397156, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012760602385969832, 'n_size': 900, 'n_batch': 113, 'duration': 148.87872958183289, 'duration_batch': 1.3175108812551584, 'duration_size': 0.16542081064648098, 'avg_pred_std': 0.09586640849577642}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002435751861194149, 'avg_role_model_std_loss': 0.36038146253469866, 'avg_role_model_mean_pred_loss': 3.095573820521539e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002435751861194149, 'n_size': 450, 'n_batch': 57, 'duration': 47.944602489471436, 'duration_batch': 0.8411333770082708, 'duration_size': 0.1065435610877143, 'avg_pred_std': 0.07143153931604486}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0011144688440577334, 'avg_role_model_std_loss': 0.1618615282109625, 'avg_role_model_mean_pred_loss': 8.490755512877918e-06, 'avg_role_model_g_mag_loss': 0.3029045389427079, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011581761015178119, 'n_size': 900, 'n_batch': 113, 'duration': 153.11266350746155, 'duration_batch': 1.354979323074881, 'duration_size': 0.17012518167495727, 'avg_pred_std': 0.0970211002349326}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002127645400275166, 'avg_role_model_std_loss': 0.5217206840770483, 'avg_role_model_mean_pred_loss': 1.6785784667897965e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002127645400275166, 'n_size': 450, 'n_batch': 57, 'duration': 45.5362389087677, 'duration_batch': 0.7988813843643456, 'duration_size': 0.10119164201948377, 'avg_pred_std': 0.07289726209924802}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001183760426776846, 'avg_role_model_std_loss': 0.12453773899971259, 'avg_role_model_mean_pred_loss': 4.877423153579243e-06, 'avg_role_model_g_mag_loss': 0.3282844849758678, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001230810873934792, 'n_size': 900, 'n_batch': 113, 'duration': 151.34659552574158, 'duration_batch': 1.3393504028826688, 'duration_size': 0.16816288391749065, 'avg_pred_std': 0.09647954430068488}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002083526839996921, 'avg_role_model_std_loss': 0.7386151516803023, 'avg_role_model_mean_pred_loss': 3.646810097101981e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002083526839996921, 'n_size': 450, 'n_batch': 57, 'duration': 45.06990456581116, 'duration_batch': 0.7907000801019501, 'duration_size': 0.10015534347958036, 'avg_pred_std': 0.06614154230868607}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0009753633572189655, 'avg_role_model_std_loss': 0.18075985179538961, 'avg_role_model_mean_pred_loss': 2.4574399071989697e-06, 'avg_role_model_g_mag_loss': 0.3026174498928918, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010182713146140384, 'n_size': 900, 'n_batch': 113, 'duration': 153.43047070503235, 'duration_batch': 1.3577917761507288, 'duration_size': 0.17047830078336929, 'avg_pred_std': 0.09725009125813974}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0027126917298947876, 'avg_role_model_std_loss': 0.44140643616923336, 'avg_role_model_mean_pred_loss': 4.97356319702471e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0027126917298947876, 'n_size': 450, 'n_batch': 57, 'duration': 46.485644578933716, 'duration_batch': 0.8155376241918195, 'duration_size': 0.10330143239763048, 'avg_pred_std': 0.060076543326048476}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0009962741199544527, 'avg_role_model_std_loss': 0.1213041851699925, 'avg_role_model_mean_pred_loss': 2.6763202339177273e-06, 'avg_role_model_g_mag_loss': 0.2919425904419687, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010380058418700678, 'n_size': 900, 'n_batch': 113, 'duration': 150.9032688140869, 'duration_batch': 1.3354271576467869, 'duration_size': 0.16767029868231878, 'avg_pred_std': 0.09730801442885294}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002843431096148884, 'avg_role_model_std_loss': 0.4800476854505124, 'avg_role_model_mean_pred_loss': 3.743590894896127e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002843431096148884, 'n_size': 450, 'n_batch': 57, 'duration': 47.381882190704346, 'duration_batch': 0.8312610910649885, 'duration_size': 0.10529307153489854, 'avg_pred_std': 0.05945909781414166}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0009301258288420892, 'avg_role_model_std_loss': 0.13480907791556043, 'avg_role_model_mean_pred_loss': 4.767580377660616e-06, 'avg_role_model_g_mag_loss': 0.26487836490074795, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009680978746877776, 'n_size': 900, 'n_batch': 113, 'duration': 152.54639387130737, 'duration_batch': 1.3499680873567024, 'duration_size': 0.16949599319034153, 'avg_pred_std': 0.09505416518819015}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0015452833087571587, 'avg_role_model_std_loss': 0.7066929681970184, 'avg_role_model_mean_pred_loss': 1.9704139299798997e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0015452833087571587, 'n_size': 450, 'n_batch': 57, 'duration': 45.19634485244751, 'duration_batch': 0.7929183307446932, 'duration_size': 0.1004363218943278, 'avg_pred_std': 0.06895738166119707}\n", + "Stopped False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'realtabformer', 'n_size': 600, 'n_batch': 75, 'role_model_metrics': {'avg_loss': 0.0008848744352386954, 'avg_g_mag_loss': 0.007162825414042118, 'avg_g_cos_loss': 6.385196543609103e-05, 'pred_duration': 3.0819342136383057, 'grad_duration': 0.6160187721252441, 'total_duration': 3.69795298576355, 'pred_std': 0.15960799157619476, 'std_loss': 0.0007001429912634194, 'mean_pred_loss': 3.047596464966773e-06, 'pred_rmse': 0.02974683977663517, 'pred_mae': 0.019926337525248528, 'pred_mape': 0.7107431888580322, 'grad_rmse': 0.6675140261650085, 'grad_mae': 0.2900869846343994, 'grad_mape': 11.930258750915527}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0008848744352386954, 'avg_g_mag_loss': 0.007162825414042118, 'avg_g_cos_loss': 6.385196543609103e-05, 'avg_pred_duration': 3.0819342136383057, 'avg_grad_duration': 0.6160187721252441, 'avg_total_duration': 3.69795298576355, 'avg_pred_std': 0.15960799157619476, 'avg_std_loss': 0.0007001429912634194, 'avg_mean_pred_loss': 3.047596464966773e-06}, 'min_metrics': {'avg_loss': 0.0008848744352386954, 'avg_g_mag_loss': 0.007162825414042118, 'avg_g_cos_loss': 6.385196543609103e-05, 'pred_duration': 3.0819342136383057, 'grad_duration': 0.6160187721252441, 'total_duration': 3.69795298576355, 'pred_std': 0.15960799157619476, 'std_loss': 0.0007001429912634194, 'mean_pred_loss': 3.047596464966773e-06, 'pred_rmse': 0.02974683977663517, 'pred_mae': 0.019926337525248528, 'pred_mape': 0.7107431888580322, 'grad_rmse': 0.6675140261650085, 'grad_mae': 0.2900869846343994, 'grad_mape': 11.930258750915527}, 'model_metrics': {'realtabformer': {'avg_loss': 0.0008848744352386954, 'avg_g_mag_loss': 0.007162825414042118, 'avg_g_cos_loss': 6.385196543609103e-05, 'pred_duration': 3.0819342136383057, 'grad_duration': 0.6160187721252441, 'total_duration': 3.69795298576355, 'pred_std': 0.15960799157619476, 'std_loss': 0.0007001429912634194, 'mean_pred_loss': 3.047596464966773e-06, 'pred_rmse': 0.02974683977663517, 'pred_mae': 0.019926337525248528, 'pred_mape': 0.7107431888580322, 'grad_rmse': 0.6675140261650085, 'grad_mae': 0.2900869846343994, 'grad_mape': 11.930258750915527}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:15:05.525518Z", + "iopub.status.busy": "2024-03-26T11:15:05.524799Z", + "iopub.status.idle": "2024-03-26T11:15:05.529569Z", + "shell.execute_reply": "2024-03-26T11:15:05.528755Z" + }, + "papermill": { + "duration": 0.024368, + "end_time": "2024-03-26T11:15:05.531493", + "exception": false, + "start_time": "2024-03-26T11:15:05.507125", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:15:05.564275Z", + "iopub.status.busy": "2024-03-26T11:15:05.563808Z", + "iopub.status.idle": "2024-03-26T11:15:05.647782Z", + "shell.execute_reply": "2024-03-26T11:15:05.646925Z" + }, + "papermill": { + "duration": 0.102813, + "end_time": "2024-03-26T11:15:05.650147", + "exception": false, + "start_time": "2024-03-26T11:15:05.547334", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:15:05.685221Z", + "iopub.status.busy": "2024-03-26T11:15:05.684927Z", + "iopub.status.idle": "2024-03-26T11:15:05.959554Z", + "shell.execute_reply": "2024-03-26T11:15:05.958496Z" + }, + "papermill": { + "duration": 0.294548, + "end_time": "2024-03-26T11:15:05.961515", + "exception": false, + "start_time": "2024-03-26T11:15:05.666967", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAEUCAYAAACCg0tBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA5cklEQVR4nO3de1xUdf4/8Nfch+twk5shaIKYIBQKYamZrGhqYW0QP1fMr2n11dYkzXAV3G2/0c2NSr+5uqX23UhyK3dXXYpIu4EQ4DWV1FBIHS4qAwyXgZnP74/DHBgZcAZhZmDez8fjPJg5533OfAaZt5/zmc9FwBhjIIQQGyC0dgEIIUSPEhIhxGZQQiKE2AxKSIQQm0EJiRBiMyghEUJsBiUkQojNoIRECLEZlJAIITZDbO0CDASdTocrV67AxcUFAoHA2sUhhNyEMYbGxkb4+/tDKOyjHsT6YcuWLSwwMJDJZDIWHR3NioqK+oz/5JNP2Lhx45hMJmNhYWHswIEDBscBGN1ef/11k8pTVVXV6zVoo40229mqqqr6/CybXUPKyclBamoqtm3bhpiYGGRlZSE+Ph7l5eXw9vbuEV9QUIDk5GRkZmZi3rx5yM7ORkJCAsrKyhAWFgYAuHr1qsE5//nPf7B06VI89thjJpXJxcUFAFBVVQVXV1dz3xIhZJA1NDQgICCA/6z2RtBZQzFZTEwMJk+ejC1btgDgbpcCAgLw3HPP4aWXXuoRn5SUBLVajf379/P77r33XkRGRmLbtm1GXyMhIQGNjY3Iz883qUwNDQ1QKBRQqVSUkAixQaZ+Rs1q1NZoNCgtLUVcXFzXBYRCxMXFobCw0Og5hYWFBvEAEB8f32t8dXU1Dhw4gKVLl/Zajra2NjQ0NBhshJChz6yEVFdXB61WCx8fH4P9Pj4+UCqVRs9RKpVmxe/evRsuLi549NFHey1HZmYmFAoFvwUEBJjzNgghNsrmvvb/4IMPsHDhQsjl8l5j0tLSoFKp+K2qqsqCJSSEDBazGrW9vLwgEolQXV1tsL+6uhq+vr5Gz/H19TU5/rvvvkN5eTlycnL6LIdMJoNMJjOn6KQPWq0W7e3t1i4GGcIkEglEItFtX8eshCSVShEVFYX8/HwkJCQA4Bq18/PzsXLlSqPnxMbGIj8/H88//zy/Ly8vD7GxsT1i33//fURFRSEiIsKcYpF+YoxBqVSivr7e2kUhw4Cbmxt8fX1vqy+g2V/7p6amYvHixZg0aRKio6ORlZUFtVqNJUuWAABSUlIwcuRIZGZmAgBWrVqF6dOnY/PmzZg7dy727NmDkpISbN++3eC6DQ0N2Lt3LzZv3tzvN0PMo09G3t7ecHR0pE6lpF8YY2hubkZNTQ0AwM/Pr9/XMjshJSUloba2Funp6VAqlYiMjERubi7fcF1ZWWnQE3PKlCnIzs7Ghg0bsH79egQHB2Pfvn18HyS9PXv2gDGG5OTkfr+ZW+nQ6nDisgq1jW2YdZePXX8AtVotn4w8PT2tXRwyxDk4OAAAampq4O3t3e/bN7P7IdkiU/s4tGi0GJ+eCwA4sWkWXOUSSxXR5rS2tqKiogJBQUH8HxMht6OlpQUXL17E6NGje3wpNSj9kIY6B6kILjKuUljT0Gbl0tgGe64lkoE1EH9LdpWQAGCEK/ftXG0jJSRCbI39JSRnLiHVNLZauSSEkJvZXULyduXubamGRPpLIBBg37591i7GgNq0aRMiIyOtXQw7TEgudMtGhr5du3bBzc1twK63Zs0akwezD6ZhMUGbOUa46G/ZKCGR4U+j0UAqld4yztnZGc7OzhYoUd+ohkR4jDE0azosvpnb8yQ3Nxf3338/3Nzc4OnpiXnz5uHChQsAuH5v69atM4ivra2FRCLBt99+C4Cbf2vu3LlwcHDA6NGjkZ2djaCgIGRlZfXr93by5Ek8+OCDcHBwgKenJ5YvX46mpib++OHDhxEdHQ0nJye4ubnhvvvuw6VLlwAAx48fx4wZM+Di4gJXV1dERUWhpKSkz9c7fPgwlixZApVKBYFAAIFAgE2bNgEAgoKC8PLLLyMlJQWurq5Yvnw5AGDdunUICQmBo6MjxowZg40bNxoMF7r5lu3JJ59EQkIC3nzzTfj5+cHT0xMrVqwY9CFGdldD8nbh2pCoUbunlnYt7kr/wuKve/pP8XCUmv6nqFarkZqaiokTJ6KpqQnp6elYsGABjh07hoULF+L111/Hq6++yn8NnZOTA39/f0ydOhUAN5qgrq4Ohw8fhkQiQWpqKt/L2FxqtRrx8fGIjY3Fjz/+iJqaGjz11FNYuXIldu3ahY6ODiQkJGDZsmX4+OOPodFoUFxczJdt4cKFuPvuu/Hee+9BJBLh2LFjkEj67h83ZcoUZGVlIT09HeXl5QBgULt58803kZ6ejoyMDH6fi4sLdu3aBX9/f5w8eRLLli2Di4sLXnzxxV5f59ChQ/Dz88OhQ4dw/vx5JCUlITIyEsuWLevX78oUdpeQ6JZt6Lt5JtEPPvgAI0aMwOnTp5GYmIjnn38e33//PZ+AsrOzkZycDIFAgLNnz+Krr77Cjz/+iEmTJgEA/va3vyE4OLhfZcnOzkZrays+/PBDODk5AQC2bNmC+fPn47XXXoNEIoFKpcK8efNw5513AgDGjx/Pn19ZWYm1a9ciNDQUAEwqh1QqhUKhgEAgMDpI/cEHH8QLL7xgsG/Dhg3846CgIKxZswZ79uzpMyG5u7tjy5YtEIlECA0Nxdy5c5Gfn08JaSDpb9nqm9vR1qGFTHz7I5SHCweJCKf/FG+V1zXHuXPnkJ6ejqKiItTV1UGn0wHgPtxhYWGYNWsWPvroI0ydOhUVFRUoLCzEX//6VwBAeXk5xGIx7rnnHv56Y8eOhbu7e7/KfubMGURERPDJCADuu+8+6HQ6lJeXY9q0aXjyyScRHx+P3/zmN4iLi0NiYiI/3is1NRVPPfUU/u///g9xcXF4/PHH+cTVX/pE211OTg7eeecdXLhwAU1NTejo6Ljl7KoTJkwwGALi5+eHkydP3lbZbsXu2pDcHCWQiLjqcl2TxsqlsS0CgQCOUrHFN3N7+M6fPx/Xr1/Hjh07UFRUhKKiIgBcAy7A3Qb94x//QHt7O7KzsxEeHo7w8PAB/32ZaufOnSgsLMSUKVOQk5ODkJAQHDlyBADXdvPTTz9h7ty5+Prrr3HXXXfh888/v63X654cAW7W1oULF+Khhx7C/v37cfToUfzhD3/gf1+9ufnWUSAQ8Ml/sNhdQhIIBF2dIxuoHWmouXbtGsrLy7FhwwbMnDkT48ePx40bNwxiHnnkEbS2tiI3NxfZ2dlYuHAhf2zcuHHo6OjA0aNH+X3nz5/vcQ1TjR8/HsePH4dareb3/fDDDxAKhRg3bhy/7+6770ZaWhoKCgoQFhaG7Oxs/lhISAhWr16NL7/8Eo8++ih27tx5y9eVSqXQarUmlbGgoACBgYH4wx/+gEmTJiE4OJhvVLc1dpeQAGAEdY4cstzd3eHp6Ynt27fj/Pnz+Prrr5GammoQ4+TkhISEBGzcuBFnzpwxmEEiNDQUcXFxWL58OYqLi3H06FEsX74cDg4O/RqLpZ/ddPHixTh16hQOHTqE5557DosWLYKPjw8qKiqQlpaGwsJCXLp0CV9++SXOnTuH8ePHo6WlBStXrsThw4dx6dIl/PDDD/jxxx8N2ph6ExQUhKamJuTn56Ourg7Nzc29xgYHB6OyshJ79uzBhQsX8M4779x2LWyw2GdCcqaG7aFKKBRiz549KC0tRVhYGFavXo033nijR9zChQtx/PhxTJ06FaNGjTI49uGHH8LHxwfTpk3DggUL+G+c+po2uTeOjo744osvcP36dUyePBm//e1vMXPmTH5VHkdHR5w9exaPPfYYQkJCsHz5cqxYsQJPP/00RCIRrl27hpSUFISEhCAxMRFz5szBH//4x1u+7pQpU/DMM88gKSkJI0aMwOuvv95r7MMPP4zVq1dj5cqViIyMREFBATZu3Gj2e7UEu5p+RG/95yeRXVSJVTODsfo3IRYooe3RTz9ibKoIe/Prr78iICAAX331FWbOnGnt4gxZff1NmfoZtbtv2YCub9qohmSfvv76azQ1NSE8PBxXr17Fiy++iKCgIEybNs3aRbN79nnLxvfWpkZte9Te3o7169djwoQJWLBgAUaMGMF3kvzoo4/4YRQ3bxMmTLBYGefMmdNrOV555RWLlcPS7LSGRI3a9iw+Ph7x8cb7Wz388MOIiYkxeuxWPagH0t/+9je0tLQYPebh4WGxclianSYkumUjxrm4uNxy/XlLGDlypLWLYBV2fsvWBp1uyLfpEzJs9Cshbd26FUFBQZDL5YiJiUFxcXGf8Xv37kVoaCjkcjnCw8Nx8ODBHjFnzpzBww8/DIVCAScnJ0yePBmVlZX9Kd4teXV+7d+hY6hvoQUSCbEVZieknJwcpKamIiMjA2VlZYiIiEB8fHyvo6ULCgqQnJyMpUuX4ujRo0hISEBCQgJOnTrFx1y4cAH3338/QkNDcfjwYZw4cQIbN24ctK+jpWIh3B259gAa9U+IDWFmio6OZitWrOCfa7Va5u/vzzIzM43GJyYmsrlz5xrsi4mJYU8//TT/PCkpif3ud78ztyg8lUrFADCVSmXyObP+8g0LXLeffVNe0+/XHcpaWlrY6dOnWUtLi7WLQoaJvv6mTP2MmlVD0mg0KC0tRVxcHL9PKBQiLi4OhYWFRs8pLCw0iAe4bzn08TqdDgcOHEBISAji4+Ph7e2NmJiYPucsbmtrQ0NDg8FmLm9afYQQm2NWQqqrq4NWq+VXqdXz8fGBUqk0eo5SqewzvqamBk1NTXj11Vcxe/ZsfPnll1iwYAEeffRRfPPNN0avmZmZCYVCwW8BAQHmvA0ANHyE9N9wnOTfVlj9Wzb9dAaPPPIIVq9ejcjISLz00kuYN28etm3bZvSctLQ0qFQqfquqqjL7dWl9NjKUDfQk/wA3Na5AIEB9ff2AXtccZvVD8vLygkgkQnV1tcH+6upqozPXAYCvr2+f8V5eXhCLxbjrrrsMYsaPH4/vv//e6DVlMhlkMpk5Re+BprIlxPaYVUOSSqWIiooyWC5Fp9MhPz8fsbGxRs+JjY3tsbxKXl4eHy+VSjF58mR+bmC9n3/+GYGBgeYUzyw0la0RjAEateU3muR/wCb5b2trw5o1azBy5Eg4OTkhJiYGhw8f5s+9dOkS5s+fD3d3dzg5OWHChAk4ePAgLl68iBkzZgDgpngRCAR48skn+/X7uB1m99ROTU3F4sWLMWnSJERHRyMrKwtqtRpLliwBwE2gPnLkSGRmZgIAVq1ahenTp2Pz5s2YO3cu9uzZg5KSEmzfvp2/5tq1a5GUlIRp06ZhxowZyM3Nxb///W+DX+RA0/fWrqOE1KW9GXjF3/Kvu/4KIHW6dVwnmuS/90n+V65cidOnT2PPnj3w9/fH559/jtmzZ+PkyZMIDg7GihUroNFo8O2338LJyQmnT5+Gs7MzAgIC8Omnn+Kxxx5DeXk5XF1d4eDg0K/fye0wOyElJSWhtrYW6enpUCqViIyMRG5uLt9wXVlZCaGwq+I1ZcoUZGdnY8OGDVi/fj2Cg4Oxb98+hIWF8TELFizAtm3bkJmZid///vcYN24cPv30U9x///0D8BaNoxrS0EWT/Buf5L+yshI7d+5EZWUl/P25/1jWrFmD3Nxc7Ny5E6+88goqKyvx2GOP8VP6jhkzhj9fP0bO29t7wNunTDZYfRIsqT/9kBpaNCxw3X4WuG4/U7e1D2LpbJPRPiM6HWNtTZbfdDqzyv7zzz+zJ554go0ePZq5uLgwJycnBoAdOHCAMcbYQw89xPdz++WXXxgAduLECcYYY/v27WNisZhptVqDa7q7u7O33nrLpNcHwD7//HPGGGOrV69mDzzwgMHx+vp6BoB98803jDHGnnzySSaTydi8efNYVlYWu3LlCh+bkZHBxGIxmzlzJsvMzGTnz583qQw7d+5kCoXCYN/+/fsZAObk5GSwicVilpiYyBhjbMeOHUwsFrMpU6aw9PR0dvz4cf78Q4cOMQDsxo0bJpXhZhbvhzScOMvE/GoX9E1bJ4GAu3Wy9EaT/A/IJP9NTU0QiUQoLS3FsWPH+O3MmTN4++23AQBPPfUUfvnlFyxatAgnT57EpEmT8O677w7Ye71ddpuQBAIB3bYNQTTJP8fYJP933303tFotampqMHbsWIOt+61dQEAAnnnmGXz22Wd44YUXsGPHDv6aAExePGAw2G1CArpNQ9JACWmooEn+OcYm+Q8JCcHChQuRkpKCzz77DBUVFSguLkZmZiYOHDgAAHj++efxxRdfoKKiAmVlZTh06BD/eoGBgRAIBNi/fz9qa2sNvim0mH7dLNqY/rQhMcbYs38vYYHr9rOd3/8ySCWzXUN5LFteXh4bP348k8lkbOLEiezw4cMG7TqMMXbw4EEGgE2bNq3H+VeuXGFz5sxhMpmMBQYGsuzsbObt7c22bdtm0uvf/FonTpxgM2bMYHK5nHl4eLBly5axxsZGxhhjSqWSJSQkMD8/PyaVSllgYCBLT09nWq2WtbW1sSeeeIIFBAQwqVTK/P392cqVK03+N3nmmWeYp6cnA8AyMjIYY4xpNBqWnp7OgoKCmEQiYX5+fmzBggV8G9rKlSvZnXfeyWQyGRsxYgRbtGgRq6ur46/5pz/9ifn6+jKBQMAWL15sUjn0BqINyS4n+dfL+Ocp7C68hP9+4E68ODt0EEtoe2iS/y40yf/AoEn+b5M3rc9ml2iSf9tl121I1Khtn2iSf9tl1zUkSkj2iSb5t112nZC8XWjEPzFEk/xbF92yAbimbkOHVmfl0ljHMPhOg9iIgfhbsuuE5Okkg1DADTa/ptZYuzgWpb/9aG5utnJJyHCh/1u6nVtbu75lEwkF8HKWoaaxDbWNbfBxtZ+vv0UiEdzc3PhR7o6Ojv3qGEgIYwzNzc2oqamBm5sbRCJRv69l1wkJ4G7bahrbOidqU1i7OBalH07Q36k3COnOzc2t14kaTWX3CcnbRYafYJ8N2wKBAH5+fvD29kZ7O61PR/pPIpHcVs1IjxKSfipbOx7PJhKJBuSPiZDbZdeN2gD1RSLElth9QqL12QixHZSQ+BoSrT5CiLXZfUKiWzZCbIfdJyR9o3ZtYxv1WibEyuw+IelrSG0dOjS0dli5NITYt34lpK1btyIoKAhyuRwxMTEoLi7uM37v3r0IDQ2FXC5HeHg4Dh48aHD8ySef5Be802+zZ8/uT9HMJpeI4CLnej/UUjsSIVZldkLKyclBamoqMjIyUFZWhoiICMTHx/fa27egoADJyclYunQpjh49ioSEBCQkJODUqVMGcbNnz8bVq1f57eOPP+7fO+oHb2pHIsQmmJ2Q/vKXv2DZsmVYsmQJ7rrrLmzbtg2Ojo744IMPjMa//fbbmD17NtauXYvx48fj5Zdfxj333IMtW7YYxMlkMvj6+vKbu7t7r2Voa2tDQ0ODwXY7RtA0JITYBLMSkkajQWlpKeLi4rouIBQiLi4OhYWFRs8pLCw0iAe4CbJujj98+DC8vb0xbtw4PPvss7h27Vqv5cjMzIRCoeC3gIAAc95GD90btgkh1mNWQqqrq4NWq+WXzdbz8fGBUqk0eo5Sqbxl/OzZs/Hhhx8iPz8fr732Gr755hvMmTOn1/Wh0tLSoFKp+K2qqsqct9ED3bIRYhtsYizbE088wT8ODw/HxIkTceedd+Lw4cNGV4GQyWSQyWQD9vp8X6QGatQmxJrMqiF5eXlBJBKhurraYH91dXWv0w74+vqaFQ8AY8aMgZeXF86fP29O8fqNHz7SRDUkQqzJrIQklUoRFRWF/Px8fp9Op0N+fj5iY2ONnhMbG2sQDwB5eXm9xgPcOlnXrl2Dn5+fOcXrNxrxT4htMPtbttTUVOzYsQO7d+/GmTNn8Oyzz0KtVmPJkiUAgJSUFKSlpfHxq1atQm5uLjZv3oyzZ89i06ZNKCkpwcqVKwEATU1NWLt2LY4cOYKLFy8iPz8fjzzyCMaOHdvryhADjYaPEGIbzG5DSkpKQm1tLdLT06FUKhEZGYnc3Fy+4bqyshJCYVeemzJlCrKzs7FhwwasX78ewcHB2LdvH8LCwgBwc/GcOHECu3fvRn19Pfz9/TFr1iy8/PLLA9pO1Bd9o7aqpR1tHVrIxDQ3ECHWYNdLaesxxjBuQy40Wh2+XzcDd7g7DkIpCbFfpn5G7X4sG8BN5Uq3bYRYHyWkTtRbmxDro4TUiWpIhFgfJaROtKw2IdZHCalT13g26q1NiLVQQurUNXyEakiEWAslpE78LRsNHyHEaighddKPZ6MaEiHWQwmpk/6Wra6pDTrdkO8rSsiQRAmpk5czl5A6dAw3mjVWLg0h9okSUieJSAgPJykA6otEiLVQQuqG+iIRYl2UkLqh3tqEWBclpG66EhJ1jiTEGighdUOrjxBiXZSQuqHVRwixLkpI3fBTkFDnSEKsghJSNzR8hBDrooTUDa3PRoh1UULqxtuVa9RWa7RQt3VYuTSE2J9+JaStW7ciKCgIcrkcMTExKC4u7jN+7969CA0NhVwuR3h4OA4ePNhr7DPPPAOBQICsrKz+FO22OMvEcJRyK47QN22EWJ7ZCSknJwepqanIyMhAWVkZIiIiEB8fj5qaGqPxBQUFSE5OxtKlS3H06FEkJCQgISEBp06d6hH7+eef48iRI/D39zf/nQwQ6hxJiPWYnZD+8pe/YNmyZViyZAnuuusubNu2DY6Ojvjggw+Mxr/99tuYPXs21q5di/Hjx+Pll1/GPffcgy1bthjEXb58Gc899xw++ugjSCSS/r2bAUDDRwixHrMSkkajQWlpKeLi4rouIBQiLi4OhYWFRs8pLCw0iAeA+Ph4g3idTodFixZh7dq1mDBhwi3L0dbWhoaGBoNtoPDLalNvbUIszqyEVFdXB61Wy69Sq+fj4wOlUmn0HKVSecv41157DWKxGL///e9NKkdmZiYUCgW/BQQEmPM2+kS3bIRYj9W/ZSstLcXbb7+NXbt2QSAQmHROWloaVCoVv1VVVQ1YeWh9NkKsx6yE5OXlBZFIhOrqaoP91dXV8PX1NXqOr69vn/HfffcdampqMGrUKIjFYojFYly6dAkvvPACgoKCjF5TJpPB1dXVYBsoNHyEEOsxKyFJpVJERUUhPz+f36fT6ZCfn4/Y2Fij58TGxhrEA0BeXh4fv2jRIpw4cQLHjh3jN39/f6xduxZffPGFue/ntlHnSEKsR2zuCampqVi8eDEmTZqE6OhoZGVlQa1WY8mSJQCAlJQUjBw5EpmZmQCAVatWYfr06di8eTPmzp2LPXv2oKSkBNu3bwcAeHp6wtPT0+A1JBIJfH19MW7cuNt9f2bTN2rX0fARQizO7ISUlJSE2tpapKenQ6lUIjIyErm5uXzDdWVlJYTCrorXlClTkJ2djQ0bNmD9+vUIDg7Gvn37EBYWNnDvYgDpa0jX1Bp0aHUQi6zezEaI3RAwxob8EhsNDQ1QKBRQqVS33Z6k0zEEb/gPtDqGI2kz4auQD1ApCbFfpn5G6b//mwiFAng5c5P90zdthFgWJSQjaCpbQqyDEpIRNJUtIdZBCckI6otEiHVQQjKCbtkIsQ5KSEbQiH9CrIMSkhEj+BH/lJAIsSRKSEZ0DR+hhESIJVFCMqL76iPDoN8oIUMGJSQj9DUkTYcODS002T8hlkIJyQi5RARXOTfMr7aJvmkjxFIoIfVCvyQStSMRYjmUkHoxwpk6RxJiaZSQeuHtSn2RCLE0Ski98Kbe2oRYHCWkXtDqI4RYHiWkXtCIf0IsjxJSL2jEPyGWRwmpF7T6CCGWRwmpF/pbtobWDrS2a61cGkLsAyWkXrg6iCEVc78eakcixDL6lZC2bt2KoKAgyOVyxMTEoLi4uM/4vXv3IjQ0FHK5HOHh4Th48KDB8U2bNiE0NBROTk5wd3dHXFwcioqK+lO0ASMQCPjOkbW0RhshFmF2QsrJyUFqaioyMjJQVlaGiIgIxMfHo6amxmh8QUEBkpOTsXTpUhw9ehQJCQlISEjAqVOn+JiQkBBs2bIFJ0+exPfff4+goCDMmjULtbW1/X9nA0DfOZKGjxBiIcxM0dHRbMWKFfxzrVbL/P39WWZmptH4xMRENnfuXIN9MTEx7Omnn+71NVQqFQPAvvrqK6PHW1tbmUql4reqqioGgKlUKnPfTp+W7f6RBa7bzz4sqBjQ6xJib/Sf6Vt9Rs2qIWk0GpSWliIuLo7fJxQKERcXh8LCQqPnFBYWGsQDQHx8fK/xGo0G27dvh0KhQEREhNGYzMxMKBQKfgsICDDnbZiMho8QYllmJaS6ujpotVp+2Ww9Hx8fKJVKo+colUqT4vfv3w9nZ2fI5XK89dZbyMvLg5eXl9FrpqWlQaVS8VtVVZU5b8Nk3jSVLSEWJbZ2AfRmzJiBY8eOoa6uDjt27EBiYiKKiorg7e3dI1Ymk0Emkw16mWj4CCGWZVYNycvLCyKRCNXV1Qb7q6ur4evra/QcX19fk+KdnJwwduxY3HvvvXj//fchFovx/vvvm1O8AUerjxBiWWYlJKlUiqioKOTn5/P7dDod8vPzERsba/Sc2NhYg3gAyMvL6zW++3Xb2qybCLpu2ai3NiGWYPYtW2pqKhYvXoxJkyYhOjoaWVlZUKvVWLJkCQAgJSUFI0eORGZmJgBg1apVmD59OjZv3oy5c+diz549KCkpwfbt2wEAarUa//M//4OHH34Yfn5+qKurw9atW3H58mU8/vjjA/hWzae/Zatr0kCnYxAKBVYtDyHDndkJKSkpCbW1tUhPT4dSqURkZCRyc3P5huvKykoIhV0VrylTpiA7OxsbNmzA+vXrERwcjH379iEsLAwAIBKJcPbsWezevRt1dXXw9PTE5MmT8d1332HChAkD9Db7x9NZCoEA0OoYrjdr4OU8+O1WhNgzAWNDf52fhoYGKBQKqFQquLq6Dui1o17OwzW1Bgd/PxV3+Q/stQmxF6Z+Rmks2y2McKHhI4RYCiWkW+hafYQatgkZbJSQboFWHyHEcigh3QINHyHEcigh3QJ1jiTEcigh3cIIWg6JEIuhhHQLtPoIIZZDCekWaIAtIZZDCekW9G1IzRot1G0dVi4NIcMbJaRbcJKJ4SQVAaBaEiGDjRKSCWiNNkIsgxKSCfiGbRo+QsigooRkghG0+gghFkEJyQQ0fIQQy6CEZAIaPkKIZVBCMgFNZUuIZVBCMsEIGs9GiEVQQjIBDbAlxDLsMyGZOWuvvoZ0Ta1Bu1Y3GCUihMDeElJTLXBgDfDBbLOSkoejFOLOFUeuNWkGq3SE2L1+JaStW7ciKCgIcrkcMTExKC4u7jN+7969CA0NhVwuR3h4OA4ePMgfa29vx7p16xAeHg4nJyf4+/sjJSUFV65c6U/R+iaWAsc+AqqOAFVFJp8mFAr4FUeoYZuQwWN2QsrJyUFqaioyMjJQVlaGiIgIxMfHo6amxmh8QUEBkpOTsXTpUhw9ehQJCQlISEjAqVOnAADNzc0oKyvDxo0bUVZWhs8++wzl5eV4+OGHb++dGSNXAGGPco9LPjDr1K7hI9SORMigYWaKjo5mK1as4J9rtVrm7+/PMjMzjcYnJiayuXPnGuyLiYlhTz/9dK+vUVxczACwS5cumVQmlUrFADCVSnXr4F9LGMtwZexPIxhrqjPp+owx9l87i1nguv0su8i0MhFCupj6GTWrhqTRaFBaWoq4uDh+n1AoRFxcHAoLC42eU1hYaBAPAPHx8b3GA4BKpYJAIICbm5vR421tbWhoaDDYTOZ/D+AXAWjbgOPZJp/mTcNHCBl0ZiWkuro6aLVafpVaPR8fHyiVSqPnKJVKs+JbW1uxbt06JCcn97qgXGZmJhQKBb8FBASY/iYEAmDSf3GPS3YCOtO+NRtBbUiEDDqb+patvb0diYmJYIzhvffe6zUuLS0NKpWK36qqqsx7obDfAlIX4PoF4OK3Jp0ywpWmsiVksJmVkLy8vCASiVBdXW2wv7q6Gr6+vkbP8fX1NSlen4wuXbqEvLy8PpfblclkcHV1NdjMInMGIpK4xyU7TTrFm6ayJWTQmZWQpFIpoqKikJ+fz+/T6XTIz89HbGys0XNiY2MN4gEgLy/PIF6fjM6dO4evvvoKnp6e5hSrf6KWcD/P7gcaq/uOBQ0fIcQSzL5lS01NxY4dO7B7926cOXMGzz77LNRqNZYs4T7gKSkpSEtL4+NXrVqF3NxcbN68GWfPnsWmTZtQUlKClStXAuCS0W9/+1uUlJTgo48+glarhVKphFKphEYziJ0QfcOAgBhA1wEc/b9bhncfPsLM7OlNCDGN2NwTkpKSUFtbi/T0dCiVSkRGRiI3N5dvuK6srIRQ2JXnpkyZguzsbGzYsAHr169HcHAw9u3bh7CwMADA5cuX8a9//QsAEBkZafBahw4dwgMPPNDPt2aCqCVcB8nS3cD9qwGhqNdQfcdIjVYHVUs73Bylg1cuQuyUgA2D/+4bGhqgUCigUqnMa09qbwE2hwKt9cD/2wuEzOozPOKPX0LV0o681dMQ7ONye4UmxI6Y+hm1qW/ZLE7iAEQu5B6b0HObGrYJGVz2nZAAYFJn4/a5L4D6vrsP0LLahAwuSkhewUDQVIDpgLIP+wyleZEIGVyUkICunttlHwLa9l7DvDs7R9LwEUIGByUkAAidBziNAJqUwM+5vYbR6iOEDC5KSAA3T9Ldv+Me99G4TauPEDK4KCHp3bMYgAC48DVw/RejIdSoTcjgooSk5zEaGDuTe1y6y2gINWoTMrgoIXWnb9w++nego2fSGdG5PltDawda27WWLBkhdoESUnfB8YCLP9B8DTjz7x6HXeViSMXcr4xqSYQMPEpI3YnEQNRi7rGRxm2BQEC9tQkZRJSQbnZPCiAQAZd+AGrLexz26eyLdLTyhqVLRsiwRwnpZq7+wLg53GMjk7fNn+gHANj85c+oqFNbsmSEDHuUkIzRT952PBvQNBscSokNQuwYT7S0a7E65xg6aCVbQgYMJSRj7nwQcBsFtKqAnz43OCQUCrA5MQIucjGOVdVj66ELViokIcMPJSRjhMKuWpKRxm1/Nwf8OYGbYO6dr8/hWFW9BQtHyPBFCak3d/8OEEqAyyXA1eM9Dj8c4Y95E/2g1TGszjmGZk2HFQpJyPBCCak3zt7A+PncYyON2wKBAP+TEA5fVzkq6tR45eAZCxeQkOGHElJf9D23T+4F2hp7HFY4SrA5MQIA8PcjlTh0tsaSpSNk2KGE1Jeg+wHPYEDTxCUlI+4b64X/um80AGDtP07gWhN1mCSkvygh9aX7sts/fgD0sh7Ci7PHIdjbGXVNbUj77CQtk0RIP/UrIW3duhVBQUGQy+WIiYlBcXFxn/F79+5FaGgo5HI5wsPDcfDgQYPjn332GWbNmgVPT08IBAIcO3asP8UaHBFPACIZUH0SuFxqNEQuESHriUhIRAJ8eboae0t/tXAhCRkezE5IOTk5SE1NRUZGBsrKyhAREYH4+HjU1BhvPykoKEBycjKWLl2Ko0ePIiEhAQkJCTh16hQfo1arcf/99+O1117r/zsZLI4eQNij3OM+Jm+b4K9A6m/GAQD++K+fUHmtuddYQohxZq/LFhMTg8mTJ2PLli0AuKW0AwIC8Nxzz+Gll17qEZ+UlAS1Wo39+/fz++69915ERkZi27ZtBrEXL17E6NGjcfTo0R6LRnbX1taGtrautpqGhgYEBASYvy6bqaqKgfd/A4jlwAtnAQd3o2FaHUPy9iMovngdkwLdkfN0LERCwcCXh5AhZlDWZdNoNCgtLUVcXFzXBYRCxMXFobCw0Og5hYWFBvEAEB8f32u8KTIzM6FQKPgtICCg39cyyR2TAZ8woKMVOL6n1zBRZy9uZ5kYJZduYNs31IubEHOYlZDq6uqg1Wr5ZbP1fHx8oFQqjZ6jVCrNijdFWloaVCoVv1VV9b2e2m0TCLrWbyvpvXEbAAI8HLHp4QkAgLfyfsapy6rBLRshw8iQ/JZNJpPB1dXVYBt04YmAxAmo+5mbmqQPj90zErMn+KJDx/B8zjGaXZIQE5mVkLy8vCASiVBdXW2wv7q6Gr6+vkbP8fX1NSveZsldgYmPc4+N9NzuTiAQ4JVHwzHCRYbzNU149T9nLVBAQoY+sxKSVCpFVFQU8vPz+X06nQ75+fmIjY01ek5sbKxBPADk5eX1Gm/T9H2STv8TUF3uM9TDSYo3fjsRALCr4CK+O1c72KUjZMgz+5YtNTUVO3bswO7du3HmzBk8++yzUKvVWLKEa2NJSUlBWloaH79q1Srk5uZi8+bNOHv2LDZt2oSSkhKsXLmSj7l+/TqOHTuG06dPAwDKy8tx7Nix22pnGhR+EcDISYCuHfggHlCe7DP8gXHeWHRvIABgzd7jqG/WWKKUhAxdrB/effddNmrUKCaVSll0dDQ7cuQIf2z69Ols8eLFBvGffPIJCwkJYVKplE2YMIEdOHDA4PjOnTsZgB5bRkaGSeVRqVQMAFOpVP15O+apO8/Y23czluHK2J/9GDv97z7Dm9s62Iw3D7HAdfvZf/+9lOl0usEvIyE2xtTPqNn9kGyRqX0cBkzLDWDvk8Avh7nnD24Apq7hvo0z4sSv9Xj0fwvQoWN4KykCC+6+Y/DLSIgNGZR+SKSTgzuw8FMg5hnu+dd/Bj5dCrS3GA2feIcbfj8zGACQvu8n/HqDenETYgwlpP4SiYE5rwHzsgChGDj1KbBzDtBwxWj4fz9wJ+4e5YbGtg688MlxaHVDvmJKyICjhHS7Ji0BUv4JOHgAV44C22cYHYQrFgnxVmIkHKUiFFVcx7IPS1BGSykRYoAS0kAIuh9Y9jUwYjzQpAR2PgSc/EfPMC8nvPxIGAQC4OuzNXj0fwuQ+NdCHDpbQ1OWEIJ+DK61RRZv1O5NawPw2TLg51zu+dQXgBkbuEUDujlX3Yi/fvsL/nnsMtq13K9/nI8Lnp4+BvMj/CER0f8TZHgx9TNKCWmg6bRA/p+AH7K45+PmAo9uB2TOPUKvqlrwwfcVyC6qhFrDDS/xV8ixdOoYPDE5AE4ysQULTsjgoYRkbcf3AP96DtBquJkCnsgG3AONhqpa2vH3I5ew84eLqOucAlfhIEFKbCAWTwmCl7PMkiUnZMBRQrIFVT8Ce/4foK4BHD2BpI+AwN6HzLS2a/FZ2WXs+O4XfplumViIxyfdgWVTxyDQ08lSJSdkQFFCshWqX4GPkwHlCW6dt3l/Ae5J6fMUrY7hy5+U2PbNBRz/lZu+RCgA5oT74ZlpdyL8DoUlSk7IgKGEZEs0amDfs9ygXAC497+5nt2OHr327gYAxhiO/HIdf/32Ag6Xdw3OvW+sJx6PCsDk0R4Y6eYw2KXvSdMMNF4F3IMAocjyr0+GHEpItkanA759HTic2bVPpgA8RgMeY7r9HAO4jwZcfA2S1ZmrDdj+7S/41/ErBp0q/RVyTArywOQgd0wK8kCIj8vAT5ur0wHVp4ALX3NbZSHXNiZ1Bu6YBATEAAHR3Myacqq9kZ4oIdmqn/YBX2UANy72HSdx5BKTx2iDZHVV5IcPf+pAwS83cOpKQ48e3y5yMaIC3TE5yAOTAt0REeAGuaQftZhGJXDhEJeAfjkEqG+aPkUo4WY9MCAAvMdzySkghts8xvRZCyT2gRKSrWtvAW5cAq7/wm03Kroe11cBrI9ZJkVSwG0UtIpA1Ij9ca7dE2UNCnxT64RyjReaIedDJSIBwkcquAQV5IGoQHd4OEmNl+dSQWct6BBQ85PhcYkTMHoqcOeD3OYxBqg5A1QVcYsgVBVx7+Fmjl5dNaiAGMA/EpBY4TaTWBUlpKFM2w7UVwLXK3omrBsXudulPrRIPXBV6Iuf2zxR3u6FKuaNSp03LjEf1MANd3q7YryvC0IElxDeWoqxDT/CT1UGka7rugwC6HwjIBw7E4KxDwJ3RANiI4msu6aaruRUVcQNpbm5rEIJN69UQDQwIrTrNtXFr0cHUjJAGOP+prRt3M+ONu7fRasBZC5c88Ago4Q0XOm0QMNlLjHpt+sVXY9brvd5ehuToIqNgKugGd6CeoNjV5gHvtNOxHe6cPygm4AbcIVQALg6SOAql0Dh0G1zlMDLSYoRLjJ4Ocvg1flzhIsMTlIRBAIB94d/9XhXgqos4rpAGCOWd96ijulxmwrXO7jBzPZKp+VuoVVVXO1ZVcn9h6X6lRsdoE8uWg3QoX/cLfn0uLW+iftorvYbNJUbBuXqP+BvgRKSvWpVGSar7glLVQXoOvjQdqEcvzjfg5/kUSgVR+Jshx9UrR1QtbRD1dIOTYeuX0WQS4RcknLuSlIjnKXwcpYiQFiHUeqT8FKdhFPTJYjqKyC4canvW1ShhOtU6t4tSXmMAcQy7lazXd35swVob+a+BWxv7nquf6zpHsf184LMlWuIlys6H7t22+fatU+u4L6E0O+TyHsvr7k6NNx/MqoqLtHUV3U9VlVx0yXfKqmYQyDibvvFUqCtEWA3/Tt7jOlMTvoE5XfbL0kJifSk7eisXVVwf5Ajo7gPdS9a27Vo6ExO+q2htR2q5nbcaG5HXVNb56ZBXVMbahvb0Kwxb4UVqVgIXycRQh1VCJHUYoyoBncwJXw6LsO99Vc4N/8Koc4Gp/4VSblvGYViQCDkNqGIa8DXPxcIuQ9/9+fCbo8Z42o+jVfBTZLaB6GYq7koRgFuAYDbKEARwM3NpU8uIim37LtIwv27iiSdz6Xd9kkNu2q0NgCVR4CL3wIXv+dqtDcnKM+xXGLSJ6h+3OJRQiJW0azpQF2jBrWdyaq2sa0rcTV2Jq6mNtQ1tvHj9/oihA6+uI5AYTWCBNUYJ63FnaIajIISEiEDEzsAEkcIpY4QyZ0gkTtB6uAEuaMrRFJHrgFd6sT9lDh2bp2PAaCtgatVtqo6Hzf0/ritcxsMYjmXYNwCun66BXY9dvGzTJ+vVhVwqRC4+F1Xgro5WXoGdyaoziTl4mP0Ut1RQiI2r0WjRV1TG66pNahrbMM1dVdt69pNP683a/pan9MohYOks41LanD76OXMtX0pHCSQiIT8JhUJIRYJ+McSMfdYLBRwbWIA157T1tiZnJq42gS/ablaj8E+HXRaLXQ6btNqdZ2PdWBMB6GLD+ReQRC7eNtm94iWeq7f2cXvgYpvOxe2uOkfYtqLwIN/6PMypn5G7bilkFibg1SEAA9HBHg43jJWq2O4rtbgmrorSXG1L02Pmti1Jg06dIy/zTzfSzu6OSSdiaprE0AkFECnY9AyBq2O2zp0XY+1nceMJ1IBABGAOgB1kIgEcJCI4CAVwVEqhlwigqNUxO9z6Hx+836RUMDlQICfU0v/egys27Gu5+iM1T8WiQQQCbj3IxIKIBYKINT/FAggFoVD6DMRYr+VkHWo4HWtDB61RXCvKYJz/VnccBoDj9v/FQPoZ0LaunUr3njjDSiVSkRERODdd99FdHR0r/F79+7Fxo0bcfHiRQQHB+O1117DQw89xB9njCEjIwM7duxAfX097rvvPrz33nsIDg7uT/HIMCQSCrjGcZdbz3yg60xG+tvD7omrrrGr3auhtR0dWgaNVod2rQ7tHTq0dz6/WbuWoV2rBTA4qxBz1+9AQ2sHgLZBeY2BowAwC8AsKNCEVc1h+K8BurLZCSknJwepqanYtm0bYmJikJWVhfj4eJSXl8Pb27tHfEFBAZKTk5GZmYl58+YhOzsbCQkJKCsrQ1hYGADg9ddfxzvvvIPdu3dj9OjR2LhxI+Lj43H69GnI5QP4bQaxC0KhAO5OUrg7SRHs42L2+ayzxtPePVlpdWjv6HreoWVo1+n4moW4Wy1DLBRCKITBz+61D5GQixUIAI1WhxaNFi3tWjRrtPzjFg33vFW/v12LFk0HH6ffr9UxCAQCCNDZng7wz9F5ByiAoNuxbs87j+trdjr9T8bQoe38eXON76YaoVbnAjfFAA4XMnd9pejoaLZixQr+uVarZf7+/iwzM9NofGJiIps7d67BvpiYGPb0008zxhjT6XTM19eXvfHGG/zx+vp6JpPJ2Mcff2xSmSy6LhshxGymfkbN6hqr0WhQWlqKuLg4fp9QKERcXBwKCwuNnlNYWGgQDwDx8fF8fEVFBZRKpUGMQqFATExMr9dsa2tDQ0ODwUYIGfrMSkh1dXXQarXw8TH8ms/Hx6fXZa+VSmWf8fqf5lwzMzMTCoWC3wICAsx5G4QQGzUkBw+lpaVBpVLxW1VVlbWLRAgZAGYlJC8vL4hEIlRXVxvsr66uhq+v8d6bvr6+fcbrf5pzTZlMBldXV4ONEDL0mZWQpFIpoqKikJ+fz+/T6XTIz89HbKzxuaJjY2MN4gEgLy+Pjx89ejR8fX0NYhoaGlBUVNTrNQkhw5S5reV79uxhMpmM7dq1i50+fZotX76cubm5MaVSyRhjbNGiReyll17i43/44QcmFovZm2++yc6cOcMyMjKYRCJhJ0+e5GNeffVV5ubmxv75z3+yEydOsEceeYSNHj2atbS0DGgLPiHEOkz9jJqdkBhj7N1332WjRo1iUqmURUdHsyNHjvDHpk+fzhYvXmwQ/8knn7CQkBAmlUrZhAkT2IEDBwyO63Q6tnHjRubj48NkMhmbOXMmKy8vN7k8lJAIsW2mfkaHxVg2lUoFNzc3VFVVUXsSITaooaEBAQEBqK+vh6KPjpTDYixbY2MjANDX/4TYuMbGxj4T0rCoIel0Oly5cgUuLi5do7J7oc/UVJvqP/odDgx7+j0yxtDY2Ah/f38I+5iqeFjUkIRCIe644w6zzqHuArePfocDw15+j33VjPSGZMdIQsjwRAmJEGIz7C4hyWQyZGRkQCa79bw6xDj6HQ4M+j32NCwatQkhw4Pd1ZAIIbaLEhIhxGZQQiKE2AxKSIQQm0EJiRBiM+wuIW3duhVBQUGQy+WIiYlBcXGxtYs0ZGzatIlb0aLbFhoaau1i2bRvv/0W8+fPh7+/PwQCAfbt22dwnDGG9PR0+Pn5wcHBAXFxcTh37px1CmsD7Coh6ZdwysjIQFlZGSIiIhAfH4+amgFYSdBOTJgwAVevXuW377//3tpFsmlqtRoRERHYunWr0eP6JcC2bduGoqIiODk5IT4+Hq2trRYuqY0Y5GlQbIq5SzgRQxkZGSwiIsLaxRiyALDPP/+cfz4QS4ANN3ZTQ+rPEk6kp3PnzsHf3x9jxozBwoULUVlZae0iDVn9WQJsuLObhNSfJZyIoZiYGOzatQu5ubl47733UFFRgalTp/LzURHz9GcJsOFuWEw/Qixjzpw5/OOJEyciJiYGgYGB+OSTT7B06VIrlowMF3ZTQ+rPEk6kb25ubggJCcH58+etXZQhqT9LgA13dpOQ+rOEE+lbU1MTLly4AD8/P2sXZUiiJcB6sqtbttTUVCxevBiTJk1CdHQ0srKyoFarsWTJEmsXbUhYs2YN5s+fj8DAQFy5cgUZGRkQiURITk62dtFsVlNTk0ENsqKiAseOHYOHhwdGjRqF559/Hn/+858RHByM0aNHY+PGjfD390dCQoL1Cm1N1v6az9L6WsKJ9C0pKYn5+fkxqVTKRo4cyZKSktj58+etXSybdujQIQagx6ZfKux2lwAbbmg+JEKIzbCbNiRCiO2jhEQIsRmUkAghNoMSEiHEZlBCIoTYDEpIhBCbQQmJEGIzKCERQmwGJSRCiM2ghEQIsRmUkAghNuP/A3DydqqwcDtnAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:15:05.997071Z", + "iopub.status.busy": "2024-03-26T11:15:05.996771Z", + "iopub.status.idle": "2024-03-26T11:16:13.043881Z", + "shell.execute_reply": "2024-03-26T11:16:13.042850Z" + }, + "papermill": { + "duration": 67.067842, + "end_time": "2024-03-26T11:16:13.046438", + "exception": false, + "start_time": "2024-03-26T11:15:05.978596", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:16:13.082681Z", + "iopub.status.busy": "2024-03-26T11:16:13.081974Z", + "iopub.status.idle": "2024-03-26T11:16:13.101589Z", + "shell.execute_reply": "2024-03-26T11:16:13.100780Z" + }, + "papermill": { + "duration": 0.039573, + "end_time": "2024-03-26T11:16:13.103438", + "exception": false, + "start_time": "2024-03-26T11:16:13.063865", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
realtabformer0.0001080.0091440.0008850.6177120.29008711.9302590.6675140.0000033.0843450.0199260.7107430.0297470.1596080.00073.702057
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "realtabformer 0.000108 0.009144 0.000885 0.617712 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss pred_duration \\\n", + "realtabformer 0.290087 11.930259 0.667514 0.000003 3.084345 \n", + "\n", + " pred_mae pred_mape pred_rmse pred_std std_loss \\\n", + "realtabformer 0.019926 0.710743 0.029747 0.159608 0.0007 \n", + "\n", + " total_duration \n", + "realtabformer 3.702057 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:16:13.136958Z", + "iopub.status.busy": "2024-03-26T11:16:13.136709Z", + "iopub.status.idle": "2024-03-26T11:16:13.589301Z", + "shell.execute_reply": "2024-03-26T11:16:13.588523Z" + }, + "papermill": { + "duration": 0.471922, + "end_time": "2024-03-26T11:16:13.591519", + "exception": false, + "start_time": "2024-03-26T11:16:13.119597", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:16:13.628282Z", + "iopub.status.busy": "2024-03-26T11:16:13.627460Z", + "iopub.status.idle": "2024-03-26T11:17:29.370436Z", + "shell.execute_reply": "2024-03-26T11:17:29.369622Z" + }, + "papermill": { + "duration": 75.764203, + "end_time": "2024-03-26T11:17:29.373160", + "exception": false, + "start_time": "2024-03-26T11:16:13.608957", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/realtabformer/all inf False\n", + "Caching in ../../../../insurance/_cache_bs_test/realtabformer/all inf False\n", + "Caching in ../../../../insurance/_cache_synth_test/realtabformer/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:17:29.411080Z", + "iopub.status.busy": "2024-03-26T11:17:29.410326Z", + "iopub.status.idle": "2024-03-26T11:17:29.429696Z", + "shell.execute_reply": "2024-03-26T11:17:29.428989Z" + }, + "papermill": { + "duration": 0.040759, + "end_time": "2024-03-26T11:17:29.431670", + "exception": false, + "start_time": "2024-03-26T11:17:29.390911", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:17:29.465417Z", + "iopub.status.busy": "2024-03-26T11:17:29.465089Z", + "iopub.status.idle": "2024-03-26T11:17:29.470361Z", + "shell.execute_reply": "2024-03-26T11:17:29.469524Z" + }, + "papermill": { + "duration": 0.024307, + "end_time": "2024-03-26T11:17:29.472225", + "exception": false, + "start_time": "2024-03-26T11:17:29.447918", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'realtabformer': 0.03380099051670792}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:17:29.505989Z", + "iopub.status.busy": "2024-03-26T11:17:29.505726Z", + "iopub.status.idle": "2024-03-26T11:17:29.905422Z", + "shell.execute_reply": "2024-03-26T11:17:29.904515Z" + }, + "papermill": { + "duration": 0.418994, + "end_time": "2024-03-26T11:17:29.907391", + "exception": false, + "start_time": "2024-03-26T11:17:29.488397", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABA/klEQVR4nO3deXhTVd4H8O+9N3vbJG1paQstLZRFQRbZRFBAcAE3dERBHYoKOA4MMsj7Kj4zKDpaZ0YZkPFldGZsZQZ1xhlwGccFV9zYQRC0LLaldKHQJWmz597z/pEmNDRpkzTtTdrf53nytLk5ufklbX8959yzcIwxBkIIkREvdwCEEEKJiBAiO0pEhBDZUSIihMiOEhEhRHaUiAghsqNERAiRHSUiQojsKBERQmRHiYgE9fjjj4PjuKies7i4GBzHYe/evSGV//3vf4+BAwdCEASMHj06qrGQ2EGJiITl6aefxptvvtktr/Xhhx/if//3fzF58mQUFRXh6aef7pbXJd1PIXcAJL48/fTTuO222zBnzpwuf61PPvkEPM/jr3/9K1QqVZe/HpEP1Yh6CIvFIncIUVdbWwutVhu1JMQYg81mi8q5OsNqtcodQsyhRBSHvH03R48exZ133onk5GRMmTIFAPD3v/8dY8eOhVarRUpKCubNm4eKigq/53/xxReYO3cucnJyoFarkZ2djV/+8pcd/pFyHAeLxYJXXnkFHMeB4zgsXLgQAFBeXo6f//znGDp0KLRaLVJTUzF37lyUlZUFPJfVasX999+P1NRU6PV6LFiwAA0NDX6vVVRUBIvF4nut4uJiAIDb7caTTz6JQYMGQa1WIzc3F48++igcDoffa+Tm5uKGG27ABx98gHHjxkGr1eLFF1/EZ599Bo7j8M9//hNr165Fv379kJSUhNtuuw0mkwkOhwMrVqxAeno6EhMTcc8997Q5d6if9bRp0zBixAjs27cPV155JXQ6HR599NF2P+feiJpmcWzu3LkYPHgwnn76aTDG8NRTT+HXv/41br/9dixatAhnz57Fxo0bceWVV+LAgQMwGo0AgDfeeANWqxUPPPAAUlNTsXv3bmzcuBGnT5/GG2+8EfT1/va3v2HRokWYMGEClixZAgAYNGgQAGDPnj34+uuvMW/ePPTv3x9lZWXYtGkTpk2bhqNHj0Kn0/mda9myZTAajXj88cdRUlKCTZs2oby83Jck/va3v+Gll17C7t278Ze//AUAcPnllwMAFi1ahFdeeQW33XYbHnroIezatQuFhYX4/vvvsW3bNr/XKSkpwfz583H//fdj8eLFGDp0qO+xwsJCaLVaPPLIIzhx4gQ2btwIpVIJnufR0NCAxx9/HDt37kRxcTHy8vKwZs0a33ND/awBoK6uDrNmzcK8efNw9913o2/fvmH+pHsBRuLOY489xgCw+fPn+46VlZUxQRDYU0895Vf28OHDTKFQ+B23Wq1tzllYWMg4jmPl5eVtXqe1hIQEVlBQ0Ob5gc75zTffMABs8+bNvmNFRUUMABs7dixzOp2+47/73e8YAPbWW2/5jhUUFLCEhAS/cx48eJABYIsWLfI7vmrVKgaAffLJJ75jAwYMYADY+++/71f2008/ZQDYiBEj/GKYP38+4ziOzZo1y6/8pEmT2IABA3z3w/msp06dygCwP/3pT20+H3IeNc3i2M9+9jPf91u3boUkSbj99ttx7tw53y0jIwODBw/Gp59+6iur1Wp931ssFpw7dw6XX345GGM4cOBARLG0PqfL5UJdXR3y8/NhNBqxf//+NuWXLFkCpVLpu//AAw9AoVDgv//9b7uv43185cqVfscfeughAMC7777rdzwvLw/XXnttwHMtWLDAL4aJEyeCMYZ7773Xr9zEiRNRUVEBt9sNILzPGgDUajXuueeedt9Xb0dNsziWl5fn+/748eNgjGHw4MEBy7b+gzt16hTWrFmDt99+269fBgBMJlNEsdhsNhQWFqKoqAiVlZVgrRb+DHTOC+NMTExEZmZm0D4lr/LycvA8j/z8fL/jGRkZMBqNKC8v9zve+jO6UE5Ojt99g8EAAMjOzm5zXJIkmEwmpKamhvVZA0C/fv3oql8HKBHFsda1EEmSwHEc3nvvPQiC0KZsYmIiAEAURVx99dWor6/Hww8/jGHDhiEhIQGVlZVYuHAhJEmKKJZf/OIXKCoqwooVKzBp0iQYDAZwHId58+ZFfM72hDrQsvVndKFAn1N7x73JNdTPOpQYiAcloh5i0KBBYIwhLy8PQ4YMCVru8OHDOHbsGF555RUsWLDAd3z79u0hvU6wBPCvf/0LBQUFeO6553zH7HY7GhsbA5Y/fvw4pk+f7rvf3NyM6upqzJ49u93XHzBgACRJwvHjx3HRRRf5jp85cwaNjY0YMGBASO+jM0L9rEnoqI+oh7j11lshCALWrl3r1ywCPP/J6+rqAJz/b9+6DGMMGzZsCOl1EhISAiYXQRDavO7GjRshimLA87z00ktwuVy++5s2bYLb7casWbPafX1volq/fr3f8XXr1gEArr/++o7eQqeF+lmT0FGNqIcYNGgQfvOb32D16tUoKyvDnDlzkJSUhNLSUmzbtg1LlizBqlWrMGzYMAwaNAirVq1CZWUl9Ho9/v3vf7fpKwpm7Nix+Oijj7Bu3TpkZWUhLy8PEydOxA033IC//e1vMBgMuPjii/HNN9/go48+QmpqasDzOJ1OzJgxA7fffjtKSkrwf//3f5gyZQpuuummdl9/1KhRKCgowEsvvYTGxkZMnToVu3fvxiuvvII5c+b41bK6SqifNQkdJaIe5JFHHsGQIUPwhz/8AWvXrgXg6Xi95pprfH/gSqUS77zzDpYvX47CwkJoNBrccsstWLZsGUaNGtXha6xbtw5LlizBr371K9hsNhQUFGDixInYsGEDBEHAli1bYLfbMXnyZHz00UdBr1j98Y9/xJYtW7BmzRq4XC7Mnz8fzz//fEh9P3/5y18wcOBAFBcXY9u2bcjIyMDq1avx2GOPhfFpdU4onzUJHccurFsSQkg3oz4iQojsKBERQmRHiYgQIjtKRIQQ2VEiIoTIjhIRIUR2cT2OSJIkVFVVISkpKeqLvBNCOo8xhqamJmRlZYHng9d74joRVVVVtZkpTQiJPRUVFejfv3/Qx+M6ESUlJQHwvEm9Xi9zNISQC5nNZmRnZ/v+VoORPRFVVlbi4YcfxnvvvQer1Yr8/HwUFRVh3LhxHT7X2xzT6/WUiAiJYR11nciaiBoaGjB58mRMnz4d7733HtLS0nD8+HEkJyfLGRYhpJvJmoh++9vfIjs7G0VFRb5j7a2oRwjpmWS9fP/2229j3LhxmDt3LtLT0zFmzBj8+c9/Dlre4XDAbDb73Qgh8U/WGtGPP/6ITZs2YeXKlXj00UexZ88eLF++HCqVCgUFBW3KFxYW+pZcCBVjDG63O+gCXSQ4QRCgUChoaATpcrIuA6JSqTBu3Dh8/fXXvmPLly/Hnj178M0337Qp73A4/Da68/bIm0ymgJ3VTqcT1dXVtLNmJ+h0OmRmZtLi7yQiZrMZBoMh6N+ol6w1oszMTFx88cV+xy666CL8+9//DlherVZDrVaHdG5JklBaWgpBEJCVlQWVSkX/2cPAGIPT6cTZs2dRWlqKwYMHtzsgjZDOkDURTZ48GSUlJX7Hjh07FpUF0J1OJyRJQnZ2dptdRklotFotlEolysvL4XQ6odFo5A4pbjRYnEhOoFpkqGT9F/fLX/4SO3fuxNNPP40TJ07g1VdfxUsvvYSlS5dG7TXov3jn0OcXHsYY9pbV4597KyBJtPhpqGT9LRs/fjy2bduG1157DSNGjMCTTz6J9evX46677pIzLEIixhiwq7QeVqeIeqtT7nDihuwjq2+44QbccMMNcodBSFTwPIe0JDUqG2w4Y7ajT2JofZq9HdW7e7nc3Nw2e4SRyBypMqHoq1JUNtgAAI1WVwfPIF6UiAiJkkaryy/5mG2UiEJFiagHcDqpLyIWeBNPv0RAKVphtlMiClWvTEROtxT05halkMu6QigbiWnTpmHZsmVYtmwZDAYD+vTpg1//+te+7Y1zc3Px5JNPYsGCBdDr9ViyZAkA4Msvv8QVV1wBrVaL7OxsLF++HBaLxXfe2tpa3HjjjdBqtcjLy8OWLVsiio8EZra7IEgOjK39N8ZUvQ7JVCV3SHFD9s5qObzw6Ymgj+X1ScCcMf1891/acRIuMfBl2P7JWswdd35htpe/KoXN6T+V5JdXD4koxldeeQX33Xcfdu/ejb1792LJkiXIycnB4sWLAQDPPvss1qxZ49vd9OTJk7juuuvwm9/8Bi+//DLOnj3rS2beScULFy5EVVUVPv30UyiVSixfvhy1tbURxUfaarK70cdyAgalA3WciAxrCYDL5A4rLvTKRBQPsrOz8Yc//AEcx2Ho0KE4fPgw/vCHP/gS0VVXXYWHHnrIV37RokW46667sGLFCgDA4MGD8fzzz2Pq1KnYtGkTTp06hffeew+7d+/G+PHjAQB//etfcdFFF3X7e+uJGGOwOkX0dZxBYpICE3JTAK1N7rDiRq9MREun5wd9jL9gFsiSKwcFLXvhjJF7J0dvCZPLLrvMb0rKpEmT8Nxzz/km7164cNy3336LQ4cO+TW3GGO+qS7Hjh2DQqHA2LFjfY8PGzYMRqMxajH3Zk5RgigxJDrPQikInoP2RkB0AYJS1tjiQa9MRCpF6F1jXVW2sxISEvzuNzc34/7778fy5cvblM3JycGxY8e6K7ReySUypCcpYeCtELiWyZ2MAbYGIDFd3uDiQK9MRPFg165dfvd37tyJwYMHQ/D+t73ApZdeiqNHjyI/P3Btb9iwYXC73di3b5+vaVZSUoLGxsaoxt1bJaoVuGt0KmDTAxyPY1YdHPWVSOlfhX6DKRF1pFdeNYsHp06dwsqVK1FSUoLXXnsNGzduxIMPPhi0/MMPP4yvv/4ay5Ytw8GDB3H8+HG89dZbWLZsGQBg6NChuO6663D//fdj165d2LdvHxYtWgStVttdb6nns5s8XzV6mJgWzQ43bE2NsoYULygRxagFCxbAZrNhwoQJWLp0KR588EHfZfpARo4cic8//xzHjh3DFVdcgTFjxmDNmjXIysrylSkqKkJWVhamTp2KW2+9FUuWLEF6Ov23jhp7o+erxgBe7dm1wm1vki+eOEJNsxilVCqxfv16bNq0qc1jZWVlAZ8zfvx4fPjhh0HPmZGRgf/85z9+x3760592Kk7isf9UA2oOlWCk04r+mQbwGs+FBokSUUgoERESBRaHGw5LE9wCA1QJ4NWevjzmoEQUCkpEhESBwyVBKdmgUHKAMgGC1nPJnjmaZY4sPlAiikGfffaZ3CGQMDncEpSSHQLPASodFNqW1SydVCMKBXVWExIFDrcIhWiDgucBpRYqbRIUPAel5ASkyOYc9iZUIyIkChxuCUrRDoFXAMoEDDT2wcDcFM+DbhugSmj/BL0c1YgIiQKnS4SiVdMMPA8oWhbPd9nlDS4OUCIiJAoMCjc0Cg5KgQOULbvGeL+6afJrR6hpRkgUzBmRDFiTAYUa4AW4RAmHq2xQWk24aLiV/tA6QDUiQqLB1bKbcEtfkILnUGvn0exww2WnnYY7QomIkGhwtiQbpWfuHsdxvu9ddkuwZ5EWlIgI6aRGqxNv7/sRR6vNgOL8JGLOm4gcVCPqCCUiQjrJ5hJhtVphd4nnr5ThfCISKRF1qHclIsYAt7P7byz0rYc3b96M1NRUOBwOv+Nz5syhCaoxyumWIDAXFDwHCOc3VOSVnu9FF10164isnfmPP/441q5d63ds6NCh+OGHH7rmBUUX8MVzXXPu9lzxkN9/yvbMnTsXy5cvx9tvv425c+cC8Oy+8e6777Y7s57Ix+GWIEhOzxiiVj9nXumZ5iG6HMGeSlrIXiMaPnw4qqurfbcvv/xS7pBkpdVqceedd/p23gCAv//978jJycG0adPkC4wE5XC1SkStakQKlQYKngMn0r5zHZF9eINCoUBGRkb3vJig9NROuluYi6cvXrwY48ePR2VlJfr164fi4mIsXLjQbzF9EjscbrGlacZ7xhG1uPKi/oAzBUikn1tHZE9Ex48fR1ZWFjQaDSZNmoTCwkLk5OQELOtwOPz6Tsxmc3gvxnEhN5HkNGbMGIwaNQqbN2/GNddcgyNHjuDdd9+VOywShNPbNFNwgNDq98tbO3JTjagjsjbNJk6ciOLiYrz//vvYtGkTSktLccUVV6CpKfDSCYWFhTAYDL5bdnZ2wHI9waJFi1BcXIyioiLMnDmzR7/XeKdU8EhUSJ7pHa1qRL7vReoj6gjHWBiXdLpYY2MjBgwYgHXr1uG+++5r83igGlF2djZMJhP0er1fWbvdjtLSUuTl5UGj0XR57NFmMpmQlZUFt9uNzZs344477pAljnj/HLvN7j8DlnPAqHlAimd/uxMV1XB+8Tz0OjX63/xY243wegGz2QyDwRDwb7Q12TurWzMajRgyZAhOnAi8JbRarYZer/e79VQGgwE/+clPkJiYiDlz5sgdDumI6PJ8bVUjanbzMNlcsDmc5x8nAcVUImpubsbJkyeRmZkpdygxobKyEnfddRfUanXHhYm8vM2v1lfNlGowcBAlRs2zDsiaiFatWoXPP/8cZWVl+Prrr3HLLbdAEATMnz9fzrBk19DQgG3btuGzzz7D0qVL5Q6HdOD9w9U4VFYLk83ldzFEpRQg8ipPIqIO63bJetXs9OnTmD9/Purq6pCWloYpU6Zg586dSEtLkzMs2Y0ZMwYNDQ347W9/i6FDh8odDulAfbMFBocLEtP41YiUAg+RV0JiIuCmxdHaI2siev311+V8+ZgVbN8yEpskp6fZxfO835gxpcBB5FQQJSs1zToQU31EhMQjye1JMoJS7XdlTKXg4eZVECVQ06wDPT4RxdDohLhEn1/HvHPJBKX/8AaVwEPiVZ7cRDWidsk+srqrKJWeKrLVaoVWq+2gNAnGavUsYeH9PIk/xhhYS/+Pd7a9l0GrxHWjcsDVWgA3JaL29NhEJAgCjEYjamtrAQA6nY7maoWBMQar1Yra2loYjUYIgiB3SDHJJTLwLWOELqwRcVyrkdY08bVdPTYRAfBNpvUmIxI+o9HYfZOS45AoMeiVIpQCB0EVYLyXt/OaElG7enQi4jgOmZmZSE9Ph8tFI1vDpVQqqSbUAa1KwK0j04HjKf7zzFrsPtWEpBozBqTZoJMhvnjRoxORlyAI9AdFuo5vVHXblR2qmyWINhcynQ5KRO3o8VfNCOly7rbTO7z4lpHWtEpj+ygREdIJlY02fP59JX481xxwrSuupbkmuaiPqD29omlGSFexONwwNzfDJort1ogkunzfLqoREdIJntUZXW0WzvcSlN5ERDWi9lAiIqQTXKJnmVie4wLWiHxNM0pE7aJEREgnuEQGgXlrRG0TkaBQgQNoHFEHqI+IkE7w1og8Wwm1bZpNu7g/+OYUcDR8pF1UIyKkE5xiSx8RF6RGpFSBAweIbkCSZIgwPlAiIqQTeI6DmnMFrRH5HZNodH8w1DQjpBOm5qcAlQbPnQA1opN1drhrm6FXC8gUnQHLEEpEhHRO6/FBAa6aNdrcsFkl8GC0k0c7qGlGSGf45pkpAL7tn5NK4CFySkgMdOWsHZSICOmET46cxtFqM5rcga+KKQQOEqds2VKIElEwlIgI6YRzjWaYbC5IfOAVLJUC17KTBzXN2kOJiJBOYC19RLwy8HbcCp6HyCkgUY2oXZSICOkEKcjC+V5KBQ+J89aIKBEFQ4mIkAgxxsBcrbYSCkDJe5pmogRqmrUjZhLRM888A47jsGLFCrlDISQkTlGCwFoWzlcFrhH1SVTj2pE5uHSAkWpE7YiJcUR79uzBiy++iJEjR8odCiEhc4kMvOQCh+A1Ip7nwKvUADhKRO2QvUbU3NyMu+66C3/+85+RnJwsdziEhEwUGTS8GwLP+Zb7CMi3kwc1zYKRPREtXboU119/PWbOnNlhWYfDAbPZ7HcjRC4GnRI3Dk/FuNzkgKOqAUCSGHadasaxM01w0brVQcnaNHv99dexf/9+7NmzJ6TyhYWFWLt2bRdHRUgY3A7P7PoAqzMCAMcBpQ1OSBYnsl1O0H65gclWI6qoqMCDDz6ILVu2QKMJ3NF3odWrV8NkMvluFRUVXRwlIR3w9vsEqRFxHAeuZQa+5LR3V1RxR7Ya0b59+1BbW4tLL73Ud0wURezYsQN//OMf4XA42uxFplaroVbT7GUSG07VWXHuxxqkMSuy2+kj8m0p5KY+omBkS0QzZszA4cOH/Y7dc889GDZsGB5++GHaEJHEPLPdhcamZuiU7sBrEbXgaCePDsmWiJKSkjBixAi/YwkJCUhNTW1znJBY5FudMch61V6+LYVob7OgZL9qRki8crklCMy7g0fwGpEvEYlUIwomJgY0en322Wdyh0BIyFy+Pc00HSQiT22JUR9RUDGViAiJJ26XHQLQYdPsmpH9obAlgxe47gsuzlDTjJAIuVsGKPK8APDB/6dr1FooeB48kwBJ7K7w4golIkIixIkOcGjZzZVrp7YjtBrGSPPNAqKmGSERmpFvBDOngKn17ZY7ftYK7pwNyVoeqaILUGq7KcL4QYmIkEiJnukdXJCZ9141ZjvQLELBSZ5ERNqgphkhkXK3P73Dy7NcLK3S2B5KRIRE6OtjVTh2pgkWsf1ZAEqBg8QrPDtOUyIKiBIRIRE6U29CncUJNx98DBEAKAUeEqegnTzaEVEi+vHHH6MdByFxx7uDh6KDPiKFwJ1vmkmUiAKJKBHl5+dj+vTp+Pvf/w67nZY2IL2PJDGwlj4iRZD1qr2UAt+ygD71EQUTUSLav38/Ro4ciZUrVyIjIwP3338/du/eHe3YCIlZLkmCgnmSSrCF873ON81ATbMgIkpEo0ePxoYNG1BVVYWXX34Z1dXVmDJlCkaMGIF169bh7Nmz0Y6TkJjiEhmEDhbO9+qfrMWM4f1wUWYS1YiC6FRntUKhwK233oo33ngDv/3tb3HixAmsWrUK2dnZWLBgAaqrq6MVJyExxemWIEjOjhfOh6dGpNXqoOB5qhEF0alEtHfvXvz85z9HZmYm1q1bh1WrVuHkyZPYvn07qqqqcPPNN0crTkJiirtlTzOB5zocRwSAdvLoQEQjq9etW4eioiKUlJRg9uzZ2Lx5M2bPng2e9+S1vLw8FBcXIzc3N5qxEhIz0vUaXD3ECKnZGXThfC+7S8T3lc0wnmtGXiY1zQKJKBFt2rQJ9957LxYuXIjMzMyAZdLT0/HXv/61U8EREss40QmB67hGJDGGH846kGt2IFd0ghYDaSuiRLR9+3bk5OT4akBejDFUVFQgJycHKpUKBQUFUQmSkJjkXXGxgz4iBc9D4jxNM9HlpAmeAUTURzRo0CCcO3euzfH6+nrk5eV1OihCYl35uWaUVNah2mRrd3VGwDPFQ+Q86UekBfQDiig5M8YCHm9ubg55jzJC4lldsxWOZjt4qDqsEXEcB15xvkZE2gorEa1cuRKA54Nds2YNdDqd7zFRFLFr1y6MHj06qgESEotEh2dGQUcL53t5L/HTlkKBhZWIDhw4AMBTIzp8+DBUqvM/AJVKhVGjRmHVqlXRjZCQGOSt2XS4OmML2mSxfWElok8//RSAZyPEDRs2QK9vf2U6Qnoq0WUHD3S4KJoXr/QkIu/8NOIvoj6ioqKiaMdBSFxxu+xQ4fxWQR25bmQONE4jFMrA/au9XciJ6NZbb0VxcTH0ej1uvfXWdstu3bq104EREssk7w4eIdaIDAk6QCEATAQkCeBpKbDWQk5EBoMBXEtb2GAwdFlAhMQDydXSWR1ijcivQ1tyAXyIz+slQk5ErZtj0Wqabdq0CZs2bUJZWRkAYPjw4VizZg1mzZoVlfMT0lWmDzJAEpPBpYT2T7mk1gp1vQ3JWgUMorPDS/69TUT1Q5vNBqvV6rtfXl6O9evX48MPPwzrPP3798czzzyDffv2Ye/evbjqqqtw880348iRI5GERUi34SUnFDzf4VpEXmX1Vpw2u9DscNPE1wAiSkQ333wzNm/eDABobGzEhAkT8Nxzz+Hmm2/Gpk2bQj7PjTfeiNmzZ2Pw4MEYMmQInnrqKSQmJmLnzp2RhEVI9/GOBwpl5j28o6tpJ49gIl6h8YorrgAA/Otf/0JGRgbKy8uxefNmPP/88xEFIooiXn/9dVgsFkyaNClgGYfDAbPZ7HcjRA4HSs/g5Nlm2FhovRve+WYSLRcbUESJyGq1IikpCQDw4Ycf4tZbbwXP87jssstQXl4e1rkOHz6MxMREqNVq/OxnP8O2bdtw8cUXByxbWFgIg8Hgu2VnZ0cSPiGdVlPXiNomB8QOdvDw8q1bTTt5BBTx4vlvvvkmKioq8MEHH+Caa64BANTW1oY9yHHo0KE4ePAgdu3ahQceeAAFBQU4evRowLKrV6+GyWTy3SoqKiIJn5BOkSQGJoa2cL6XUuA861ZLoEQUQESJaM2aNVi1ahVyc3MxceJEX1Pqww8/xJgxY8I6l0qlQn5+PsaOHYvCwkKMGjUKGzZsCFhWrVZDr9f73Qjpbi5JgkJqSUQhjiNSCjxE395m1DS7UEQjq2+77TZMmTIF1dXVGDVqlO/4jBkzcMstt3QqIEmS4HDQxEASuzwL53sWOBNU2pCeoxA4SNQ0CyriNZoyMjKQkZHhd2zChAlhnWP16tWYNWsWcnJy0NTUhFdffRWfffYZPvjgg0jDIqTLOd0SBBbawvleg9ISkT0sC5q6RtpkMYCIEpHFYsEzzzyDjz/+GLW1tZAkye/xUHeCra2t9e32YTAYMHLkSHzwwQe4+uqrIwmLkG7hdHuaZgLPhTwwUaMUoEnQAY08Nc0CiCgRLVq0CJ9//jl++tOfIjMz0zf1I1y0pjWJR06XCEFyQVDw4Y2Q9u3kQYnoQhElovfeew/vvvsuJk+eHO14CIl52UYFsnKNnp1bQxzQaHG4UXbGDkODFf2zqGl2oYiumiUnJyMlJSXasRASFzi3Awqeh0qhOF/L6YDdJeJwtRXVJjt1VgcQUSJ68sknsWbNGr/5ZoT0Gt6mlUIV0uqMAKAQeM84Irp8H1BETbPnnnsOJ0+eRN++fZGbmwul0v+/wv79+6MSHCGxqPRMHaSzzdAbNUgL8TmqlpHVEgMkt7NzWyz3QBElojlz5kQ5DELix9kGM8QmB6QELuREpGiZ9Ap4thSiROQvokT02GOPRTsOQuKG6PSuzhj61lkKnoMkeP7cJFpAv42IE3NjYyP+8pe/YPXq1aivrwfgaZJVVlZGLThCYpHoXZ0xjETk2dtM3fJ8mjlwoYhqRIcOHcLMmTNhMBhQVlaGxYsXIyUlBVu3bsWpU6d8axUR0hNJLjsEAEKI88y8+JYrbBLt5NFGRDWilStXYuHChTh+/Ljfzq6zZ8/Gjh07ohYcIbHI2zRThDjPzGvWqByM7G+Alhe7Iqy4FlGNaM+ePXjxxRfbHO/Xrx9qamo6HRQhscy7cL4QRtMMANKMSYBK4dnJg7GQL/33BhHViNRqdcDVEY8dO4a0tFCvIxASn0S3JxGFuhaRT+udPGhQo5+IEtFNN92EJ554Ai6X58PkOA6nTp3Cww8/jJ/85CdRDZCQWDNtYBIuzTHCoE8M63nHztpQ0WiDxemmQY0XiCgRPffcc2hubkZaWhpsNhumTp2K/Px8JCUl4amnnop2jITEFBVzQa0QoFDpwnre9zVNKG9wweKgRHShiPqIDAYDtm/fjq+++grffvstmpubcemll2LmzJnRjo+Q2OO2eb6GuTeZguch8QqInuHVXRBY/Ao7EUmShOLiYmzduhVlZWXgOA55eXnIyMgAYyziJUEIiQd2l4hTp89BI1mRowzvqtn5LYWoRnShsJpmjDHcdNNNWLRoESorK3HJJZdg+PDhKC8vx8KFCzu9TCwhsc7qFFHbYMIZsx1QhNdZrfROfKUthdoIq0ZUXFyMHTt24OOPP8b06dP9Hvvkk08wZ84cbN68GQsWLIhqkITECqfTBYG5IAh82IlIIXCw8UqIzEpXzS4QVo3otddew6OPPtomCQHAVVddhUceeQRbtmyJWnCExBqXw9M/5FkmNpIaEW2yGEhYiejQoUO47rrrgj4+a9YsfPvtt50OipBY5bRbPN8o1AAf3kVnTx+Rd00iqhG1FlbTrL6+Hn379g36eN++fdHQ0NDpoAiJVS67ZzFAPsyOagAY0jcJeUMyoKtvpER0gbASkSiKUCiCP0UQBLjddFmS9FwuZ0siCnOeGQAkaZRAUiLQJFDT7AJhJSLGGBYuXAi1OvD4CdoYkfR0LrunjyicJUD80E4eAYWViAoKCjosQ1fMSE82MkMNt8kIpPUJ+7lNdheqzjmQ3GRHOg1o9BNWIioqKuqqOAiJCwrJAYVCALQJYT/XZHPhYKUFQy02pFONyA8tnUtIOFqWAAl3egfQsoA+p4AogZpmF5A1ERUWFmL8+PFISkpCeno65syZg5KSEjlDIqRdxyrPorzOgiYx/GmaypadPES6fN+GrIno888/x9KlS7Fz505s374dLpcL11xzDSwWi5xhERJU1bkGVJnssLPQNlZsTaU4P6CRUY3IT0Sz76Pl/fff97tfXFyM9PR07Nu3D1deeaVMURESHHN5Lt8rtUlhP1fZ0jRjAESXU94/vhgTU5+FyWQCgKDbWTscDr8hAoFWiSSkq0gSA9eSiFSa8DurlQIHxrfsbeZyxNYfn8xiprNakiSsWLECkydPxogRIwKWKSwshMFg8N2ys7O7OUrSmzlFCQrR01mt1oVfI+I4DrzSs1ys6KKmWWsxk4iWLl2K7777Dq+//nrQMqtXr4bJZPLdKioqujFC0tvZHQ4IkhMCz0FQh18jAoBrLsnG8Cw91LSTh5+YqB0uW7YM//nPf7Bjxw70798/aDm1Wh10VDchXc1p81xEEfjwlwDx6peqBzRKgLlpJ49WZK0RMcawbNkybNu2DZ988gny8vLkDIeQdjlszZ5vVLrIE4h3Jw9Gy8W2JmuNaOnSpXj11Vfx1ltvISkpybcnmsFggFYb/qRCQrpSlo4hLccISRf+9A6vsgYn1CYbDFoldKLz/NyzXk7WGtGmTZtgMpkwbdo0ZGZm+m7/+Mc/5AyLkIAEtxUahQBdgj7icxyqasKP9U402d2AmyaJe8laI2KMyfnyhITH1bJ7hzK8bYRaUwk83LzKs5MHDWr0iYnOakLiQVlNLbh6CxKNAlIjPIdKwUHkVRCZg2pErcTM5XtCYl1tXSOqGu1ocEXer6MSBIhcS42IEpEPJSJCQiQ6PFfNVBEsAeKlFFpqRBIDREpEXpSICAmR5PCMI1Jpw9vzvjWVomUGvsQAN/UReVEiIiRErKVGpNYZIj6HUuDh5lSenTzc9miFFveos5qQEIiiBN7lSUSaRGPE58lJ1aHP4CwknauhplkrlIgICYHN1gyOSeAAaDoxjkivUUKfogfMCmqatUJNM0JCYGtqBABwqgRwQif/f3vnqVHTzIdqRISEIFXhhD7HCJc2rVPncbhFVNQ7kWS2o28q1Yi8qEZESAh4VzPUCgGJ+uROncfukvBVWRPK6ixUI2qFEhEhoXB6Z95HfukeANQKHiKvgsQAyUWd1V6UiAgJQWllDcrrLKh3d249LO/lewAQXVQj8qJEREgIzp2rRZXJjgYpwq2mWwg8B17pSWaikxKRFyUiQkLAbA0AAE1S5GsReQlKz1pbbpfDs0AaoURESEeYJILZPTvGaJM611kNAIqW5Y5FUaKlQFpQIiKkA/ZmMyRRBAOHJGPgra7CoVaqwcDBLUl05awFjSMipANNprOebzQGKBWd/5OZlN8HuoY0JAkuWgqkBSUiQjpgMdUBAIQEY1TOl52iA4wGwFp/ftXHXo6aZoR0wN5SI1ImRLouYwA0zcMP1YgI6cAwvROuAclwDYzOdlf1FiesFiDJ5oKBakQAKBER0iHOcg4qgYcqOSMq5ys9Z0FllR0XcXYYqEYEgJpmhLRPdAH2Rs/3ndjPrDW1goebV3tWaaQaEQBKRIS0y1xfgx+qTTjVJAGqyNeqbk2j9CQit0SrNHpRIiKkHY1nKtBgdaHSlRS1ferVCgFuXgO3yACXNSrnjHeUiAhph+XsKQCAMrl/1M6pVnqbZhJAE18ByJyIduzYgRtvvBFZWVngOA5vvvmmnOGQXsbhFrGnrB4uUQpaxlbnSURJ6blRe11fjUhigJv6iACZE5HFYsGoUaPwwgsvyBkG6SVO1DbjXPP5kcwHTzXiy+Pn8K99p2F1utuUd9ktcJlrAQDp/aJz6R4431nt2XWaEhEg8+X7WbNmYdasWSGXdzgccDjO/yKZzeauCIv0QKfqrPjv4WokqhW4c2IONEoBqYkqaJQCakx2bN1fidvG9odGKfieU1P2vWcBM20qjEZj1GJRK3hMHZGNdC4JHHVWA4izPqLCwkIYDAbfLTs7W+6QSBwwWV1451AVRIkhw6CBSvD82uenJ2He+GzoVALONjnw9rdVfs202rIjAIDErKHgotRRDQAcx2Fov3Qk61TgJbdniEAvF1eJaPXq1TCZTL5bRUWF3CGRGCdKDP/9rhpOt4QsowbXXNwXPM95OoklEckJKtwyph9UCh6VDTb851AV3KIE5rIj2VIKngP6DR4V/cAUaoBr+fOjsUTxNbJarVZDre7cUp2kd9n5Yx1qTHaolTyuuzgNitM7gaoDgN3sSQTJA5CePQFzxvTDtv2nUXbOiq9O1mGqthQDU9Ton5kBVdbAqMdVbbaDswF63g2d2w4g8r3SeoK4SkSEhKPaZMOesnoAwDX5iTAcfRVorj1fgElAfSlQX4p+ybmYk38Z3vlRQrbGAZR9CQBQ5V4WtfFDre0vb4T2rAsX6V3Q0VgiSkSk59r1Yz0YA4anq5Bf/R+g+Syg1AL5M4A+QwBHs6d2VLUfaChD/8Zy3KtNh6KswbNyoj4TyBjZJbHpVAKcgg5u0QE4LV3yGvFE1kTU3NyMEydO+O6Xlpbi4MGDSElJQU5OjoyRkZ5g1iUZ2HOiBhPM2wHLWc8UjTF3A7qWVRYVamDwTKD/WKB0B3DmKNTWM57HEtOB4bcCfNd0o2pVAiy8Fi6xjhIRZE5Ee/fuxfTp0333V65cCQAoKChAcXGxTFGRnkLNA1McXwKWak/SGTXvfBJqTZsMXHwzkDcVMFd5ak3GHIAX2paNEq1SgFPQwiWx83um9WKyJqJp06aB0S4GJMpON1jRz6AG9/3bQP2PgKAARt7uqeW0R2v03LqBTiXAJejgdkmAk/qIqI+I9CinG6x4Y08Fxjl3YYruNDhB4WliGaI3VywatCoBLl7rGbdETbP4GkdESHsYY/jqxDnkmHajn60EHM8DF90IpA6SO7Q2dCoFnIIOLpGaZgAlItKDnDxrATu1C9lN36J/sg4Yci2QfpHcYQWUqFZgysU5GJyeSDUiUCIiPYQkMRw9+A0GNO5ChkEL1ZAZQNYYucMKSqXgMSQ7A0adyjOyWgq+AkBvQImI9AjHjx1Bn9PboeA5ZAyfAuRcJndIHVMmeAZLMglw9e5aESUiEvdc5lqY9rwOjklIzb0EqiFXyx1SSGqanKi2Kz1LkNh790oSlIhIfHM74T70b2jggisxEzmT7+iyQYjRtq+8AYfPMZhsLsDRuxMRXb4n8Ysx4Nh70LoaccnAfjAPXwCFKn4mRSdqFDALiXC6G6lGJHcAhESs6gBw5ijA8eCG3wJDFBcv6w5JGgWcigQ43FKvrxFRIiLxqakGjh8+RGWjDa4BVwDG+FskL0mtgENIhFOUALtJ7nBkRYmIxB+XHTiyDZV1TfjW1gfvmQbIHVFEEjUKOBSJcLopEVEiIvGFMaDkXVhM51BhU+JEyjSMyw0wkTUOGLRKOIQkON0SRGuD5731UpSISHw5vRfsbAnK6+04ljoDg7LSkGXUyh1VRLRKAUxrBAMHh93Wq6d60FUzEj8aK4CTn6Cu2Ylv1WNh02ZgSn509qOXA8dxuHpEP6QiG2rWDFjrAHWS3GHJgmpEJD44moGjb8LpdmO/PRM1icMxPjcFBp1S7sg6JT89Ccl9MiHwnCcR9VKUiEjsczuB7/4FOJpxolmD7/VTkKbXYEJefPYNtZGQ6vlq6b2JiJpmJLZJInD0TcBcDSg1GDD1TuSUuXDF4DRPLSLO2V0iqiwaaBptyEo+J3c4sqFERGKX2wF8txVifSl4QQnukrlIMPTFLclyBxY9dpeI7RXA6AYrMlKrwUtS3ExRiabe945JfDBXA/tegbn6OA5VW/FdysyYW2UxGgxaJSRdGlxQwma19tp+IqoRkdjSdAbs1DdoqjiM6kYbauwKlPSZBb7RgGGiBKXQs/53chyHvgYtLOo0NDtMSDBXAolpcofV7SgRkdhgOg136VeoP3UUNWY7LA4RdbpBKMucjGHZfXF5fmqPS0JeGXoNqlTpMNvPoW/jKSBrtNwhdTtKREReljrg5MdA3UlU11txutGOOt1AnMkajZwBAzE/x4jUxPiZUR+JAX0ScFSTjcZz30KqO9kr+4koERFZ1DeaUH3oI/S3fA+DRgB4AYb88fi8PgfD8nIwI8sArarr9hWLJZl6DVyJWbCfU6O5uQl60ykgOVfusLoVJSLSbWxOESU1Jpz54RvoKr+GQnKgNlEFQ/9LgUFXQa9LwV2MgeuCveZjGc9zGJiehIbaAbA4KqCvOkiJSA4vvPACfv/736OmpgajRo3Cxo0bMWHCBLnDij2MAc1ngNrvgcZyz4xtSQQTVGhSJKPCmYQKMQVXjB2JhEQ9AOBEbRO+qzTDoFVCr1XCoFXCqFMiUa2AWuDAgQEc71k7uUtCZjhR24yjVSY0nD6G7IZvoHc1gAOgTc6A8pLrgMEjfOV7WxLyuiwvFVzf2dAf3gyc/cHTZPUOdOwFZE9E//jHP7By5Ur86U9/wsSJE7F+/Xpce+21KCkpQXp6Bztz9gaMAU01nl/Osz8AtkbPYTBYHCLqLU7UWxywucoAAAYAovUTIDkdMPSDvY6D44wV9ZIDZsmBGtEOlWiFUrJCJdowMisRiRoVoNSgyqHFj1YNXJpUuLV94NamgikTwHMcBJ7D+Nxkz64T8Ix/AQCNsm3zSZTY+cGGTMKR7w5AW70bQx01SFQrkNI3FakjZkCTM67X9YUEY9ApAV0/IDUfqDsB/PAOMOpOQKGSO7RuwTGZ93yeOHEixo8fjz/+8Y8AAEmSkJ2djV/84hd45JFH2n2u2WyGwWCAyWSCXq/vjnC7FmOA2w44mjw1H3MVUHcCzGYCA8BzHCAoUKvsjx0NRtQxAyROAaVkR5K7HrlqM7L5BvQRmqFqucJkcbrRbHfD4ZZgd4m+r27J82MfnW2EtiWZVDRYcbrB5heSS9DCqkyFQ0jAFRdnI0WfBPAK/FDbjP2nzFAqFUjSqKBSKsA4DhanBJvNhjnDk6G2nwUaK1BbVwebS0QffQIS8sYDAyZ79pcnbdlNaNzxJ9TUNSI9KwdJw6ZDmZILKDVyRxaRUP9GZa0ROZ1O7Nu3D6tXr/Yd43keM2fOxDfffNOmvMPhgMPh8N03m0NcXrO+1HNlJlDOZQwNNieO1zQBaP048xXPTdWhr97zi2C2OfFDtbmlJEPrLwCQnazxLUvRZHfhaJUZaFPW832/ZA36t5S1OkUcqTgHjokXhgcXBPQZcDGGDB8PpAyCwgFUfF0GlYJHbmoC8tMTkdtHB7WipXbisnmSmLkKCS4rEkQXoFADCg2gSgBUiXAJGjh5HdRaHcDcgMuKxHPV6Fd/Brz1HHjrWfB2ExiTwFAPxuqRWFsL1HkSnK7eirxG/6QFALqWr5YfkqBuqT2lpyQDfUcA2eMBjSHgj4h4OBSJeIdNQa79v2g4cRLciZNQCBwkQQ1OUEFQqnDpgPNNtqNVJjQ5/H9n0FIZVXAcxg5I9jW7f6hp8izUH8TEVnP3jp1pQoP1fFkOaDu375LbAU10KgCyJqJz585BFEX07dvX73jfvn3xww8/tClfWFiItWvXhv9CbgfQfDb441YnREtT0IeZzun7j8RsLoi2dhKgww20/GJwTjckR/A1ZpiTAa6W3xqXG2632xMur4ZVmQyLqg/M6iyYNP0xIjMVQ9I9n1OKArhtbH9kGjRQBBpbo9R6tlluZ6tlZcvNR6NHclIGkvNaHXM7ActZz81p8dTW3A6AichJF5ElirA6XLA6nC2xS1DzQIJWB21iEqBLBfRZgCEb4HvHFbDOUisE3HjFeOw/ngnzya+Q0HQSanczINoB2ME7ecBy/j+fZDFBsrsDnkvkOcByPkmxJjOkIImIA4DmVmWbmyBZnL77POf/uKdQ9DaFlL2PKByrV6/GypUrfffNZjOys0NYq9iYDYya5/nerzPU873WJSGn5UPnuLZlErVKQO35s1W7ReTa3K2ezbU6LYdEjQJQKQBwUIkS8i/8wXOc7z+WTqVsKQuoJIYhDgamSgB4BVpHoFYK0F3QF5OdokOXU6gAQz/PLdDDAPQtNxI9Rp0KV40aCIwaCKvTDavVCtFmhtvlhAJuIMk7roqhb74DKW52QWXfc4fjABjON+nSBjmgd7fTE2M8XzbV4kCiiwV9HICndh0lsiaiPn36QBAEnDlzxu/4mTNnkJGR0aa8Wq2GWh3B4DZVApCSF/RhDYDAf2qBy2aG+rIA0oyhlVUASO2da2KRduhUCuhUesAYON2nhTEBuE8Yq6aEUzYaZL1koVKpMHbsWHz88ce+Y5Ik4eOPP8akSZNkjIwQ0p1kb5qtXLkSBQUFGDduHCZMmID169fDYrHgnnvukTs0Qkg3kT0R3XHHHTh79izWrFmDmpoajB49Gu+//36bDmxCSM8l+ziizuhx44gI6WFC/RulYa2EENlRIiKEyI4SESFEdrJ3VneGt3sr5KkehJBu5f3b7KgrOq4TUVOTZ1pGSKOrCSGyaWpqgsEQfJ5hXF81kyQJVVVVSEpKipt1bLzTUioqKnrUlT56X/Glu94XYwxNTU3IysoC386SL3FdI+J5Hv37x+cWM3q9vkf9YnvR+4ov3fG+2qsJeVFnNSFEdpSICCGyo0TUzdRqNR577LHIVhGIYfS+4kusva+47qwmhPQMVCMihMiOEhEhRHaUiAghsqNERAiRHSWiLvbUU0/h8ssvh06ng9FoDOk5jDGsWbMGmZmZ0Gq1mDlzJo4fP961gYapvr4ed911F/R6PYxGI+677z40NwffsQQApk2bBo7j/G4/+9nPuini4F544QXk5uZCo9Fg4sSJ2L17d7vl33jjDQwbNgwajQaXXHIJ/vvf/3ZTpOEJ530VFxe3+dloNN24lxojXWrNmjVs3bp1bOXKlcxgMIT0nGeeeYYZDAb25ptvsm+//ZbddNNNLC8vj9lstq4NNgzXXXcdGzVqFNu5cyf74osvWH5+Pps/f367z5k6dSpbvHgxq66u9t1MJlM3RRzY66+/zlQqFXv55ZfZkSNH2OLFi5nRaGRnzpwJWP6rr75igiCw3/3ud+zo0aPsV7/6FVMqlezw4cPdHHn7wn1fRUVFTK/X+/1sampqui1eSkTdpKioKKREJEkSy8jIYL///e99xxobG5larWavvfZaF0YYuqNHjzIAbM+ePb5j7733HuM4jlVWVgZ93tSpU9mDDz7YDRGGbsKECWzp0qW++6IosqysLFZYWBiw/O23386uv/56v2MTJ05k999/f5fGGa5w31eov59dhZpmMaa0tBQ1NTWYOXOm75jBYMDEiRMD7n4rh2+++QZGoxHjxo3zHZs5cyZ4nseuXbvafe6WLVvQp08fjBgxAqtXr4bVau3qcIPy7jTc+rNub6dhwPPeW5cHgGuvvTZmfjZAZO8LAJqbmzFgwABkZ2fj5ptvxpEjR7ojXABxPum1J6qpqQGAgLvfeh+TW01NDdLT0/2OKRQKpKSktBvjnXfeiQEDBiArKwuHDh3Cww8/jJKSEmzdurWrQw4o3J2GAc97j+WfDRDZ+xo6dChefvlljBw5EiaTCc8++ywuv/xyHDlypFsmllONKAKPPPJIm469C2/BfuCxrKvf15IlS3DttdfikksuwV133YXNmzdj27ZtOHnyZBTfBYnEpEmTsGDBAowePRpTp07F1q1bkZaWhhdffLFbXp9qRBF46KGHsHDhwnbLDBw4MKJze3e4PXPmDDIzz+8pe+bMGYwePTqic4Yq1PeVkZGB2tpav+Nutxv19fUBd+gNZuLEiQCAEydOYNCgQWHH21nh7jQMeH4+4ZSXQyTv60JKpRJjxozBiRMnuiLENigRRSAtLQ1paWldcu68vDxkZGTg448/9iUes9mMXbt24YEHHuiS1/QK9X1NmjQJjY2N2LdvH8aOHQsA+OSTTyBJki+5hOLgwYMA4Jdwu1PrnYbnzJkD4PxOw8uWLQv4nEmTJuHjjz/GihUrfMe2b98eUzsTR/K+LiSKIg4fPozZs2d3YaStyNZN3kuUl5ezAwcOsLVr17LExER24MABduDAAdbU1OQrM3ToULZ161bf/WeeeYYZjUb21ltvsUOHDrGbb745Ji/fjxkzhu3atYt9+eWXbPDgwX6X70+fPs2GDh3Kdu3axRhj7MSJE+yJJ55ge/fuZaWlpeytt95iAwcOZFdeeaVcb4Ex5rnMrVarWXFxMTt69ChbsmQJMxqNvkvXP/3pT9kjjzziK//VV18xhULBnn32Wfb999+zxx57LGYv34fzvtauXcs++OADdvLkSbZv3z42b948ptFo2JEjR7olXkpEXaygoIABaHP79NNPfWUAsKKiIt99SZLYr3/9a9a3b1+mVqvZjBkzWElJSfcH3466ujo2f/58lpiYyPR6Pbvnnnv8kmtpaanf+zx16hS78sorWUpKClOr1Sw/P5/9z//8j+zjiBhjbOPGjSwnJ4epVCo2YcIEtnPnTt9jU6dOZQUFBX7l//nPf7IhQ4YwlUrFhg8fzt59991ujjg04byvFStW+Mr27duXzZ49m+3fv7/bYqVlQAghsqOrZoQQ2VEiIoTIjhIRIUR2lIgIIbKjREQIkR0lIkKI7CgREUJkR4mIECI7SkQkrhQXF/stufv444/7TQZeuHChb34ViR+UiEhACxcuDLqm9NKlS8FxnN9M/a5IALm5uVi/fr3fsTvuuAPHjh0L+pwNGzaguLjYd3/atGl+E1RJbKJERILKzs7G66+/DpvN5jtmt9vx6quvIicnR5aYtFptm0XZWjMYDCFvUkBiByUiEtSll16K7OxsvxUUt27dipycHIwZM6ZT5w5UU5kzZ46vljVt2jSUl5fjl7/8pW9RNqBt0+xCrWtmCxcuxOeff44NGzb4zlFaWor8/Hw8++yzfs87ePAgOI7rtvV3iD9KRKRd9957L4qKinz3X375Zdxzzz1d/rpbt25F//798cQTT6C6uhrV1dVhn2PDhg2YNGkSFi9e7DtHTk5Om/cEAEVFRbjyyiuRn58frbdAwkCJiLTr7rvvxpdffony8nKUl5fjq6++wt13393lr5uSkgJBEJCUlISMjIyIVkA0GAxQqVTQ6XS+cwiCgIULF6KkpMS3z5fL5cKrr76Ke++9N9pvg4SIVmgk7UpLS8P111+P4uJiMMZw/fXXo0+fPnKH1SlZWVm4/vrr8fLLL2PChAl455134HA4MHfuXLlD67WoRkQ6dO+996K4uBivvPJK1GoNPM/jwqWwXC5XVM4dikWLFvk64ouKinDHHXdAp9N12+sTf5SISIeuu+46OJ1OuFwuXHvttVE5Z1paml+/jyiK+O677/zKqFQqiKLYqdcJdo7Zs2cjISEBmzZtwvvvv0/NMplR04x0SBAEfP/9977vgzGZTL4F8b1SU1ORnZ3dpuxVV12FlStX4t1338WgQYOwbt06NDY2+pXJzc3Fjh07MG/ePKjV6oiahLm5udi1axfKysqQmJiIlJQU8Dzv6ytavXo1Bg8eHFOL3/dGVCMiIdHr9dDr9e2W+eyzzzBmzBi/29q1awOWvffee1FQUIAFCxZg6tSpGDhwIKZPn+5X5oknnkBZWRkGDRoU8a4pq1atgiAIuPjii5GWloZTp075HrvvvvvgdDq75SogaR+tWU16rS+++AIzZsxARUVFm11RSfeiRER6HYfDgbNnz6KgoAAZGRnYsmWL3CH1etQ0I73Oa6+9hgEDBqCxsRG/+93v5A6HgGpEhJAYQDUiQojsKBERQmRHiYgQIjtKRIQQ2VEiIoTIjhIRIUR2lIgIIbKjREQIkd3/A/UZSjkTBCR9AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:17:29.944342Z", + "iopub.status.busy": "2024-03-26T11:17:29.944053Z", + "iopub.status.idle": "2024-03-26T11:17:30.290546Z", + "shell.execute_reply": "2024-03-26T11:17:30.289644Z" + }, + "papermill": { + "duration": 0.367556, + "end_time": "2024-03-26T11:17:30.292775", + "exception": false, + "start_time": "2024-03-26T11:17:29.925219", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+EElEQVR4nO3deXxU1d348c+9s2ZIMkkgIQlJCMii7ChCcQOVimCtaKtWbQVRu2Fb5aFF+nuKorXYujzyqA+1toC+qtJF1LYu1OJCpYII4sYmkUAgCYHsyez33t8fkxkyZJtMlsmQ7/v1mhfJzJ073yGZb84595zvUQzDMBBCiDhS4x2AEEJIIhJCxJ0kIiFE3EkiEkLEnSQiIUTcSSISQsSdJCIhRNxJIhJCxJ053gF0ha7rlJaWkpKSgqIo8Q5HCHEKwzCor68nNzcXVW273ZPQiai0tJT8/Px4hyGE6EBJSQl5eXltPp7QiSglJQUIvsnU1NQ4RyOEOFVdXR35+fnhz2pb4p6Ijh49ytKlS3n99ddxuVyMGDGCtWvXMmXKlA6fG+qOpaamSiISog/raOgkromourqa888/n4svvpjXX3+dzMxMvvjiC9LT0+MZlhCil8U1Ef36178mPz+ftWvXhu8bNmxYHCMSQsRDXC/f/+1vf2PKlClce+21ZGVlMXnyZJ5++uk2j/d6vdTV1UXchBCJL64toi+//JLVq1ezePFifv7zn7N9+3Z+/OMfY7VamT9/fovjV65cyYoVKzr1GoZhEAgE0DStu8IW/YjFYsFkMsU7jNOeEs/CaFarlSlTpvCf//wnfN+Pf/xjtm/fzvvvv9/ieK/Xi9frDX8fGpGvra1tdbDa5/NRVlaGy+XqmTcgTnuKopCXl0dycnK8Q0lIdXV1OJ3ONj+jIXFtEeXk5DBmzJiI+8466yxefPHFVo+32WzYbLaozq3rOgcPHsRkMpGbm4vVapVJj6JTDMPg+PHjHDlyhJEjR0rLqAfFNRGdf/757Nu3L+K+/fv3M3To0C6f2+fzoes6+fn5OByOLp9P9E+ZmZkUFxfj9/s7lYiqG32kD7D2YGSnl7gOVt91111s3bqVX/3qVxw4cIDnn3+e3/3udyxatKjbXqO9aeVCdKSzrWjDMPiwuIo/f1iCrks5+GjF9VN67rnn8tJLL/HCCy8wbtw47r//fh577DFuuummeIYlRMwMA7YdrMLl06hy+eIdTsKI+8zqr33ta3zta1+LdxhCdAtVVchMsXG02s2xOg+DkqMb0+zvpN/STxQXF6MoCrt27eqV82zZsoXx48djsViYN29el14zUXxeWsvaLQc5Wu0GoMblj3NEiUMSUT/1zjvvoCgKNTU1PXL+xYsXM2nSJA4ePMi6det65DX6mhqXPyL51LklEUVLElEC8PkSb6yhqKiISy65hLy8PNLS0mI6R2++79DE164IJZ4hyWDRXNR5JBFFq18mIl9Ab/MW0PSoj/VHcWwsZs6cyR133MGdd97JoEGDmD17Np999hlz5swhOTmZwYMH853vfIcTJ06En/PGG29wwQUXkJaWxsCBA/na175GUVFRq+cvLi7m4osvBiA9PR1FUViwYEGnzrN3717OO+887HY748aN49133w2fW1EUKisrWbhwIYqihFtE7777LlOnTsVms5GTk8Pdd98d8eFv7X2HWm4bN25k8uTJJCUlcckll1BRUcHrr7/OWWedRWpqKjfeeGPExFVd11m5ciXDhg0jKSmJiRMn8te//jX8eOi8r7/+Oueccw42m4333nsvpp9XSJ3Hj0n3ck7Fi0wuXY9eW9ql8/UncR+sjocn3z7Q5mPDBg1g3uQh4e9/t7kIv9b6Zdi89CSunXKyMNuaLQdx+yKXktz11VExxfjMM8/wgx/8gC1btlBTU8Mll1zCbbfdxv/8z//gdrtZunQp1113HW+99RYAjY2NLF68mAkTJtDQ0MDy5cu5+uqr2bVrV4spDPn5+bz44ot84xvfYN++faSmppKUlNSp8/z0pz/lscceY8yYMTz66KNceeWVHDx4kPz8fMrKyhg9ejT33Xcf119/PU6nk6NHjzJ37lwWLFjAs88+y969e7n99tux2+3ce++9rb5vgLKyMgDuvfdennjiCRwOB9dddx3XXXcdNpuN559/noaGBq6++moef/xxli5dCgSXA/3xj3/kt7/9LSNHjmTz5s18+9vfJjMzkxkzZoRf7+677+bhhx9m+PDhXa76UO8JMKjxAE6Ll0pFI9u1D/hKl87ZX/TLRJQIRo4cyW9+8xsAfvnLXzJ58mR+9atfhR9fs2YN+fn57N+/n1GjRvGNb3wj4vlr1qwhMzOT3bt3M27cuIjHTCYTGRkZAGRlZUV0naI9zx133BE+dvXq1bzxxhv84Q9/4Gc/+xnZ2dkoioLT6SQ7OxuA//u//yM/P58nnngCRVE488wzKS0tZenSpSxfvjyc5Jq/bziZiH75y19y/vnnA3DrrbeybNkyioqKGD58OADf/OY3efvtt1m6dCler5df/epX/Otf/2L69OkADB8+nPfee4+nnnoqIhHdd999fPWrX+34B9IBwzBw+TQGe4+RnGJmamEGJLm7fN7+ol8mokUXj2jzMfWU+WvfveiMNo89da7bwvO7r4TJOeecE/76448/5u233251vVNRURGjRo3iiy++YPny5Wzbto0TJ06g68Fu4eHDh1skovZEe57QBxzAbDYzZcoU9uzZ0+Z59+zZw/Tp0yMmCJ5//vk0NDRw5MgRCgoKWrzv5iZMmBD+evDgwTgcjnASCt33wQcfAHDgwAFcLleLBOPz+Zg8eXLEfdEU4IuGT9PRdINk33EsoRnYnhrQ/GCydMtrnM76ZSKymqMfGuupYzsyYMCA8NcNDQ1ceeWV/PrXv25xXE5ODgBXXnklQ4cO5emnnyY3Nxdd1xk3blynB3y76zyxav6+m7NYTn6YFUWJ+D50XyhpNjQ0APDqq68yZMiQiONOXavY1ut1ll8zyEqx4FRdmJSmxZ2GAe5qSM7qltc4nfXLRJRozj77bF588UUKCwsxm1v+yCorK9m3bx9PP/00F154IUCHA69Wa3AdVPPyKJ05z9atW7nooosACAQC7NixgzvuuKPN1wstZjYMI9wq2rJlCykpKe0WVY/FmDFjsNlsHD58OKIb1pOSbWZumjQQ3KmgqOx3OfBWHSUjr5QhIyURdaRfXjVLNIsWLaKqqoobbriB7du3U1RUxMaNG7nlllvQNI309HQGDhzI7373Ow4cOMBbb73F4sWL2z3n0KFDURSFf/zjHxw/fpyGhoZOnefJJ5/kpZdeYu/evSxatIjq6moWLlzY5uv98Ic/pKSkhB/96Efs3buXV155hXvuuYfFixd3+3rAlJQUlixZwl133cUzzzxDUVERO3fu5PHHH+eZZ57p1teK4KkN/mtPpdZIosEbwF1f03OvdxqRRJQAcnNz2bJlC5qmcdlllzF+/HjuvPNO0tLSUFUVVVVZv349O3bsYNy4cdx111089NBD7Z5zyJAhrFixgrvvvpvBgwdzxx13dOo8Dz74IA8++CATJ07kvffe429/+xuDBg1q9/Vee+01PvjgAyZOnMj3v/99br31Vv77v/+7S/83bbn//vv5xS9+wcqVKznrrLO4/PLLefXVV3u2FLGnJviv3YlqC+5aEfDU99zrnUbiWhitq9oruuTxeDh48CDDhg3DbrfHKUKR6KL9Pdp5uJryTzYxwfcxeWO+wkcnFDz73sI57GzGXHJjL0bctyREYTQhTheN3gDexnoCJgOsA1BtwStnhldaRNGQRCREN/D6dSy6G7NFAcsATEnBq3qGtyHOkSUGSURCdANvQMeiezCpClgdmJOaunE+aRFFQwarhegG3oCGWXNjVlWwJGFNSsGsKlh0H+ixrTnsT6RFJEQ38AZ0LJoHk2oGywCGpw1ieGFwGQ0BN1i7Z+Lk6UpaREJ0A59fw9ysa4aqgrmpeL7fE9/gEoAkIiG6gdMcwG5WsJgUsDTtGhP6NyCLXzsiXTMhusG8cengSgezDVQTfk3n01I3FlctZ411yQetA9IiEqI7+JuKsjWNBZlVhQqPSoM3gN8jOw13RBJRHzRz5kzuvPPOeIcR1tfi6ZN8TcnGEiwwpyhK+Gu/pzFeUSUMSUSnqUSsc52oalw+/rbjS3aX1YE5KXy/EkpEXmkRdUQSUR+zYMEC3n33XVatWoWiKCiKQlFREbfeemu4/vLo0aNZtWpVi+fNmzePBx54gNzcXEaPHg3Af/7zHyZNmoTdbmfKlCm8/PLLLbYDaq8edmvxFBcX99Z/R0Jw+zVcLhcev3byShknE5EmiahDcU1E9957b/iXO3Q788wze+4FDQMCvt6/dWJd8apVq5g+fTq33347ZWVllJWVkZeXR15eHn/5y1/YvXs3y5cv5+c//zl//vOfI567adMm9u3bx5tvvsk//vEP6urquPLKKxk/fjw7d+7k/vvvD9d0DgnVw548eTIffvghb7zxBseOHeO6665rM578/HzESb6AjsnwY1YVMJ0svKZagl9rfrlq1pG4D+aPHTuWf/3rX+HvWyv81W00P/z7kZ47f1su/K+Iv5TtcTqdWK1WHA5HuN4zwIoVK8JfDxs2jPfff58///nP4YQBwWqDv//978NFz37729+iKApPP/00drudMWPGcPToUW6//fbwc5544okO62G3Fo84yRvQMem+4ByiZj9n1RJc5qH5vfEKLWHEPRGZzWb5BY/Ck08+yZo1azh8+DButxufz8ekSZMijhk/fnw4CQHs27ePCRMmRJSvmDp1asRzoqmHLdrn9TdLRM1aRGarHbOqoGgyXteRuCeiL774gtzcXOx2O9OnT2flypXhQurdzmQJtk56WxeLp69fv54lS5bwyCOPMH36dFJSUnjooYfYtm1bxHGx1F+Oph62aJ83oDV1zdTgPKImF52VB74MSFbaebaAOCeiadOmsW7dOkaPHk1ZWRkrVqzgwgsv5LPPPiMlJaXF8V6vF6/3ZDO3rq6ucy+oKFF3keLJarVG1JLesmUL5513Hj/84Q/D97W1eWJzo0eP5o9//CNerzdcNH779u0Rx3RUD7u1eEQkX6hrZlbA1Oz3K9Q6CkiLqCNxHayeM2cO1157LRMmTGD27Nm89tpr1NTUtBiEDVm5ciVOpzN8O10HTQsLC9m2bRvFxcWcOHGCkSNH8uGHH7Jx40b279/PL37xixYJpTU33ngjuq7z3e9+lz179rBx40YefvhhgHAB+47qYbcWjy6rySNYzCrJZj24vKNZiyj8tSZjRB3pU5fv09LSGDVqFAcOtL4T67Jly6itrQ3fSkpKejnC3rFkyRJMJhNjxowhMzOT2bNnc80113D99dczbdo0KisrI1pHbUlNTeXvf/87u3btYtKkSfy///f/WL58OUB43KijetitxXP48OGee/MJ6NzCDC4clkJeuiOiRXSgys/usjqOnKjt1JXT/qhP1axuaGigoKCAe++9lx//+McdHi81qzvvueee45ZbbqG2tja8zbRoW9S/R+//X3AXj3PmQ2ouALsOHsP91kMMSrYy8hv3JsSwQHdLiJrVS5YsCW/oV1payj333IPJZOKGG26IZ1inlWeffZbhw4czZMgQPv74Y5YuXcp1110nSai7hbpfza+aWWwYKGi6EXy8HyaiaMU1ER05coQbbriByspKMjMzueCCC9i6dSuZmZnxDOu0Ul5ezvLlyykvLycnJ4drr72WBx54IN5hnVbe+LSM3OIKhmYk4WyWbKwWE5pqRdO14IC1rZ2T9HNxTUTr16+P58v3Cz/72c/42c9+Fu8wTmtVDY04vX50wx7RIrKYVDTVgm5oEJDiaO3pU4PVQiQi3RfslqmqGjFnzGJS0BQrmq7LlbMOSCISoov0QDDJmCy24Fy1JlazSkC1ounIXKIOnPaJqA9dFBQJKJrfn9BaMpMl8qqa1aSiq9ZgbpIWUbvivsSjp1gswSayy+WSK0QiZqG6TiaTqdXHDcPAaBr/Ca22D3EmWbh8YgFKRSMEJBG157RNRCaTibS0NCoqKgBwOBzh2cRCREPXdY4fP47D4Whz+YtfM1A1P9CyRaQozWZay8LXdp22iQgIr+oPJSMhOktVVQoKCtr8I6bpBqkWDYtJwWRt5fp8aPBaElG7TutEpCgKOTk5ZGVl4ff74x2OSEBWqzW81KU1SVYT10zIgi8yIteZNfngcD0p5XUMzXTj6MlAE9xpnYhCTCZTm318IbosPKu65czpsgYdze0nx+eVRNSO0/6qmRA9LtByeUeI2jTTWqo0tk8SkRBdcLTGzbt7jvLliYZW15IpTd013S9jRO3pF10zIXpKozdAXUMDbk1rt0Wky+X7dkmLSIguCFZn9LconB9isoQSkbSI2iOJSIgu8GvBMrGqorTaIgp3zSQRtUsSkRBd4NcMTEaoRdQyEZnMVhSQeUQdkDEiIbog1CIKbiXUsms2c0weakMGikwfaZe0iIToAp/WNEaktNEislhRUEALgGw60CZJREJ0gaoo2BR/my2iiPt0md3fFumaCdEFM0ZkwFFn8JtWWkRFlR4CFQ2k2kzkaL5WjxGSiITomubzg1q5albjDuB26agYoEmLqC3SNROiK8LrzMzQyuJYq0lFUyzoBnLlrB2SiITogrc+P8LusjrqA61fFTObFHTF0rSlkCSitkgiEqILTtTUUev2o6uWVh+3mJSmnTyka9YeSURCdIHRNEakWlrfBdasqmiKGV1aRO2SRCREF+htFM4PsZhVdCXUIpJE1BZJRELEyDAMDH+zrYRaYVGDXTNNR7pm7egziejBBx9EURTuvPPOeIciRFR8mo7JaCqcb229RTQo2cbsCQWcPTRNWkTt6BPziLZv385TTz3FhAkT4h2KEFHzawaq7keh7RaRqiqoVhugSCJqR9xbRA0NDdx00008/fTTpKenxzscIaKmaQZ2NYBJVcLlPloV3slDumZtiXsiWrRoEVdccQWzZs3q8Fiv10tdXV3ETYh4cTosXDl2IFMK01udVQ2g6wbbDjew/1g9fqlb3aa4ds3Wr1/Pzp072b59e1THr1y5khUrVvRwVEJ0QsAbXF3fSnVGAEWBg9U+9EYf+X4frc82EnFrEZWUlPCTn/yE5557Dru99YG+Uy1btoza2trwraSkpIejFKIDoXGfNlpEiqKgNK3A132e3ooq4cStRbRjxw4qKio4++yzw/dpmsbmzZt54okn8Hq9LfYis9ls2Gyyeln0DYcrXZz4spxMw0V+O2NE4S2FAjJG1Ja4JaJLL72UTz/9NOK+W265hTPPPJOlS5fKhoiiz6vz+Kmpb8BhCbRei6iJIjt5dChuiSglJYVx48ZF3DdgwAAGDhzY4n4h+qJwdcY26lWHhLcUkr3N2hT3q2ZCJCp/QMdkhHbwaLtFFE5EmrSI2tInJjSGvPPOO/EOQYio+cN7mtk7SETB1pIhY0Rt6lOJSIhEEvB7MEGHXbPLJuRhdqejmpTeCy7BSNdMiBgFmiYoqqoJ1Lb/ptttSZhVFdXQQdd6K7yEIolIiBgpmheFpt1clXZaO6Zm0xhlvVmrpGsmRIwuHZGGUZeBYUtt97gvjrtQTrhJT1IZqPnBktRLESYOSURCxEoLLu9Q2lh5H1Je54EGDbOiBxORaEG6ZkLEKtD+8o6QYLlYqdLYHklEQsToP/tL2X+snkat/VUAFpOCrpqDO05LImqVJCIhYnSsqpbKRh8Bte05RAAWk4qumGUnj3bElIi+/PLL7o5DiIQT2sHD3MEYkdmknOya6ZKIWhNTIhoxYgQXX3wxf/zjH/F4pLSB6H903cBoGiMyt1GvOsRiUpsK6MsYUVtiSkQ7d+5kwoQJLF68mOzsbL73ve/xwQcfdHdsQvRZfl3HbASTSluF80NOds2QrlkbYkpEkyZNYtWqVZSWlrJmzRrKysq44IILGDduHI8++ijHjx/v7jiF6FP8moGpg8L5IXnpSVw6dghn5aRIi6gNXRqsNpvNXHPNNfzlL3/h17/+NQcOHGDJkiXk5+dz8803U1ZW1l1xCtGn+AI6Jt3XceF8gi2ipCQHZlWVFlEbupSIPvzwQ374wx+Sk5PDo48+ypIlSygqKuLNN9+ktLSUq666qrviFKJPCTTtaWZSlQ7nEQGyk0cHYppZ/eijj7J27Vr27dvH3LlzefbZZ5k7dy6qGsxrw4YNY926dRQWFnZnrEL0GVmpdr46Kg29wddm4fwQj19jz9EG0k40MCxHumatiSkRrV69moULF7JgwQJycnJaPSYrK4s//OEPXQpOiL5M0XyYlI5bRLphsPe4l8I6L4WaDykG0lJMiejNN9+koKAg3AIKMQyDkpISCgoKsFqtzJ8/v1uCFKJPClVc7GCMyKyq6Eqwa6b5fbLAsxUxjRGdccYZnDhxosX9VVVVDBs2rMtBCdHXHTrRwL6jlZTVututzgjBJR6aEkw/mhTQb1VMydkwjFbvb2hoiHqPMiESWWWDC2+DBxVrhy0iRVFQzSdbRKKlTiWixYsXA8H/2OXLl+NwOMKPaZrGtm3bmDRpUrcGKERfpHmDKwo6KpwfErrEL1sKta5Tieijjz4Cgi2iTz/9FKv15A/AarUyceJElixZ0r0RCtEHhVo2HVZnbCKbLLavU4no7bffBoIbIa5atYrU1PYr0wlxutL8HlTosChaiGoJJqLQ+jQRKaYxorVr13Z3HEIklIDfg5WTWwV15PIJBdh9aZgtrY+v9ndRJ6JrrrmGdevWkZqayjXXXNPusRs2bOhyYEL0ZXpoB48oW0TOAQ4wm8DQQNdBlVJgzUWdiJxOJ0pTX9jpdPZYQEIkAt3fNFgdZYsoYkBb94Ma5fP6iagTUfPuWHd1zVavXs3q1aspLi4GYOzYsSxfvpw5c+Z0y/mF6CkXn+FE19JRMqL7o7yvwoWtyk16khmn5uvwkn9/E1P70O1243K5wt8fOnSIxx57jH/+85+dOk9eXh4PPvggO3bs4MMPP+SSSy7hqquu4vPPP48lLCF6jar7MKtqh7WIQoqrXByp89PgDcjC11bElIiuuuoqnn32WQBqamqYOnUqjzzyCFdddRWrV6+O+jxXXnklc+fOZeTIkYwaNYoHHniA5ORktm7dGktYQvSe0HygaFbeE5pdLTt5tCXmCo0XXnghAH/961/Jzs7m0KFDPPvss/zv//5vTIFomsb69etpbGxk+vTprR7j9Xqpq6uLuAkRDx8dPEbR8QbcRnSjG6H1ZrqUi21VTInI5XKRkpICwD//+U+uueYaVFXlK1/5CocOHerUuT799FOSk5Ox2Wx8//vf56WXXmLMmDGtHrty5UqcTmf4lp+fH0v4QnRZeWUNFfVetA528AgJ162WnTxaFXPx/JdffpmSkhI2btzIZZddBkBFRUWnJzmOHj2aXbt2sW3bNn7wgx8wf/58du/e3eqxy5Yto7a2NnwrKSmJJXwhukTXDQwtusL5IRaTEqxbrSOJqBUxJaLly5ezZMkSCgsLmTZtWrgr9c9//pPJkyd36lxWq5URI0ZwzjnnsHLlSiZOnMiqVataPdZms5GamhpxE6K3+XUds96UiKKcR2QxqWjhvc2ka3aqmGZWf/Ob3+SCCy6grKyMiRMnhu+/9NJLufrqq7sUkK7reL2yMFD0XcHC+cECZyZrUlTPMZsUdOmatSnmGk3Z2dlkZ2dH3Dd16tROnWPZsmXMmTOHgoIC6uvref7553nnnXfYuHFjrGEJ0eN8AR2TEV3h/JAzMpPJPzMXe2WNbLLYipgSUWNjIw8++CCbNm2ioqICXdcjHo92J9iKiorwbh9Op5MJEyawceNGvvrVr8YSlhC9whcIds1MqhL1xES7xYR9gANqVOmatSKmRHTbbbfx7rvv8p3vfIecnJzw0o/OkprWIhH5/Bom3Y/JrHZuhnR4Jw9JRKeKKRG9/vrrvPrqq5x//vndHY8QfV5+mpncwrTgzq1RTmhs9AYoPubBWe0iL1e6ZqeK6apZeno6GRkZ3R2LEAlBCXgxqypWs/lkK6cDHr/Gp2Uuymo9MljdipgS0f3338/y5csj1psJ0W+EulZma1TVGQHMJjU4j0gu37cqpq7ZI488QlFREYMHD6awsBCLJfKvws6dO7slOCH6ooPHKtGPN5CaZiczyudYm2ZW6wboAV/Xtlg+DcWUiObNm9fNYQiROI5X16HVe9EHKFEnInPTolcIbikkiShSTInonnvu6e44hEgYmi9UnTH6rbPMqoJuCn7cdCmg30LMibmmpobf//73LFu2jKqqKiDYJTt69Gi3BSdEX6SFqjN2IhEF9zazNT1fVg6cKqYW0SeffMKsWbNwOp0UFxdz++23k5GRwYYNGzh8+HC4VpEQpyPd78EEmKJcZxaiNl1h02UnjxZiahEtXryYBQsW8MUXX0Ts7Dp37lw2b97cbcEJ0ReFumbmKNeZhcyZWMCEPCdJqtYTYSW0mFpE27dv56mnnmpx/5AhQygvL+9yUEL0ZaHC+aZOdM0AMtNSwGoO7uRhGFFf+u8PYmoR2Wy2Vqsj7t+/n8zMaK8jCJGYtEAwEUVbiyis+U4eMqkxQkyJ6Otf/zr33Xcffn/wP1NRFA4fPszSpUv5xje+0a0BCtHXzByewtkFaThTkzv1vP3H3ZTUuGn0BWRS4yliSkSPPPIIDQ0NZGZm4na7mTFjBiNGjCAlJYUHHnigu2MUok+xGn5sZhNmq6NTz9tTXs+haj+NXklEp4ppjMjpdPLmm2+yZcsWPv74YxoaGjj77LOZNWtWd8cnRN8TcAf/7eTeZGZVRVfNaMHp1T0QWOLqdCLSdZ1169axYcMGiouLURSFYcOGkZ2djWEYMZcEESIRePwah4+cwK67KLB07qrZyS2FpEV0qk51zQzD4Otf/zq33XYbR48eZfz48YwdO5ZDhw6xYMGCLpeJFaKvc/k0KqprOVbnAXPnBqstoYWvsqVQC51qEa1bt47NmzezadMmLr744ojH3nrrLebNm8ezzz7LzTff3K1BCtFX+Hx+TIYfk0ntdCIymxTcqgXNcMlVs1N0qkX0wgsv8POf/7xFEgK45JJLuPvuu3nuuee6LTgh+hq/Nzg+FCwTG0uLSDZZbE2nEtEnn3zC5Zdf3ubjc+bM4eOPP+5yUEL0VT5PY/ALsw3Uzl10Do4RhWoSSYuouU51zaqqqhg8eHCbjw8ePJjq6uouByVEX+X3BIsBqp0cqAYYNTiFYaOycVTVSCI6RacSkaZpmM1tP8VkMhEIyGVJcfry+5oSUSfXmQGk2C2Qkgz1JumanaJTicgwDBYsWIDN1vr8CdkYUZzu/J7gGFFnSoBEkJ08WtWpRDR//vwOj5ErZuJ0NiHbRqA2DTIHdfq59R4/pSe8pNd7yJIJjRE6lYjWrl3bU3EIkRDMuhez2QRJAzr93Fq3n11HGxnd6CZLWkQRpHSuEJ3RVAKks8s7oKmAvmJG05Gu2SnimohWrlzJueeeS0pKCllZWcybN499+/bFMyQh2rX/6HEOVTZSr3V+maalaScPTS7ftxDXRPTuu++yaNEitm7dyptvvonf7+eyyy6jsbExnmEJ0abSE9WU1nrwGNFtrNic1XxyQqMhLaIIMa2+7y5vvPFGxPfr1q0jKyuLHTt2cNFFF8UpKiHaZviDl+8tSSmdfq6lqWtmAJrfF98PXx/Tp/4vamtrAdrcztrr9UZMEWitSqQQPUXXDZSmRGS1d36w2mJSMNSmvc383r714YuzPjNYres6d955J+effz7jxo1r9ZiVK1fidDrDt/z8/F6OUvRnPk3HrAUHq22OzreIFEVBtQTLxWp+6Zo112cS0aJFi/jss89Yv359m8csW7aM2tra8K2kpKQXIxT9ncfrxaT7MKkKJlvnW0QAl43PZ2xuKjbZySNCn2gd3nHHHfzjH/9g8+bN5OXltXmczWZrc1a3ED3N5w5eRDGpnS8BEjJkYCrYLWAEZCePZuLaIjIMgzvuuIOXXnqJt956i2HDhsUzHCHa5XU3BL+wOmJPIKGdPAwpF9tcXFtEixYt4vnnn+eVV14hJSUlvCea0+kkKanziwqF6Em5DoPMgjR0R+eXd4QUV/uw1bpxJllwaL6Ta8/6ubi2iFavXk1tbS0zZ84kJycnfPvTn/4Uz7CEaJUp4MJuNuEYkBrzOT4prefLKh/1ngAEZJF4SFxbRIZhxPPlhegcf9PuHZbObSPUnNWkElCtwZ08ZFJjWJ8YrBYiERSXV6BUNZKcZmJgjOewmhU01YpmeKVF1EyfuXwvRF9XUVlDaY2Han/s4zpWkwlNaWoRSSIKk0QkRJQ0b/CqmTWGEiAhFlNTi0g3QJNEFCKJSIgo6d7gPCJrUuf2vG/Oam5aga8bEJAxohBJREJEyWhqEdkczpjPYTGpBBRrcCePgKe7Qkt4MlgtRBQ0TUf1BxORPTkt5vMUDHQwaGQuKSfKpWvWjCQiIaLgdjegGDoKYO/CPKJUu4XUjFSoM0vXrBnpmgkRBXd9DQCKdQCKqYt/v0Pr1KRrFiYtIiGiMNDsI7UgDX9SZpfO4w1olFT5SKnzMHigtIhCpEUkRBRUfwM2s4nk1PQuncfj19lSXE9xZaO0iJqRRCRENHyhlfexX7oHsJlVNNWKboDul8HqEElEQkTh4NFyDlU2UhXoWj2s0OV7AM0vLaIQSURCROHEiQpKaz1U6zFuNd3EpCqolmAy03ySiEIkEQkRBcNdDYA9JfZaRCEmS7DWVsDvDRZIE5KIhOiIoWsYnuCOMUkpXRusBjA3lTvWNF1KgTSRRCREBzwNdeiahoFCSlrrW111hs1iw0AhoOty5ayJzCMSogP1tceDX9idWMxd/8hMHzEIR3UmKSa/lAJpIolIiA401lYCYBqQ1i3ny89wQJoTXFUnqz72c9I1E6IDnqYWkWVArHUZWyHLPCJIi0iIDpyZ6sM/NB3/8O7Z7qqq0YerEVLcfpzSIgIkEQnRIaXxBFaTijU9u1vOd/BEI0dLPZyleHBKiwiQrpkQ7dP84KkJft2F/cyas5lVAqotWKVRWkSAJCIh2lVXVc7esloO1+tgjb1WdXN2SzARBXSp0hgiiUiIdtQcK6Ha5eeoP6Xb9qm3mU0EVDsBzQC/q1vOmejimog2b97MlVdeSW5uLoqi8PLLL8czHCFaaDx+GABLel63ndNmCXXNdJCFr0CcE1FjYyMTJ07kySefjGcYop/yBjS2F1fh1/Q2j3FXBhNRSlZht71uuEWkGxCQMSKI81WzOXPmMGfOnHiGIPqRAxUNpDksDEoOrvXadbiG/xRVcqCigasm5eKwRn4c/J5G/HUVAGQN6Z5L93BysDq467QbU7edOXEl1BiR1+ulrq4u4iZENA5Xunjt0zL+tqsUj18DYGCyFbvFRHmthw07j4bvDykv3hMsYJY0kLS0tG6LxWZWmTEunzOzU1BksBpIsES0cuVKnE5n+Jafnx/vkEQCqHX5+fsnpWi6QbbTjtUU/LUfkZXCt87Nx2E1cbzey98+Lo3oplUUfw5Acu5olG4aqAZQFIXRQ7JId1hR9UBwikA/l1CJaNmyZdTW1oZvJSUl8Q5J9HGabvDaZ2X4Ajq5aXYuGzMYVVWCg8S6RvoAK1dPHoLVrHK02s0/PikloOkYfg/pjQdRFRgycmL3B2a2gdL08ZO5RIk1s9pms2Gzda1Up+hftn5ZSXmtB5tF5fIxmZiPbIXSj8BTF0wE6UPJyp/KvMlDeGnnEYpPuNhSVMmMpIMMz7CRl5ONNXd4t8dVVudBcUOqGsAR8ACx75V2OkioRCREZ5TVutleXAXAZSOSce5+HhoqTh5g6FB1EKoOMiS9kHkjvsLfv9TJt3uh+D0ArIVf6bb5Q83tPFRD0nE/Z6X6cchcovgmooaGBg4cOBD+/uDBg+zatYuMjAwKCgriGJk4HWz7sgrDgLFZVkaU/QMajoMlCUZcCoNGgbch2Doq3QnVxeTVHGJhUhbm4upg5cTUHMie0COxOawmfCYHAc0LvsYeeY1EEtdE9OGHH3LxxReHv1+8eDEA8+fPZ926dXGKSpwu5ozPZvuBcqbWvQmNx4NLNCZ/GxxNVRbNNhg5C/LOgYOb4dhubK5jwceSs2DsNaD2zDBqktVEo5qEX6uUREScE9HMmTMxpHi46CE2FS7wvgeNZcGkM/FbJ5NQc0npMOYqGDYD6kqDraa0AlB7boZPksWEz5SEXzdO7pnWj8kYkTjtHKl2McRpQ9nzN6j6EkxmmHBdsJXTnqS04K0XOKwm/CYHAb8OPhkjkkQkTitHql38ZXsJU3zbuMBxBMVkDnaxnN23Vqw7JFlN+NWk4Lwl6Zol1jwiIdpjGAZbDpygoPYDhrj3oagqnHUlDDwj3qG14LCa8Zkc+DXpmoEkInEaKTreiHF4G/n1H5OX7oBRsyHrrHiH1apkm5kLxhQwMitZWkRIIhKnCV032L3rfYbWbCPbmYR11KWQOzneYbXJalYZlZ9NmsManFmtt10BoD+QRCROC1/s/5xBR97ErCpkj70ACr4S75A6ZhkQnCxp6ODv360iSUQi4fnrKqjdvh7F0BlYOB7rqK/GO6SolNf7KPNYcPkCwSUn/ZgkIpHYAj4Cn7yIHT/+5BwKzr++xyYhdrcdh6r59IRBrdsP3v6diOTyvUhchgH7XyfJX8P44UOoG3szZmviLIpOtpupMyXjC9RIiyjeAQgRs9KP4NhuUFSUsVfj7MbiZb0hxW7GZx6AN6D3+xaRJCKRmOrL8e79J0dr3PiHXghpiVckL8VmxmtKxqfp4KmNdzhxJYlIJB6/Bz5/iaOV9XzsHsTrtUPjHVFMku1mvOZkfAFJRJKIRGIxDNj3Ko21JyhxWziQMZMpha0sZE0AziQLXlMKvoCO5qoOvrd+ShKRSCxHPsQ4vo9DVR72D7yUM3IzyU1LindUMUmymDCS0jBQ8Hrc/Xqph1w1E4mjpgSK3qKywcfHtnNwJ2VzwYju2Y8+HhRF4avjhjCQfGxGA7gqwZYS77DiQlpEIjF4G2D3y/gCAXZ6cihPHsu5hRk4HZZ4R9YlI7JSSB+Ug0lVgomon5JEJPq+gA8++yt4GzjQYGdP6gVkptqZOiwxx4ZaGDAw+G9j/01E0jUTfZuuwe6Xoa4MLHaGzriRgmI/F47MDLYiEpzHr1HaaMde4yY3/US8w4kbSUSi7wp44bMNaFUHUU0WlPHXMsA5mKvT4x1Y9/H4Nd4sgUnVLrIHlqHqesIsUelO/e8di8RQVwY7nqGu7As+KXPxWcasPldlsTs4kyzojkz8WHC7XP12nEhaRKJvqT+Gcfh96ks+pazGTbnHzL5Bc1BrnJyp6VhMp9ffTkVRGOxMotGWSYO3lgF1RyE5M95h9TpJRKJvqD1C4OAWqg7vprzOQ6NXo9JxBsU553Nm/mDOGzHwtEtCIdmpdkqtWdR5TjC45jDkTop3SL1OEpGIr8ZKKNoElUWUVbk4UuOh0jGcY7mTKBg6nBsK0hiYnDgr6mMxdNAAdtvzqTnxMXplUb8cJ5JEJOKiqqaWsk/+RV7jHpx2E6gmnCPO5d2qAs4cVsCluU6SrD23r1hfkpNqx5+ci+eEjYaGelJrD0N6YbzD6lWSiESvcfs09pXXcmzv+ziO/gez7qUi2Yoz72w44xJSHRncZBgoPbDXfF+mqgrDs1KorhhKo7eE1NJdkoji4cknn+Shhx6ivLyciRMn8vjjjzN16tR4h9X3GAY0HIOKPVBzKLhiW9cwTFbqzemU+FIo0TK48JwJDEhOBeBART2fHa3DmWQhNcmCM8lCmsNCss2MzaSgYICiBmsn90jIBgcqGthdWkv1kf3kV79Pqr8aBUhKz8Yy/nIYOS58fH9LQiFfGTYQZfBcUj99Fo7vDXZZQxMd+4G4J6I//elPLF68mN/+9rdMmzaNxx57jNmzZ7Nv3z6ysjrYmbM/MAyoLw/+ch7fC+6a4N0YNHo1qhp9VDV6cfuLAXACmustSM8C5xA8lQreYy6qdC91updyzYNVc2HRXVg1NxNyk0m2W8Fip9SbxJcuO377QAJJgwgkDcSwDEBVFEyqwrmF6cFdJwjOfwGwW1p2nzTdODnZ0ND5/LOPSCr7gNHecpJtZjIGD2TguEuxF0zpd2MhbXE6LOAYAgNHQOUB2Pt3mHgjmK3xDq1XKEacN5+fNm0a5557Lk888QQAuq6Tn5/Pj370I+6+++52n1tXV4fT6aS2tpbU1NTeCLdnGQYEPOCtD7Z86kqh8gCGuxYDUBUFTGYqLHlsrk6j0nCiK2YsuoeUQBWFtjry1WoGmRqwNl1havQFaPAE8AZ0PH4t/G9AD/7YJ+WnkdSUTEqqXRypdkeE5Dcl4bIMxGsawIVj8slITQHVzN6KBnYersNiMZNit2K1mDEUhUafjtvtZt7YdGye41BTQkVlJW6/xqDUAQwYdi4MPT+4v7xoyVNLzebfUl5ZQ1ZuASlnXowloxAs9nhHFpNoP6NxbRH5fD527NjBsmXLwvepqsqsWbN4//33Wxzv9Xrxer3h7+vqoiyvWXUweGWmtZxrGFS7fXxRXg80f9wIH1440MHg1OAvQp3bx96yuqYjDZr/A5Cfbg+Xpaj3+NldWgctjg1+PSTdTl7TsS6fxuclJ1AM7dTw8GNi0NAxjBp7LmScgdkLJf8pxmpWKRw4gBFZyRQOcmAzN7VO/O5gEqsrZYDfxQDND2YbmO1gHQDWZPwmOz7VgS3JAUYA/C6ST5QxpOoYqusEqus4qqcWw9AxqMIwqkiuqIDKYIJzVLkYVhOZtAAcTf827k3B1tR6yspIh8HjIP9csDtb/RGJIK85mb8bF1DoeY3qA0UoB4owmxR0kw3FZMVksXL20JNdtt2ltdR7I39naGqMmhWFc4amh7vde8vrg4X62zCt2dq9/cfqqXadPFaBlmv7xl8H9u5pAMQ1EZ04cQJN0xg8eHDE/YMHD2bv3r0tjl+5ciUrVqzo/AsFvNBwvO3HXT60xvo2HzYcvvBfJMPtR3O3kwC9AWj6xVB8AXRv2zVmDJ8B/qbfGn+AQCAQDFe14bKk02gdRJ0tl1p7HuNyBjIqK/j/lGGGb56TR47Tjrm1uTWWpOA2y+1stWxpuoXZU0lPySZ9WLP7Aj5oPB68+RqDrbWAFwyNgiyNXE3D5fXj8vqaYtexqTAgyUFScgo4BkJqLjjzQe0fV8C6ymY2ceWF57LzixzqirYwoL4IW6ABNA/gQfWp0HjyL5/eWIvuCbR6Lk1VoPFkkjLq69DbSEQKQEOzYxvq0Rt94e9VJfLx4EHdtylk3MeIOmPZsmUsXrw4/H1dXR35+VHUKk7Lh4nfCn4dMRga/DrJr1PQ9J+uKC2PSU6ygC34sbUFNArdgWbPVpqdViHZbgarGVCwajojTv3BK0r4L5bDamk6Fqy6wSivgWEdAKqZ5hHYLCYcp4zF5Gc46HFmKziHBG+tPQykNt1E90lzWLlk4nCYOByXL4DL5UJz1xHw+zATgJTQvCqDwSO8ZASMUxr7wW8UBXCe7NJlnuElNdDOSEzayWMHNnpJ9httPg4EW9fdJK6JaNCgQZhMJo4dOxZx/7Fjx8jOzm5xvM1mw2aLYXKbdQBkDGvzYTvQ+ket9WNzon1ZIDMtumPNwMD+WRNLtMNhNeOwpkJa6+k+sxMLgAd1ompKZ47tDnG9ZGG1WjnnnHPYtGlT+D5d19m0aRPTp0+PY2RCiN4U967Z4sWLmT9/PlOmTGHq1Kk89thjNDY2csstt8Q7NCFEL4l7Irr++us5fvw4y5cvp7y8nEmTJvHGG2+0GMAWQpy+4j6PqCtOu3lEQpxmov2MyrRWIUTcSSISQsSdJCIhRNzFfbC6K0LDW1Ev9RBC9KrQZ7OjoeiETkT19cFlGVHNrhZCxE19fT1OZ9vrDBP6qpmu65SWlpKSkpIwdWxCy1JKSkpOqyt98r4SS2+9L8MwqK+vJzc3F7Wdki8J3SJSVZW8vMTcYiY1NfW0+sUOkfeVWHrjfbXXEgqRwWohRNxJIhJCxJ0kol5ms9m45557Yqsi0IfJ+0osfe19JfRgtRDi9CAtIiFE3EkiEkLEnSQiIUTcSSISQsSdJKIe9sADD3DeeefhcDhIS0uL6jmGYbB8+XJycnJISkpi1qxZfPHFFz0baCdVVVVx0003kZqaSlpaGrfeeisNDW3vWAIwc+ZMFEWJuH3/+9/vpYjb9uSTT1JYWIjdbmfatGl88MEH7R7/l7/8hTPPPBO73c748eN57bXXeinSzunM+1q3bl2Ln43d3ot7qRmiRy1fvtx49NFHjcWLFxtOpzOq5zz44IOG0+k0Xn75ZePjjz82vv71rxvDhg0z3G53zwbbCZdffrkxceJEY+vWrca///1vY8SIEcYNN9zQ7nNmzJhh3H777UZZWVn4Vltb20sRt279+vWG1Wo11qxZY3z++efG7bffbqSlpRnHjh1r9fgtW7YYJpPJ+M1vfmPs3r3b+O///m/DYrEYn376aS9H3r7Ovq+1a9caqampET+b8vLyXotXElEvWbt2bVSJSNd1Izs723jooYfC99XU1Bg2m8144YUXejDC6O3evdsAjO3bt4fve/311w1FUYyjR4+2+bwZM2YYP/nJT3ohwuhNnTrVWLRoUfh7TdOM3NxcY+XKla0ef9111xlXXHFFxH3Tpk0zvve97/VonJ3V2fcV7e9nT5GuWR9z8OBBysvLmTVrVvg+p9PJtGnTWt39Nh7ef/990tLSmDJlSvi+WbNmoaoq27Zta/e5zz33HIMGDWLcuHEsW7YMl8vV0+G2KbTTcPP/6/Z2Gobge29+PMDs2bP7zM8GYntfAA0NDQwdOpT8/HyuuuoqPv/8894IF0jwRa+no/LycoBWd78NPRZv5eXlZGVlRdxnNpvJyMhoN8Ybb7yRoUOHkpubyyeffMLSpUvZt28fGzZs6OmQW9XZnYYh+N778s8GYntfo0ePZs2aNUyYMIHa2loefvhhzjvvPD7//PNeWVguLaIY3H333S0G9k69tfUD78t6+n1997vfZfbs2YwfP56bbrqJZ599lpdeeomioqJufBciFtOnT+fmm29m0qRJzJgxgw0bNpCZmclTTz3VK68vLaIY/Nd//RcLFixo95jhw4fHdO7QDrfHjh0jJ+fknrLHjh1j0qRJMZ0zWtG+r+zsbCoqKiLuDwQCVFVVtbpDb1umTZsGwIEDBzjjjDM6HW9XdXanYQj+fDpzfDzE8r5OZbFYmDx5MgcOHOiJEFuQRBSDzMxMMjMze+Tcw4YNIzs7m02bNoUTT11dHdu2beMHP/hBj7xmSLTva/r06dTU1LBjxw7OOeccAN566y10XQ8nl2js2rULICLh9qbmOw3PmzcPOLnT8B133NHqc6ZPn86mTZu48847w/e9+eabfWpn4lje16k0TePTTz9l7ty5PRhpM3EbJu8nDh06ZHz00UfGihUrjOTkZOOjjz4yPvroI6O+vj58zOjRo40NGzaEv3/wwQeNtLQ045VXXjE++eQT46qrruqTl+8nT55sbNu2zXjvvfeMkSNHRly+P3LkiDF69Ghj27ZthmEYxoEDB4z77rvP+PDDD42DBw8ar7zyijF8+HDjoosuitdbMAwjeJnbZrMZ69atM3bv3m1897vfNdLS0sKXrr/zne8Yd999d/j4LVu2GGaz2Xj44YeNPXv2GPfcc0+fvXzfmfe1YsUKY+PGjUZRUZGxY8cO41vf+pZht9uNzz//vFfilUTUw+bPn28ALW5vv/12+BjAWLt2bfh7XdeNX/ziF8bgwYMNm81mXHrppca+fft6P/h2VFZWGjfccIORnJxspKamGrfccktEcj148GDE+zx8+LBx0UUXGRkZGYbNZjNGjBhh/PSnP437PCLDMIzHH3/cKCgoMKxWqzF16lRj69at4cdmzJhhzJ8/P+L4P//5z8aoUaMMq9VqjB071nj11Vd7OeLodOZ93XnnneFjBw8ebMydO9fYuXNnr8UqZUCEEHEnV82EEHEniUgIEXeSiIQQcSeJSAgRd5KIhBBxJ4lICBF3koiEEHEniUgklHXr1kVUurz33nsj1uAtWLAgvKxBJA5JRKJVCxYsaLOU66JFi1AUJWKBbE8kgMLCQh577LGI+66//nr279/f5nNWrVrFunXrwt/PnDkzYl2Y6JskEYk25efns379etxud/g+j8fD888/T0FBQVxiSkpKalELqTmn0xl1bXDRd0giEm06++yzyc/PjyhctmHDBgoKCpg8eXKXzt1aS2XevHnhVtbMmTM5dOgQd911V7gWErTsmp2qectswYIFvPvuu6xatSp8joMHDzJixAgefvjhiOft2rULRVF6reyFiCSJSLRr4cKFrF27Nvz9mjVruOWWW3r8dTds2EBeXh733XcfZWVllJWVdfocq1atYvr06dx+++3hcxQUFLR4TwBr167loosuYsSIEd31FkQnSCIS7fr2t7/Ne++9x6FDhzh06BBbtmzh29/+do+/bkZGBiaTiZSUFLKzs2MqPOZ0OrFarTgcjvA5TCYTCxYsYN++feHtdfx+P88//zwLFy7s7rchoiSF0US7MjMzueKKK1i3bh2GYXDFFVcwaNCgeIfVJbm5uVxxxRWsWbOGqVOn8ve//x2v18u1114b79D6LWkRiQ4tXLiQdevW8cwzz3Rbq0FVVU6tQOP3+7vl3NG47bbbwgPxa9eu5frrr8fhcPTa64tIkohEhy6//HJ8Ph9+v5/Zs2d3yzkzMzMjxn00TeOzzz6LOMZqtaJpWpdep61zzJ07lwEDBrB69WreeOMN6ZbFmXTNRIdMJhN79uwJf92W2tracB3qkIEDB5Kfn9/i2EsuuYTFixfz6quvcsYZZ/Doo49SU1MTcUxhYSGbN2/mW9/6FjabLaYuYWFhIdu2baO4uJjk5GQyMjJQVTU8VrRs2TJGjhzZp2pO90fSIhJRSU1NJTU1td1j3nnnHSZPnhxxW7FiRavHLly4kPnz53PzzTczY8YMhg8fzsUXXxxxzH333UdxcTFnnHFGzJsVLFmyBJPJxJgxY8jMzOTw4cPhx2699VZ8Pl+vXAUU7ZNSsaLf+ve//82ll15KSUlJi80IRe+SRCT6Ha/Xy/Hjx5k/fz7Z2dk899xz8Q6p35Oumeh3XnjhBYYOHUpNTQ2/+c1v4h2OQFpEQog+QFpEQoi4k0QkhIg7SURCiLiTRCSEiDtJREKIuJNEJISIO0lEQoi4k0QkhIg7SURCiLj7/54T7faL8Sd3AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:17:30.331463Z", + "iopub.status.busy": "2024-03-26T11:17:30.331157Z", + "iopub.status.idle": "2024-03-26T11:17:30.563234Z", + "shell.execute_reply": "2024-03-26T11:17:30.562107Z" + }, + "papermill": { + "duration": 0.253956, + "end_time": "2024-03-26T11:17:30.565357", + "exception": false, + "start_time": "2024-03-26T11:17:30.311401", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAErCAYAAAChCiRcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA7iElEQVR4nO3de1wU9f4/8NcuC8v9fhNEFyMFEzW1PHgEvIB3vl7LyNSQo5WZKaip5S07kibmyTz5rU6apWYJkiFeEEHQFA3NowkaCKlcBEHuuiy7n98f/na+rCywq8Be5v18PHjAzHxm9j27O28+M/OZz0fAGGMghBAjJ9R1AIQQ0hko2RFCeIGSHSGEFyjZEUJ4gZIdIYQXKNkRQniBkh0hhBco2RFCeIGSHSGEFyjZEUJ4weCS3fbt2yGRSGBubo7Bgwfj/PnzLZb96quvEBgYCAcHBzg4OCAkJKTV8oQQ42VQyW7//v2IiorCmjVrcPHiRfTr1w+jR49GaWmp2vJpaWkIDw9Hamoqzp49Cy8vL4waNQqFhYWdHDkhRNcEhtQRwODBg/HCCy/g888/BwAoFAp4eXnhnXfewfLly9tcXy6Xw8HBAZ9//jlmzZrV0eESQvSISNcBaKqhoQFZWVlYsWIFN08oFCIkJARnz57VaBv19fWQyWRwdHRssYxUKoVUKuWmFQoFKioq4OTkBIFA8OQ7QAhpd4wx1NTUwMPDA0Jh6yeqBpPs7t27B7lcDjc3N5X5bm5uyMnJ0Wgb7733Hjw8PBASEtJimZiYGKxbt+6pYiWEdK7bt2+ja9eurZYxmGT3tD7++GP88MMPSEtLg7m5eYvlVqxYgaioKG66qqoK3bp1Q35+PmxsbDojVJ2QyWRITU3F8OHDYWpqqutwyFPiy+dZU1MDb29vjY5Ng0l2zs7OMDExwd27d1Xm3717F+7u7q2uu3nzZnz88cc4ceIE+vbt22pZsVgMsVjcbL6joyNsbW21D9xAyGQyWFpawsnJyagPDr7gy+ep3DdNLjEZzN1YMzMzDBw4ECkpKdw8hUKBlJQUBAQEtLjepk2bsH79ehw9ehSDBg3qjFAJIXrIYGp2ABAVFYXZs2dj0KBBePHFF7F161bU1dUhIiICADBr1ix4enoiJiYGALBx40asXr0ae/fuhUQiQUlJCQDA2toa1tbWOtsPQkjnM6hkN336dJSVlWH16tUoKSlB//79cfToUe6mxa1bt1TuyHzxxRdoaGjAtGnTVLazZs0arF27tjNDJ4TomEElOwBYsGABFixYoHZZWlqaynRBQUHHB0QIMQgGc82OEEKeBiU7QggvGNxpLCGkufr6epXG9bUPpPj1Sh4cnH+DtYVqUypfX19YWlp2dog6R8mOpzQ9OPh6YBianJwcDBw4sNn8TWrKZmVlYcCAAR0flJ6hZMdTmh4cfD0wDI2vry+ysrK46evFlYj66Qq2vOSPXl3sm5XlI0p2PKXpwcHXA8PQWFpaqvxTEv5VDnHGA/j16Yf+3Z10GJn+oGTHU3RwGL78e3WokzaqXZZXVsf9FolaPsytxCJ4O1t1SHz6hpIdIQYo/14dhm9Oa7Nc9IErbZZJXTKMFwmPkh1PtFYLAKgmYGiUn+XW6f3h49r80ce6B1Ikpp3FhGEBsLJo3rEFAOSW1mLR/t9b/V4YE0p2PKBpLQCgmoCh8XG1Rh9Pu2bzZTIZSlyAAd0djLrXE21QsuOBtmoBANUEDI1U/hBC80LkV1+H0Lz5Z9rY2IiixiJkV2S3WFPPr66F0LwQUvlDAM0TprGhZMcDygPDxNxF7YEBABaiRng4FMHCpgTCFg4OE3N+HRz6rKjuL1h5b8PKNgbL+/fRf7e63MobKKrrj4Fwa7WcMaBkxwOaHhgAHRyGwsOqO+ry38G/pvfHM2pq642NjThz+gz+PvTvLdbs8kpr8e7+3+ExvHtHh6sXKNnxQFsHBkAHh6ERm5hD8dAT3ra90NtJ/TW7fFE+/Bz9Wrxmp3hYBcXDMohNWh6mwJhQsuOBtg4MgA4OYvyo1xNCCC9QsiOE8AIlO0IIL1CyI4TwAiU7QggvULIjhPACJTtCCC9QsiOE8AIlO0IIL1CyI4TwAiU7QggvULIjhPACJTtCCC9QsiOE8AIlO0IIL1CyI4TwAiU7QggvGFyy2759OyQSCczNzTF48GCcP9/6wAo//fQTfH19YW5uDn9/fyQlJXVSpIQQfWJQyW7//v2IiorCmjVrcPHiRfTr1w+jR49GaWmp2vK//vorwsPDERkZiUuXLmHSpEmYNGkSrl692smRE0J0zaCS3ZYtWzB37lxERESgd+/e2LFjBywtLfHNN9+oLf+vf/0LY8aMwdKlS+Hn54f169djwIAB+Pzzzzs5ckKIrhnMgDsNDQ3IysrCihUruHlCoRAhISE4e/as2nXOnj2LqKgolXmjR49GQkJCi68jlUohlUq56erqagCPBqSRyWRPsQe609jYyP1uaR+U81vbR022QzpHzYNH39HLtyq4z6WpuodS/FYGON8sg5V5C4Oel9UBMOzPU5u4DSbZ3bt3D3K5HG5uquOVurm5IScnR+06JSUlasuXlJS0+DoxMTFYt25ds/nHjx+HpaXlE0Sue7drAUCE06dP4y/1IylykpOT22U7pGOdvSsAYIL3f77WSikRvsu91Oa2Lpw9jb8s2i20TlVfX69xWYNJdp1lxYoVKrXB6upqeHl5YdSoUbC1tdVhZE/uj6JqbL5yDkOHDsVzHur3QSaTITk5GaGhoS0OpajJdkjn+FtdA/yzS9HDxQoWpibNlt8oqcKyg9nYNNkPPd3VD58JAFZiE0icrDoy1A6lPPPShMEkO2dnZ5iYmODu3bsq8+/evQt3d3e167i7u2tVHgDEYjHE4ubVflNT0xaTgL6TMQEAIOduXYsDYNc9eHTa415UCysL9ac9BRUPAQAikchg3wtj4WZvihkB3m2W6+luh/7dnTohIt3Q5ntoMMnOzMwMAwcOREpKCiZNmgQAUCgUSElJwYIFC9SuExAQgJSUFCxatIibl5ycjICAgE6IWH/kldYCAJbHX2mjpAjf5V5oc3tWYoP52hDCMahvbVRUFGbPno1BgwbhxRdfxNatW1FXV4eIiAgAwKxZs+Dp6YmYmBgAwLvvvovg4GDExsZi/Pjx+OGHH/Dbb7/hyy+/1OVudLpRzz2qyT7jaq32lAcArhdXIfrAFcRO80evLq2d9ojg7Wy4pz2Evwwq2U2fPh1lZWVYvXo1SkpK0L9/fxw9epS7CXHr1i0Ihf/XmmbIkCHYu3cvPvjgA6xcuRLPPvssEhIS0KdPH13tgk44WpnhlRe7tVpGeUfvGRcr9PFsOdkRYqgMKtkBwIIFC1o8bU1LS2s276WXXsJLL73UwVERQvSdQTUqJoSQJ0XJjhDCC5TsCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCCwaT7CoqKjBjxgzY2trC3t4ekZGRqK2tbbX8O++8g169esHCwgLdunXDwoULUVVV1YlRE9L5KioqMG3UUNz6VzimjRqKiooKXYekF0S6DkBTM2bMQHFxMZKTkyGTyRAREYF58+Zh7969assXFRWhqKgImzdvRu/evfHXX3/hzTffRFFREQ4cONDJ0RPSOdzd3XH37l1uOu9GDpycnODm5oaSkhIdRqYHmAG4du0aA8AuXLjAzTty5AgTCASssLBQ4+38+OOPzMzMjMlkMo3XqaqqYgBYVVWVVjEbmksF91j39xLZpYJ7ug6FPCE3NzcGoMUfNzc3XYfY7rQ5PrWu2d28eRM9evRov2yrgbNnz8Le3h6DBg3i5oWEhEAoFCIzMxOTJ0/WaDtVVVWwtbWFSNTybkulUkilUm66uroaACCTySCTyZ5wD/RfY2Mj99uY99NYVVRUqNTo1Ll79y7u3r0LR0fHToqq42nzXdU62fn4+CA4OBiRkZGYNm0azM3Ntd2E1kpKSuDq6qoyTyQSwdHRUeOq+b1797B+/XrMmzev1XIxMTFYt25ds/nHjx+HpaWl5kEbmNu1ACDCuXPnUHhV19EQbc2fP5/728TEBHK5XO20v78/Pv/8c4jF4k6PsSPU19drXFbrZHfx4kXs3LkTUVFRWLBgAaZPn47IyEi8+OKL2m4Ky5cvx8aNG1stk52drfV2H1ddXY3x48ejd+/eWLt2batlV6xYgaioKJV1vby8MGrUKNja2j51LPpILpfj27gk1F07A1P/v2H06HEwMTHRdVhEC2VlZdzfTRPd49NlZWWQSCR4/vnnOy22jqQ889KEgDHGnuRFGhsbcejQIezatQtHjx5Fz549MWfOHMycORMuLi4abaOsrAzl5eWtlunRowe+//57REdH4/79+yqvb25ujp9++qnV09iamhqMHj0alpaWSExM1LomWl1dDTs7O+4U2NjEx8cjOjoaBQUF3DyJRILY2FhMmTJFd4ERrZibm3OXXwIDAxE+YyZu3LmHnl2dsW/Pd8jIyAAAmJqaorKy0mjOUrQ6Pp/2AuHDhw/Zli1bmFgsZgKBgInFYjZz5kxWVFT0tJvmKG9Q/Pbbb9y8Y8eOtXmDoqqqiv3tb39jwcHBrK6u7ole25hvUMTFxTGBQMAsLCxULmRbWFgwgUDA4uLidB0i0ZC/vz/3+VVUVLDk5GQWFRXFkpOTWUVFBbfM399f16G2K22OzydOdhcuXGBvvfUWc3BwYF27dmXvv/8+u3nzJktPT2cjR45kL7zwwpNuWq0xY8aw559/nmVmZrLTp0+zZ599loWHh3PL79y5w3r16sUyMzMZY4/ehMGDBzN/f3+Wm5vLiouLuZ/GxkaNX9dYk11jYyNzdXVlANj48ePZ8g83Msex77LlH25k48ePZwCYq6urVu8V0Z3p06e3eidW+TN9+nRdh9quOjTZxcbGsj59+jBTU1M2ceJE9ssvvzC5XK5S5vbt28zExETbTbeqvLychYeHM2tra2Zra8siIiJYTU0Ntzw/P58BYKmpqYwxxlJTU1v8wPPz8zV+XWNNdocPH2YAmEQiYR4eHirvj4eHB5NIJAwAO3z4sK5DJRpYuXKlRslu5cqVug61XXVosvPx8WEbNmxo9TRVKpWyXbt2abtpvWSsyS4yMlKjgyMyMlLXoRINHDt2TKPP89ixY7oOtV11aDu75ORkdOvWDUKh6pNmjDHcvn0b3bp1g5mZGWbPnq3tpkknUra16tu3L7788kucO/8bTmScRUhgAP724iDMnTsXV65cMao2WYTftE52zzzzDIqLi5u1e6uoqIC3t3ez295EP7m5uQF4dDfr5Zdfxq1btwAAiT/tQbdu3SAQCFTKEf2mvNuqSblRo0Z1cDT6Setkx1poqVJbW9spDYxJ+3B3dwcAFBQUNKul37lzBwqFQqUc0W9Nn/oRCAQqx2nT6abl+EbjZKdsaCsQCLB69WqVdjpyuRyZmZno379/uwdIOoamSYySnWFoWrPz9PTEnTt31E5rWgM0Rhonu0uXLgF4VLO7cuUKzMzMuGVmZmbo168flixZ0v4Rkg6hvNzQtDGqEmMM5ubmePjwIV2WMBBNG4UXFhaqLGs63bQc32ic7FJTUwEAERER+Ne//mWUTxPwifI//MOHD+Hq6gpfX1/cu3cPzs7OyMnJQWlpKVeOr9d4DEnTS0iPX2pqOs3nS01aX7PbuXNnR8RBOpnympy9vT1KS0u55KZkb2+PyspKrhzRbyNHjsR//vMfjcrxlUbJbsqUKdi1axdsbW3bfF4yPj6+XQIjHUvZpKSyslLtBe3KykqVckS/OTk5qUz7+vqid+/euHbtGnJyclosxycaJTs7OzuuKYKdnV2HBkQ6h7OzM/e3i4sLZsyYgbq6OlhZWWHPnj1cTa9pOaK/lE2HlHJyclSSXEvl+ESjZNf01JVOY41DZmYm93dpaSk+/fTTFstRA3H9d+/ePQCAhYUFHjx40Gy5cr6yHB8ZzBgUpH0VFRVxf7fWLqtpOaK/rKysAAAPHjyAi4sLgoODUVFRAUdHR5w6dYrr705Zjo80SnbPP/88dxrblosXLz5VQKRzWFtbc3+3dhrbtBzRX0OHDsXPP/8M4NFTMU0HlWp6B3bo0KGdHpu+0CjZTZo0qYPDIJ3N398fACAUCmFmZqZyGuvl5QWhUAiFQsGVI/qtb9++3N+Pt5t8+PCh2nJ8o1GyW7NmTUfHQTqZsjtrhULRrBHqnTt3uNNYbbq9JrrTVo/f2pYzRgYzSDZpX02fh22tEerjz80S/aTsmMPT07PZ+CEikQienp4q5fhIo2+yo6MjdxfHwcEBjo6OLf4QwxAUFAQAEIvFzQ4OExMTbvQpZTliGLy9vVFbW4vNmzdj3Lhx2Lx5M2pqaiCRSHQdms5pdBr76aefwsbGhvtb05sVRH8pa2zqesGQy+XcM7FUszMMyhtKp0+fxrRp0+Dt7Y3GxkYUFBRg2rRpOHPmjEo5PtIo2TVtZ/X66693VCykE2k63q6m5YhudenSBcCjkcUOHz7MzT9+/Dg3PyMjgyvHR1q3szMxMVHbeWd5eTlcXV2plwwDoWw/Z2ZmplKTAx5d4xEKhWhoaKB2dgYiMDAQtra2yMjIgKurK/z8/FBWVgYXFxdkZ2cjIyMDtra2CAwM1HWoOtNunXdKpVKVbp+Ifrt8+TIAoKGhAePHj8fo0aNx48YN9OzZE8eOHeNqB8pyRL/J5XLU1tYCgNqOHYBHHezK5XLeDoCucbL77LPPADxqXf/111+rNDaVy+VIT0+Hr69v+0dIOoTywFBS/hN7/J/Z4+WIfvr3v//N9VCjbCOpZGJiArlcDoVCgX//+99YtGiRjqLULY2TnbLRKWMMO3bsUPnvYGZmBolEgh07drR/hKRDeHh4AHjUldPRo0dVrvOYmJhwXTwpyxH9duPGDQCPOm64c+cOMjIycOTIEYwdOxaBgYHw9PREeXk5V46PNE52+fn5AIDhw4cjPj4eDg4OHRYU6XgBAQH44osvUFlZCVdX1xYfFwsICNBxpEQTxcXFAICxY8dCLBYjODgYdXV1CA4OhqmpKcaMGYM9e/Zw5fhI62t2yh6LiWFrWmO7d++eyuNiTWvtVLMzDMqxQo4ePQqpVIqMjAykp6fDysoKgYGBOHbsmEo5PtI62c2ZM6fV5d98880TB0M6n6enZ7P/9owxeHp6NnuMjOivXr16AQDKyspgaWnJXbPbsmWLyjU8ZTk+0jrZ3b9/X2VaJpPh6tWrqKysxIgRI9otMNKxlKepRUVFGDt2LMzMzJCXl4dnnnkGDQ0NOHLkiEo5ot/mz5+P6OhoKBSKZl3pN71xMX/+fF2Epxe0TnYHDx5sNk+hUOCtt97CM8880y5BkY6nbFz66quvYv/+/WhsbAQAXLlyBSKRCOHh4di7dy+vG6EaEhMTE5ibm6O+vr7FMubm5rxtdgIAAtZSwzktXb9+HcOGDTO6C6DV1dWws7NDVVWVUY2oJpfL0aVLF5SVlWHcuHEQi8XIzc2Fj48PpFIpkpKS4OrqiqKiIl4fIIYiJSUFISEhbZY7ceKEUQ26o83x2W49Fefl5XG1A2IYlM84JyUlcfOuXLmiq3DIU1A+FgY8uiPr4+OD69evo1evXsjNzeUuSxw/ftyokp02tE52UVFRKtOMMRQXF+Pw4cM0VoEBycjI4K7HtdQItbS0FBkZGRg2bJiOoiSaOnHiBIBHd88TExMhl8uRlJSEcePGwcTEBF5eXigqKuLK8ZHWye7SpUsq00KhEC4uLoiNjW3zTi3RH7dv3wbwqEv227dvN2uE6uXlhbKyMq4c0W81NTUA0OKpnLLXImU5PqJ2djylHF0sMjJSbSPUiIgIbNq0CZmZmZg5c6aOoyVt6datG/7880/k5ORg4sSJWLZsGR48eIBz585h06ZNuH79OleOrwyms7KKigrMmDEDtra2sLe3R2RkpMbPbTLGMHbsWAgEAiQkJHRsoAZCeV/q4sWLapsqKGvw7XT/inSwpUuXcn+npKQgKCgI4eHhCAoKQkpKitpyfGMwyW7GjBn4448/kJycjMTERKSnp2PevHkarbt161bqcPQxzz77LAAgOTkZkyZNwrlz57iawKRJk7hrO8pyRL+FhITA0tISAJqNG6uctrS01OiOrdFiBuDatWsMALtw4QI378iRI0wgELDCwsJW17106RLz9PRkxcXFDAA7ePCgVq9dVVXFALCqqqonCV1vSaVSJhKJmJ2dHZNIJAwA9+Pt7c3s7OyYSCRiUqlU16ESDcXFxal8jo//xMXF6TrEdqfN8WkQg2SfPXsW9vb2GDRoEDcvJCQEQqEQmZmZmDx5str16uvr8eqrr2L79u0aPxMolUpVuipXjq4lk8kgk8meYi/0i0AgwMKFC7FlyxaYmppiypQpqKyshL29PdLT01FVVYWoqCgIBAKj2m9jFhYWhv3792Pp0qW4desWN7979+7YtGkTwsLCjO6z1GZ/2i3Z3blzBx9++CG+/PLL9tokp6SkpFnPyCKRCI6Ojq12G7548WIMGTIEEydO1Pi1YmJisG7dumbzjx8/zp0mGIugoCCcPn0a58+fR3x8vMqyF198EUFBQSpt8Ij+E4vF+PTTT3Ht2jXcv38fDg4O6N27N0xMTIzys2ztiZHHtVuyKy8vx3/+8x+tkt3y5cuxcePGVstkZ2c/UTyHDh3CyZMnmzWVacuKFStU2hJWV1fDy8sLo0aNMqonKIBHj/6dP39e7bLz589DKpW2WGsm+m3MmDFITk5GaGgoTE1NdR1Oh9FmXGOdnsZGR0e3OYBPjx494O7u3uyB9MbGRlRUVLR4enry5Enk5eXB3t5eZf7UqVMRGBiItLQ0teuJxWJuGMGmTE1NjepLI5fLuRs8rq6u+PDDDyEWiyGVSrF69WqUlpZi3rx5mDp1Kj0uZsCM7Xv7OG32TafJzsXFBS4uLm2WCwgIQGVlJbKysjBw4EAAj5KZQqHA4MGD1a6zfPly/OMf/1CZ5+/vj08//RRhYWFPH7yBO3nyJKqqquDg4IDCwkIwxrgW95GRkXB1dcX9+/dx8uRJhIaG6jpcQp6aQTQ98fPzw5gxYzB37lycP38eZ86cwYIFC/DKK69wnUsWFhbC19eXOy1zd3dHnz59VH6AR40qvb29dbYv+uK7774DAHz44YcQiVT/54lEIqxdu1alHCGGTuOa3ZQpU1pdXllZ+bSxtGrPnj1YsGABRo4cCaFQiKlTp3KDAAGP7spcv35dqwuWfKZ8bKilxK8cQZ7PjxcR46JxsrOzs2tz+axZs546oJY4Ojpi7969LS6XSCRttvZvazmfBAYGIiEhAe+//z7Gjh2rskyhUGDVqlVcOUKMQbv1Z2esjLU/u4aGBlhYWEChUGDChAlYtmwZCgsL4enpiU2bNiExMRFCoRAPHjyg8YANkEwm467BGvMNCp30Z0cMi5mZGaKjo/HJJ58gKSkJiYmJ3DLl3dfo6GhKdMRoaJzsNO2+iQbcMRybNm0C8GhQlsctXbqUW06IMdA42e3atQvdu3fH888/T9e+jMimTZvw0UcfYdu2bTh58iRGjBiBd955h2p0xOhonOzeeust7Nu3D/n5+YiIiMBrr70GR0fHjoyNdBIzMzMsXLgQPj4+Rn+Nh/CXxu3stm/fjuLiYixbtgy//PILvLy88PLLL+PYsWNU0yOE6D2tGhWLxWKEh4cjOTkZ165dw3PPPYf58+dDIpFo3JEmIYTowhM/QSEUCiEQCMAYg1wub8+YCCGk3WmV7KRSKfbt24fQ0FD07NkTV65cweeff45bt27B2tq6o2IkhJCnpvENivnz5+OHH36Al5cX5syZg3379sHZ2bkjYyOEkHajcbLbsWMHunXrhh49euDUqVM4deqU2nKPdwJJCCH6QONkN2vWLBq0hhBisLRqVEwIIYbKIPqzI4SQp0XJjhDCC5TsCDEycrkcp06dQnp6Ok6dOkXtYP8/SnaEGJH4+Hj4+PggNDQUW7ZsQWhoKHx8fKiVBCjZEWI04uPjMW3aNPj7+yMjIwP79u1DRkYG/P39MW3aNN4nPEp2hBgBuVyO6OhoTJgwAQkJCRg8eDAsLCwwePBgJCQkYMKECViyZAmvT2kp2RFiBDIyMlBQUICVK1dCKFQ9rIVCIVasWIH8/HxkZGToKELdo2RHiBEoLi4GAG7I0Mcp5yvL8RElO0KMQJcuXQAAV69eVbtcOV9Zjo8o2RFiBAIDAyGRSLBhwwYoFAqVZQqFAjExMfD29ub10JiU7AgxAiYmJoiNjUViYiImTZqEc+fO4cGDBzh37hwmTZqExMREbN68mRs5jo9oKEWChoYGbsCd3NxcGnDHQE2ZMgUHDhxAdHQ0goKCuPne3t44cOAApkyZosPodI8GyW6DsQ6SrbRs2TJ8+umnaGxs5OaJRCIsXryYhlI0UHK5HKmpqThy5AjGjh2L4cOHG22NjgbJJhpZtmwZPvnkk2ZNFRQKBT755BMAoIRngExMTBAcHIy6ujoEBwcbbaLTFl2z46mGhgbExsYCAMaOHavS4n7s2LEAgNjYWDQ0NOgyTELaDSU7ntq2bRsUCgX69u2LQ4cOqbS4P3ToEPz9/aFQKLBt2zZdh0pIu6Bkx1OnT58GAGzYsEFti/uPPvpIpRwhho6SHU8pR4PLz89Xu7ygoEClHCGGjpIdT82cORMAsGbNGpU7sQDQ2NiIdevWqZQjxNAZTLKrqKjAjBkzYGtrC3t7e0RGRqK2trbN9c6ePYsRI0bAysoKtra2CAoKwoMHDzohYv02cuRI2NraoqKiAl27dsXXX3+NiooKfP311+jatSsqKipga2uLkSNH6jpUQtqFwTQ9mTFjBoqLi5GcnAyZTIaIiAjMmzcPe/fubXGds2fPYsyYMVixYgW2bdsGkUiEy5cvN7tGxUcmJibYuXMnpk6ditLSUsyfP59bphxFbufOndRsgRgPZgCuXbvGALALFy5w844cOcIEAgErLCxscb3BgwezDz744Kleu6qqigFgVVVVT7UdfRUXF8e6devGAHA/3bt3Z3FxcboOjTyFhoYGlpCQwBoaGnQdSofS5vg0iJrd2bNnYW9vj0GDBnHzQkJCIBQKkZmZicmTJzdbp7S0FJmZmZgxYwaGDBmCvLw8+Pr64p///CeGDh3a4mtJpVJIpVJuurq6GgAgk8kgk8naca/0Q1hYGMaNG4e0tDQkJycjNDQUw4YNg4mJiVHuL18oPztj/wy12T+DSHYlJSVwdXVVmScSieDo6IiSkhK169y8eRMAsHbtWmzevBn9+/fH7t27MXLkSFy9ehXPPvus2vViYmK4i/NNHT9+HJaWlk+5J/otKCgIUqkUx44d03UopJ0kJyfrOoQOVV9fr3FZnSa75cuXY+PGja2Wyc7OfqJtK7u5eeONNxAREQEAeP7555GSkoJvvvkGMTExatdbsWIFoqKiuOnq6mp4eXlh1KhRRvlsrJJMJuNqdqamproOhzwlvnyeyjMvTeg02UVHR+P1119vtUyPHj3g7u6O0tJSlfmNjY2oqKiAu7u72vWUnRT27t1bZb6fnx9u3brV4uuJxWKIxeJm801NTY36S6PEl/00ZnK5HL/++ivS09NhZWVl1B0BaPNd1Wmyc3FxgYuLS5vlAgICUFlZiaysLAwcOBAAcPLkSSgUCgwePFjtOhKJBB4eHrh+/brK/Bs3bnDPfhJibOLj4xEdHc01Ct+yZQskEgliY2N538WTQbTB8PPzw5gxYzB37lycP38eZ86cwYIFC/DKK6/Aw8MDAFBYWAhfX1+cP38ewKPmE0uXLsVnn32GAwcOIDc3F6tWrUJOTg4iIyN1uTuEdAgaSrENnXB3uF2Ul5ez8PBwZm1tzWxtbVlERASrqanhlufn5zMALDU1VWW9mJgY1rVrV2ZpackCAgJYRkaGVq9r7E1PlPjSVMFYNTY2MolEwsLCwphcLlf5POVyOQsLC2Pe3t6ssbFR16G2K6NregIAjo6OrTYglkgkYGr6IV2+fDmWL1/ekaERonPKoRT37dsHoVCoMj6scijFIUOGICMjA8OGDdNdoDpkEKexpGPJ5XKcOnUK6enpOHXqFK8HUjZUNJRi2yjZ8Vx8fDx8fHwQGhqKLVu2IDQ0FD4+PnR9x8A0HUpR3T8vGkqRxqBokzGPQaG8oD1u3Dh4e3vjxo0b6NmzJ/Lz85GUlESDtBgQuVwOHx8fODs7o7S0VKV5Vbdu3eDq6ory8nL8+eefRtUMRZvjk5JdG4w12SkPDhMTExQUFKicupqYmEAikUChUBjdwWHMmo4p0nTsWOX00qVLjW5MEW2OTzqN5SnlBe28vDw4Oztjx44d2LlzJ3bs2AFnZ2fk5eUhPz8fGRkZug6VaEAul2PXrl0A0KxRvLm5OQDg22+/5fX1WEp2PHX79m0AgKurK+7cuYM5c+bAwcEBc+bMwZ07d7hnkZXliH5LS0tDWVkZhg4dioqKCmzevBnjxo3D5s2bUV5ejqFDh6K0tBRpaWm6DlVnKNnxVGZmJgBgzpw5EIlUWyCJRCLuMT5lOaLflEksJCQEfn5+WLJkCZKSkrBkyRL4+flxnbBSsiO8o7xUm5WVpXJ9B3jUicKlS5dUyhHDsHbtWrVPUKjryYdvKNnxlLKLq+TkZEyaNAnnzp3DgwcPcO7cOUyaNInrGqilrrCIfgkKCgIAODg4ID4+XmVozPj4eDg4OKiU4yNKdjw1f/58iEQi2NnZ4b///S+CgoIQHh6OoKAgXLlyBXZ2dhCJRCrdtRP9pRxq4P79+5g8ebLKP6/Jkyfj/v37KuX4yGAeFyPty8zMDIsXL8Ynn3wCc3NzLFq0CPX19bC0tMS+fftQVVWFpUuXwszMTNehEg0ou0ATCARISUlBYmIit8zS0hICgQCMsWZdpfEJJTseU7a5+vTTT7F161ZuvkgkMso2WcZM+WTEhg0b8L//+79cF08A4Obmhrlz52LlypX0BIWug9BnxtqouKmGhgZs27YNJ0+exIgRI/DOO+9Qjc7AKBuJ+/v7Iy4uDqdOncKRI0cwduxYBAcHY+rUqbh69arRNRLX5vikmh2BmZkZFi5cCB8fH4wbN456KjZAJiYmiI2NxbRp0zB16lQsXboUL7zwAsRiMaZOnYrExEQcOHDAqBKdtijZEWIkpkyZggMHDiA6Olrlrqu3tzc95wy6G0uI0Xn8ytTj7Sj5ipIdIUZC2YtN3759VRoV9+3bl7plByU7QoyCXC5HdHQ0JkyYgISEBJVGxQkJCZgwYQKWLFlCHQEQQgybsheblStXNms4rOyWne+92FCyI9QtuxGgbtnbRsmO5+Lj4/HMM8+odMv+zDPP8P76jqFp2i27OtQtOyU7XouPj8fUqVNVuvAGgFu3bmHq1KmU8AxIYGAgJBIJNmzYoLYXm5iYGHh7eyMwMFBHEeoeJTueksvlmDNnDgDAxcVFpadiFxcXAI/6uqNTWsOgbFScmJiothebxMREbN68mdeNig1mkGxdMdZBso8fP84AMAcHByaTyVQGVZbJZMzBwYEBYMePH9d1qEQLcXFxTCKRMADcj7e3N4uLi9N1aB1Cm+OTanY89d133wEAPvzwQ7U9Fa9du1alHDEMU6ZMQW5uLpKTkxEVFYXk5GT8+eefvH96AqDHxXirpqYGwKNHidSRSCQq5YjhMDExQXBwMOrq6hAcHMzvU9cmqGbHU8oL1e+//z5kMplK0xOZTIZVq1aplCPE0FEXT20w1i6eGhoaYGFhAYVCAXNzczx8+JBbppwWCoV48OABdfdkgGQyGZKSkoy+FxsaN5a0yczMDGFhYQCgkuiaToeFhVGiI0aDkh1PyeVypKamtlomNTWVmp4Qo0HJjqdSUlJQXV0NR0dH1NXVqQyqXFdXB0dHR1RXVyMlJUXXoRLSLijZ8ZSyScm6detgaWmJhQsXYt68eVi4cCEsLS2xZs0alXKEGDqDSXYVFRWYMWMGbG1tYW9vj8jISNTW1ra6TklJCWbOnAl3d3dYWVlhwIABiIuL66SI9ZvyvWur6Ulb7zEhhsJgkt2MGTPwxx9/IDk5GYmJiUhPT8e8efNaXWfWrFm4fv06Dh06hCtXrmDKlCl4+eWXudHu+Wzo0KEAgJUrV6p9lvKDDz5QKUcMB/Vi04IOf56jHVy7do0BYBcuXODmHTlyhAkEAlZYWNjielZWVmz37t0q8xwdHdlXX32l8Wsb6+NiUqmUCYVCBoCNHz+epaens3379rH09HQ2fvx4BoAJhUImlUp1HSrRQlxcHOvevbvK42Ldu3enx8UYYwbxBMXZs2dhb2+PQYMGcfNCQkIgFAqRmZmJyZMnq11vyJAh2L9/P8aPHw97e3v8+OOPePjwIYYNG9bia0mlUkilUm66uroawKN2SzKZrH12SA8IBAIsWrQIW7ZswZEjR3D48GFumbLF/aJFiyAQCIxqv43ZwYMHMX36dFhYWKjMLy0txdSpU7F///4WjxVDpc130yCSXUlJCVxdXVXmiUQiODo6oqSkpMX1fvzxR0yfPh1OTk4QiUSwtLTEwYMH4ePj0+I6MTExWLduXbP5x48fh6Wl5ZPvhB4KCgrCzZs3cejQIZX5jDFMmjQJQUFBSEpK0lF0RBtyuRxz584FADQ2NqosU07PmzcPIpHIqB4fq6+v17isTpPd8uXLsXHjxlbLZGdnP/H2V61ahcrKSpw4cQLOzs5ISEjAyy+/jIyMDPj7+6tdZ8WKFYiKiuKmq6ur4eXlhVGjRhnVExRK48aNQ0NDA7Zv345Tp04hODgYb7/9NjUmNjAnT55UOQtpSjldVVUFKysrjBgxotPj6yjKfdaETpNddHQ0Xn/99VbL9OjRA+7u7igtLVWZ39jYiIqKCri7u6tdLy8vD59//jmuXr2K5557DgDQr18/ZGRkYPv27dixY4fa9cRiMcRicbP5pqamRvvYjampKRYvXoxevXoZ/eNFxio9PZ3728zMDIsWLYK3tzfy8/OxdetWNDQ0cOVGjx6tqzDbnTbfVZ0mOxcXF66jyNYEBASgsrISWVlZGDhwIIBH/8kUCgUGDx6sdh1l9fbxwUdMTExoHE1idG7evAng0fe7pqYGAoEASUlJiIyMxPr167nnoJXl+Mggmp74+flhzJgxmDt3Ls6fP48zZ85gwYIFeOWVV+Dh4QEAKCwshK+vL86fPw8A8PX1hY+PD9544w2cP38eeXl5iI2NRXJyMiZNmqTDvSGk/d24cQMA4ObmprZ/QuUZkLIcHxlEsgOAPXv2wNfXFyNHjsS4ceMwdOhQfPnll9xymUyG69evczU6U1NTJCUlwcXFBWFhYejbty92796Nb7/9FuPGjdPVbhDSIQQCAQCgqKgIEydOVOmWfeLEiSgqKlIpx0cGcTcWABwdHbF3794Wl0skErDHeqt69tln6YkJwgsvvPACsrKyADx67jkxMZFb1rQVwQsvvNDpsekLg6nZEUJatmXLlhaXNa0EtFbO2FGyI8QIWFhYYOLEiQCABw8eqCxTTk+cOLFZg2M+oWRHiJFISEjgEt7jJk6ciISEhM4NSM9QsiPEiCQkJKC+vh5vvvkm+vfvjzfffBP19fW8T3SAAd2gIIRoxsLCAp999hkvxqDQBtXsCCG8QMmOEMILlOwIIbxA1+zaoGyjpE3vCoZIJpOhvr4e1dXVdI3HCPDl81Qel48/UKAOJbs21NTUAAC8vLx0HAkhpCU1NTWws7NrtYyAaZISeUyhUKCoqAg2NjZG/Vyhst++27dvG2W/fXzDl8+TMYaamhp4eHg06+HocVSza4NQKETXrl11HUansbW1NeqDg2/48Hm2VaNTohsUhBBeoGRHCOEFSnYEwKPu6NesWaO2S3pieOjzbI5uUBBCeIFqdoQQXqBkRwjhBUp2hBBeoGRnAAoKCiAQCPD77793ynbOnDkDf39/mJqa0khsxGhQsjNAaWlpEAgEqKys7JDtR0VFoX///sjPz8euXbs65DVIc8OGDcOiRYt0HQZH3+J5WpTsOphyJHZDkpeXhxEjRqBr166wt7d/om105n4zxtDY2Nhpr6fPDPH71mkYaVfBwcHs7bffZu+++y5zcnJiw4YNY1euXGFjxoxhVlZWzNXVlb322musrKyMW+fIkSPs73//O7Ozs2OOjo5s/PjxLDc3l1uen5/PALBLly5xfzf9mT17tlbb2bdvHwsICGBisZg999xzLC0tTWV505+dO3cyxhhLS0tjL7zwAjMzM2Pu7u7svffeYzKZrNX9Tk1NZQDY0aNHWf/+/Zm5uTkbPnw4u3v3LktKSmK+vr7MxsaGhYeHs7q6Om5bcrmcbdiwgUkkEmZubs769u3LfvrpJ265crtJSUlswIABzNTUlKWmprbnx9jpZs+e3ey9z83NZXPmzOHeh549e7KtW7c2W2/ixInso48+Yl26dGESiYQxxtiZM2dYv379mFgsZgMHDmQHDx7kvkNKrX0v1cWTn5/fWW9Hh6Bk186Cg4OZtbU1W7p0KcvJyWHnzp1jLi4ubMWKFSw7O5tdvHiRhYaGsuHDh3PrHDhwgMXFxbE///yTXbp0iYWFhTF/f38ml8sZY6rJrrGxkcXFxTEA7Pr166y4uJhVVlZqtZ2uXbuyAwcOsGvXrrF//OMfzMbGht27d481Njay4uJiZmtry7Zu3cqKi4tZfX09u3PnDrO0tGTz589n2dnZ7ODBg8zZ2ZmtWbOmxf3OycnhktLf/vY3dvr0aXbx4kXm4+PDgoOD2ahRo9jFixdZeno6c3JyYh9//DG3rY8++oj5+vqyo0ePsry8PLZz504mFou5pKzcbt++fdnx48dZbm4uKy8v7+iPtkNVVlaygIAANnfuXFZcXMyKi4vZw4cP2erVq9mFCxfYzZs32ffff88sLS3Z/v37ufVmz57NrK2t2cyZM9nVq1fZ1atXWVVVFXN0dGSvvfYa++OPP1hSUhLr2bOnSrK7f/9+q99LdfE0Njbq4q1pN5Ts2llwcDB7/vnnuen169ezUaNGqZS5ffs2l6zUKSsrYwDYlStXGGOqyY6x/zvY79+/32osLW2naWKRyWSsa9eubOPGjdw8Ozs7rkbHGGMrV65kvXr1YgqFgpu3fft2Zm1tzSXSx/e7aZwnTpzg5sXExDAALC8vj5v3xhtvsNGjRzPGGHv48CGztLRkv/76q8q2IiMjWXh4uMp2ExISWt1/QxMcHMzefffdVsu8/fbbbOrUqdz07NmzmZubG5NKpdy8L774gjk5ObEHDx5w87766iuV75Am30tN4jEkdM2uAwwcOJD7+/Lly0hNTYW1tTX34+vrC+DRtTEA+PPPPxEeHo4ePXrA1tYWEokEAHDr1i2tXlfT7QQEBHB/i0QiDBo0CNnZ2S1uNzs7GwEBASpdXP39739HbW0t7ty5o3a/m+rbty/3t5ubGywtLdGjRw+VeaWlpQCA3Nxc1NfXIzQ0VOU92717N/d+KQ0aNKjFmI3F9u3bMXDgQLi4uMDa2hpffvlls8/T398fZmZm3PT169fRt29fmJubc/NefPFFlXU0+V4aG+riqQNYWVlxf9fW1iIsLAwbN25sVq5Lly4AgLCwMHTv3h1fffUVPDw8oFAo0KdPH60vNrfXdp5U0/1uqmlPuQKBoFnPuQKBAAqFAsCj9wsADh8+DE9PT5Vyjz/n2dLrGYsffvgBS5YsQWxsLAICAmBjY4NPPvkEmZmZKuWe5H3Q5HtpbCjZdbABAwYgLi4OEokEIlHzt7u8vBzXr1/HV199hcDAQADA6dOnW92m8r+4XC5/ou2cO3cOQUFBAIDGxkZkZWVhwYIFLb6en58f4uLiwBjjandnzpyBjY1Nu/f117t3b4jFYty6dQvBwcHtum19Z2ZmpvKZnjlzBkOGDMH8+fO5eZrUunr16oXvv/8eUqmU+wdx4cIFlTJtfS/VxWPo6DS2g7399tuoqKhAeHg4Lly4gLy8PBw7dgwRERGQy+VwcHCAk5MTvvzyS+Tm5uLkyZOIiopqdZvdu3eHQCBAYmIiysrKUFtbq9V2tm/fjoMHDyInJwdvv/027t+/jzlz5rT4evPnz8ft27fxzjvvICcnBz///DPWrFmDqKioNnuH1ZaNjQ2WLFmCxYsX49tvv0VeXh4uXryIbdu24dtvv23X19I3EokEmZmZKCgowL179/Dss8/it99+w7Fjx3Djxg2sWrWqWdJS59VXX4VCocC8efOQnZ2NY8eOYfPmzQDA/bNq63upLh5l7dtQUbLrYB4eHjhz5gzkcjlGjRoFf39/LFq0CPb29hAKhRAKhfjhhx+QlZWFPn36YPHixfjkk09a3aanpyfWrVuH5cuXw83NDQsWLNBqOx9//DE+/vhj9OvXD6dPn8ahQ4fg7Ozc6uslJSXh/Pnz6NevH958801ERkbigw8+eKr3piXr16/HqlWrEBMTAz8/P4wZMwaHDx+Gt7d3h7yevliyZAlMTEzQu3dvuLi4YPTo0ZgyZQqmT5+OwYMHo7y8XKWW1xJbW1v88ssv+P3339G/f3+8//77WL16NQBw1/Ha+l6qi0fba8j6hrp4IoQH9uzZg4iICFRVVcHCwkLX4egEXbMjxAjt3r0bPXr0gKenJy5fvoz33nsPL7/8Mm8THUDJjhCjVFJSgtWrV6OkpARdunTBSy+9hH/+85+6Dkun6DSWEMILdIOCEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmO6I3XX38dAoGA6xnFzc0NoaGh+Oabb7R6LnPXrl1P3J3803j99ddpgCI9RsmO6JUxY8aguLgYBQUFOHLkCIYPH453330XEyZMoHEmyNPRadehhDShHE/hcSkpKQwA++qrrxhjjMXGxrI+ffowS0tL1rVrV/bWW2+xmpoaxtj/9WLc9EfZffzu3bvZwIEDmbW1NXNzc2Ph4eHs7t273OtUVFSwV199lTk7OzNzc3Pm4+PDvvnmG275rVu32EsvvcTs7OyYg4MD+5//+R9uXIY1a9Y0e11DHxfD2FDNjui9ESNGoF+/foiPjwcACIVCfPbZZ/jjjz/w7bff4uTJk1i2bBkAYMiQIdi6dStsbW1RXFyM4uJiLFmyBAAgk8mwfv16XL58GQkJCSgoKMDrr7/Ovc6qVatw7do1HDlyBNnZ2fjiiy+43mBkMhlGjx4NGxsbZGRk4MyZM7C2tsaYMWPQ0NCAJUuW4OWXX+ZqpsXFxRgyZEjnvlGkdbrOtoQotVSzY4yx6dOnMz8/P7XLfvrpJ+bk5MRN79y5k9nZ2bX5ehcuXGAAuFphWFgYi4iIUFv2u+++azYOh1QqZRYWFuzYsWNtxk90j2p2xCCwJr0knzhxAiNHjoSnpydsbGwwc+ZMlJeXo76+vtVtZGVlISwsDN26dYONjQ3XE7Kyn7a33noLP/zwA/r3749ly5bh119/5da9fPkycnNzYWNjw43Z4OjoiIcPHxrtmA3GhpIdMQjZ2dnw9vZGQUEBJkyYgL59+yIuLg5ZWVnYvn07gNYHiK6rq8Po0aNha2uLPXv24MKFCzh48KDKemPHjsVff/2FxYsXo6ioCCNHjuROgWtrazFw4ED8/vvvKj83btzAq6++2sF7T9oDdfFE9N7Jkydx5coVLF68GFlZWVAoFIiNjeV61P3xxx9VyqsbOyEnJwfl5eX4+OOP4eXlBQD47bffmr2Wi4sLZs+ejdmzZyMwMBBLly7F5s2bMWDAAOzfvx+urq6wtbVVG6exjdlgbKhmR/SKVCpFSUkJCgsLcfHiRWzYsAETJ07EhAkTMGvWLPj4+EAmk2Hbtm24efMmvvvuO+zYsUNlGxKJBLW1tUhJScG9e/dQX1+Pbt26wczMjFvv0KFDWL9+vcp6q1evxs8//4zc3Fz88ccfSExMhJ+fHwBgxowZcHZ2xsSJE5GRkYH8/HykpaVh4cKF3HCSEokE//3vf3H9+nXcu3cPMpmsc940ohldXzQkRGn27Nlcsw2RSMRcXFxYSEgI++abb7jBuBljbMuWLaxLly7MwsKCjR49mu3evbvZoOFvvvkmc3JyUml6snfvXiaRSJhYLGYBAQHs0KFDzQaO9vPzYxYWFszR0ZFNnDiR3bx5k9tmcXExmzVrFnN2dmZisZj16NGDzZ07l1VVVTHGGCstLWWhoaHM2tqamp7oIeq8kxDCC3QaSwjhBUp2hBBeoGRHCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF6gZEcI4YX/B/kwtWbCr0CFAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:17:30.607050Z", + "iopub.status.busy": "2024-03-26T11:17:30.606703Z", + "iopub.status.idle": "2024-03-26T11:17:30.909236Z", + "shell.execute_reply": "2024-03-26T11:17:30.907541Z" + }, + "papermill": { + "duration": 0.326798, + "end_time": "2024-03-26T11:17:30.911418", + "exception": false, + "start_time": "2024-03-26T11:17:30.584620", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABRxUlEQVR4nO2dd3iTZffHP9ndLaODUTaUPURAloKAoICKA0VEQBwIqMCLAg6GIPC+IKKo4GAqiIgo/JQhIIJMGWVToFCgzAKlu818fn+kSZM2bZOSNk17f64r19U8ecbJ0+Sbc59z7nPLJEmSEAgEAg8i97QBAoFAIIRIIBB4HCFEAoHA4wghEggEHkcIkUAg8DhCiAQCgccRQiQQCDyOECKBQOBxlJ424F4wmUxcu3aNwMBAZDKZp80RCAS5kCSJ1NRUqlatilyev9/j1UJ07do1IiMjPW2GQCAohPj4eKpXr57v614tRIGBgYD5TQYFBXnYGoFAkJuUlBQiIyOt39X88GohsgzHgoKChBAJBKWYwkInIlgtEAg8jhAigUDgcYQQCQQCj+PVMSJvQZIkDAYDRqPR06YIBG5FoVCgVCrvuXxGCFExo9PpuH79OhkZGZ42RSAoFvz8/KhSpQpqtbrI5xBCVIyYTCbi4uJQKBRUrVoVtVotCi8FZQZJktDpdNy6dYu4uDjq169fYNFiQXhUiGrVqsWlS5fybB8xYgRffvmlByyyR2cwEX35Lrtjb3Pw0l26NQpnWKfazh+v02EymYiMjMTPz68YLRUIPIOvry8qlYpLly6h0+nw8fEp0nk8KkQHDhywi5ucOHGCHj168Oyzz3rEHpNJIuZGKrtjb7Mr9jb/xiWSqc+x79+4RPo0r0J4kGs3u6i/EgKBN+COz7dHhSg0NNTu+axZs6hbty4PPfRQidkQn5hhFZ695+9wJ11n93rlADUd6lbm7M1UYm6k8v3eS4zrGVVi9gkE5YFSEyPS6XT88MMPjB07Nt84ilarRavVWp+npKS4fJ276Tr2XrjDrtjb7I69zaU79kFkP7WCdrUr0rFeZTrVr0xUuHlC7aYTNxj+wyFW7L/EqIfr4aNSuHxtgUDgmFIjRL/99htJSUkMGTIk331mzpzJ1KlTXT53zI0Ufo2+yu7Y25y8loLtAkoKuYxWkSFW4WlRPQS1Mq+r2aNxONUr+HLlbia/Rl9lQNsaLttR3rl48SK1a9cmOjqali1bFvt5du/ezfDhw4mJiaF379789ttvRb5meeXvv/+ma9eu3L17l5CQkGK7TqkRokWLFvHoo49StWrVfPeZOHEiY8eOtT63TKgrjJjrqXy944L1eVR4YLbwVKJt7UoEaAq/DQq5jCEdajH9j9Ms3hXH820iRQbsHinuD/nYsWNp2bIlGzduJCAgwO3nF7iPUiFEly5dYuvWraxdu7bA/TQaDRqNxuXzd6hXiWdbV6dT/cq0r1uJsMCiRfafaxPJvK3nOJeQxj/nbvNgg9DCDyoj6HS6e6oT8QTnz59n+PDhBbafKIySfN+SJGE0GlEq3fu19Ib/XalI5yxZsoSwsDB69+5dLOcPC/Rh9rMteKJltSKLEECgj4pn7zd/qBfvjivSOSRJIkNn8MjDlUV9u3TpwqhRoxg9ejSVK1emZ8+enDhxgkcffZSAgADCw8MZNGgQt2/fth6zadMmOnXqREhICJUqVaJPnz6cP3/e4fkvXrxI165dAahQoQIymcw6LHf2PDExMXTo0AEfHx+aNm3Kjh07rOeWyWTcuXOHl19+GZlMxtKlSwHYsWMHbdu2RaPRUKVKFSZMmIDBYCjwff/999/IZDI2b95Mq1at8PX15eGHHyYhIYGNGzfSqFEjgoKCeOGFF+wKV00mEzNnzqR27dr4+vrSokUL1qxZY33dct6NGzfSunVrNBoNu3btKvR/M336dMLCwggMDOSVV15hwoQJdkPUIUOG8OSTT/Lxxx9TtWpVoqLMyZXvv/+e+++/n8DAQCIiInjhhRdISEiwO/eGDRto0KABvr6+dO3alYsXLxZqjzvwuEdkMplYsmQJgwcPdvsvQXEwtENtlu65yN9nbhGbkEq9sIL7rOQmU2+k8aTNxWRdwZz6qCd+aufv8bJly3jjjTfYvXs3SUlJPPzww7zyyit8+umnZGZmMn78ePr3789ff/0FQHp6OmPHjqV58+akpaUxadIk+vXrx5EjR/KkeCMjI/nll194+umnOXPmDEFBQfj6+rp0nnfeeYd58+bRuHFj5s6dS9++fYmLiyMyMpLr168TFRXFRx99xHPPPUdwcDBXr17lscceY8iQISxfvpyYmBheffVVfHx8mDJlisP3DXD9+nUApkyZwhdffIGfnx/9+/enf//+aDQaVq5cSVpaGv369WP+/PmMHz8eMMc0f/jhBxYuXEj9+vXZuXMnL774IqGhoXaZ4QkTJjBnzhzq1KlDhQoVCvyfrFixgo8//pivvvqKjh07smrVKj755BNq17avb9u2bRtBQUFs2bLFuk2v1zNt2jSioqJISEhg7NixDBkyhA0bNgDmvl5PPfUUI0eO5LXXXuPgwYP85z//KfyD4gY8/s3funUrly9f5uWXX/a0KU5Ro5IfPRqF8+epmyzZfZGP+zXztEnFRv369fnf//4HmH+FW7VqxYwZM6yvL168mMjISM6ePUuDBg14+umn7Y5fvHgxoaGhnDp1iqZNm9q9plAoqFixIgBhYWF2MSJnzzNq1CjrvgsWLGDTpk0sWrSId999l4iICGQyGcHBwURERADw1VdfERkZyRdffIFMJqNhw4Zcu3aN8ePHM2nSJKvI2b5vyBGi6dOn07FjRwCGDRvGxIkTOX/+PHXq1AHgmWeeYfv27YwfPx6tVsuMGTPYunUr7du3B6BOnTrs2rWLr7/+2k6IPvroI3r06FH4PwSYP38+w4YNY+jQoQBMmjSJP//8k7S0NLv9/P39+e677+yGZLbfsTp16vD555/Tpk0b0tLSCAgIYMGCBdStW5dPPvkEgKioKI4fP85///tfp2y7FzwuRI888ohLQ4bSwMudavPnqZv8cvgK4x6JooK/8+NvX5WCUx/1LEbrCr62K7Ru3dr699GjR9m+fbvDoO/58+dp0KAB586dY9KkSezfv5/bt29jMpkAuHz5ch4hKghnz2P5ggMolUruv/9+Tp8+ne95T58+Tfv27e2SDB07diQtLY0rV65Qo0aNPO/blubNm1v/Dg8Px8/PzypClm3//vsvALGxsWRkZOQRGJ1OR6tWrey23X///fnanJszZ84wYsQIu21t27a1eqUWmjVrlicudOjQIaZMmcLRo0e5e/eu3X1t3Lgxp0+fpl27dnbH2N7j4sTjQuSNtKtdkSZVgzh5LYUfD1xmRJd6Th8rk8lcGh55En9/f+vfaWlp9O3b1+GvY5UqVQDo27cvNWvW5Ntvv6Vq1aqYTCaaNm2KTqfLc0xBuOs8RcX2fduiUqmsf8tkMrvnlm2WL7fFQ/njjz+oVq2a3X65Ey75Xe9eyH3O9PR0evbsSc+ePVmxYgWhoaFcvnyZnj17lth9LYhSEaz2NmQyGS93NI/Jl++5hN5o8rBFxc99993HyZMnqVWrFvXq1bN7+Pv7c+fOHc6cOcMHH3xAt27daNSoEXfv3i3wnJZfbNtpPq6cZ9++fda/DQYDhw4dolGjRvler1GjRuzdu9fOA9+9ezeBgYH3lFlzROPGjdFoNFy+fDnP/bqXBR+ioqI4cOCA3bbczx0RExPDnTt3mDVrFp07d6Zhw4Z5AtWNGjWyenQWbO9xcSKEqIj0aVGF0EANN1Ky2HD8uqfNKXZGjhxJYmIiAwYM4MCBA5w/f57NmzczdOhQjEYjFSpUoFKlSnzzzTfExsby119/2dV8OaJmzZrIZDJ+//13bt26RVpamkvn+fLLL/n111+JiYlh5MiR3L17t8BY44gRI4iPj+fNN98kJiaGdevWMXnyZMaOHev2+YCBgYGMGzeOMWPGsGzZMs6fP8/hw4eZP38+y5YtK/J533zzTRYtWsSyZcs4d+4c06dP59ixY4XWtNWoUQO1Ws38+fO5cOEC69evZ9q0aXb7DB8+nHPnzvHOO+9w5swZVq5cac02FjdCiIqIRqlg0AM1AVi8K87r4lyuUrVqVXbv3o3RaOSRRx6hWbNmjB49mpCQEORyOXK5nFWrVnHo0CGaNm3KmDFjmD17doHnrFatGlOnTmXChAmEh4czatQol84za9YsZs2aRYsWLdi1axfr16+ncuXKBV5vw4YN/Pvvv7Ro0YLhw4czbNgwPvjgg3u6N/kxbdo0PvzwQ2bOnEmjRo3o1asXf/zxR54MlysMHDiQiRMnMm7cOO677z7i4uIYMmRIobPeQ0NDWbp0KT///DONGzdm1qxZzJkzx26fGjVq8Msvv/Dbb7/RokULFi5caJecKE5kkhd/g1JSUggODiY5Odkjq3jcTtPSYdZf6AwmfnmjPa1rVrR7PSsri7i4OGrXrl3k9ggCQWH06NGDiIgIvv/+e49cv6DPubPfUeER3QOVAzT0a2kORC7eddGzxgjKBRkZGcydO5eTJ08SExPD5MmT2bp1K4MHD/a0afeEEKJ7ZGinWgBsPHGdK3dFO1jBvdGkSRMCAgIcPlasWIFMJmPDhg08+OCDtG7dmv/7v//jl19+oXv37p42/Z7wjjxyKaZhRBCd6lVmV+xtlu+9xHuP5Z+1EQgKY8OGDej1eoevhYeH4+vry9atW0vYquJHCJEbeLlTLXbF3ubHfy/zdrf6+Dsxm18gcETNmjU9bYJHEEMzN9ClQRh1KvuTmmVgzaErnjZHIPA6hBC5AblcxtCOtQBYsjsOk8lrE5ECgUcQQuQmnrqvOkE+Si7eyeCvmITCDxAIBFaEELkJf42SAe3MkyaL2qtIICivCCFyIy+1r4VCLmPP+TucuuZ6Y3+BoLwihMiNVAvxpVdTc++bJeXcK6pVqxbz5s2zPpfJZKJ5vSBfhBC5GctKsOuOXONuuufbK5QWrl+/zqOPPuppMwSlFCFEbua+GhVoGRmCzmhi/dGrnjan1BAREVGkhQ/ciSRJdv2pBaUHIUTFgMUrWn/0Wt5Z+ZIEunTPPFyc35yamsrAgQPx9/enSpUqfPrpp3Tp0oXRo0e7fE9sh2aW5vZr166la9eu+Pn50aJFC/bu3Wt3zK5du+jcuTO+vr5ERkby1ltvkZ6ebn29sGbwRW1OLyh5PF4CfPXqVcaPH8/GjRvJyMigXr16LFmyxKX2maWNXk0jqBLsQ1KGngyd0f5FfQbMyH/ttmLlvWugdr4b4NixY9m9ezfr168nPDycSZMmcfjw4XtaHNGW999/nzlz5lC/fn3ef/99BgwYQGxsLEqlkvPnz9OrVy+mT5/O4sWLuXXrFqNGjWLUqFEsWbIEKLwZvAVXmtMLPINHheju3bt07NiRrl27snHjRkJDQzl37pzXf1hUCjkvta/FD7vPka51bRmf0kJqairLli1j5cqVdOvWDTAv+1TQApiuMm7cOOsSUlOnTqVJkybExsbSsGFDZs6cycCBA63eV/369fn888956KGHWLBgAT4+PoU2g7fgSnN6gWfwqBD997//JTIy0voLB9xT06jSxIC2kfy8/wI6o3kds+yVckDlZ/ZMPIHKz+ldL1y4gF6vp23bttZtwcHB1jWy3IFtM3pL3+uEhAQaNmzI0aNHOXbsGCtWrLDuI0kSJpOJuLg4GjVqVGgzeAve7F2XFzwqROvXr6dnz548++yz7Nixg2rVqjFixAheffVVh/trtVq0Wq31eUpK6a3VCfFT80iTcMBAUoaeSsHZL8hkLg2PyjK5m9EDds3nX3/9dd566608x9WoUcOlZvDF0Zxe4F48Gqy+cOECCxYsoH79+mzevJk33niDt956K9+evjNnziQ4ONj6uJcm5CXBU/eZm6alaQ1o9cZC9i5d1KlTB5VKZdeYPTk5mbNnz5bI9e+77z5OnTqVp/F8vXr1UKvVTjWDF3gPHhUik8nEfffdx4wZM2jVqhWvvfYar776KgsXLnS4/8SJE0lOTrY+4uPjS9hi16hR0R8flfkW3/GymqLAwEAGDx7MO++8w/bt2zl58iTDhg1DLpcX2qjdHYwfP549e/YwatQojhw5wrlz51i3bh2jRo0CnGsGL/AePCpEVapUsRvLg3lJk8uXLzvcX6PREBQUZPco7QRk9yZKTNdhMHnXskNz586lffv29OnTh+7du9OxY0caNWpUIv23mzdvzo4dOzh79iydO3emVatWTJo0yRosd6YZvMB78Gjz/BdeeIH4+Hj++ecf67YxY8awf/9+9uzZU+jxnm6eXxiWpuIEhqFHQZVgX0IDPVvUdy+kp6dTrVo1PvnkE4YNG+ZpcwSlBK9vnj9mzBj27dvHjBkziI2NZeXKlXzzzTeMHDnSk2a5nQp+5qDsnTStV6Xyo6Oj+fHHH61rcg0cOBCAJ554wsOWCcoaHhWiNm3a8Ouvv/Ljjz/StGlTpk2bxrx586wf+LJCkI8KpVyOzmgiJdNxP+LSypw5c2jRogXdu3cnPT2df/75h9OnT+fb4N22fkcgcBaPV1b36dOHPn36eNqMYkUul1HRX01Caha303QE+6k9bZJTtGrVikOHDuXZnpmZyZEjR0reIEGZxeNCVF6oFKDmVpqWdJ2BDJ0BP7X33npfX1/q1avnaTMEZQgx6bUEkCQJlUJOiK85ViTagwjKEu6IewohKkYslcMZGeaFFy3LDGkN3pXGFwgKwvL5tq2UdxXvHR94AQqFgpCQEGvFr6RQIRl0aCUjWVni1gu8G0mSyMjIICEhgZCQEBQKRZHPJb4NxUxEhLl1bEJCAnqjiYQULXIZkOZb8IECgZcQEhJi/ZwXFSFExYxMJqNKlSqEhYWRnJ7FG/PNxZv/92Ynrw5YCwRgHo7diydkQXwTSgiFQkHFIH+SdTLStAbuaqFiUPFPlRAIvAERrC5hwoPMUzxuJmd52BKBoPQghKiEiQg2e0E3UoQQCQQWhBCVMOFBQogEgtwIISphIrKFSAzNBIIchBCVMGJoJhDkRQhRCZMzNNMWsqdAUH4QQlTCWIZmN5IzPWyJQFB6EEJUwliGZrdStRiMYs6ZQABCiEqcygEaFHIZJglup4lZ+AIBCCEqcRRyGWHZfatFwFogMHNPUzzS0tKsC+JZKI1N7Esb4UE+XE/O4kZyFpTupdkEghLBZY8oLi6O3r174+/vT3BwMBUqVKBChQqEhIR4/Zr1JYW1lkh4RAIBUASP6MUXX0SSJBYvXkx4ePg9LbY3ZcoUpk6darctKiqKmJiYIp/TGxC1RAKBPS4L0dGjRzl06BBRUVFuMaBJkyZs3bo1xyBl2W8IEC6qqwUCO1z+1rdp04b4+Hi3CZFSqbznpkreRkSwCFYLBLa4LETfffcdw4cP5+rVqzRt2jRPn9rmzZu7dL5z585RtWpVfHx8aN++PTNnzqRGjRoO99VqtWi1ORXJKSkprppfKrBWVwuPSCAAiiBEt27d4vz58wwdOtS6TSaTIUkSMpkMo9Ho9LnatWvH0qVLiYqK4vr160ydOpXOnTtz4sQJAgMD8+w/c+bMPDElbyTCZga+5b4JBOUZmeTiWiCNGzemUaNGvPvuuw6D1TVr1iyyMUlJSdSsWZO5c+c6XFvdkUcUGRlZ6LrapY0MnYHGkzYDcGzKIwT5FH31A4GgNJOSkkJwcHCh31GXPaJLly6xfv36YllgLyQkhAYNGhAbG+vwdY1Gg0ajcft1Sxo/tZIgHyUpWQZuJmcJIRKUe1yuI3r44Yc5evRocdhCWloa58+fp0qVKsVy/tKESOELBDm47BH17duXMWPGcPz4cZo1a5YnWP344487fa5x48bRt29fatasybVr15g8eTIKhYIBAwa4apbXER7kw9mbaSJgLRBQBCEaPnw4AB999FGe11wNVl+5coUBAwZw584dQkND6dSpE/v27SM0NNRVs7wOUV0tEOTgshDlnlt2L6xatcpt5/I2xNBMIMjBpRiRXq9HqVRy4sSJ4rKn3JBTSyQ6NQoELgmRSqWiRo0aLg2/BI7JqSUSnRoFApezZu+//z7vvfceiYmJxWFPucE6NBMekUDgeozoiy++IDY2lqpVq1KzZk38/f3tXj98+LDbjCvLWIZmd9K16I0mVArRo05QfnFZiJ588sliMKP8UclfjUohQ2+USEjVUi3E19MmCQQew2Uhmjx5cnHYUe6Qy2WEBfpwNSmTG8lZQogE5ZoiN/85dOgQp0+fBsw9hVq1auU2o8oLEcFmIRK1RILyjstClJCQwPPPP8/ff/9NSEgIYJ6s2rVrV1atWlUuihHdRYRoByIQAEXImr355pukpqZy8uRJEhMTSUxM5MSJE6SkpPDWW28Vh41llnBRXS0QAEXwiDZt2sTWrVtp1KiRdVvjxo358ssveeSRR9xqXFlHdGoUCMy47BGZTKY8E13BXOzozukf5QGLR3RdDM0E5ZwitQF5++23uXbtmnXb1atXGTNmDN26dXOrcWUdMfFVIDDjshB98cUXpKSkUKtWLerWrUvdunWpXbs2KSkpzJ8/vzhsLLPkVFebW8YKBOUVl2NEkZGRHD58mK1bt1rXH2vUqBHdu3d3u3FlHcvQTGswkZypJ8RP7WGLBALPUKQ6IplMRo8ePejRo4e77SlX+KgUVPBTcTdDz42ULCFEgnJLkYRo27ZtbNu2jYSEhDwB6sWLF7vFsPJCeJCPWYiSs2gY4T0LAAgE7sTlGNHUqVN55JFH2LZtG7dv3+bu3bt2D4FrWOJEImAtKM+47BEtXLiQpUuXMmjQoOKwp9wRIRqkCQSue0Q6nY4OHTq43ZBZs2Yhk8kYPXq0289dmgkPEi1jBQKXheiVV15h5cqVbjXiwIEDfP311y4vV10WEEMzgaAIQ7OsrCy++eYbtm7dSvPmzfNUWc+dO9el86WlpTFw4EC+/fZbpk+f7qo5Xk+EqK4WCFwXomPHjtGyZUuAPE30i7KG+8iRI+nduzfdu3cvl0IkJr4KBEUQou3bt7vt4qtWreLw4cMcOHDAqf21Wi1abU5QNyUlxW22eArL0CwxXYfWYESjVHjYIoGg5PFYo+T4+HjefvttVqxYgY+Pj1PHzJw5k+DgYOsjMjKymK0sfir4qVArzf+GhBSROROUTzwmRIcOHSIhIYH77rsPpVKJUqlkx44dfP755yiVSodLFk2cOJHk5GTrIz4+3gOWuxeZTGaztJAYngnKJ0VuFXuvdOvWjePHj9ttGzp0KA0bNmT8+PEoFHmHKBqNBo1GU1ImlhgRQT5cTsxwulNjhs7Aq8sP0jUqjFc61ylm6wSC4sdjQhQYGEjTpk3ttvn7+1OpUqU828s64S6m8KMvJ7E79g4Xb2cIIRKUCVwemu3cuRODwZBnu8FgYOfOnW4xqrwREZTdqdFJjyg1y3z/b6VqRfsQQZnAZY+oa9euXL9+nbCwMLvtycnJdO3a9Z6Wo/7777+LfKw342p1dZrWLEQ6o4mkDD0V/MWsfYF347JHJEmSw3qhO3fu5Fn1VeActg3SnCFdm+ORJqSKTJvA+3HaI3rqqacAc5ZnyJAhdkFjo9HIsWPHimUOWnnA1axZmp0QZREVEVgsdgkEJYXTQhQcHAyYPaLAwEB8fXNWJlWr1TzwwAO8+uqr7rewHGAZmiWkaPP1OG2xEyJReyQoAzgtREuWLAGgVq1ajBs3TgzD3IhFiHRGE4npOioFFFyikJYlhmaCsoXLMaLJkycLEXIzaqWcygHmgLMzw7P0XEMzgcDbcVmIbt68yaBBg6hatSpKpRKFQmH3EBQNVya/popgtaCM4XL6fsiQIVy+fJkPP/yQKlWqFGnGvSAvEUE+nLyW4lSnRluP6JaIEQnKAC4L0a5du/jnn3+srUAE7sFSXe3M0Cx31kwg8HZcHppFRkaKat5iwLrqqxO1RGliaCYoY7gsRPPmzWPChAlcvHixGMwpv1g7NTrjEdlkzTJ0RjthEgi8EZeHZs899xwZGRnUrVsXPz+/PK1iExMT3WZcecI68dUJjyg9l/AkpGQREBpQLHYJBCWBy0I0b968YjBD4Gx1tckkka4zz+cL9FGSmmUgIVVLHSFEAi/GZSEaPHhwcdhR/JhMIPdYH7hCsQhRcqaeLL0RH5XjUoh0XY43VCc0gKPxSSJOJPB6ivTNPH/+PB988AEDBgwgISEBgI0bN3Ly5Em3Guc2MpPgu4fh1DpPW5IvQb5KfFTmf0dBk1/TtWZvSCmXEVnBPM0mQXR2FHg5LgvRjh07aNasGfv372ft2rWkpaUBcPToUSZPnux2A93C/oVwLRpWD4bdn0EpzPrJZDKqBJuFpaDhWZpWD4C/RklYoNmLuiU8IoGX47IQTZgwgenTp7NlyxbU6pw+OA8//DD79u1zq3Fuo/M4aPMqIMGWSfD7aDDqPW1VHsKzG6QVVF2dlu0RBWiUhDmxv0DgDbgsRMePH6dfv355toeFhXH79m23GOV2FEp4bDb0mgXI4NBSWNkfspI9bZkd1oB1AUMzS+o+QKMkLNAsRCJGJPB2XBaikJAQrl+/nmd7dHQ01apVc4tRxYJMBg+8Ac+vBJUfnP8LFveCpNKzEogz1dWWmqEAn5yhmRAigbfjshA9//zzjB8/nhs3biCTyTCZTOzevZtx48bx0ksvFYeN7qXhYzB0AwREQMIp+K4bXD3saasAJz2ibCHytxmaiWC1wNtxWYhmzJhBw4YNiYyMJC0tjcaNG/Pggw/SoUMHPvjgA5fOtWDBApo3b05QUBBBQUG0b9+ejRs3umqS61RtBa9ug7AmkHYTljwGp38v/usWgjO1RJZixgCNwjo0S8kykKUveq9wgcDTuCxEarWab7/9lvPnz/P777/zww8/EBMTw/fff+9yG5Dq1asza9YsDh06xMGDB3n44Yd54oknSqYMILg6vLwJ6nUHQyb89CLs+cKjGTVnqqutQzONkmDfnFViReZM4M0UeV2zGjVqUKNGjXu6eN++fe2ef/zxxyxYsIB9+/bRpEmTezq3U/gEwYCfYOM7cHAx/Pk+JF6AR/9nDnCXMBaPKCFVi8kkIZfnbbFiOzSTyWSEBmi4mpRJQmoWkRX9StRegcBdOPVtGzt2LNOmTcPf35+xY8cWuO/cuXOLZIjRaOTnn38mPT2d9u3bO9xHq9Wi1eb88qekpBTpWnYolNB7LlSsC39+AAcXQdIleGaJWahKkNBADTIZGEwSt9O11mC0LZasWaDG/K8LC8oWItGXSODFOCVE0dHR6PV669/5UZQmacePH6d9+/ZkZWUREBDAr7/+SuPGjR3uO3PmTKZOneryNQpFJoMOo6BCTfjlVYjdCksehRd+Mg/hSgiVQk5ogIaEVC03kx0LUbqNRwQQLjJngjKAU0K0fft2h3+7g6ioKI4cOUJycjJr1qxh8ODB7Nixw6EYTZw40c4jS0lJITIy0n3GNOoLQ/+Alc/DzRPwbTezGFVt6b5rFEJEsA8JqVpupGTRjOA8r9um74GczJlokCbwYjw+C1StVlOvXj1at27NzJkzadGiBZ999pnDfTUajTXDZnm4nWqtzRm10EaQdsPsGZ0pgUxeNoWt+mobrAZyihrF0EzgxTjlEVkWV3SGtWvXFtkYAJPJZBcH8gghNWDYZvh5iLnw8ccB5qrsB4YX+6UL69SYnkeIxNBM4P045REFBwdbH0FBQWzbto2DBw9aXz906BDbtm2zLsLoLBMnTmTnzp1cvHiR48ePM3HiRP7++28GDhzo2rsoDnyC4YXV0HoIIMGm8bDhHTAWbzfEiEKqq1NzxYhCg8Q0D0EJc3AJrH/LraUuTnlElsUVAcaPH0///v1ZuHChtW7IaDQyYsQIl4dKCQkJvPTSS1y/fp3g4GCaN2/O5s2b6dGjh0vnKTYUKugzz5xR2/Ih/PsN3L0EzywGTfE0IgsvpLo6r0dkFqJbIkYkKG5MJtg6GfZ8bn7eoCc07O2WU7tcLLN48WJ27dplV7yoUCgYO3YsHTp0YPbs2U6fa9GiRa5evuSRyaDjW+aM2trX4NxmWNLL7C0FVXX75Qqrrrad9Ao5Q7M76ToMRhNKhcfDfoKyiC4Dfn0dTq83P+/6PkQ95rbTu/ypNRgMxMTE5NkeExODyWRyi1GlksZPwJA/wD8Ubhw3Z9SuH3P7ZSKCs1t7OPCIbNvEWoZmlfzVKOQyJAlup+ncbo9AQFoCLOtjFiGFGp76Fh561/wj7SZc9oiGDh3KsGHDOH/+PG3btgVg//79zJo1i6FDh7rNsFJJ9fvhlW3mFiK3Ysyz959dYnZR3YRlaJaqNZCuNVgFB+zbxAZmp+/lchmVA9TcTNGSkJpljTEJBG4hIQZWPAvJl8G3grl7Rc0Obr+My0I0Z84cIiIi+OSTT6ztQKpUqcI777zDf/7zH7cbWOqoUBNe3gyrX4K4HfDj89Drv9DuNbecPtBHhb9aQbrOyI2ULOraNMW3tIlVyGVolDnObFigj1mIRApf4E7Obzd3NdUmQ8U6MHANVKpbLJdyeWgml8t59913uXr1KklJSSQlJXH16lXeffddlye9ei2+IfDiL9BqEEgm81y1jRPA5J4Z8BH5TH61tIkNyJ5nZkE0SBO4ncPLYcUzZhGq0d48EigmEYJ7LGgstqJCb0ChgsfnQ7fsPt37F8CqgaBNu+dT55fCt20Ta8u9VlebTJJYvVdgxmSCrVNg/ZtgMkCz/vDSOvCrWKyXLdIU8zVr1rB69WouX76MTmcfID18uHQ0GSsRZDLoPBYq1IJfh8PZjbD0MfOM/qAqRT5tftXVloyZv8be8wy9h6JGg9FEn/m7qByg4YdX2hXFXEFZQZ9p/hyf+s38/KEJ0GWCW4PS+eGyR/T5558zdOhQwsPDiY6Opm3btlSqVIkLFy7w6KOPFoeNpZ+mT8GQ38GvElw/au76eONEkU+XX3V17ukdFu5lmseF2+nE3EhlV+xt9MYynPUUFEzaLVjW1yxCchU8uRC6TiwREYIiCNFXX33FN998w/z581Gr1bz77rts2bKFt956i+Tk0tWMvkSJbGseR1duAClXYXFPOLe1SKeyDM2u5yNE/vkIUVGKGn1tFnLM0Ikuj+WSW2fMP55XDoBPCLz0G7QcUKImuCxEly9fpkMHc/rO19eX1NRUAAYNGsSPP/7oXuu8jYq1YdifUKsz6NLMaf4D37l8GsvQLPcyQZaqakvq3kJYUNGHZgqb5muZQojKH3E7YVEPcw+uCrXgla1Qq1OJm+GyEEVERJCYmAiYuzRa1jKLi4sTAU8w11q8uBZaDgTJCH/8Bza/71JGLb/qaqtHpM7PIzJ3dnQF271t65QE5YDoFfB9P/OyWpHtsj36+h4xxWUhevjhh1m/3lzmPXToUMaMGUOPHj147rnnHK53Vi5RquGJL+HhD83P935hrjvSpTt1uGVoditVi8EmbpO7F5GFygFmITKYJBIzXKuutv3xyNAKj6hcIEnw13RYN8KcGWvyFLy0Hvwre8wkl7Nm33zzjXUqx8iRI6lUqRJ79uzh8ccf5/XXX3e7gV6LTAYPjjO7u7+NgJjfYWlvc0YtMLzAQysHaFDIZRhNErfTdFZhyj3h1YJaKaeiv5rEdB0JKVqrMDmDrRObITyiso8+C9aNhBNrzM87jzPPG5N7do6iS1c3GAxMnz6dGzduWLc9//zzfP7557z55pt2S1ALsmn2DAxeD74V4Vq0OSh481SBhyjk5qb4YD88y0nf5/39yClqdC1gbS9EwiMq06TfgeVPmEVIrjR77d0+9LgIgYtCpFQq+d///ofBIH45XaLGA+YgYKV6kBxvzqjFbivwEGtRo03mLL/0PRQ9YC3ZRIlEjKgMczvW/CMYvw80weY4ZqsXPW2VFZelsFu3buzYsaM4bCnbVKoLw7ZAzY6gTTFPJDy4JN/dIxxkzixC4VCIbALWrmASHpFHWL73Iq8sO8Dw7w9x8loxl71c3G0Wobtx5u6jr2yBOg8V7zVdxOUY0aOPPsqECRM4fvw4rVu3xt/f3+71xx9/3G3GlTn8KsKgX83d7Y6tgt9Hm9dR6z41j3vsaJqH3mBWDbUy7+9HTlGjq0Mz22C18IhKgkydkSnrT1p/BHxUcuY936p4LnZ0FawbBSY9VG8Dz/8IAaHFc617wGUhGjFiBOB4/TKZTIbRKH5VC0SpgX4LzbOZ/55h7nZ39yL0+xrUOQskOurUaMhOEigcLLxoO/H19PUUMnQGWtcsfH6Qffpe/O9KguRMvZ0neikxw/0XkST4exbsmGV+3vhJ8+dO5ev+a7kBl4dmJpMp34cQISeRyaDLeHODKYXa3HBqWR9zA6psLA3SbIXImP3pVTgou7fEiK7czeTRz/7h6QV7ScnSF2qKrUdU1ILGhNQsxv50hMOX7xbp+PJGaq7/S7y7hcigNXdTtIhQpzHmBUNLqQhBKVhOqFzTPHtms28FuHrI3PUxwdz90lF1tcEiRIr8PaLjV3PiDalZhQ+1bLNmRQ1WT/jlOGujr/LUV3uKdHx5w/IDEeKnAsydNd1WOpGRCMufhGM/gUwBfT+H7lNKRWasIJwemmVmZrJt2zb69OkDmFfgsF32R6FQMG3aNHx8nO8QOHPmTNauXUtMTAy+vr506NCB//73v0RFRbnwFrycmh3MFa0rnjHHixY9Av2XERHUBjDHiCRJMg97s4VI6XBolve+G5yYxGo7NCtqQeO5hNQiHVdeSck0i071Cr6YTBIpWQbiEzOJigi8txPfOW9OgiSeB00Q9F8GdR92g8XFj9MyuWzZMr7++mvr8y+++II9e/YQHR1NdHQ0P/zwAwsWLHDp4jt27GDkyJHs27ePLVu2oNfreeSRR0hPd64CucxQqa5ZjGq0NzeiWvEM1eLMBWcZOqN1CSHr0MyREAXlLWLUGwuf7mGSCk7f/7DvEv+cu1XgOYxOXKe8kq418NnWc5y5kSPWFo8oyEdFjUrmuODlex2eXdoL33U3i1BwDfOcRy8RIXBBiFasWMFrr9m3Q125ciXbt29n+/btzJ49m9WrV7t08U2bNjFkyBCaNGlCixYtWLp0KZcvX+bQoUMunadM4FfRPExr1h9MBjQb3uYDn9XIMFnbgeR4RHn/bT4qhd1MesCpth62QzOtwX7/k9eS+eC3Ewxa9G+B5zC4OL+tPPH5tnN8uvUsPefttG5LyTQLUaCPkhoV3SBEx9fA8schMxGq3meuWQtrdE92lzROC1FsbCzNmjWzPvfx8UFu84Vo27Ytp04VXDFcGJY2IhUrOs72aLVaUlJS7B5lCqUGnvrG3JAKeIXfmK+aT0JiEmATI3LgEQFk6u2HVs4Ika1HlHsol2YTY9Ia8h+2GYUQ5UvMjbzD1pTs+xrkoyIyW4iKFLCWJNgxG34ZBkYdNOxjXmmmkClEpRGnhSgpKckuJnTr1i1q1aplfX6vS0WbTCZGjx5Nx44dadq0qcN9Zs6cabfqbGRkZJGvV2qRycwNqZ5ciAElfRT7abh5IKTdKjBG5AhXPaJ/zt22y5yF+OVM2SloqSKj6LqQL7lbtoDN0MxXRWSFIgqRQWeew7h9uvl5hzeh//d2JSDehNNCVL16dU6cyL/r4LFjx6hevXqRDRk5ciQnTpxg1apV+e4zceJEkpOTrY/4+PgiX6/U03IA39aaS5LkT6Ukc9fH6sbLQP4eUW6ciRHZYjBJvPmj41a/BRVKihhR/gT6qPJsswSrg3xURRuaZd6FH56CoyvNmbHec+GR6aU+M1YQTlv+2GOPMWnSJLKy8n4gMzMzmTp1Kr17F2352VGjRvH777+zffv2AsVMo9FYG/aXh8b9GVUf4CndVO6oq0LSJb7TT6S9/CRKB+l7R7g6NAPYejrB4X4FTR0RMaL8CfItyCPKiRHF381wrp9XYhx81wMu/gPqQBi4GtoMc6vNnsBpIXrvvfdITEwkKiqK2bNns27dOtatW8f//vc/oqKiuHv3Lu+9955LF5ckiVGjRvHrr7/y119/Ubt2bZffQFkmPMiHC1JVpkV8DpHtCCSD5apZVDjjXFLA1aFZntdskvsFTaYVMaL8CbLxiLKyY3iW+q5AHxVVQ3yRyyBLb+JWWiGhjcv7zXPG7pyDoOowbDPU615stpckTtcRhYeHs2fPHt544w0mTJhgVW+ZTEaPHj346quvCA93LUg2cuRIVq5cybp16wgMDLS2FwkODsbXt/RWgZYUlomvsek+MHw9G6f341HZHsK3/wcM1wvtI6MzFC4QzkpIwR6RaLqfH/7qnExmSpYeH5XCmjUL8lGiVsqpEuzL1aRM4hMzHNaDAXDiF/j1DTBqoUpLeOEnCIwogXdQMrg016x27dps2rSJxMREYmNjAahXr16+Wa7CsNQddenSxW77kiVLGDJkSJHOWZbIaQWiBZUP/zG9SawUxpvK3+CfOebZ1E98BSrHH15nBCL30Cw/CvKIhEOUP7YLYaZk6gkL9LELVgNEVjQL0eXEjLzzAyUJds2FbR+Zn0f1hqe/BbX9ZPOSxGSSuJGSRdUQ9zkLRVrXrGLFitZ17+8F0eO6YCzTPO6ka9EbTRhMMj4x9ufFR7tQYds4869k8hXzeuQO2nze89DM5jVX24sIzNh+xpOzPSHbYDVAjYp+7LuQSHxipv3BRr25Q0P0D+bnD4yER6aB3HMrKl9LymTs6iNcuZvJxrc7OwzGFwXvDbOXAyr5q1EpZEiS2SOxeDj6ZgPMja18giF+v7mi9va5PMfrCxiaHYlPIjFd5/SPQVGWKhLYD32tQmQTrAYcZ84yk+CHp80iJJPDY3Og1wyPitD6o9foNW8n+y4kkpiu48RV99XxCSEqxcjlMmvM4EZylnUIpJDLzI2thm2BkJrmIdp33WknO213vC4fj2j/hTs8+eVuOszaVmCMSHhE947tPUzJNJClN6LLrmC3eBORuYXo7kXznMO4HaAOMPc5b/tqSZptR3KmntGronnrx2hSsgy0iAzhj7c6075uJbddQwhRKccSJ7qalOO2W6d4hEaZ56hVbwNZSXyvnkE/+T/W/fKb9LrjrHnuWJbe5HTrj1tpWjGULgKmXEMzS8ZMJoPA7E6bdtXVVw5me7hnILAqDN0IDR4pecOz2XfhDo999g+/HbmGXAZvd6vPmuHtqV3ZvTEqIUSlHEvm7OrdHCGyawMSEAqD/w8aP4laZuRT9QLGKNcAUr4FjXKbAOrtwlLG2eiNEkkZhfc3EuRPcqbeOiwL0CiRZxemWoZmLdN2Ii3tDem3IKIZvLoNqjT3iK06g4lZG2MY8O0+riZlUqOiHz8P78CYHg1QKdwvG0UKVgtKjnBrw7Oc+EGeKR4qX3hmCV9OMjFSuZ63lWupIbvJDf0ch+e0PbwgIZJyDdwSUrVU8BcrtbiC/dBMb5O6zwnyVvJTMUr9B+PkK8AA1O8JzywGTUAJW2smNiGVt1cd4eQ1cwzoufsj+bBvY4e90t2FEKJSjqVTo+3QzOEUD7mc2YbnuSSF87FyMf0Uu7lydAS0X2ue2W+DzM4jyjuHTG80MWPDaSrlEp1bqdp775lTzrAVc7NHlJ0xy07dY9Qj2zDOLELAlQaDqP78Zx4JSkuSxPf7LvHxH6fRGkyE+KmY9VQzejWtUuzXFkJUygl3NDRz0CrWwmpjV65KlVmgmkf1lOx11AauMfc8ysZuaOYgCL3qQDxLdl/Msz0hNYvkTD0BGqXT893KOgajiVUH4nmgTkXqheUVaVuPyBwjymkBQlYyrB4MF7ZjQs5H+hepW/s/DPKACCWkZvHummP8fcYcP+xcvzJznm1h/fwVNyJGVMqxxoiyPSK5DGtsIT92m5rxtG4qSeoIc9fH77rBpZw2rnZDs/S8HtHlO44b0x2JT6LF1D/p//VeV99GmWXF/st88NsJus/d6fD13Ol7Sw1RLUUiLOoJF7aDyo/VdWex1NiL+LuZDs9TnPx58ga95v3D32duoVbKmdK3McuGti0xEQIhRKUeS9bMsuaYo6ZoFhYNvp8KfioaVwninFSd7xp+B9Vam2drL38CjpnnqNkKmSOPKL8g9/K9lwA4dEk0ybdwsJB7YZs1S8kykJKlp7nsPB9cHwW3TkNABAzdSFYdc2bs8p1iWNEjH9K1Bib8cozXvj9EYrqORlWC+P3NTgzpWLvQHzt3I4SolJP7V6mgIVG3RuEc/rAHPRqb5/wlyUNg8O/Q6HFz46y1r8Lf/0Vm8zvtKFjtTEV2YrqOWRtjiE1Ic/KdlE1MhcxvyR2sDr+6hZ/U0wgyJkJ4U3NmrGpL97WMdZLoy3fp/fk/rDoQj0wGrz9Yh99GdqBBuGdigCJGVMrxUSkI8VNZU+eFxWZkMpl1AUa9QTI3ynp2GWybArs/g79n0C38KPPojw4VdxwMzZwRoolrj7H55E0W74qzbnO2YVtZwvnOAxJPZP7KE2e/Ry6TiKvQgdovrwaN+Ytv2yDNslhCcWAwmvhy+3k+/+scRpNElWAfPunfgg51804RKkmER+QFRNh4Rc4EiVXZdUZ6y6RXuRx6fAR95oFMQdTNP1iunkUwaQ6/SAYnGp0du2Ju62tbvV3S7nxpoLBJw5IkocDINOUS3pUtR47E94bu/NXyM6sIAVTPFqJUrcE6FcTdXLqTzrNf7+XTrWcxmiT6tqjKprcf9LgIgRAir8ASJwLnvA5LwVmeWM/9Q2Hgz+gU/jwgP81a9WRqyG7mOT6/qSGOrmFLefSICnOIlPo0FqnmMEi5FZMkY5H/q3xoGEqgn/3MdV+1wro2nbuHZ5IksfpgPI999g/Rl5MI1CiZ91xL5g9oRbCfeyat3itCiLwAVz0ipUWIDA4EpV43/u/+pVyRKlNXfp1f1ZNoLTtjt4szQzOVgy6R5TGlX+C0l+QrPH3sVboojpIpqRmuH80CbU9A5rBzo1tW9MjF3XQdb/xwmHfXHCNdZ6Rt7YpsHN2ZJ1tVc9s13IEQIi/ANmDtjNehtgzN8hGU5MD69NNO5aipDpVkqazUzKCvPCe978zQ7PytvCn+8ugR5btwwLUj8G03QtPPkSCF0F83iT9NbawFpEEO2mfkzDlzTwp/59lb9Jy3k00nb6BSyBjfqyE/vvqAdRhYmhDBai/AdmjmaLnp3FiHZvmMG+QyuEUFntd9wDzVV/RUHGS++gtq6BP40viEw6GZRinPs+5ZbhRe3Ly9qDi8xWc2wpqXQZ/Bbb+69Et8i6uE2u1iray2Ic8s/CKSpTcya2MMS/dcBKBuqD+fPd+KptWC7+m8xUn5++R4IRF2HlHh/zJVQUMzcoLKmfjwhn40GwKfAeAd1WpmK7/GZMib0q/oxByzwjyiY1eSWLbnYpmaxZ8nfb9vIfw4APQZUPdhVjX7Lo8IgeNlhmrcyxpn2Zy8lkzf+busIvRS+5r8/mbnUi1CIDwiryC8qFmzfIZmtqlhE3L+qvEWe44EMlW5lGeVO2l4O5mBjCSFnEmXFf3VXE8uuDma3mhi6v+d5JHGEQ571Tz+xW4AKvirebxF1ULfhzdgyZopMMKGd+Dfb8wvtB4Cj81Bt+2Cw+McDc3uJUZkMkl8t+sCszefQW+UqBygYfYzzenaMMzlc3kCj3pEO3fupG/fvlStWhWZTMZvv/3mSXNKLUXPmuXjEeU6RVighh+MPRimf4c0yYdm+qOsVU8h0iaj5oxHdCddx5LdFxnw7b4C9zvrYPVTb8VokvAji29Uc3NEqMc0c6mEQpVv4zlHHlFkRXMm7VpSZr69pByRnKFn4Hf7mbEhBr1RokfjcDaP7uw1IgQeFqL09HRatGjBl19+6UkzSj0V/FTWIkXnPKJ80vfZyHMVy1nSxn+bWvKsbjK3ZJWpJ7/Gr+rJ3Cc7m21D+Wn/ka410GveTmZuPF3oviGG2/ysnko3RTRa1NB/OXR8y9z5DMc9wf3VCmtm05bwQB/UCjkGk1So92nL8r0X2XvhDr4qBTOfasY3g1pTKUDj9PGlAY8K0aOPPsr06dPp16+fJ80o9chkMsKDzB8sd3hEuc8QarOEzWmpJm/4/o/jplpUlqXwo/pjesv3OeUROUvuPkeljd+PXSPmRipf73A8rLJy/Rgz7rxNE/klbklBPKf9ABo/YbeLo/fqKFAN5thd9WyvyJU4kWUo90aXugxoW6PYqrKLExGs9hKqBJk/oO6IEeUmLMj+1/OKwZxu3mJsjUam50v15/S4sxLnV0HzbpzK/p3dDIt7Ucl0h7OmavTTTeOIVM+6iKIFRx6Ro/iQBduVX53FMk3H4tl6I14lRFqtlpSUFLtHeSE8O07kUtYsn6FZ7q25P8AZOgOZ+PC6fgyLDI8C0PHSF8xSfosSg4uWO7h+Kdcz20URHWb4/v0Wfnwe9OkcVbXkGd0UrkjmzFjuScCO0vuO4kMWLHPOnAlY307TsvpgPCeumqfbVPay4ZgtXpU1mzlzJlOnTvW0GR4hIttrcS1G5Ngjsp0fFahR4quyb8SVmf2rbkLONMMgLklhTFV9z/PKv6kuu8UI/WhSKLh5+vYzCaw+EM/H/Zq5dVhXEvjZtETN0pvwtQiTyYhu43uoDyw0P281iCmXnyElNae48+zNVLtUuStDM7DNnDkuajx/K42tp26y5dRNDl2+axV1hVxG/XDPtJZ1B14lRBMnTmTs2LHW5ykpKURGRnrQopLDksJXOlHQaPni307TcjtNm+eX0vZHvnKgJk9MIbcntdzYkwE9OlNj+0g6KU6yRjaFl/XvWr0ARwxdcgAwN4mf/WwL++sX+g48i5+NRxR/N4Nfo6/Sv3lFQv8cScDFP80vdJsMncag/XyX3bFncmcEHQ7NCvCIcqXwjSaJ6Mt32XLaLD4XclW0N60WRPdG4fRpXoWalTy3+uu94lVCpNFo0Gi81/28F1rXrIBMhlP9YiKCfWhRPZijV5LZeOIGgx6oafe67XCjcoCT3kpUT57dPJnF6tk0kF/lV/WHvKIbx1GpXoGHuZL9KY2MXnWE29cv0XvvHGrL4tBKKmZo3mZqZ/MPYm6v88xNeyFyJLrOeEQXbqXx7pqjbDudYNeqRaWQ8UCdSvRoHE73RuFuXfbZk3hUiNLS0oiNjbU+j4uL48iRI1SsWJEaNWp40LLSR6saFTj4fnenhzl9mlfl6JVk/u/otTxCZBu3qBygKbSVBZg9m1NSLZ7UfsQi9RyayC/xk3oao/Uj2WRybfnx0h4jsq2WNt04wa+a2VST3UGrrsCA1Lc5omvAewYjGqUiz3SY3DVSjtqsFBSsttQSpWYZWH3wCmCOKXWNCqNH43Aeigot8HhvxaPB6oMHD9KqVStatWoFwNixY2nVqhWTJk3ypFmllkoBeYdR+dG7uXnlhQMXE7mebB9vsPeINE4199Jk1zHdoBL9dZPYZmyFj0zPQvU8XlP8H/kNuHbF3i61UzquJmXarY4CsO30TZ77xlyQ+ZD8KD+rp1JNdodYU1U2d1jBYakBJimnpasu1zSaa8lZ1n5Caw9fYZFN4zgLBQWrA31UPNWqGrUr+zOkQy1+GNaOQx/04PMBrejbomqZFCHwsBB16dIFSZLyPJYuXepJs8oEVUN8aVOrApIEfxy7bvdabo/IIjIFobbZJx1fXtOPZanB3Gf5PdWPzFAuyjejNmODfWFgaagj0hqMdJz1Fx1n/WUnJsOWHQRgoGIri1SzCZRlssfYmKd0U7irzmmdYek+kFuIAM5lD8/Grj7q8NoFDc0A5j7Xku3jujDl8SZ0ql/Z7t6XVcr+OyzH9M2ez/V/uYToTnrOpNZKAWoqBWiY8GjDAs+lUdpn1owomGIYwhT9S5gkGS8o/2KxajaB5E07f/tPXq/A01hW0wBI0+b8LcPEe8oVfKxajFJm4mfDgwzWTyCFALt4UNxtsxBZOhKM6lqPLlHm4H3uOFFuyqpXcy8IISrDPNq0CnIZHI1PsvZC/nrHeb7cft66jyWjNvyhunnS+Lbk96u81NiL1/RjyZA0PKg4zhr1FKpxy71vpKTQZbBA9RmvKf8AYLa+P+8YXkefHUq1bYNy4Za5XsjiEQ1oV4Oo7ERCnsxZLhw1RSvvCCEqw4QGaqxZtnMJqXy47gQzN8bY7VO7ck7Kt6AhQEH1S1tNrXlWN4mbUghR8iv8pplEc9l5u31s41Bf77hg54WUClJvwtLe9FIcQCspeUs3ii+NT2I7IcZ2GBZ3Ox1JkqzBao1Sbl0Ft1AhEh5RHoQQlXF8sr2cl5ce5Id9l5HJ4MM+jdnxTheWDGljt4T0vcQiTkq1eVI7jdOmGoTKkvlJPY2e8gPW10euOGy3vyVu5Ow0FHdjG/OXJ5wyL0J57TB3pUAG6t5jvalDnmNsM2QXbqfbPVcr5VbRP3sztcAEQEHB6vKKEKIyjm0gWqOUs2DgfQzrVJualfzztIlQ28wIrxLs+iqf16nEM7rJbDe2wFemY4FqHsMUfwASm07esNt33/k7jP3pCM2n/MktB4s8uov8MnYWHeokP07Qj30gOR4q1mUgH3NQchwvS7fx4hLTdWyPSbA+Vyvk1AsLQC6Duxn6Ar2iwoLV5REhRGUcWy/n+TaR9GpaJd99bUWrdc0KvP5gHZevl44vr+jH8b2hO3KZxIeqFUxTLjE3DrPhwu101kZfJVNvZPXBeJev4wz/xiXS8qMtrD18hTmbzzDwu312Htjzir9Yqvovcl0qSaFt4JWt3FDm37DNsracheE/5Hh5aoUcH5WCWtnVzbtjb+d7HuER5UXckTKObbarefWQAve1Fa0AjbLI7SSMKPjQMJSLUgTvK1cwSLmVSNktRunfJI28jdtz90dyF/2/3gvYp9H/OHYdo9FIlysLmKX6DoC1xk5MiH+V3UZ/h6uTWLibkXcxSguW9rtREYFcuJ3O7vP5C1HuDKRAeERlHtv+Rc2rF9y32HaOVXiQT55Ojq4hY5HxMYbrR5MpqemiOMrP6o+owp08e7p78Y/baVo++r9TDl+bu+EIPuuGUenIVwB8qn+asfo30KGizcdbuZmS/zCxICGyYIkT7YnN+z4F+SOEqIxj29emTmjBs7MftokZVQvxdcs6ZX+a2tBfN4kEKYRG8sv8pvmQJjL7uqLcHtG1pMx7qsYev+YYi3fnXMNy+kok85l2Er0V/6KTFIzWjeAz49PkbRXnmCt3zVXYlvlgjrAE/51ZpFKQgxCiMs5Zm+K6woSlf5ucTgaVA9VuGzIdl+rQTzuVGFMk4bIkflZ/RDf5IevrtpdZc+gKHWb9xcS1x4t8vWPZ/XksSBLUk13hN/UkWsljuSsF8KLuPX4zdXLpvJYYkW2mMTfOTEoW5EUIURnHEhdq4ESvmrBAHyb3bUzvZlXoVC/UrbGbq4TyrG4yO43N8JNp+VY1l6GKjYBkd505m82rzq464L4Adgf5CfNiAPJbxJnCeUo3lX+lRkU+X6MChKhWJb9yMSXD3Yg7VsaZ278FwzrVZvnL7Zzaf2jH2nw58D7USjmVA93b0CwVP17Wv8MKQzfkMonJqu+ZolzGwQvmSuyLt9O5kXLvbUOSbbJbzyr+ZpnqvwTJMjhgasBTuqnESflnDp2hdmhOEejDDcP4c8yD1udKhZx6hQyBBXkRWbMyTs1K/nzYp3GRju1/fyTRl5PoXL+y2+wxoOR9w8tclMKZqPyRIco/qXEugbirP9F1/iG7fQ9dukvrmhVcOv+fJ2+gM5qQYWKccjUjlesBWGfswLv618wrbRSB+mEBnMtuA+urUtC7eRUOXkzk0/4tCfazrwuKigjk1PXy08bYHQiPSJAvKoWcOc+24ImW5lnnDXMNSeY915L973WjV5MIlr/sSk8iGd8a+/CG/m0yJTUPK46Q+fUjROTKqD29YI/TZzx9PYWdZ28xce1xNOiYr/rCKkKfGfrxtn6k0yKkVsjzrJZiOyVFrZTzxYBW7JnQLY8IQcFxImdWYSmPCI9I4DQ/vd6enw/Gs/PcbbpGhfJES/PCmAsHtQbgzzEP8sinO50+32ZTW57XVeQ79Rwayy/xm2YSw3TvcFKq5bJtj372DwAVSWGl+hNay8+hkxRM1L/KL6YHCznansmPN+b+mhXpOS/nvdh2mlQp5MhkMvIrOQoPyr+LaEGB7vKMECKB0wT7qnilcx1e6ey44rooGaOjUj366aaxWPU/Gsivslo9lTf1b/KX6T6Xz1VXdpXFqtnUlCeQLPkxXD+GvaYmTh8fM60XZ26k0qxacJGWfbbQsZ55KBsaqOHrQa1Z9E8c3RqF8fux60x93Hl7yhNCiARuZdnLbTkan8SnW8863RL2ihTKM7opfKn6jM6KE3yr+oSphpdYbuzJTwcu81yb/NsGS5LEznO3eUB+iq9VcwmWZXDZFMpQ/bucl6rle1xuOtevjI9KQYvIEKDgCcCGQjpahgf58M+7XfHXKKnor+a+geY411P3VXfanvKGiBEJ3MpDDUJ5q1t94mb2ZtoTzv/6p+DPUP27rDJ0QSGT+Ei1jEnK5Uz85Wi+q56u2H+J2hM3sH7pHJarZhIsy+CQqT5P6qa5JEID29VgfC/7ia4FCZExn/XibIms6Od1yyh5EuERCYqNQe1r8UzrSHxUclIyDRy5koRCJuPFRfsd7m9AyQTDq1yUIpigWsXLyk1EyhLo+T8dWyc8ZrdixYGLibz/63HGKNfwtvJXAH43PsB/9MOdDkprlHJipvVyOKeuICGKKEJnAkHBlAqP6Msvv6RWrVr4+PjQrl07/v33X0+bJHATvmoFMpmMYD8VDzUIpVP9ypyf8VgBR8hYaHycEbq30EoqeigO85P6I56c9YvdXi8u3ME81ZdWEfrS8Dhv6kcVKkIrXmnHzKea8dR91Yie1CPfib1+Nt0qVQoZnz7XglWvPcDsZ5rbLaAocA8yycNLLPz000+89NJLLFy4kHbt2jFv3jx+/vlnzpw5Q1hYWIHHpqSkEBwcTHJyMkFBQSVkscAdnLiaTJ/5uwrc5z7ZWb5Rz6WyLIVrUkU+D/+Yj4cPYPGfB2mxZyRt5WfQSwreMwzjZ2MXh+d4qEEo9cMCaFkjhPZ1KlHJhWWZLU3w64UFFLkTQXnH2e+ox4WoXbt2tGnThi+++AIAk8lEZGQkb775JhMmTCjwWCFEZYdPt5zls23n8myPlN1kiWo29eTXSJN8mCMNYjDrqS2/SYrkx3D9aPaYmuY57vthbelcP/+VaAUlg1cIkU6nw8/PjzVr1vDkk09atw8ePJikpCTWrVtnt79Wq0WrzWnTYFlyWghR2SI5U0+LqX9anweRxgLVZ3RUnLRuizeFMlT/DrGSORM1/KG6vP5gHYJ8VSRn6kWguJTgrBB5NFh9+/ZtjEYj4eHhdtvDw8OJiYnJs//MmTOZOnVqSZkn8BDBviouzuoNQKbOyJI9cQzZNJ6PpUX0V+7giKkur+jGEVolkn9fbkNYoH3wWIiQ9+FVWbOJEycyduxY63OLRyQou/iqFYzoUo8RXephNPbl/Nlj1KvVkIO+InNVlvCoEFWuXBmFQsHNmzfttt+8eZOIiIg8+2s0GjQa54ONgrKFQiGnbqOWnjZDUAx4NH2vVqtp3bo127Zts24zmUxs27aN9u3be9AygUBQknh8aDZ27FgGDx7M/fffT9u2bZk3bx7p6ekMHTrU06YJBIISwuNC9Nxzz3Hr1i0mTZrEjRs3aNmyJZs2bcoTwBYIBGUXj9cR3QuijkggKN04+x0tFVM8BAJB+UYIkUAg8DhCiAQCgcfxeLD6XrCEt1JSRKNygaA0YvluFhaK9mohSk01z44W1dUCQekmNTWV4OD826d4ddbMZDJx7do1AgMDy3SbBstUlvj4+HKfHRT3woy33AdJkkhNTaVq1arI5flHgrzaI5LL5VSvXn76AAcFBZXqD11JIu6FGW+4DwV5QhZEsFogEHgcIUQCgcDjCCHyAjQaDZMnTxadBxD3wkJZuw9eHawWCARlA+ERCQQCjyOESCAQeBwhRAKBwOMIIRIIBB5HCFEpJTExkYEDBxIUFERISAjDhg0jLS2twGO++eYbunTpQlBQEDKZjKSkpJIx1s24uvLvzz//TMOGDfHx8aFZs2Zs2LChhCwtXly5DydPnuTpp5+mVq1ayGQy5s2bV3KGugEhRKWUgQMHcvLkSbZs2cLvv//Ozp07ee211wo8JiMjg169evHee++VkJXu56effmLs2LFMnjyZw4cP06JFC3r27ElCQoLD/ffs2cOAAQMYNmwY0dHRPPnkkzz55JOcOHGihC13L67eh4yMDOrUqcOsWbMcLjxR6pEEpY5Tp05JgHTgwAHrto0bN0oymUy6evVqocdv375dAqS7d+8Wo5XFQ9u2baWRI0danxuNRqlq1arSzJkzHe7fv39/qXfv3nbb2rVrJ73++uvFamdx4+p9sKVmzZrSp59+WozWuR/hEZVC9u7dS0hICPfff791W/fu3ZHL5ezfv9+DlhUvOp2OQ4cO0b17d+s2uVxO9+7d2bt3r8Nj9u7da7c/QM+ePfPd3xsoyn3wdoQQlUJu3LhBWFiY3TalUknFihW5ceOGh6wqfgpa+Te/933jxg2X9vcGinIfvB0hRCXIhAkTkMlkBT4cLbUtEJR1vLoNiLfxn//8hyFDhhS4T506dYiIiMgTlDQYDCQmJnpnINJJXF35FyAiIsKl/b2BotwHb0d4RCVIaGgoDRs2LPChVqtp3749SUlJHDp0yHrsX3/9hclkol27dh58B8VLUVb+bd++vd3+AFu2bPHqlYLL5QrIno6WCxzTq1cvqVWrVtL+/fulXbt2SfXr15cGDBhgff3KlStSVFSUtH//fuu269evS9HR0dK3334rAdLOnTul6Oho6c6dO554C0Vi1apVkkajkZYuXSqdOnVKeu2116SQkBDpxo0bkiRJ0qBBg6QJEyZY99+9e7ekVCqlOXPmSKdPn5YmT54sqVQq6fjx4556C27B1fug1Wql6OhoKTo6WqpSpYo0btw4KTo6Wjp37pyn3oJLCCEqpdy5c0caMGCAFBAQIAUFBUlDhw6VUlNTra/HxcVJgLR9+3brtsmTJ0tAnseSJUtK/g3cA/Pnz5dq1KghqdVqqW3bttK+ffusrz300EPS4MGD7fZfvXq11KBBA0mtVktNmjSR/vjjjxK2uHhw5T5YPg+5Hw899FDJG14ERBsQgUDgcUSMSCAQeBwhRAKBwOMIIRIIBB5HCJFAIPA4QogEAoHHEUIkEAg8jhAigUDgcYQQCQQCjyOESOB2hgwZ4rCzQK9evTxtmqCUImbfC4qFXr16sWTJErtt+a1KqtfrUalUdtt0Oh1qtdrl6xb1OIFnER6RoFjQaDRERETYPSpUqACATCZjwYIFPP744/j7+/Pxxx8zZcoUWrZsyXfffUft2rXx8fEB4PLlyzzxxBMEBAQQFBRE//797dpj5HecwLsQQiTwCFOmTKFfv34cP36cl19+GYDY2Fh++eUX1q5dy5EjRzCZTDzxxBMkJiayY8cOtmzZwoULF3juuefszpX7OIH3IYZmgmLh999/JyAgwG7be++9Z11h5IUXXmDo0KF2r+t0OpYvX05oaChg7it0/Phx4uLiiIyMBGD58uU0adKEAwcO0KZNG4fHCbwPIUSCYqFr164sWLDAblvFihWtf9suDGChZs2admJy+vRpIiMjrSIE0LhxY0JCQjh9+rRViHIfJ/A+hBAJigV/f3/q1atX4OvObHP2WgLvRsSIBKWWRo0aER8fT3x8vHXbqVOnSEpKonHjxh60TOBuhEckKBa0Wm2epW+USiWVK1d2+hzdu3enWbNmDBw4kHnz5mEwGBgxYgQPPfSQw6GdwHsRHpGgWNi0aRNVqlSxe3Tq1Mmlc8hkMtatW0eFChV48MEH6d69O3Xq1OGnn34qJqsFnkK0ihUIBB5HeEQCgcDjCCESCAQeRwiRQCDwOEKIBAKBxxFCJBAIPI4QIoFA4HGEEAkEAo8jhEggEHgcIUQCgcDjCCESCAQeRwiRQCDwOEKIBAKBx/l/LT7suJCLTo0AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019598, + "end_time": "2024-03-26T11:17:30.952725", + "exception": false, + "start_time": "2024-03-26T11:17:30.933127", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 3006.118528, + "end_time": "2024-03-26T11:17:33.693958", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/realtabformer/2/mlu-eval.ipynb", + "output_path": "eval/insurance/realtabformer/2/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/insurance/realtabformer/2", + "path_prefix": "../../../../", + "random_seed": 2, + "single_model": "realtabformer" + }, + "start_time": "2024-03-26T10:27:27.575430", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/insurance/realtabformer/model.pt b/insurance/realtabformer/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c227c1c9bdd7035d2a679e3c874ba668af7d1c4 --- /dev/null +++ b/insurance/realtabformer/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ad88a05d944e07adc9302f8d8ad9e3718eb20dfc57ab9f223af6392daf326d7 +size 43501485 diff --git a/insurance/realtabformer/params.json b/insurance/realtabformer/params.json new file mode 100644 index 0000000000000000000000000000000000000000..90dceb7b341a233b4d0fb13b25888dbbc4448b9c --- /dev/null +++ b/insurance/realtabformer/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.77, "head_final_mul": "identity", "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "loss_balancer_beta": 0.75, "loss_balancer_r": 0.95, "dataset_size": 2048, "batch_size": 8, "epochs": 100, "n_warmup_steps": 100, "Optim": "diffgrad", "fixed_role_model": "realtabformer", "mse_mag": true, "mse_mag_target": 0.1, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "leakyrelu", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "relu6", "tf_activation_final": "leakyhardsigmoid", "tf_num_inds": 32, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "relu", "ada_activation_final": "softsign", "head_d_hid": 128, "head_n_layers": 9, "head_n_head": 64, "head_activation": "rrelu", "head_activation_final": "softsign", "models": ["realtabformer"], "max_seconds": 3600} \ No newline at end of file diff --git a/insurance/tab_ddpm_concat/eval.csv b/insurance/tab_ddpm_concat/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..9c75aefac67d4a843e4033c34f5831e2a37377e6 --- /dev/null +++ b/insurance/tab_ddpm_concat/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tab_ddpm_concat,0.00222469786672832,0.6476468851148562,0.023898356467834673,1.9961369037628174,0.10473108291625977,0.9842270016670227,0.1544250249862671,4.775751949637197e-05,3.285273790359497,0.10506823658943176,0.9151608347892761,0.1545909345149994,0.059609778225421906,0.702870786190033,5.2814106941223145 diff --git a/insurance/tab_ddpm_concat/history.csv b/insurance/tab_ddpm_concat/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..ad7aada2fba69f67da594a33fb831007be9ace7c --- /dev/null +++ b/insurance/tab_ddpm_concat/history.csv @@ -0,0 +1,9 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.024075572021215016,2.3508909012960273,0.002771060421534274,0.0029211476824194606,0.0,0.0,0.0,0.0,0.050037135023028693,900,225,310.30886697769165,1.3791505199008518,0.34478762997521295,0.06582206586789753,0.013342676964950645,2.0058086246364546,0.0007188991684725465,0.0,0.0,0.0,0.0,0.0,0.013342676964950645,450,113,89.34503555297852,0.7906640314422877,0.19854452345106338,0.04214210955805219 +1,0.013382578615047451,3.458099740034493,0.0009716474906536288,3.707941207620833e-05,0.0,0.0,0.0,0.0,0.016183423791856815,900,225,311.0531280040741,1.3824583466847737,0.34561458667119344,0.04208948137238622,0.014266901978052273,4.464227094694783,0.0011343707141915163,0.0,0.0,0.0,0.0,0.0,0.014266901978052273,450,113,89.86122393608093,0.7952320702308047,0.19969160874684652,0.01992809144054762 +2,0.013254342839516337,3.1204963845842646,0.0006813948405281079,0.0001075446810379314,0.0,0.0,0.0,0.0,0.0524565856976551,900,225,312.3241837024689,1.3881074831220839,0.34702687078052097,0.0423520589288738,0.012265912334292403,1.3157129772379734,0.0005628797040111236,0.0,0.0,0.0,0.0,0.0,0.012265912334292403,450,113,90.2688856124878,0.7988396956857327,0.2005975235833062,0.03948800182546926 +3,0.012562644095063054,3.5180461831355214,0.0006036552235798092,6.118627109875282e-06,0.0,0.0,0.0,0.0,0.012832146921718959,900,225,312.1873142719269,1.3874991745418972,0.3468747936354743,0.04454210979657041,0.013842605696991086,0.963626003249711,0.0003593607491630893,0.0,0.0,0.0,0.0,0.0,0.013842605696991086,450,113,91.40894317626953,0.8089287006749516,0.20313098483615452,0.04356716709021907 +4,0.013598391496020162,3.5109803807122972,0.000997141477106349,9.703711192640993e-06,0.0,0.0,0.0,0.0,0.02892149961404761,900,225,311.90593457221985,1.386248598098755,0.3465621495246887,0.04308184798806906,0.01243379472349059,2.2559976406838214,0.0005069997021732793,0.0,0.0,0.0,0.0,0.0,0.01243379472349059,450,113,91.92064929008484,0.8134570733635826,0.20426810953352187,0.030669460783968063 +5,0.012828913821463681,3.2254131293075554,0.0009589220510274338,2.19366033302827e-05,0.0,0.0,0.0,0.0,0.06420697301956049,900,225,315.50258231163025,1.402233699162801,0.35055842479070026,0.04400004963131828,0.012932158961290649,2.3544916858941582,0.0008117084715219698,0.0,0.0,0.0,0.0,0.0,0.012932158961290649,450,113,95.47646307945251,0.8449244520305532,0.21216991795433893,0.025758121650979186 +6,0.012587066907209293,3.38541289440383,0.0006065711585707823,7.494021996131374e-05,0.0,0.0,0.0,0.0,0.01373073202017824,900,225,317.88066506385803,1.412802955839369,0.3532007389598423,0.04219078679051664,0.011988562133143811,1.5825580488287612,0.0005653089338077846,0.0,0.0,0.0,0.0,0.0,0.011988562133143811,450,113,92.79533505439758,0.8211976553486512,0.20621185567643907,0.03183364872284959 +7,0.012521472344815265,2.8174274650184175,0.0006091171978181395,2.9657912544078297e-05,0.0,0.0,0.0,0.0,0.04151430725343137,900,225,315.3764338493347,1.4016730393303765,0.3504182598325941,0.04545953687590858,0.013241176685518843,1.917701503053899,0.0007860712953399703,0.0,0.0,0.0,0.0,0.0,0.013241176685518843,450,113,90.48038101196289,0.8007113363890521,0.20106751335991754,0.037769694870288396 diff --git a/insurance/tab_ddpm_concat/mlu-eval.ipynb b/insurance/tab_ddpm_concat/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..0c83dfce63b61312f7237a942a39b99411397481 --- /dev/null +++ b/insurance/tab_ddpm_concat/mlu-eval.ipynb @@ -0,0 +1,2389 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:18:06.857147Z", + "iopub.status.busy": "2024-03-26T13:18:06.856394Z", + "iopub.status.idle": "2024-03-26T13:18:06.890084Z", + "shell.execute_reply": "2024-03-26T13:18:06.889443Z" + }, + "papermill": { + "duration": 0.048217, + "end_time": "2024-03-26T13:18:06.891959", + "exception": false, + "start_time": "2024-03-26T13:18:06.843742", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:18:06.917815Z", + "iopub.status.busy": "2024-03-26T13:18:06.917088Z", + "iopub.status.idle": "2024-03-26T13:18:06.924119Z", + "shell.execute_reply": "2024-03-26T13:18:06.923293Z" + }, + "papermill": { + "duration": 0.02215, + "end_time": "2024-03-26T13:18:06.926111", + "exception": false, + "start_time": "2024-03-26T13:18:06.903961", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:18:06.949693Z", + "iopub.status.busy": "2024-03-26T13:18:06.949058Z", + "iopub.status.idle": "2024-03-26T13:18:06.952857Z", + "shell.execute_reply": "2024-03-26T13:18:06.952060Z" + }, + "papermill": { + "duration": 0.017505, + "end_time": "2024-03-26T13:18:06.954692", + "exception": false, + "start_time": "2024-03-26T13:18:06.937187", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:18:06.978222Z", + "iopub.status.busy": "2024-03-26T13:18:06.977682Z", + "iopub.status.idle": "2024-03-26T13:18:06.981437Z", + "shell.execute_reply": "2024-03-26T13:18:06.980639Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017655, + "end_time": "2024-03-26T13:18:06.983299", + "exception": false, + "start_time": "2024-03-26T13:18:06.965644", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:18:07.006635Z", + "iopub.status.busy": "2024-03-26T13:18:07.006132Z", + "iopub.status.idle": "2024-03-26T13:18:07.011429Z", + "shell.execute_reply": "2024-03-26T13:18:07.010656Z" + }, + "papermill": { + "duration": 0.018988, + "end_time": "2024-03-26T13:18:07.013248", + "exception": false, + "start_time": "2024-03-26T13:18:06.994260", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "3200873d", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:18:07.038893Z", + "iopub.status.busy": "2024-03-26T13:18:07.038295Z", + "iopub.status.idle": "2024-03-26T13:18:07.043515Z", + "shell.execute_reply": "2024-03-26T13:18:07.042777Z" + }, + "papermill": { + "duration": 0.02003, + "end_time": "2024-03-26T13:18:07.045419", + "exception": false, + "start_time": "2024-03-26T13:18:07.025389", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"tab_ddpm_concat\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 42\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/tab_ddpm_concat/42\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011123, + "end_time": "2024-03-26T13:18:07.067793", + "exception": false, + "start_time": "2024-03-26T13:18:07.056670", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:18:07.091461Z", + "iopub.status.busy": "2024-03-26T13:18:07.090970Z", + "iopub.status.idle": "2024-03-26T13:18:07.099681Z", + "shell.execute_reply": "2024-03-26T13:18:07.098936Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.022689, + "end_time": "2024-03-26T13:18:07.101519", + "exception": false, + "start_time": "2024-03-26T13:18:07.078830", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/tab_ddpm_concat/42\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:18:07.125083Z", + "iopub.status.busy": "2024-03-26T13:18:07.124431Z", + "iopub.status.idle": "2024-03-26T13:18:09.067763Z", + "shell.execute_reply": "2024-03-26T13:18:09.066818Z" + }, + "papermill": { + "duration": 1.957446, + "end_time": "2024-03-26T13:18:09.070040", + "exception": false, + "start_time": "2024-03-26T13:18:07.112594", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:18:09.096552Z", + "iopub.status.busy": "2024-03-26T13:18:09.096090Z", + "iopub.status.idle": "2024-03-26T13:18:09.108701Z", + "shell.execute_reply": "2024-03-26T13:18:09.107746Z" + }, + "papermill": { + "duration": 0.027876, + "end_time": "2024-03-26T13:18:09.110775", + "exception": false, + "start_time": "2024-03-26T13:18:09.082899", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:18:09.134993Z", + "iopub.status.busy": "2024-03-26T13:18:09.134733Z", + "iopub.status.idle": "2024-03-26T13:18:09.142137Z", + "shell.execute_reply": "2024-03-26T13:18:09.141430Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021664, + "end_time": "2024-03-26T13:18:09.144024", + "exception": false, + "start_time": "2024-03-26T13:18:09.122360", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:18:09.168074Z", + "iopub.status.busy": "2024-03-26T13:18:09.167417Z", + "iopub.status.idle": "2024-03-26T13:18:09.259597Z", + "shell.execute_reply": "2024-03-26T13:18:09.258882Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.10655, + "end_time": "2024-03-26T13:18:09.261756", + "exception": false, + "start_time": "2024-03-26T13:18:09.155206", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:18:09.287291Z", + "iopub.status.busy": "2024-03-26T13:18:09.287014Z", + "iopub.status.idle": "2024-03-26T13:18:13.819637Z", + "shell.execute_reply": "2024-03-26T13:18:13.818864Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.548263, + "end_time": "2024-03-26T13:18:13.822252", + "exception": false, + "start_time": "2024-03-26T13:18:09.273989", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-26 13:18:11.480090: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-26 13:18:11.480144: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-26 13:18:11.481702: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:18:13.847446Z", + "iopub.status.busy": "2024-03-26T13:18:13.846918Z", + "iopub.status.idle": "2024-03-26T13:18:13.853485Z", + "shell.execute_reply": "2024-03-26T13:18:13.852798Z" + }, + "papermill": { + "duration": 0.021141, + "end_time": "2024-03-26T13:18:13.855384", + "exception": false, + "start_time": "2024-03-26T13:18:13.834243", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:18:13.881072Z", + "iopub.status.busy": "2024-03-26T13:18:13.880796Z", + "iopub.status.idle": "2024-03-26T13:18:22.377706Z", + "shell.execute_reply": "2024-03-26T13:18:22.376783Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.512615, + "end_time": "2024-03-26T13:18:22.380216", + "exception": false, + "start_time": "2024-03-26T13:18:13.867601", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-26T13:18:22.408364Z", + "iopub.status.busy": "2024-03-26T13:18:22.407491Z", + "iopub.status.idle": "2024-03-26T13:18:22.414134Z", + "shell.execute_reply": "2024-03-26T13:18:22.413314Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.022796, + "end_time": "2024-03-26T13:18:22.416000", + "exception": false, + "start_time": "2024-03-26T13:18:22.393204", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 36,\n", + " 'realtabformer': (19, 551, Embedding(551, 800), True),\n", + " 'lct_gan': 29,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:18:22.440788Z", + "iopub.status.busy": "2024-03-26T13:18:22.440531Z", + "iopub.status.idle": "2024-03-26T13:18:22.445231Z", + "shell.execute_reply": "2024-03-26T13:18:22.444372Z" + }, + "papermill": { + "duration": 0.019242, + "end_time": "2024-03-26T13:18:22.447038", + "exception": false, + "start_time": "2024-03-26T13:18:22.427796", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:18:22.472114Z", + "iopub.status.busy": "2024-03-26T13:18:22.471447Z", + "iopub.status.idle": "2024-03-26T13:18:47.088831Z", + "shell.execute_reply": "2024-03-26T13:18:47.087875Z" + }, + "papermill": { + "duration": 24.632647, + "end_time": "2024-03-26T13:18:47.091343", + "exception": false, + "start_time": "2024-03-26T13:18:22.458696", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/tab_ddpm_concat/all inf False\n", + "../../../../ml-utility-loss/aug_test/insurance 0\n", + "Caching in ../../../../insurance/_cache_bs_test/tab_ddpm_concat/all inf False\n", + "../../../../ml-utility-loss/bs_test/insurance 0\n", + "Caching in ../../../../insurance/_cache_synth_test/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/insurance 600\n", + "600\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:18:47.120367Z", + "iopub.status.busy": "2024-03-26T13:18:47.120078Z", + "iopub.status.idle": "2024-03-26T13:18:47.441414Z", + "shell.execute_reply": "2024-03-26T13:18:47.440386Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.338698, + "end_time": "2024-03-26T13:18:47.443539", + "exception": false, + "start_time": "2024-03-26T13:18:47.104841", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 64,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.6896836352825375,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'loss_balancer_beta': 0.7520229775744602,\n", + " 'loss_balancer_r': 0.9706519501751338,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.08030439779404704,\n", + " 'n_warmup_steps': 85,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'tab_ddpm_concat',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.Sigmoid,\n", + " 'tf_d_inner': 256,\n", + " 'tf_n_layers_enc': 5,\n", + " 'tf_n_head': 128,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 256,\n", + " 'ada_n_layers': 8,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU6,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 32,\n", + " 'head_activation': torch.nn.modules.activation.ReLU6,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['tab_ddpm_concat'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.13044551835398707,\n", + " 'multiply': True,\n", + " 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "# params = {\n", + "# **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "# }\n", + "params = getattr(PARAMS, dataset_name).BEST_DICT[gp][gp_multiply][single_model]\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).DEFAULTS,\n", + " **params,\n", + "}\n", + "if isinstance(params, (list, tuple)):\n", + " params = params[param_index]\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:18:47.471234Z", + "iopub.status.busy": "2024-03-26T13:18:47.470950Z", + "iopub.status.idle": "2024-03-26T13:19:40.041991Z", + "shell.execute_reply": "2024-03-26T13:19:40.040994Z" + }, + "papermill": { + "duration": 52.600589, + "end_time": "2024-03-26T13:19:40.057322", + "exception": false, + "start_time": "2024-03-26T13:18:47.456733", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_train/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/insurance [400, 0]\n", + "Caching in ../../../../insurance/_cache_aug_val/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/insurance [0, 200]\n", + "Caching in ../../../../insurance/_cache_bs_train/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/insurance [100, 0]\n", + "Caching in ../../../../insurance/_cache_bs_val/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/insurance [0, 50]\n", + "Caching in ../../../../insurance/_cache_synth/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/insurance [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-26T13:19:40.085752Z", + "iopub.status.busy": "2024-03-26T13:19:40.085367Z", + "iopub.status.idle": "2024-03-26T13:19:40.498539Z", + "shell.execute_reply": "2024-03-26T13:19:40.497593Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.429458, + "end_time": "2024-03-26T13:19:40.500584", + "exception": false, + "start_time": "2024-03-26T13:19:40.071126", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['tab_ddpm_concat'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:19:40.530743Z", + "iopub.status.busy": "2024-03-26T13:19:40.530230Z", + "iopub.status.idle": "2024-03-26T13:19:40.534913Z", + "shell.execute_reply": "2024-03-26T13:19:40.534144Z" + }, + "papermill": { + "duration": 0.022314, + "end_time": "2024-03-26T13:19:40.536860", + "exception": false, + "start_time": "2024-03-26T13:19:40.514546", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:19:40.563978Z", + "iopub.status.busy": "2024-03-26T13:19:40.563736Z", + "iopub.status.idle": "2024-03-26T13:19:40.570696Z", + "shell.execute_reply": "2024-03-26T13:19:40.569782Z" + }, + "papermill": { + "duration": 0.022847, + "end_time": "2024-03-26T13:19:40.572696", + "exception": false, + "start_time": "2024-03-26T13:19:40.549849", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "8696065" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:19:40.599573Z", + "iopub.status.busy": "2024-03-26T13:19:40.599304Z", + "iopub.status.idle": "2024-03-26T13:19:40.700993Z", + "shell.execute_reply": "2024-03-26T13:19:40.700178Z" + }, + "papermill": { + "duration": 0.117684, + "end_time": "2024-03-26T13:19:40.703249", + "exception": false, + "start_time": "2024-03-26T13:19:40.585565", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 12] --\n", + "├─Adapter: 1-1 [2, 1071, 12] --\n", + "│ └─Sequential: 2-1 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 256] 3,328\n", + "│ │ │ └─ReLU6: 4-2 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-4 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-6 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-8 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-10 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-12 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-14 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-8 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-15 [2, 1071, 256] 65,792\n", + "│ │ │ └─LeakyHardtanh: 4-16 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 12] (recursive)\n", + "│ └─Sequential: 2-2 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-9 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 256] (recursive)\n", + "│ │ │ └─ReLU6: 4-18 [2, 267, 256] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 256] (recursive)\n", + "│ │ │ └─ReLU6: 4-20 [2, 267, 256] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 256] (recursive)\n", + "│ │ │ └─ReLU6: 4-22 [2, 267, 256] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 256] (recursive)\n", + "│ │ │ └─ReLU6: 4-24 [2, 267, 256] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 256] (recursive)\n", + "│ │ │ └─ReLU6: 4-26 [2, 267, 256] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─ReLU6: 4-28 [2, 267, 256] --\n", + "│ │ └─FeedForward: 3-15 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 267, 256] (recursive)\n", + "│ │ │ └─ReLU6: 4-30 [2, 267, 256] --\n", + "│ │ └─FeedForward: 3-16 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 267, 256] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-32 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 16384] --\n", + "│ └─Encoder: 2-3 [2, 64, 256] --\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 128, 64, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 128, 64, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─Sigmoid: 7-6 [2, 64, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 128, 1071, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 128, 1071, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─Sigmoid: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 65,792\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 128, 64, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 128, 64, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─Sigmoid: 7-18 [2, 64, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 128, 1071, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 128, 1071, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─Sigmoid: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-11 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 65,792\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 128, 64, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 128, 64, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─Sigmoid: 7-30 [2, 64, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 128, 1071, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 128, 1071, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─Sigmoid: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-17 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 65,792\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 128, 64, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 128, 64, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─Sigmoid: 7-42 [2, 64, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 128, 1071, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 128, 1071, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─Sigmoid: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 65,792\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 64, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-9 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-26 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 128, 64, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 128, 64, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─Sigmoid: 7-54 [2, 64, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-27 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 128, 1071, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 128, 1071, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─Sigmoid: 7-60 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-28 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-29 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 1071, 256] 65,792\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-11 [2, 64, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-31 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-32 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 128, 64, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 128, 64, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─Sigmoid: 7-66 [2, 64, 256] --\n", + "│ └─Encoder: 2-4 [2, 64, 256] (recursive)\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 128, 64, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 128, 64, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Sigmoid: 7-72 [2, 64, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 128, 267, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 128, 267, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Sigmoid: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-37 [2, 267, 256] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-39 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-40 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 128, 64, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 128, 64, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Sigmoid: 7-84 [2, 64, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 128, 267, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 128, 267, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Sigmoid: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-42 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-43 [2, 267, 256] --\n", + "│ │ │ │ │ └─Linear: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-40 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-45 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-46 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 128, 64, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 128, 64, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Sigmoid: 7-96 [2, 64, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 128, 267, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 128, 267, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Sigmoid: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-48 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-49 [2, 267, 256] --\n", + "│ │ │ │ │ └─Linear: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-41 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-18 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-51 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-52 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 128, 64, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 128, 64, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Sigmoid: 7-108 [2, 64, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-53 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-109 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-110 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-111 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-112 [2, 128, 267, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-19 [2, 128, 267, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-113 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Sigmoid: 7-114 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-19 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-54 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-55 [2, 267, 256] --\n", + "│ │ │ │ │ └─Linear: 6-56 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-42 [2, 64, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-20 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-57 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-58 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-115 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-116 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-117 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-118 [2, 128, 64, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-20 [2, 128, 64, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-119 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Sigmoid: 7-120 [2, 64, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-59 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-121 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-122 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-123 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-124 [2, 128, 267, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-21 [2, 128, 267, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-125 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Sigmoid: 7-126 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-21 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-60 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-61 [2, 267, 256] --\n", + "│ │ │ │ │ └─Linear: 6-62 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-22 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-63 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-64 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-127 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-128 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-129 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-130 [2, 128, 64, 2] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-22 [2, 128, 64, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-131 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Sigmoid: 7-132 [2, 64, 256] --\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-19 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 4,194,560\n", + "│ │ │ └─ReLU6: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-20 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-51 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-52 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 256] --\n", + "│ │ │ └─Linear: 4-53 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-54 [2, 256] --\n", + "│ │ └─FeedForward: 3-25 [2, 256] --\n", + "│ │ │ └─Linear: 4-55 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-56 [2, 256] --\n", + "│ │ └─FeedForward: 3-26 [2, 1] --\n", + "│ │ │ └─Linear: 4-57 [2, 1] 257\n", + "│ │ │ └─Softsign: 4-58 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 8,696,065\n", + "Trainable params: 8,696,065\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 25.21\n", + "========================================================================================================================\n", + "Input size (MB): 0.13\n", + "Forward/backward pass size (MB): 233.93\n", + "Params size (MB): 34.78\n", + "Estimated Total Size (MB): 268.84\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:19:40.734118Z", + "iopub.status.busy": "2024-03-26T13:19:40.733838Z", + "iopub.status.idle": "2024-03-26T14:22:50.465777Z", + "shell.execute_reply": "2024-03-26T14:22:50.464807Z" + }, + "papermill": { + "duration": 3789.764992, + "end_time": "2024-03-26T14:22:50.483197", + "exception": false, + "start_time": "2024-03-26T13:19:40.718205", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 600]\n", + "Creating model of type \n", + "[*] Embedding False True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.024075572021215016, 'avg_role_model_std_loss': 2.3508909012960273, 'avg_role_model_mean_pred_loss': 0.002771060421534274, 'avg_role_model_g_mag_loss': 0.0029211476824194606, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.050037135023028693, 'n_size': 900, 'n_batch': 225, 'duration': 310.30886697769165, 'duration_batch': 1.3791505199008518, 'duration_size': 0.34478762997521295, 'avg_pred_std': 0.06582206586789753}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013342676964950645, 'avg_role_model_std_loss': 2.0058086246364546, 'avg_role_model_mean_pred_loss': 0.0007188991684725465, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013342676964950645, 'n_size': 450, 'n_batch': 113, 'duration': 89.34503555297852, 'duration_batch': 0.7906640314422877, 'duration_size': 0.19854452345106338, 'avg_pred_std': 0.04214210955805219}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013382578615047451, 'avg_role_model_std_loss': 3.458099740034493, 'avg_role_model_mean_pred_loss': 0.0009716474906536288, 'avg_role_model_g_mag_loss': 3.707941207620833e-05, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.016183423791856815, 'n_size': 900, 'n_batch': 225, 'duration': 311.0531280040741, 'duration_batch': 1.3824583466847737, 'duration_size': 0.34561458667119344, 'avg_pred_std': 0.04208948137238622}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.014266901978052273, 'avg_role_model_std_loss': 4.464227094694783, 'avg_role_model_mean_pred_loss': 0.0011343707141915163, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.014266901978052273, 'n_size': 450, 'n_batch': 113, 'duration': 89.86122393608093, 'duration_batch': 0.7952320702308047, 'duration_size': 0.19969160874684652, 'avg_pred_std': 0.01992809144054762}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013254342839516337, 'avg_role_model_std_loss': 3.1204963845842646, 'avg_role_model_mean_pred_loss': 0.0006813948405281079, 'avg_role_model_g_mag_loss': 0.0001075446810379314, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0524565856976551, 'n_size': 900, 'n_batch': 225, 'duration': 312.3241837024689, 'duration_batch': 1.3881074831220839, 'duration_size': 0.34702687078052097, 'avg_pred_std': 0.0423520589288738}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012265912334292403, 'avg_role_model_std_loss': 1.3157129772379734, 'avg_role_model_mean_pred_loss': 0.0005628797040111236, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012265912334292403, 'n_size': 450, 'n_batch': 113, 'duration': 90.2688856124878, 'duration_batch': 0.7988396956857327, 'duration_size': 0.2005975235833062, 'avg_pred_std': 0.03948800182546926}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012562644095063054, 'avg_role_model_std_loss': 3.5180461831355214, 'avg_role_model_mean_pred_loss': 0.0006036552235798092, 'avg_role_model_g_mag_loss': 6.118627109875282e-06, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012832146921718959, 'n_size': 900, 'n_batch': 225, 'duration': 312.1873142719269, 'duration_batch': 1.3874991745418972, 'duration_size': 0.3468747936354743, 'avg_pred_std': 0.04454210979657041}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013842605696991086, 'avg_role_model_std_loss': 0.963626003249711, 'avg_role_model_mean_pred_loss': 0.0003593607491630893, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013842605696991086, 'n_size': 450, 'n_batch': 113, 'duration': 91.40894317626953, 'duration_batch': 0.8089287006749516, 'duration_size': 0.20313098483615452, 'avg_pred_std': 0.04356716709021907}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013598391496020162, 'avg_role_model_std_loss': 3.5109803807122972, 'avg_role_model_mean_pred_loss': 0.000997141477106349, 'avg_role_model_g_mag_loss': 9.703711192640993e-06, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.02892149961404761, 'n_size': 900, 'n_batch': 225, 'duration': 311.90593457221985, 'duration_batch': 1.386248598098755, 'duration_size': 0.3465621495246887, 'avg_pred_std': 0.04308184798806906}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01243379472349059, 'avg_role_model_std_loss': 2.2559976406838214, 'avg_role_model_mean_pred_loss': 0.0005069997021732793, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01243379472349059, 'n_size': 450, 'n_batch': 113, 'duration': 91.92064929008484, 'duration_batch': 0.8134570733635826, 'duration_size': 0.20426810953352187, 'avg_pred_std': 0.030669460783968063}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012828913821463681, 'avg_role_model_std_loss': 3.2254131293075554, 'avg_role_model_mean_pred_loss': 0.0009589220510274338, 'avg_role_model_g_mag_loss': 2.19366033302827e-05, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.06420697301956049, 'n_size': 900, 'n_batch': 225, 'duration': 315.50258231163025, 'duration_batch': 1.402233699162801, 'duration_size': 0.35055842479070026, 'avg_pred_std': 0.04400004963131828}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.012932158961290649, 'avg_role_model_std_loss': 2.3544916858941582, 'avg_role_model_mean_pred_loss': 0.0008117084715219698, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.012932158961290649, 'n_size': 450, 'n_batch': 113, 'duration': 95.47646307945251, 'duration_batch': 0.8449244520305532, 'duration_size': 0.21216991795433893, 'avg_pred_std': 0.025758121650979186}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012587066907209293, 'avg_role_model_std_loss': 3.38541289440383, 'avg_role_model_mean_pred_loss': 0.0006065711585707823, 'avg_role_model_g_mag_loss': 7.494021996131374e-05, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01373073202017824, 'n_size': 900, 'n_batch': 225, 'duration': 317.88066506385803, 'duration_batch': 1.412802955839369, 'duration_size': 0.3532007389598423, 'avg_pred_std': 0.04219078679051664}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.011988562133143811, 'avg_role_model_std_loss': 1.5825580488287612, 'avg_role_model_mean_pred_loss': 0.0005653089338077846, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011988562133143811, 'n_size': 450, 'n_batch': 113, 'duration': 92.79533505439758, 'duration_batch': 0.8211976553486512, 'duration_size': 0.20621185567643907, 'avg_pred_std': 0.03183364872284959}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.012521472344815265, 'avg_role_model_std_loss': 2.8174274650184175, 'avg_role_model_mean_pred_loss': 0.0006091171978181395, 'avg_role_model_g_mag_loss': 2.9657912544078297e-05, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.04151430725343137, 'n_size': 900, 'n_batch': 225, 'duration': 315.3764338493347, 'duration_batch': 1.4016730393303765, 'duration_size': 0.3504182598325941, 'avg_pred_std': 0.04545953687590858}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.013241176685518843, 'avg_role_model_std_loss': 1.917701503053899, 'avg_role_model_mean_pred_loss': 0.0007860712953399703, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013241176685518843, 'n_size': 450, 'n_batch': 113, 'duration': 90.48038101196289, 'duration_batch': 0.8007113363890521, 'duration_size': 0.20106751335991754, 'avg_pred_std': 0.037769694870288396}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.01243473827111302, 'avg_role_model_std_loss': 3.305539116058347, 'avg_role_model_mean_pred_loss': 0.0006290159570787659, 'avg_role_model_g_mag_loss': 2.9187890662190814e-05, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 3.4580708610135478, 'n_size': 900, 'n_batch': 225, 'duration': 313.4266664981842, 'duration_batch': 1.3930074066585965, 'duration_size': 0.3482518516646491, 'avg_pred_std': 0.04256669734501176}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.011888014511342691, 'avg_role_model_std_loss': 1.1363639000046413, 'avg_role_model_mean_pred_loss': 0.00047483680290026815, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011888014511342691, 'n_size': 450, 'n_batch': 113, 'duration': 87.56288599967957, 'duration_batch': 0.7748927964573413, 'duration_size': 0.19458419111039904, 'avg_pred_std': 0.039356029522399194}\n", + "Time out: 3648.186664581299/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tab_ddpm_concat', 'n_size': 600, 'n_batch': 150, 'role_model_metrics': {'avg_loss': 0.023898356550683578, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.0018930683376068675, 'pred_duration': 3.248927593231201, 'grad_duration': 1.9813952445983887, 'total_duration': 5.23032283782959, 'pred_std': 0.059609778225421906, 'std_loss': 0.702870786190033, 'mean_pred_loss': 4.775751585839316e-05, 'pred_rmse': 0.1545909345149994, 'pred_mae': 0.10506824404001236, 'pred_mape': 0.9151608347892761, 'grad_rmse': 0.1544250249862671, 'grad_mae': 0.10473109036684036, 'grad_mape': 0.9842270016670227}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.023898356550683578, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.0018930683376068675, 'avg_pred_duration': 3.248927593231201, 'avg_grad_duration': 1.9813952445983887, 'avg_total_duration': 5.23032283782959, 'avg_pred_std': 0.059609778225421906, 'avg_std_loss': 0.702870786190033, 'avg_mean_pred_loss': 4.775751585839316e-05}, 'min_metrics': {'avg_loss': 0.023898356550683578, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.0018930683376068675, 'pred_duration': 3.248927593231201, 'grad_duration': 1.9813952445983887, 'total_duration': 5.23032283782959, 'pred_std': 0.059609778225421906, 'std_loss': 0.702870786190033, 'mean_pred_loss': 4.775751585839316e-05, 'pred_rmse': 0.1545909345149994, 'pred_mae': 0.10506824404001236, 'pred_mape': 0.9151608347892761, 'grad_rmse': 0.1544250249862671, 'grad_mae': 0.10473109036684036, 'grad_mape': 0.9842270016670227}, 'model_metrics': {'tab_ddpm_concat': {'avg_loss': 0.023898356550683578, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.0018930683376068675, 'pred_duration': 3.248927593231201, 'grad_duration': 1.9813952445983887, 'total_duration': 5.23032283782959, 'pred_std': 0.059609778225421906, 'std_loss': 0.702870786190033, 'mean_pred_loss': 4.775751585839316e-05, 'pred_rmse': 0.1545909345149994, 'pred_mae': 0.10506824404001236, 'pred_mape': 0.9151608347892761, 'grad_rmse': 0.1544250249862671, 'grad_mae': 0.10473109036684036, 'grad_mape': 0.9842270016670227}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T14:22:50.517886Z", + "iopub.status.busy": "2024-03-26T14:22:50.517135Z", + "iopub.status.idle": "2024-03-26T14:22:50.521488Z", + "shell.execute_reply": "2024-03-26T14:22:50.520635Z" + }, + "papermill": { + "duration": 0.024304, + "end_time": "2024-03-26T14:22:50.523503", + "exception": false, + "start_time": "2024-03-26T14:22:50.499199", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T14:22:50.555762Z", + "iopub.status.busy": "2024-03-26T14:22:50.554926Z", + "iopub.status.idle": "2024-03-26T14:22:50.634146Z", + "shell.execute_reply": "2024-03-26T14:22:50.633338Z" + }, + "papermill": { + "duration": 0.097822, + "end_time": "2024-03-26T14:22:50.636515", + "exception": false, + "start_time": "2024-03-26T14:22:50.538693", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T14:22:50.672271Z", + "iopub.status.busy": "2024-03-26T14:22:50.671383Z", + "iopub.status.idle": "2024-03-26T14:22:50.963724Z", + "shell.execute_reply": "2024-03-26T14:22:50.962810Z" + }, + "papermill": { + "duration": 0.312342, + "end_time": "2024-03-26T14:22:50.965803", + "exception": false, + "start_time": "2024-03-26T14:22:50.653461", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAESCAYAAABU2qhcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABI9ElEQVR4nO2de3gT15n/vyPJknyV71d8BYxtbGwugRgSaBpvTJYmcZoCZWkgbAoND25I3NAsaQJpsxs32ZBNUvIrhTYk3Q2FTZqkLWVJiINTLiYEG4O5mKuxjW35BpZs2dZ1fn+MZyQZWZbkkTSSzud55gGPjkZnbOmrc97zvt9D0TRNg0AgEASAyNsdIBAIBBYiSAQCQTAQQSIQCIKBCBKBQBAMRJAIBIJgIIJEIBAEAxEkAoEgGCTe7gAfmEwmtLe3Izw8HBRFebs7BAJhFDRNo7+/H8nJyRCJxh4H+YUgtbe3IzU11dvdIBAI49Da2opJkyaN+bhfCFJ4eDgA5mYjIiK83BsCgTAatVqN1NRU7rM6Fn4hSOw0LSIigggSgSBgxgupkKA2gUAQDESQCASCYCCCRCAQBINfxJAcxWg0Qq/Xe7sbBB9HKpXaXbomuE5ACBJN01Aqlejr6/N2Vwh+gEgkQmZmJqRSqbe74ncEhCCxYhQfH4+QkBCSPElwGTYJt6OjA2lpaeS9xDN+L0hGo5ETo5iYGG93h+AHxMXFob29HQaDAUFBQdz53gEtaACxYTLvdc7H8fuJMBszCgkJ8XJPCP4CO1UzGo3cOZ3BhIe3H8P33jmKYb1xrKcSxsHvBYmFDK0JfGHrvXSxQ422viEo1cNo7h30Qq/8g4ARJALBnZy52cf9v6lnwHsd8XGIIBEIPFDf0sf9/3qPxnsd8XGIIBHsQlEUPvvsM293g1defvllFBUV8XrNessRUjcRJFchgkQQPO+//z4iIyN5u95zzz2Hqqoq3q6nGtTjuoUINZERkssQQSL4DTqdzqF2YWFhvKaAnG3rAwBIREywmwiS6wSkINE0jUGdwSuHsxsFHzx4EPfccw8iIyMRExOD733ve7h27RoAYP78+Xj++eet2nd3dyMoKAj/+Mc/AAAdHR1YsmQJgoODkZmZiT179iAjIwNvvfWWS7+7hoYGfPe730VwcDBiYmKwbt06DAyYg7jV1dWYO3cuQkNDERkZiQULFqC5uRkAcObMGdx3330IDw9HREQEZs+ejVOnTtl9verqaqxZswYqlQoURYGiKLz88ssAgIyMDLzyyitYtWoVIiIisG7dOgDA888/j+zsbISEhCArKwsvvfSSVcnQ6CnbE088gbKyMrzxxhtISkpCTEwMNmzY4HCZERs/+s60OABAr0YH1SApUXIFv0+MtMWQ3oi8LZ975bUv/KoUIVLHf+0ajQYVFRWYMWMGBgYGsGXLFjz66KOor6/HypUr8frrr+PXv/41txS9b98+JCcn49577wUArFq1Cj09PaiurkZQUBAqKirQ1dXlUt81Gg1KS0tRXFyMb7/9Fl1dXfjxj3+M8vJyvP/++zAYDCgrK8PatWvxpz/9CTqdDidPnuT6tnLlSsycORO//e1vIRaLUV9fb5VYaIv58+fjrbfewpYtW3Dp0iUAzAiH5Y033sCWLVuwdetW7lx4eDjef/99JCcno6GhAWvXrkV4eDh+/vOfj/k6hw8fRlJSEg4fPoyrV69i+fLlKCoqwtq1a8f9vbArbPMnx6KhTYVOtRZNvRoUhUSO+1yCNQEpSL7EY489ZvXze++9h7i4OFy4cAHLli3DM888g6NHj3ICtGfPHqxYsQIURaGxsRFffvklvv32W8yZMwcA8Pvf/x5Tp051qS979uzB8PAw/vjHPyI0NBQAsH37djz00EN47bXXEBQUBJVKhe9973uYPHkyACA3N5d7fktLCzZt2oScnBwAcKgfUqkUCoUCFEUhMTHxjse/+93v4mc/+5nVuRdffJH7f0ZGBp577jns3bvXriBFRUVh+/btEIvFyMnJwZIlS1BVVTWuINE0jfrWPgBAYWokMmNDGUHqGUBRauS490ewJiAFKThIjAu/KvXaazvDlStXsGXLFnzzzTfo6emByWQCwHy48/Pz8cADD+DDDz/Evffei6amJtTU1OB3v/sdAODSpUuQSCSYNWsWd70pU6YgKirKpb5fvHgRhYWFnBgBwIIFC2AymXDp0iUsXLgQTzzxBEpLS/FP//RPKCkpwbJly5CUlAQAqKiowI9//GP893//N0pKSrB06VJOuFyFFVpL9u3bh3feeQfXrl3DwMAADAbDuE6i06dPh1hs/tskJSWhoaFh3Ndv6xtCz4AOEhGF6ckRyIoLw4nrt6yC3ATHCcgYEkVRCJFKvHI4mzH+0EMP4datW9i1axe++eYbfPPNNwDMAdyVK1fi448/hl6vx549e1BQUICCggLef2eOsnv3btTU1GD+/PnYt28fsrOzceLECQBM7Ob8+fNYsmQJvvrqK+Tl5eHTTz+d0OtZiiMA1NTUYOXKlfjnf/5n7N+/H6dPn8YvfvGLcQPeo6eOFEVx4m8PdnSUmxQBeZAYWbFMf0gukmsEpCD5Cr29vbh06RJefPFF3H///cjNzcXt27et2jzyyCMYHh7GwYMHsWfPHqxcuZJ7bNq0aTAYDDh9+jR37urVq3dcw1Fyc3Nx5swZaDTmD9uxY8cgEokwbdo07tzMmTOxefNmHD9+HPn5+dizZw/3WHZ2Np599ll88cUX+P73v4/du3eP+7pSqdSqbswex48fR3p6On7xi19gzpw5mDp1KhdUdwdnRgSJnZ5ljggSyUVyDSJIAiYqKgoxMTHYuXMnrl69iq+++goVFRVWbUJDQ1FWVoaXXnoJFy9exIoVK7jHcnJyUFJSgnXr1uHkyZM4ffo01q1bh+DgYJdq+1auXAm5XI7Vq1fj3LlzOHz4MH7605/i8ccfR0JCApqamrB582bU1NSgubkZX3zxBa5cuYLc3FwMDQ2hvLwc1dXVaG5uxrFjx/Dtt99axZjGIiMjAwMDA6iqqkJPTw8GB8euFZs6dSpaWlqwd+9eXLt2De+8886ER2H2sIwfARaC1KNxekWV4KIgvfvuu8jIyIBcLse8efNw8uRJu+0/+ugj5OTkQC6Xo6CgAAcOHLijzcWLF/Hwww9DoVAgNDQUd911F1paWlzpnt8gEomwd+9e1NbWIj8/H88++yz+8z//8452K1euxJkzZ3DvvfciLS3N6rE//vGPSEhIwMKFC/Hoo49yK05yudzp/oSEhODzzz/HrVu3cNddd+EHP/gB7r//fmzfvp17vLGxEY899hiys7Oxbt06bNiwAT/5yU8gFovR29uLVatWITs7G8uWLcODDz6IX/7yl+O+7vz58/HUU09h+fLliIuLw+uvvz5m24cffhjPPvssysvLUVRUhOPHj+Oll15y+l4dwWA0oaFNBcA8QkqNDoFYRGFIb0SnWuuW1/VraCfZu3cvLZVK6ffee48+f/48vXbtWjoyMpLu7Oy02f7YsWO0WCymX3/9dfrChQv0iy++SAcFBdENDQ1cm6tXr9LR0dH0pk2b6Lq6Ovrq1av0X/7ylzGvORqVSkUDoFUq1R2PDQ0N0RcuXKCHhoacvVW/pLW1lQZAf/nll97uis/Cvqcamrvp9Of30/lbD9JGo4l7/Dv/eZhOf34/fexqtxd7KSzsfUYtcVqQ5s6dS2/YsIH72Wg00snJyXRlZaXN9suWLaOXLFlidW7evHn0T37yE+7n5cuX0z/60Y+c7QoHEaSxqaqqov/yl7/Q169fp48dO0YvWLCAzsjIoHU6nbe75rOw76mPTlyj05/fT6/cdcLq8TW7T9Lpz++n/+fEDS/1UHg4KkhOTdl0Oh1qa2tRUlLCnROJRCgpKUFNTY3N59TU1Fi1B4DS0lKuvclkwt///ndkZ2ejtLQU8fHxmDdvnt2CTq1WC7VabXUQbKPX6/HCCy9g+vTpePTRRxEXF8clSX744YcICwuzeUyfPt1jfXzwwQfH7Merr77qsX44S6OyHwBQmKqwOk8C267jVB5ST08PjEYjEhISrM4nJCSgsbHR5nOUSqXN9kqlEgDQ1dWFgYEB/PrXv8a///u/47XXXsPBgwfx/e9/H4cPH8aiRYvuuGZlZaVDsQcCI/6lpbZzrh5++GHMmzfP5mPjZVDzye9//3sMDQ3ZfCw6Otpj/XCWxg7mi7BwUqTVecvANsE5vJ4YyeZ6PPLII3j22WcBgAtG7tixw6Ygbd682Wq1id03nOAc4eHh4+617glSUlK83QWnMdE0mm4xgjM6IzuLCJLLOCVIsbGxEIvF6OzstDrf2dlpM60fABITE+22j42NhUQiQV5enlWb3NxcHD161OY1ZTIZZDJipE7wHnqDCaCBZIUc8RHWK5aZcYwgtdwahN5oQpCYZNc4ilO/KalUitmzZ1t5yZhMJlRVVaG4uNjmc4qLi+/wnjl06BDXXiqV4q677uIKJ1kuX76M9PR0Z7pHIHgMnZEZ2RelRd7xWEK4HMFBYhhMNG7etj0VJdjG6SlbRUUFVq9ejTlz5mDu3Ll46623oNFosGbNGgBMdXlKSgoqKysBABs3bsSiRYuwbds2LFmyBHv37sWpU6ewc+dO7pqbNm3C8uXLsXDhQtx33304ePAg/va3v6G6upqfuyQQeEZnYJIeR8ePAEAkopAZG4oLHWpc7x7gYkqE8XFakJYvX47u7m5s2bIFSqUSRUVFOHjwIBe4bmlpsdpmeP78+dizZw9efPFFvPDCC5g6dSo+++wz5Ofnc20effRR7NixA5WVlXj66acxbdo0/PnPf8Y999zDwy0SCPyjZ0dIY1T0Z8YxgkTiSM5B0bTv57er1WooFAqoVKo7qrqHh4fR1NSEzMxMl7KTCYTR9GsGceJMI35Z3YXPf1aCUNmd3+vbvriE33x1Ff8yLw2vPuq9YmehYO8zagmJthHs4o8m/xOF3QgyIybUphgBJBfJVYggEQQP3yb/AGONS1EU+vr6nH7u0Igg5SaN/U1PcpFcgwgSgeAk2hFBykkcO4eLFSSlehgarcEj/fIHAlOQaBrQabxzEJN/3kz+tVotnnvuOaSkpCA0NBTz5s2zWpltbm7GQw89hKioKISGhmL69Ok4cOAAbty4gfvuuw8AY/FCURSeeOIJh+6fpmkM65mAdk7i2COkyBApokOlAIAbvWSU5Chez9T2CvpB4NVk77z2C+2A1PFlYGLyP7bJf3l5OS5cuIC9e/ciOTkZn376KRYvXoyGhgZMnToVGzZsgE6nwz/+8Q+EhobiwoULCAsLQ2pqKv785z/jsccew6VLlxAREYHg4GCHfgdagwkmmoaIAjLGWc7PjA3FLY0OTT0aTE9W2G1LYAhMQfIhiMm/bZP/lpYW7N69Gy0tLUhOZr5cnnvuORw8eBC7d+/Gq6++ipaWFjz22GOcpW9WVhb3fLZGLj4+3qn41KCOma4FiUUQi+yb3GXGhqK2+TYJbDtBYApSUAgzUvHWazsBMfm3TUNDA4xGI7Kzs63Oa7VabhPIp59+GuvXr8cXX3yBkpISPPbYY5gxY4ZLr8cypGPiQVLJ+NEOEth2nsCMIVEUM23yxkFM/nkx+R8YGIBYLEZtbS3q6+u54+LFi3j77bcBAD/+8Y9x/fp1PP7442hoaMCcOXPwm9/8ZkL3x46QpA7UpxHDf+cJTEHyEYjJP4Mtk/+ZM2fCaDSiq6sLU6ZMsTosp3apqal46qmn8Mknn+BnP/sZdu3axV0TgMObBwCAyWQOaAc5MkIaKbK93j1A/LUdhAiSgCEm/wy2TP6zs7OxcuVKrFq1Cp988gmamppw8uRJVFZW4u9//zsA4JlnnsHnn3+OpqYm1NXV4fDhw9zrpaeng6Io7N+/H93d3VYrhWMxpDeCBg2xSATJOPEjgEmcpChAPWzAbbK1tmN4wL3S7fizhe2hQ4fo3NxcWiaT0TNmzKCrq6tpAPSnn37KtTlw4AANgF64cOEdz29vb6cffPBBWiaT0enp6fSePXvo+Ph4eseOHQ69/ujXOnv2LH3ffffRcrmcjo6OpteuXUv39/fTNE3TSqWSLisro5OSkmipVEqnp6fTW7ZsoY1GI63Vaukf/vCHdGpqKi2VSunk5GS6vLzc4b/LU089RcfExNAA6K1bt9I0TdM6nY7esmULnZGRQQcFBdFJSUn0o48+Sp89e5amaZouLy+nJ0+eTMtkMjouLo5+/PHH6Z6eHu6av/rVr+jExESaoih69erV4/ahSz1Mn2m9TV9t73X4PTW/sopOf34//W1Tr0P36a84amFLatkCjJs3byI1NRVffvkl7r//fm93x6do6R1E35AOMXIK6u52h95Tj//hGxy50oPXfzADy+YEromgo7VsgbnKFkB89dVXGBgYQEFBATo6OvDzn/8cGRkZWLhwobe75nMM6pkVNnmQCI66uGfGhuLIlR6y0uYgJIbk5xCTf34wGE3QGZiAtjxI7PDzSJGtc5ARkp9DTP75gV3ul0nEEIsc/x4nuUjOQQQpgCEm/44zOFJQGyJ1fHQEAFmxTJlLU68GJhMNkQOrc4FMwEzZ/CB2T/AiQzqzIDnzXkqJCkaQmILOYEK7ivhrj4ffCxI79RgcHPRyTwi+Ck3TGBwpGQmWirksebF4/NGSWEQhPYZM2xzF76dsYrEYkZGRXIV7SEiIS0mBhMBFpzfCoNOCoiiY9Fp0d3cjJCQEEoljH5/M2FBc7RpAU48G906Nc3NvfRu/FyQAXCmBq7YbhMBmUGfALY0eMokIrYMyiEQipKWlOfzFxtW0kZW2cQkIQaIoCklJSYiPj4deT1L4Cc6x/asr+PR0Fx6dlYLyGZmQSqVWO+uMB1lpc5yAECQWsVjs0LyfQLDkaJMabf1GTE2KdinbPytuZKWNCNK4+H1QO9DZf7Ydc//jS5xuca3CP9DRGoy40M7kZY+1B9t4sCOkm7cHoTU47i4QiBBB8nP+dLIFXf1afHq6zdtd8UkaO/qhM5oQFRKEtGjnzPVYYsOkCJdJYKKZejjC2BBB8mNMJhpnb6oAgPuWJzhHfWsfAKAwNdLl1VmKoszeSGTaZhciSH7MjV4N+oeZ/JmLHWqYTCQ51FnOsII0KXJC1yGBbccgguTHsKMjANDojGi+RaYLzsKOkIrSIid0HVJk6xhEkPyYMzf7rH4+366y3ZBgE9WgnptikRGSZyCC5MewIyRFMFM+Q+JIzsEKenpMCLfpo6uwRbYkhmQflwTp3XffRUZGBuRyOebNm4eTJ0/abf/RRx8hJycHcrkcBQUFOHDggNXjTzzxBLcrKXssXrzYla4RRjAYTdyI6LFZkwAA54kgOQVf8SMAyIhlVuh6BrRQD5Pk3LFwWpD27duHiooKbN26FXV1dSgsLERpaemYZRnHjx/HihUr8OSTT+L06dMoKytDWVkZzp07Z9Vu8eLF6Ojo4I4//elPrt0RAQBwuXMAw3oTwmUSLJnB7It2oYMIkjNw8SMX848sCZcHIS5cBgC4QUZJY+K0IL355ptYu3Yt1qxZg7y8POzYsQMhISF47733bLZ/++23sXjxYmzatAm5ubl45ZVXMGvWLGzfvt2qnUwmQ2JiIne4upmhPYwmGufbVfhLvf/n5JwdmW7kpyiQlxQBEQV092vR1T/s3Y75CDRNc1O2Qh4ECSBxJEdwSpB0Oh1qa2tRUlJivoBIhJKSEtTU1Nh8Tk1NjVV7gHExHN2+uroa8fHxmDZtGtavX4/e3t4x+6HVaqFWq60OR7il0WHJO0fxzL56vx82nxmJH81IVSBYKubKF0gcyTFu3h5Cz4AOEhGF6cljm9I7AymyHR+nBKmnpwdGoxEJCQlW5xMSEqBUKm0+R6lUjtt+8eLF+OMf/4iqqiq89tpr+Prrr/Hggw+OuYlfZWUlFAoFd6SmOrabQ1y4DKnRwaBpc3zAX2FHSGz8Iy+J+VCROJJjsKOj3KQIpzy07UFGSOMjiFW2H/7wh3j44YdRUFCAsrIy7N+/H99++y2qq6tttt+8eTNUKhV3tLa2Ovxas9KYqWBdcx8PPRcmw3ojLin7AQAzJikAgPuWJ3EkxzjDY/yIhRTZjo9TghQbGwuxWIzOzk6r852dnVbbF1uSmJjoVHsAyMrKQmxsLK5evWrzcZlMhoiICKvDUWaOvMHq/LjY9EKHGgYTjZhQKVIigwEAeawgkRGSQ1iWjPBFZizZWns8nBIkqVSK2bNno6qqijtnMplQVVWF4uJim88pLi62ag8Ahw4dGrM9wGxm2Nvbi6SkJGe65xCz0pkR0umW235bSnF25MM0Y5KCq79ip2w3ejUY0Bq81TWfwGA0oaGNicHxOUJKiw6BiGKy5rv7tbxd159wespWUVGBXbt24YMPPsDFixexfv16aDQarFmzBgCwatUqbN68mWu/ceNGHDx4ENu2bUNjYyNefvllnDp1CuXl5QCAgYEBbNq0CSdOnMCNGzdQVVWFRx55BFOmTBlz+56JwMQERFAPG/w2SY1NiJxhkT8TEyZDYoQcNA00kmmbXS519jMpE3IJF4jmA6lEhNQRxwB/fe9NFKcFafny5XjjjTewZcsWFBUVob6+HgcPHuQC1y0tLejo6ODaz58/H3v27MHOnTtRWFiIjz/+GJ999hny8/MBMKZpZ8+excMPP4zs7Gw8+eSTmD17No4cOQKZTMbTbZoJEoswIyUSgP9O28zL1Qqr8ySO5BhnWhlBL5wUyfu2RSSwbR+XHCPLy8u5Ec5obAWily5diqVLl9psHxwcjM8//9yVbrjMzPRInLxxC6dbbvvdfuv9w+b6qxmjMozzkiNQ1diF821EkOxR38p8UY0WdD7IjA1F9aVuIkhjIIhVNk8zM9V/V9oa2lSgaSAlMhixYdYjTDJCcgzLERLfkFwk+wSkIM1KjwQAXO7q97sESXP86M5v97wk5twlZT/0RpNH++UrDGgNuNzFpEzwGdBmyWR3su0Z4P3a/kBAClJ8uByTopgEybOt/mXJwSZEjp6uAUBqdDDCZRLojCZc6yYfCFs03GRGmMkKOeIjnDf0Hw/WObLl1iAM5EvhDgJSkACLBEk/C2ybpxt3jpAoikLuyLSNxJFsw5ch21gkRcghk4igN9Jo6yNba48mYAVp5sgbzp8EqXdAy73J820IEkDiSOPBp+WILUQiypwgSQLbdxCwgsSOkE639PlNgiQbP8qKC0WEPMhmG3NNm39NVfmCT8uRsSB2tmMTsIKUmxQBmUQE1ZAeTb3+8cY4M6qg1hbTk5mR04V2NSlfGIVSNQylehgiirFtcRdZcSQXaSwCVpCkEhG3ElXX7B/TNnsrbCxT4sMQJKagHjbg5m0Sw7CEHR1lJ4QjVOa+TZ0zOTtbsrAwmoAVJMAysN3n3Y7wAE3TdlfYWKQSEbITwgEQK5LRsCNMd07XADJls0dACxIb2PaHbabbVcMOG4qxcSQS2LamfuSLyd2CxCZHtquGMaQjW2tbEtCCxI6QLnX2+3wF/FmL6cZ4hmLcShsJbHMYTTRX4c+n5YgtokKliAxhFh1u+En8ki8CWpDiI+RIifQPB0nWstaR+qs8i8A2geFa9wAGtAaESMXclNadkCJb2wS0IAFmfyRfD2w7Ej9iyU1iPnDtqmHc1ujc2CvfgQ1o56coIOa5wt8WRJBsE/CC5A8OkiYTjQYHVthYwuVBSI9hfHlIHImBFaSZbp6usZAiW9sEvCBxDpKtfT6bl9PUq0G/1gCZxQraeLBxJJIgyXDGDZa19iBFtrYJeEHKG0mQ7BvU++zwmZ2uTU+OQJDYsT8pt9JG4kgY1hvRqHRfhb8tyJTNNgEvSFKJCAUjWbm+mo/EFtQ6Ej9iYTO2SS4ScK5NBaOJRly4DEkK/iv8bcFurX17UE/ieBYEvCABFoFtH40jnR3DstYe7C4k17oHMKwP7FyYeouCWnZTBHcTIpVw4ucvpUt8QAQJFoFtH1xp0xtN3CjHmRFSfLgMsWFSmGhw05VAhQtou8lyZCy4mjYS2OYgggTzCOmyDyZIXu7sh9ZgQrhMgswYx3fIoCgKuSSOBMCxomR3QOJId0IECUDCSIKkiTZnPPsKbEFtwSSF0ztkmONIgbvS1jugRestpsh4hhtM/e1BimzvhAjSCL5q2OZMQuRo8ohZGzc6mmzHQ8pdkFykOyGCNMJMH638t2dZOx5sLlJjRz+MfmJS5yzmgtooj782O2W70avxG5PAiUIEaYRZFpX/vpIgOaw34lInE5Ce4UL+TEZMKIKDxBjSGwM2jlF/k90y27PTNQCYFBUMiYjCsN4EpXrY468vRIggjTA9WQGpRITbg3rc6B30dncc4ny7GkYTjdgwKZJdyJ8Riyiuri0Q40g0TXMZ2t4YIUnEIqSNlPAE6hfCaIggjWCVIOkjy/+W8SNX82cCOY50o3cQqiE9pBIRpiW6v8LfFlk+bPh/W6PDoI7fVWkiSBbM8rHAtiOWteMxPYCtSNgts/OTIyCVeOej4Mvukb/7x3UU/vIL/L/qq7xdkwiSBb4W2OYjf8ayps1XYmd8wS0IeKh+zRa+XGR79Go39EYayYpg3q5JBMkCzkFSqYZG4AmS6mE9t1w8kRHStMRwiEUUejU6dKq1fHXPJzjtgS2PxsNXkyN7B7RchcCCKbG8XdclQXr33XeRkZEBuVyOefPm4eTJk3bbf/TRR8jJyYFcLkdBQQEOHDgwZtunnnoKFEXhrbfecqVrEyJRIUeyQg4TbR59CJVzI9O1lMhgxITJXL6OPEiMySMlDBc6AiewrTUYcXHkA+VNQWLLR1pvD0Fn8J2ttY9d6wVNAzmJ4YgLd/39NxqnBWnfvn2oqKjA1q1bUVdXh8LCQpSWlqKrq8tm++PHj2PFihV48skncfr0aZSVlaGsrAznzp27o+2nn36KEydOIDk52fk74YmZrD+SwKdtzljWjgeXsR1A22tf7OiHzmhCVEgQ0qJDvNaP+HAZQqViGE00Wm/7xuouABy90g0AuHcqf6MjwAVBevPNN7F27VqsWbMGeXl52LFjB0JCQvDee+/ZbP/2229j8eLF2LRpE3Jzc/HKK69g1qxZ2L59u1W7trY2/PSnP8WHH36IoCDPZsxa4iuFthPJ0B5NIO5CYmnI5qkKf1tQFIVMHyuypWkaR6/0AADumRrH67WdEiSdTofa2lqUlJSYLyASoaSkBDU1NTafU1NTY9UeAEpLS63am0wmPP7449i0aROmT58+bj+0Wi3UarXVwRe+4iDJxwobi9k9MnAEydJyxNv4Wk3b9R4N2lXDkEpEmJsRzeu1nRKknp4eGI1GJCQkWJ1PSEiAUqm0+RylUjlu+9deew0SiQRPP/20Q/2orKyEQqHgjtTUVGduwy7TkyMgFYtwS6NDs0ATJHsGtGjrGwJFgcudmghsLlLLrUGoh/UTvp4vwCVEethyxBa+FthmR0d3ZUQhWGp/yy1n8foqW21tLd5++228//77Dg+dN2/eDJVKxR2tra289UcmESM/hfmACjUfiZ2uZcWGIpyHgtDIEClSIpml24sBMEpSDeq5REQhjJB8rcj2CDtdm8LvdA1wUpBiY2MhFovR2dlpdb6zsxOJiYk2n5OYmGi3/ZEjR9DV1YW0tDRIJBJIJBI0NzfjZz/7GTIyMmxeUyaTISIiwurgE/MW28IUJHNBbSRv18wNoDgSu4KaHhOC6FCpdzsD3xoh6Y0mnLjeC4D/gDbgpCBJpVLMnj0bVVVV3DmTyYSqqioUFxfbfE5xcbFVewA4dOgQ1/7xxx/H2bNnUV9fzx3JycnYtGkTPv/8c2fvhxe4BMnmPq+8/niYA9r8FYQGUhzpjIDiRwCQMSJIXf1awRsE1rf2YUBrQHSolFsM4ROJs0+oqKjA6tWrMWfOHMydOxdvvfUWNBoN1qxZAwBYtWoVUlJSUFlZCQDYuHEjFi1ahG3btmHJkiXYu3cvTp06hZ07dwIAYmJiEBMTY/UaQUFBSExMxLRp0yZ6fy4xKz0SANCoVGNQZ0CI1Olfk9ugadoc0OYxf4araQsAQaoXQEKkJYrgIMSGSdEzoMONHg3yeYgLugt2urZgSqzThoCO4HQMafny5XjjjTewZcsWFBUVob6+HgcPHuQC1y0tLejo6ODaz58/H3v27MHOnTtRWFiIjz/+GJ999hny8/P5uwueSVIEI4lNkGwVVrJgW98QejU6SEQUr99Q7AjpSle/TyXoOQtN0+aSG4EIEmCetgm9yJbLP+IxO9sSl776y8vLUV5ebvOx6urqO84tXboUS5cudfj6N27ccKVbvDIrLQp/b+hAXcttFE+OGf8JHoIdHU1LDIc8iL8VjpTIYCiCg6Aa0uNKVz+XLOlv3Lw9hJ4BRtBZERYCmbGh+PbGbUHnIqmG9FxC7j1uiB8BAlhlEyozLQzbhMQZHhMiLaEo84jLn+NI7O8vNymCV0GfKL5QZFtzrRdGE42suFAkR/JXUGsJEaQxsKz8F1KC5NkJWNaORyDEkcyWtZFe7cdofGGl7ehV907XACJIY5KfYk6QbLkljARJk4nGuTbnd6l1lOkBIEhCjB8B5iLb6z0aQX0BWuKuchFLiCCNgUwixnSBJUg29WrQrzVAHiRCdkIY79e3dI/0R9N5vdGEhjbWQzvSu50ZRXpMCCgK6B82oFeAW2u33hrEjd5BSEQU7s7it1zEEiJIdpglsHwkNv9oerICEjH/f7rJcWGQSkQY0Bp8qvLcUS539mNYb0K4XMJlRwsFmUSMSVFMXEaI07ajV5nR0cy0SF6qA8aCCJIdhLZXG5uCwGdCpCVBYhGmJTDe0v44bbMsqHVHDs1E4Ypsu4UX2D4ystzvjnIRS4gg2YEdITUq+3k3M3eFsx7Y8tmfM7bNliPCTGkQquG/0UTj2FWmXMRdy/0sRJDskBwZjMQIOYwmc3a0t9AbTZxIuGuEBPj3LiT1XtzyyBGEavh/rk0F1ZAe4XKJW1Z3LSGCNA5sGYm3p22XO/uhNTDxj4wY98U/zCMkYWWoT5QBrQFXupipkLs/VK4i1KV/Nn40f3KMW2KXlhBBGgehBLYtDdncGf/ISYwARQGdai16BvzH9P/szT7QNJCskCM+wvlNNT0BK0jNvYOC2tqcix+5cbmfhQjSOLCB7fpW726xzadlrT1CZRJkjozA/CmwzS4ICMGQbSySI4MhlYigM5rQ3jfk7e4AADRaA2pH7JzdmRDJQgRpHKYnKxAkptAzoEPrLe+9Sc64MUN7NLl+GEdiN4UUiuWILcQiChkjW2sLJbB9sukW9EYak6KCkR7j/s0QiCCNgzxIzBWaeiuONKw34lJnPwD3j5AA/1xp40ZIAkuIHI05sC2MpX/WbuTeqbEe2QyBCJIDeNtB8ny7GkYTjdgwGZIU7o9/mHez9Y/AtlI1DKV6GCIKgvYaAiyLbIUxQmLr19ydf8RCBMkBvJ0gac4/UnjkW4odEV7v0Qgi/2qisMv92QnhCJUJx2zPFkLKRepUD+Ny5wAoClgwxTMWPESQHIDdGuliRz+GdEaPv755hS3SI68XFy5DXLgMNM0khfo6QnOItAdbZCuEERJbTDsjRYHIEM94jxNBcoBkhRwJEbKRBMk+j78+54HkwQxjf4ojnfEhQWJjSG19QxjWe/7LzxLzcr/7V9dYiCA5AEVRFnGkPo++tnpYz22P48kVInMcybcFyfJLRGiWI7aIDpUiQi4BTcOrtjc0TeMoWy7iofgRQATJYbwV2D43Ml2bFBXs0S172DiSrwe2r3UPQKMzIkQqRvZI4bCQYbbW9n6RbaOyHz0DWgQHiblqBU9ABMlBzJa2nnWQZD2MPZ0/w9a0NSr7YTD6ruk/Gz/KT1FALMAKf1sIIbDNxo/mZUVDJvGc1S8RJAfJT2ETJLW4edtzCZLu2IPNEdKjQxAqFUNrMAlixcdVWEGa6QPTNRYhFNkeucrmH3luugYQQXIYeZAYeV5IkPT0ChuLSERxu9n6cqGt2XIk0qv9cAZvF9kO6434xo2709qDCJITzGLzkZo9I0g9A1q09Q2BooACL1So+7rH9pDOyKUt+MIKG4u3Bam2+Ta0BhMSImSYGs+/VbI9iCA5wUwPr7Sx07XJcWEI80JCX56PL/2fb1fBaKIRF+6ZDHe+YAWpV6ODalDv8de33J3WE4m4lhBBcgJ2hHSxQ+2RBEl3W9aOB7fS1qEW7E4Y9rC0rPX0B2sihMokSIiQAWA2dvA03HZHHp6uAUSQnCIlMhjx4TIYTDS3e4U78YRlrT2mJoRBIqLQN6hHu2rYK32YCFxAW8CWI2NhnrZ5dum/d0DLjYgXeMBuZDREkJzAOkHSvXEkmqatTNm8gUwixpSRGIIvxpEsR0i+Bldk6+GVtmPXekHTQE5iOOLDPT/NJYLkJJylrZsD2219Q+jVMHvQs6td3oCdtvnaSptleoYnS274wlu5SEeveG+6BhBBcho2sH261b0JkuzoKCcp3Kt70Pvq9trscv/kuFBEuHEfMXfhjSJbmqY9sjutPYggOUlBigISEYXufvcmSJ7xkGXtePhqke0Zge8wMh6WS/+eWlC43qNBu2oYUrEIczPctzutPVwSpHfffRcZGRmQy+WYN28eTp48abf9Rx99hJycHMjlchQUFODAgQNWj7/88svIyclBaGgooqKiUFJSgm+++caVrrkdxkHS/Vtsn/WgZa092OliW9+QV5agXeU0J0i+N10DgNToEIhFFAZ1RnSqPbPZAjs6mpMRhWCpd0blTgvSvn37UFFRga1bt6Kurg6FhYUoLS1FV1eXzfbHjx/HihUr8OSTT+L06dMoKytDWVkZzp07x7XJzs7G9u3b0dDQgKNHjyIjIwMPPPAAuru7Xb8zN8JN29yUj2Qy0Tg3sopXkBLpltdwFEVwEFKjmS2ez3f4RhyJpmmfHyEFiUVIi2b9tT2z0ma2q/XOdA1wQZDefPNNrF27FmvWrEFeXh527NiBkJAQvPfeezbbv/3221i8eDE2bdqE3NxcvPLKK5g1axa2b9/OtfmXf/kXlJSUICsrC9OnT8ebb74JtVqNs2fP2rymVquFWq22OjyJux0kr/do0K81QB4kQnaCZzNlbeFrViRNPRqohw2QSkSYlij8Cv+x8GTGtt5owgkvlYtY4pQg6XQ61NbWoqSkxHwBkQglJSWoqamx+Zyamhqr9gBQWlo6ZnudToedO3dCoVCgsLDQZpvKykooFAruSE1NdeY2Jgy79H+hXe0WEy02/2h6ssLtG/M5gtmKxDcEiY2/5SdHQCrx/u/PVTxZZFvf2ocBrQHRoVLuC8gbOPXX6unpgdFoREJCgtX5hIQEKJVKm89RKpUOtd+/fz/CwsIgl8vxX//1Xzh06BBiY20r9ebNm6FSqbijtbXVmduYMJOighHnxgRJb+cfjYYbIfnItkj1I1NpXyqotYUnR0jsdG3+5Bi3bkQ6HoL5+rjvvvtQX1+P48ePY/HixVi2bNmYcSmZTIaIiAirw5MwCZKRANyTj3TGyxnao5mewvx+r3QNeN1W1RHqb/rGlkfjkeVBQfJ2/hGLU4IUGxsLsViMzs5Oq/OdnZ1ITEy0+ZzExESH2oeGhmLKlCm4++678Yc//AESiQR/+MMfnOmeR3FXxrbeaOKmRkIZISVGyBEVEgSjicaVTmHsFzYWWoMRF0d+f74uSJkjuUgttwahd6NJnmpIzxkBeiv/iMUpQZJKpZg9ezaqqqq4cyaTCVVVVSguLrb5nOLiYqv2AHDo0KEx21teV6sV7t7ylpX/fOaJXFL2Q2swIVwuQcbIltbehqIon8nY/rbpNnRGE6JCgrhVKl8lIVyO4CAxDCbarTlvNdd6YTTRyIoLRUpksNtexxGcnrJVVFRg165d+OCDD3Dx4kWsX78eGo0Ga9asAQCsWrUKmzdv5tpv3LgRBw8exLZt29DY2IiXX34Zp06dQnl5OQBAo9HghRdewIkTJ9Dc3Iza2lr867/+K9ra2rB06VKebpN/ZkwyJ0i28bgPu2X8yJtz+dHk+cj22ruOXAcALJmR5FMV/rYQiShkeKDIlqvu90Ix7WicNtlZvnw5uru7sWXLFiiVShQVFeHgwYNc4LqlpQUikVnn5s+fjz179uDFF1/ECy+8gKlTp+Kzzz5Dfn4+AEAsFqOxsREffPABenp6EBMTg7vuugtHjhzB9OnTebpN/mEcJCNw9qYKdS19mBTFz7fxWYFkaI/GFzK2z7er8PXlbogoYN29k73dHV7Iig3FxQ41rndr8N0c97yGt8tFLHHJ9au8vJwb4Yymurr6jnNLly4dc7Qjl8vxySefuNINrzMrLYoRpObbeLgwmZdrmk39hRE/YmFX2i52qGEy0YIavbH87mt2dJSMtBjfnq6xuLumrfXWIG70DkIsonB3lnfKRSwRzCqbL2LeiYSfwPaQzojLnYzlqtBGSFlxYZAHiTCoM+KGF0zDxqOldxD7z7YDAJ5alOXl3vCHu5f+j46Y+c9MjUS4AIqQiSBNAHal7TxPCZIXOhjL1dgw4VmuikUUpiUKN46068h1mGhgYXYcF4D3B9wtSEe45X7vT9cAIkgTYlJUMGLDmATJczwkSJ6xKKgVYkBWqHGkngEt/vcUkxy7fpF/xI5YWEHqUA1jUGfg9dpGE41j7O60Xs4/YiGCNAGsEiR5mLYJNaDNItSatveP3YDWYEJhaqQg4iB8Ehki5XYs5nuUdK5NBdWQHuFyiWBilkSQJsis9JF8pOa+CV+LW/IXqGWGEEdIA1oD/lhzAwAzOhLiyHKiuGvaxsaPirNiBFEzCRBBmjDsjqh1LbcnlCCpGtJzdqVCKRkZTU5iBEQUM0Xq6heG6f+fvmmBetiArLhQPJCXMP4TfBB3FdkeEUi5iCVEkCbIjEmRkIgodPVrJ7QzBxuDmhQVzA3RhUawVIysOMYORQijJK3BiN8fZZb6n1o4WZCpCHzgjhGSRmtA7UgdphDyj1iIIE2QYKmYc1WcSKGt0Apqx0JIcaS/nG5Hp1qLhAgZHpnJTx6YEHGH4f/JplvQG2lMigpGhoBytogg8QAfge2zXt4U0lGEsr22yURjxz+uAQCevCcTMon3NkJwN2yR7fXuAd7qJs3ukJ7fndYeRJB4gA1sT8TSVugrbCxCqWn74kInrndrECGXYMXcNK/2xd2wRdbqYQNu8+Rrztav3TNFONM1gAgSL8xMZRMkVS4lSHaPxJ8oCigQ+AiJnbI19WgwoOU3L8ZRaJrGb79mRkePF6cLIsPYnciDxFwVPh9Ftp3qYVzuHABFMYZsQoIIEg+kRgcjNkwKvZF2yZ6DHR1NjgtDmMyl8kKPERMmQ2IEk0Xe6KVR0onrt3CmtQ8yiQhPzM/0Sh88TRY3bZt4HIktpi1IUSBKYAsoRJB4gKIosz+SC/lIZwRmWTse3s5H2jEyOlo6ZxLiwmVe6YOn4XOlTYjL/SxEkHhiIg6SZ31khY3Fm7vZ+qPFiCPwJUg0TeMoWy4isPgRQASJN8yV/31OPY+macGZ+o8HN0Lywj5t/mgx4gisIE10ytao7EfPgBbBQWLMSo/koWf8QgSJJ2ZMUkAsoqBUD6PdCQfJm7eHcEujg0REcflMQicviRHOy8oBt3o9j8ZfLUYcISuWSUht6tXAZHJ96Z+NH83LihZkqgQRJJ4IkUqQm8RsSujMtI0dHeUkhUMeJLw3iC1So4MRLpNAZzThapfnTP/91WLEEVKighEkpqAzmNCuct0y+chI/do9ArCrtQURJB6Z5UJg21fyjyyhKAq5Ho4j+bPFiCOIRRTSYyYWRxrWG/ENtzut8OJHABEkXmEF6XSr4yMkc8mIb33je3qlzZ8tRhxlooHt2ubb0BpMiA+XCWKLdlsQQeIRNrB9vk0NrWH8BEmTica5NnYPtkg39ox/zLvZuj+wbW0xkiWoUgdPkjXBwPYRzsxfWOUilhBB4pG06BDEhEqhM5o4obHH9Z4BDGgNkAeJMDVemN9YY8HGcC60q3ndl84W1hYjtjckDQQmOkLitjsSYP4RCxEkHrFMkHTE+J+1rM1PVgjGIMtRpsSHIUhMQT1scOsmhpYWIz9ZmOW3FiOOMBFB6h3QctPrBQINaANEkHiHze1wZKXNFwPaLFKJCNkJzKqiO+NIlhYjZTNT3PY6vgBb9X/z9qBDIQFLjl3rBU0DOYnhiA8X1gYSlhBB4hm20NaRBEluDzaBWtaOhzmO5B5BCiSLEUeIC5MhTCaBiWb2U3OGo1fY6n7hjo4AIki8U5jKJEh2qIbRYSdfRGcwcR9kXxwhAZbeSO4JbAeSxYgjUBTlUpEtTdNcQuS92cJc7mchgsQzIVIJchJHEiTt5CNd7uyHzmBChFwiKMc+Z8izCGzzTaBZjDiKK3Gk6z0atKuGIRWLMDdD2CkTRJDcgCOFtmcs4kdCXYIdDzYzvV01jNsaHa/XDkSLEUdwRZDY0dGcjCgES4U97SWC5AbYwLa9lTZfsay1R7g8COkjozu+40iBaDHiCK4U2VrmHwkdIkhugA1sn7OTIHnGh1fYLDFnbPMXRwpUixFHYItsHTX81xtNOMGWiwjQbmQ0LgnSu+++i4yMDMjlcsybNw8nT5602/6jjz5CTk4O5HI5CgoKcODAAe4xvV6P559/HgUFBQgNDUVycjJWrVqF9vZ2V7omCNJjQhA9kiBpa0l8SGfElZGiVF9dYWNxxy4kgWox4ggZsczvo2dAC/Xw+P7a9a19GNAaEBUSxH15CBmnBWnfvn2oqKjA1q1bUVdXh8LCQpSWlqKrq8tm++PHj2PFihV48skncfr0aZSVlaGsrAznzp0DAAwODqKurg4vvfQS6urq8Mknn+DSpUt4+OGHJ3ZnXsRqi20bWyOdb1fBaKIRF262g/VV2IxtvnKRLC1GfrIwsCxGHCFcHsRNYW84MEpip2sLpsT6RFKp04L05ptvYu3atVizZg3y8vKwY8cOhISE4L333rPZ/u2338bixYuxadMm5Obm4pVXXsGsWbOwfft2AIBCocChQ4ewbNkyTJs2DXfffTe2b9+O2tpatLS0TOzuvIg5Y7vvjse4/KNJCp8NaLOw7pHXugdc2uBgNJYWI/kpvj16dBfOBLaPCtiu1hZOCZJOp0NtbS1KSkrMFxCJUFJSgpqaGpvPqampsWoPAKWlpWO2BwCVSgWKohAZGWnzca1WC7VabXUIDbOD5J0jJF/O0B5NfLgMsWFSmGjGjXAiWFqMBJoBmzM4WmSrGtJzX35C2p3WHk4JUk9PD4xGIxISrPdQT0hIgFKptPkcpVLpVPvh4WE8//zzWLFiBSIibM95KysroVAouCM1NdWZ2/AIhZMiIaKYJXHlqC22fc2y1h4UZXa6nGhg29JipDhLWNvzCAlHR0g113phNNHIig3ltlESOoJaZdPr9Vi2bBmTFPfb347ZbvPmzVCpVNzR2trqwV46RqhMgpzEkS22LUZJqiE990byhxESYF357yrEYsRxHBUkbjNIH5muAU4KUmxsLMRiMTo7O63Od3Z2IjHRti1EYmKiQ+1ZMWpubsahQ4fGHB0BgEwmQ0REhNUhRLhCW4vAdsPI6Cg1OhjRAtsTy1XyeDBrIxYjjsOWjzT1aOxav3DlIj4yXQOcFCSpVIrZs2ejqqqKO2cymVBVVYXi4mKbzykuLrZqDwCHDh2yas+K0ZUrV/Dll18iJsY/hutmB8k+7py/5B9Zwi4nNyrVMLpgQE8sRpwjLToUIooZVXYPaG22ab01iBu9gxCLKJ9y2HR6ylZRUYFdu3bhgw8+wMWLF7F+/XpoNBqsWbMGALBq1Sps3ryZa79x40YcPHgQ27ZtQ2NjI15++WWcOnUK5eXlABgx+sEPfoBTp07hww8/hNFohFKphFKphE7HbzmCp2FX2hraVNAZmN05zvqoZa09MmJCERwkxrDe5NJWz8RixDmkEhFSo5l8pKYxAttHR8z8Z6ZG+lQdoNOCtHz5crzxxhvYsmULioqKUF9fj4MHD3KB65aWFnR0dHDt58+fjz179mDnzp0oLCzExx9/jM8++wz5+fkAgLa2Nvz1r3/FzZs3UVRUhKSkJO44fvw4T7fpHTLYBEmDiQv4mgPakV7sGb+IRRRX1+bstI1YjLjGeHEkdndaX4ofAYBLG8mXl5dzI5zRVFdX33Fu6dKlWLp0qc32GRkZbrdA9RYURWFmaiSqGrtQ19KHlKhgdKiGQVHwuxybvOQI1LX04UK7Go8UOT7KIRYjrpEZG4rqS902S0iMJhrHrrK7i/iWIAlqlc0fmZVutrRlC2qnxIUhTObSd4Fg4VbanCiyJRYjrmMvF+lcmwqqIT3CZRKf2Z6dxb8+FQJkZmokACZjm30T+dN0jSUvybzSRtO0Q8v2rMWIlFiMOE0mu5OtjZgdGz8qnhzjc17tvtVbH6QwlUmQbOsbwqGLXSPn/Gu6BgDTEsMhFlG4pdGhU2175Wc0rMXIMmIx4jSsv3bLrUEYRm1nfsTHykUsIYLkZkJlEkwbSZC86OOWtfaQB4kxeeRD4kjGNrEYmRhJEXLIJCLojTTa+sxWyRqtAbUjeW++Ui5iCREkD8BW/gNAkNi8IuVvOJOxTSxGJoZIRJnN2iwC2yebbkFvpJESGeyT1shEkDwAmyAJADmJEX67tG0ZR7IHsRjhB27p3yKwfYTLzhbu7rT2IILkAWZajJD8oaB2LLhdSMZZaSMWI/xgKxfJvDut703XACJIHiEzNhRRIcyStq8twzoDW9PWcmtwTDdDYjHCH6MFqVM9jMudA6AoYP5k3yy/IoLkASiKwrqFk1GYGol/yksY/wk+SmSIlLO5uDjGtI1YjPBHVhy79M8IEltMW5CiQJSPFm4TQfIQ678zGX/ZsMBn3yiOkmsnjkQsRviFzWtr6xvCsN5oLhcR+O609iCCROAVe3EkYjHCL1GhUkSOhAKaejQ4OlIu4mv1a5YQQSLwyljeSMRixD2wcaT/O6dEz4AWwUFizE6PGudZwoUIEoFX2BHS1a5+znIFIBYj7oIVpD+dZDbEmJcV7dNpJUSQCLySEhkMRXAQ9EYalzsZ039iMeI+2DhSdz9TruPL8SOACJJ7MZkAdQfQehJo+Bio/xNw+4a3e+VWKIoybx45EkdiLUbCicUI77BFtiy+mn/EQqr9J4LJCKjbAVUr0NcC9LUCfc3mn1U3AaMN18uoTCDrO8yRuRAI8R2LUUfIS45AzfVeXBip/GctRlYRixHeYadsALMlVXZCmJ3WwocIkj2MekDdZiE2LSNCMyI86nbAZLB/DUoMRKQAkalM27Za4HYTUNsE1O4GQAHJRWaBSr0bCPLh3WwHunC/qBYKyZe4+3wXOjWTkdMuh1iSgSfm3uPt3vkd7NbaADNd8/VUCor2A7tGtVoNhUIBlUrl3A4kBi0ziulrNguO5Winvx2gTfavIQoCFJMYwYlMAxRpzL/sz+HJgNhC97X9QPNx4Nph4Ho10H3R+noSOZBWbBaoxBmASKAza20/0F7PiGxbLdB+mvn9jQUlBuKmAYkF5iOhAAglCZITobiyCh2qYfzX8kI8OnOSt7tjE0c/o4ElSNp+4G8bzYIzYHuzSiskckCRaiE4qUBkuvnnsARANIEgbb8SuP41I07XDwP9HdaPB0cDWYvMAhWV4fprTQSDDug6PyI+dcy/3ZcAjH77UKDjpuGTzgScN6YhgbqF6aJmFIe0QTx85y6+AJgRpKVIJRYAkRnCFWKB8efamzh6tQevPlqAYKmHFgwMOuDWNaC7EUgqAqLtG+wRQbKFyQj8e7z1NCso1CwunOCkmY/QOMBTw2CaBnouM+J07TBw4yigG7U9dVQGkHWfe+NPJhNw67p55NNWCygbAKMN4zVFKpA8E0iZzRzJRYAsHA/95iga2hhfpIcKk/GbHxYxYqtsAJRnR/5tYF7HFtJwIDHfWqTicoU9nTVoAUoEiP0oTmYpPF2NzIi++xLQe9X8OfrnN4C5a+1ehgjSWJzazXyI2elVSLTnBMdZjHpmNHK9mjlunhwVs6KApEJGnCbf53r8qV9pLT7tp4FhGyZr8sgR4Zk1Ij6zgHDbtXn/9uez2PstM33b/9N7xq7qH1YDneetharrom3xszXlS5zBryjrh4ChPmC4z/l/DcOMIIUnm7/QotKtv+AiUoQpWEY90HuNEZyuRkaAuhuthWc00nDm7zF3HVC43O7liSD5I2z8iRWorgvWj0vkQNrd5hGUrfjTsMo67tNWx8TKRiORM2KXPMssQtFZDov3x7U38dxHZ/CdaXF4f81c5+7TqAd6rowaTZ0Fhhyc8iXkA0HBrgmLLSHkE26RI836YIVrdMyRbyyFp/sSI/6OCk98DhCXw4xU43OY+3Dw/UAEKRBwJP6UuZARlp7LjAD1XMEdcR9KxLzJUmaZRz/xeRP6JjeaaHxxXokFU2MRwcdSP00zq5rsVI8VqttNE7/2aCgRMxoMjnT+X4OWiVHevmFelbU8xhM8SgwoUkbilGkW/7IjrGTHYpac8IyMdDwkPGNBBCnQsIw/Xa8Gmo7cGX9iiUwzT7lSZjOCJfPR/BWbU74LTLxQrmBEIjjKOWGRhbtnGm8yAZousziNFi1Vq+28NUtEkpFV3TRrwZLILEY8bIzHticVJzxxOSPik8v8rJjktvAFEaRAx2gA2uuY4Hj3RSA22yxCYb6dzTsuJiMzyhFqbHAsTCZm5ZcTqWYL8WpmUlTGEhlbSMNGRjqeE56xcPQzShIj/RWxBEidyxyBxkTSMLyJSMRMySKSmVjgaExGZppuJVgjoqUfZr50uOlWjleEZ6IQQSIQfAXRSHxJkQKkF3u7N26BZJ4RCATBQASJQCAIBpcE6d1330VGRgbkcjnmzZuHkydP2m3/0UcfIScnB3K5HAUFBThw4IDV45988gkeeOABxMTEgKIo1NfXu9ItAoHg4zgtSPv27UNFRQW2bt2Kuro6FBYWorS0FF1dXTbbHz9+HCtWrMCTTz6J06dPo6ysDGVlZTh37hzXRqPR4J577sFrr73m+p0QCASfx+ll/3nz5uGuu+7C9u3bAQAmkwmpqan46U9/in/7t3+7o/3y5cuh0Wiwf/9+7tzdd9+NoqIi7Nixw6rtjRs3kJmZidOnT6OoqMjhPpFlfwJB2Dj6GXVqhKTT6VBbW4uSkhLzBUQilJSUoKamxuZzampqrNoDQGlp6ZjtHUGr1UKtVlsdBALB93FKkHp6emA0GpGQYF1QmZCQAKXStpWHUql0qr0jVFZWQqFQcEdqaqrL1yIQCMLBJ/OQNm/ejIqKCu5nlUqFtLQ0MlIiEAQK+9kcL0LklCDFxsZCLBajs7PT6nxnZycSE21v/JeYmOhUe0eQyWSQyWTcz+zNkpESgSBs+vv7oVCMYUUDJwVJKpVi9uzZqKqqQllZGQAmqF1VVYXy8nKbzykuLkZVVRWeeeYZ7tyhQ4dQXMxfpmlycjJaW1sRHh4+rqewWq1GamoqWltbAyIAHmj3CwTePfvC/dI0jf7+fiQnJ9tt5/SUraKiAqtXr8acOXMwd+5cvPXWW9BoNFizZg0AYNWqVUhJSUFlZSUAYOPGjVi0aBG2bduGJUuWYO/evTh16hR27tzJXfPWrVtoaWlBezvjy3Pp0iUAzOjKkZGUSCTCpEnOeQlHREQI9o/nDgLtfoHAu2eh36+9kREH7QK/+c1v6LS0NFoqldJz586lT5w4wT22aNEievXq1Vbt//d//5fOzs6mpVIpPX36dPrvf/+71eO7d++mwZj0WB1bt251pXt2UalUNABapVLxfm0hEmj3S9OBd8/+dL9+YT/iDIGWsxRo9wsE3j370/0GXC2bTCbD1q1brYLi/kyg3S8QePfsT/cbcCMkAoEgXAJuhEQgEIQLESQCgSAYiCARCATBQASJQCAIBiJIBAJBMAScIDnrdumrVFZW4q677kJ4eDji4+NRVlbGZcAHAr/+9a9BUZRVyZI/0tbWhh/96EeIiYlBcHAwCgoKcOrUKW93y2UCSpCcdbv0Zb7++mts2LABJ06cwKFDh6DX6/HAAw9Ao9F4u2tu59tvv8Xvfvc7zJgxw9tdcSu3b9/GggULEBQUhP/7v//DhQsXsG3bNkRFRXm7a67j3URxzzJ37lx6w4YN3M9Go5FOTk6mKysrvdgrz9DV1UUDoL/++mtvd8Wt9Pf301OnTqUPHTpEL1q0iN64caO3u+Q2nn/+efqee+7xdjd4JWBGSK64XfoTKpUKABAdHe3lnriXDRs2YMmSJXe4lPojf/3rXzFnzhwsXboU8fHxmDlzJnbt2uXtbk2IgBEkV9wu/QWTyYRnnnkGCxYsQH5+vre74zb27t2Luro6zmnC37l+/Tp++9vfYurUqfj888+xfv16PP300/jggw+83TWX8UnHSIJzbNiwAefOncPRo0e93RW30draio0bN+LQoUOQy+Xe7o5HMJlMmDNnDl599VUAwMyZM3Hu3Dns2LEDq1ev9nLvXCNgRkiuuF36A+Xl5di/fz8OHz7stGeUL1FbW4uuri7MmjULEokEEokEX3/9Nd555x1IJBIYjUZvd5F3kpKSkJeXZ3UuNzcXLS0tXurRxAkYQbJ0u2Rh3S75dK8UCjRNo7y8HJ9++im++uorZGZmertLbuX+++9HQ0MD6uvruWPOnDlYuXIl6uvrIRaLvd1F3lmwYMEdqRyXL19Genq6l3rEA96OqnuSvXv30jKZjH7//ffpCxcu0OvWraMjIyNppVLp7a7xzvr162mFQkFXV1fTHR0d3DE4OOjtrnkMf19lO3nyJC2RSOj/+I//oK9cuUJ/+OGHdEhICP0///M/3u6aywSUING0fbdLfwI2HDgB0Lt37/Z21zyGvwsSTdP03/72Nzo/P5+WyWR0Tk4OvXPnTm93aUIQPyQCgSAYAiaGRCAQhA8RJAKBIBiIIBEIBMFABIlAIAgGIkgEAkEwEEEiEAiCgQgSgUAQDESQCASCYCCCRCAQBAMRJAKBIBiIIBEIBMHw/wEDdaIL7IlcTwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T14:22:51.001098Z", + "iopub.status.busy": "2024-03-26T14:22:51.000815Z", + "iopub.status.idle": "2024-03-26T14:25:18.235966Z", + "shell.execute_reply": "2024-03-26T14:25:18.235130Z" + }, + "papermill": { + "duration": 147.255503, + "end_time": "2024-03-26T14:25:18.238462", + "exception": false, + "start_time": "2024-03-26T14:22:50.982959", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T14:25:18.275307Z", + "iopub.status.busy": "2024-03-26T14:25:18.274964Z", + "iopub.status.idle": "2024-03-26T14:25:18.296544Z", + "shell.execute_reply": "2024-03-26T14:25:18.295637Z" + }, + "papermill": { + "duration": 0.042259, + "end_time": "2024-03-26T14:25:18.298695", + "exception": false, + "start_time": "2024-03-26T14:25:18.256436", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tab_ddpm_concat0.0022250.6476470.0238981.9961370.1047310.9842270.1544250.0000483.2852740.1050680.9151610.1545910.059610.7028715.281411
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "tab_ddpm_concat 0.002225 0.647647 0.023898 1.996137 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss \\\n", + "tab_ddpm_concat 0.104731 0.984227 0.154425 0.000048 \n", + "\n", + " pred_duration pred_mae pred_mape pred_rmse pred_std \\\n", + "tab_ddpm_concat 3.285274 0.105068 0.915161 0.154591 0.05961 \n", + "\n", + " std_loss total_duration \n", + "tab_ddpm_concat 0.702871 5.281411 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T14:25:18.334423Z", + "iopub.status.busy": "2024-03-26T14:25:18.334113Z", + "iopub.status.idle": "2024-03-26T14:25:18.836686Z", + "shell.execute_reply": "2024-03-26T14:25:18.835544Z" + }, + "papermill": { + "duration": 0.522875, + "end_time": "2024-03-26T14:25:18.839357", + "exception": false, + "start_time": "2024-03-26T14:25:18.316482", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T14:25:18.879701Z", + "iopub.status.busy": "2024-03-26T14:25:18.879316Z", + "iopub.status.idle": "2024-03-26T14:27:59.172372Z", + "shell.execute_reply": "2024-03-26T14:27:59.171102Z" + }, + "papermill": { + "duration": 160.316923, + "end_time": "2024-03-26T14:27:59.175222", + "exception": false, + "start_time": "2024-03-26T14:25:18.858299", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/tab_ddpm_concat/all inf False\n", + "Caching in ../../../../insurance/_cache_bs_test/tab_ddpm_concat/all inf False\n", + "Caching in ../../../../insurance/_cache_synth_test/tab_ddpm_concat/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T14:27:59.213902Z", + "iopub.status.busy": "2024-03-26T14:27:59.213592Z", + "iopub.status.idle": "2024-03-26T14:27:59.233426Z", + "shell.execute_reply": "2024-03-26T14:27:59.232711Z" + }, + "papermill": { + "duration": 0.042022, + "end_time": "2024-03-26T14:27:59.235440", + "exception": false, + "start_time": "2024-03-26T14:27:59.193418", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T14:27:59.270371Z", + "iopub.status.busy": "2024-03-26T14:27:59.270030Z", + "iopub.status.idle": "2024-03-26T14:27:59.275866Z", + "shell.execute_reply": "2024-03-26T14:27:59.274953Z" + }, + "papermill": { + "duration": 0.025491, + "end_time": "2024-03-26T14:27:59.277807", + "exception": false, + "start_time": "2024-03-26T14:27:59.252316", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tab_ddpm_concat': 0.02689460172200294}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T14:27:59.313941Z", + "iopub.status.busy": "2024-03-26T14:27:59.313611Z", + "iopub.status.idle": "2024-03-26T14:27:59.670242Z", + "shell.execute_reply": "2024-03-26T14:27:59.669345Z" + }, + "papermill": { + "duration": 0.377364, + "end_time": "2024-03-26T14:27:59.672396", + "exception": false, + "start_time": "2024-03-26T14:27:59.295032", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASsAAAE8CAYAAACYd5eXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABDlUlEQVR4nO3deXhTZb4H8O/Jni5JW7pDNyiL7AUKAy4F5bLIVSqOC6JQEGFGGERGrzD3AqLjII6yKILOPAOFEXHGGVwGR5BBllGw7IvgsNOWli6UNmmbNut7/zjNadIkbRKaJml+n+fJ0+Ysye+0za/v+5534RhjDIQQEuBE/g6AEELcQcmKEBIUKFkRQoICJStCSFCgZEUICQqUrAghQYGSFSEkKFCyIoQEBUpWhJCgQMmqE0pPT8d///d/+/Q9OI7Dq6++2uZxr776KjiO82ksJDRQsgpAhw4dwquvvoqamhp/h0KCwIYNG5Cfn+/vMHyOklUAOnToEFasWEHJiriFkhUhhAQQSlYB5tVXX8XLL78MAMjIyADHceA4DtevX8fmzZtx//33Iz4+HnK5HH379sXGjRtdvtY333yDwYMHQ6FQoG/fvtixY4fH8ej1erz44ouIi4tDZGQkHn74Ydy4ccPpsd999x2ys7OhUCjQo0cPfPjhh06P4zgO8+fPx7Zt29C7d28oFAoMHToUBw8edPhZcByHixcv4umnn4ZarUZcXByWLl0KxhiKi4sxefJkqFQqJCYm4p133vH4+gDg66+/Rk5ODiIjI6FSqZCdnY2PP/7Y7phPP/0UQ4cOhVKpRGxsLJ5++mmUlJTYHZOXl4eIiAiUlJQgNzcXERERiIuLw0svvQSz2Wx3rMViwbp16zBgwAAoFArExcVhwoQJOHbsmHCMO7/v9PR0nDt3DgcOHBD+VkaPHu3VzyHgMRJQTp8+zaZOncoAsDVr1rA///nP7M9//jOrq6tj2dnZLC8vj61Zs4a99957bNy4cQwAW79+vd1rpKWlsV69erGoqCi2ePFitnr1ajZgwAAmEonYN99841E8Tz/9NAPAnnrqKbZ+/Xo2ZcoUNnDgQAaALV++XDjuzJkzTKlUstTUVLZy5Ur2+uuvs4SEBOFYWwBY//79WWxsLHvttdfYqlWrWFpaGlMqlezs2bPCccuXL2cA2ODBg9nUqVPZhg0b2KRJkxgAtnr1ata7d2/2y1/+km3YsIHdfffdDAA7cOCAR9e3efNmxnEc69+/P3vjjTfY+++/z2bPns2eeeYZu2MAsOzsbLZmzRq2ePFiplQqWXp6OquurhaOmzFjBlMoFKxfv35s1qxZbOPGjezRRx9lANiGDRvs3jcvL48BYBMnTmRr165lb7/9Nps8eTJ77733hGPc+X1/9tlnrFu3bqxPnz7C34qnv+NgQckqAP3+979nANi1a9fstut0Oodjx48fz7p37263LS0tjQFgf//734VtGo2GJSUlsaysLLfjOHXqFAPAnn/+ebvtTz31lEOyys3NZQqFghUWFgrbzp8/z8RisdNkBYAdO3ZM2FZYWMgUCgV75JFHhG3WZDVnzhxhm8lkYt26dWMcx7E333xT2F5dXc2USiWbMWOG29dXU1PDIiMj2YgRI1hDQ4PdPovFwhhjzGAwsPj4eNa/f3+7Y3bu3MkAsGXLlgnbZsyYwQCw1157ze61srKy2NChQ4Xn3377LQPAFixY4BCT9X0Zc//33a9fP5aTk+PGFQc3qgYGEaVSKXyv0Whw69Yt5OTk4OrVq9BoNHbHJicn45FHHhGeq1QqTJ8+HSdPnkRZWZlb7/fPf/4TALBgwQK77QsXLrR7bjabsXv3buTm5iI1NVXYftddd2H8+PFOX3vkyJEYOnSo8Dw1NRWTJ0/G7t27HapMs2fPFr4Xi8UYNmwYGGN49tlnhe1RUVHo3bs3rl696ta1AcCePXtQW1uLxYsXQ6FQ2O2zdrc4duwYKioq8Pzzz9sdM2nSJPTp0wdfffWVw+v+4he/sHt+77332sX197//HRzHYfny5Q7n2nbz8OT3HQooWQWR77//HmPHjkV4eDiioqIQFxeH3/zmNwDg8MebmZnp0L+pV69eAIDr16+79X6FhYUQiUTo0aOH3fbevXvbPa+srERDQwN69uzp8Botj7VydmyvXr2g0+lQWVlpt902AQKAWq2GQqFAbGysw/bq6mrXF9TClStXAAD9+/d3eUxhYSEA59fRp08fYb+Vtf3JVnR0tF1cV65cQXJyMmJiYlqNz5PfdyiQ+DsA4p4rV67ggQceQJ8+fbB69WqkpKRAJpPhn//8J9asWQOLxeLvEH1GLBa7tQ0AmJ9n6XYVl6dC+fftCiWrAOSsx/c//vEP6PV6fPnll3YljX379jl9jcuXL4MxZvdaFy9eBMDfQXJHWloaLBYLrly5YleyuHDhgt1xcXFxUCqVuHTpksNrtDzWytmxFy9eRFhYmEPJxFesJcYff/wRmZmZTo9JS0sDwF/H/fffb7fvwoULwn5P33f37t24ffu2y9KVJ7/vUBkhQNXAABQeHg4Adp1Crf+xbUsOGo0GmzdvdvoapaWl+Oyzz4TnWq0WW7duxeDBg5GYmOhWHBMnTgQAvPvuu3bb165da/dcLBZj/Pjx+Pzzz1FUVCRs/+mnn7B7926nr3348GGcOHFCeF5cXIwvvvgC48aNa7fSSVvGjRuHyMhIrFy5Eo2NjXb7rD/nYcOGIT4+Hh988AH0er2w/+uvv8ZPP/2ESZMmefy+jz76KBhjWLFihcM+6/t68vsODw8PiQ7EVLIKQNaG5//93//Fk08+CalUivvuuw8ymQwPPfQQ5s6di7q6Ovzxj39EfHw8bt686fAavXr1wrPPPoujR48iISEBmzZtQnl5ucvk5szgwYMxdepUbNiwARqNBqNGjcLevXtx+fJlh2NXrFiBXbt24d5778Xzzz8Pk8mE9957D/369cOZM2ccju/fvz/Gjx+PBQsWQC6XY8OGDcLrdBSVSoU1a9Zg9uzZyM7OxlNPPYXo6GicPn0aOp0OW7ZsgVQqxapVqzBz5kzk5ORg6tSpKC8vx7p165Ceno4XX3zR4/cdM2YMnnnmGbz77ru4dOkSJkyYAIvFgn//+98YM2YM5s+fj3Hjxrn9+x46dCg2btyI3/72t8jMzER8fLxDKbBT8N+NSNKa119/nXXt2pWJRCKhG8OXX37JBg4cyBQKBUtPT2erVq1imzZtcujmkJaWxiZNmsR2797NBg4cyORyOevTpw/79NNPPY6joaGBLViwgHXp0oWFh4ezhx56iBUXFzt0XWCMsQMHDrChQ4cymUzGunfvzj744AOh+4EtAGzevHnso48+Yj179mRyuZxlZWWxffv22R1nPbeystJu+4wZM1h4eLhDrDk5Oaxfv34eX+OXX37JRo0axZRKJVOpVGz48OFs+/btdsf85S9/YVlZWUwul7OYmBg2bdo0duPGDbficvYzMJlM7Pe//z3r06cPk8lkLC4ujk2cOJEdP37cLi53ft9lZWVs0qRJLDIykgHotN0YOMZo3UDSsTiOw7x587B+/Xp/h0KCCLVZEUKCArVZhai2OoYqlUqo1eoOiqb9VVZWOnQutSWTydrs50QCCyWrEJWUlNTq/hkzZgT1tCPZ2dkOHTZt5eTkYP/+/R0XELljlKxC1J49e1rdn5yc7LP37ohm0m3btqGhocHl/ujoaJ/HQNoXNbATQoICNbATQoJCp68GWiwWlJaWIjIyMmSGJRASLBhjqK2tRXJyMkSi1stOnT5ZlZaWIiUlxd9hEEJaUVxcjG7durV6TKdPVpGRkQD4H4ZKpfJzNIQQW1qtFikpKcLntDWdPllZq34qlYqSFSEByp0mGmpgJ4QEBUpWhJCgQMmKEBIUOn2blTsYYzCZTK2OJSPOicViSCQS6hZCfC7kk5XBYMDNmzeh0+n8HUrQCgsLQ1JSEmQymb9DIZ1YSCcri8WCa9euQSwWIzk5GTKZjEoIHmCMwWAwoLKyEteuXUPPnj3b7NhHiLdCOlkZDAZYLBakpKQgLCzM3+EEJaVSCalUisLCQhgMBof19zq7cm0jvrt0C/f0jEWCKrSuvaPRv0GASgN3KJR/fgcuVqLotg7fX77l71A6vdD9KyOkHZRU89PQFFZRm6evUbIihAQFSlakTenp6Q5rBRL7SQQfG9b6IFxy50K6gZ2QO2GyMMRGyKA3WRAfSY3rvkbJKkQYDAbqB9XOpGIRnhmZ7u8wQoZfq4EHDx7EQw89hOTkZHAch88//1zYZzQa8corr2DAgAEIDw9HcnIypk+fjtLSUp/HZTBZXD5MZovbxxrdPNYbo0ePxvz58zF//nyo1WrExsZi6dKlQtUkPT0dr7/+OqZPnw6VSoU5c+YAAL777jvce++9UCqVSElJwYIFC1BfXy+8bkVFBR566CEolUpkZGRg27ZtXsUXKhhjOF54G4evVHn9uyTu8WvJqr6+HoMGDcKsWbMwZcoUu306nQ4nTpzA0qVLMWjQIFRXV+OFF17Aww8/jGPHjvk0rvf3OS6PbpURG47crK7C8z8cvAKj2fk09t2ilXhsWPPEf5u+v4YGg+OQnhf/q5dXcW7ZsgXPPvssjhw5gmPHjmHOnDlITU3Fc889BwB4++23sWzZMixfvhwAcOXKFUyYMAG//e1vsWnTJlRWVgoJz7qsfF5eHkpLS7Fv3z5IpVIsWLAAFRUVXsUXCjiOw/eXq2C2MPTrqoJMQs3AvuLXZDVx4kRMnDjR6T61Wu2wAsv69esxfPhwFBUVITU11el5er0eer1eeK7Vatsv4ACTkpKCNWvWgOM49O7dG2fPnsWaNWuEZHX//ffj17/+tXD87NmzMW3aNCxcuBAA0LNnT7z77rvIycnBxo0bUVRUhK+//hpHjhxBdnY2AOBPf/oT7rrrrg6/tmBQVKXD/osVMFv4f1ZGKln5VFC1WWk0GnAch6ioKJfHrFy5EitWrLij95k3JtPlPlGL0Thz7uvh8tiWI3dm3Z1xJ2E5+NnPfmY3PGjkyJF45513hAHZw4YNszv+9OnTOHPmjF3VjjEmDDu6ePEiJBIJhg4dKuzv06dPqz/vUNZoMqOqziA8N1looShfCppk1djYiFdeeQVTp05tdcbPJUuWYNGiRcJz67SpnvCkKO+rY9tDeHi43fO6ujrMnTsXCxYscDg2NTUVFy9e7KjQOgVnbZLEd4IiWRmNRjz++ONgjGHjxo2tHiuXyyGXyzsoMv8qKCiwe/7DDz+gZ8+eEIvFTo8fMmQIzp8/j8xM5yXHPn36wGQy4fjx40I18MKFC6ipqWnXuDsLU4u2SipZ+VbAtwZaE1VhYSH27NlD86jbKCoqwqJFi3DhwgVs374d7733Hl544QWXx7/yyis4dOgQ5s+fj1OnTuHSpUv44osvMH/+fABA7969MWHCBMydOxcFBQU4fvw4Zs+eDaVS2VGXFFRMFvuSlJmSlU8FdLKyJqpLly7hX//6F7p06eLvkALK9OnT0dDQgOHDh2PevHl44YUXhC4KzgwcOBAHDhzAxYsXce+99yIrKwvLli2zWyp+8+bNSE5ORk5ODqZMmYI5c+YgPj6+Iy4n6LQsWVGy8i2/VgPr6upw+XJzN4Fr167h1KlTiImJQVJSEn7+85/jxIkT2LlzJ8xmM8rKygAAMTEx1MERgFQqxdq1a51Wja9fv+70nOzsbHzzzTcuXzMxMRE7d+602/bMM8/cUZydlbXa1y1aibszYxEdRn+TvuTXZHXs2DGMGTNGeG5tGJ8xYwZeffVVfPnllwCAwYMH2523b98+jB49uqPCJMQpiYhDpEKC1JgwJEdRVdnX/JqsRo8ebTcYtKXW9hHibyO6d8GI7tQ00VGC4m4gcbR//35/h0CaFFXpUFmnR9coJRLVNKDZVwK6gZ2QYPCfMi0OXqxEYVV92wcTr1HJihAv7ftPBcq1jdA1jfc0U7OFT1HJihAvVdUbcFPTKHRZsFAHdp+iZEWIl6zTBcml/MeoZSdR0r4oWRHiJWNTiUreNObTQtVAn6JkRYiXhJKVRNz0nJKVL1GyIsRL1rYqhZRKVh2B7gYS4iXrDLF9ElW4K0mFCDl9nHyJfrqEeEkuEYGBISZCBpVC6u9wOj1KVrYYA8xG/7y3WOo4tagLW7duxYsvvojS0lK7ubtyc3MRGRmJP//5z76KktiYdU/7zvxKWkfJypbZCPz7Hf+8972/BiTujdp/7LHHsGDBAnz55Zd47LHHAPCr0nz11VetzqhAfEOjM6Lotg5KmRiZ8RH+DqfTogb2IKRUKvHUU08JK9IAwEcffYTU1FSajcIPymsb8a+fynGiqNrfoXRqVLKyJZbyJRx/vbcHnnvuOWRnZ6OkpARdu3ZFfn4+8vLy7BaQIL7TaDTjy1OlEIs4DEqJAkCzhPgaJStbHOd2VczfsrKyMGjQIGzduhXjxo3DuXPn8NVXX/k7rJBhMFtQUtMAiYhDVmoUAMBMHdh9ipJVEJs9ezbWrl2LkpISjB071uNVfIj3zE3dFkQiDuKm9dmon5VvUZtVEHvqqadw48YN/PGPf8SsWbP8HU5Isc6wIBFxEHGUrDoCJasgplar8eijjyIiIgK5ubn+DiekWHuvi0UcRNaSFS0Y4VOUrIJcSUkJpk2bFjJrJQYKk22yarqnQUMDfYvarIJUdXU19u/fj/3792PDhg3+DifkWGySVXSYDA8NSoLMxeKypH1QsgpSWVlZqK6uxqpVq9C7d29/hxNyGANkEhGkYhEUUjEy4yP9HVKnR8kqSLlaF5B0jNQuYZg3JtPfYYQUv7ZZHTx4EA899BCSk5PBcRw+//xzu/2MMSxbtgxJSUlQKpUYO3YsLl265J9gCXHBYLLgp5ta/HRT6+9QOjW/Jqv6+noMGjQI77//vtP9b731Ft5991188MEHKCgoQHh4OMaPH4/GxsZ2jYN6Ht+ZUP/5NRjN2PVjGf51vtzfoXRqfq0GTpw4ERMnTnS6jzGGtWvX4v/+7/8wefJkAPxsAwkJCfj888/x5JNP3vH7S6X8EBedTgelklbU9ZZOpwPQ/PMMBVcr63D6Rg26RoWhb7IKAK1u42sB22Z17do1lJWVYezYscI2tVqNESNG4PDhwy6TlV6vh16vF55rta6L5mKxGFFRUaioqAAAhIWF0dg6DzDGoNPpUFFRgaioKIhD6G6YpsGI67d0kInFQtcFxvifCf0N+UbAJquysjIAQEJCgt32hIQEYZ8zK1euxIoVK9x+n8TERAAQEhbxXFRUlPBzDBXW3upimx7sAN9ZVCKmZOULAZusvLVkyRIsWrRIeK7ValsdM8dxHJKSkhAfHw+j0U8T7wUxqVQaUiUqK+viEC2TFXVi952ATVbW/9Tl5eVISkoStpeXl2Pw4MEuz5PL5V715haLxSH5oSPesQ63kdgMZAZofKAvBexwm4yMDCQmJmLv3r3CNq1Wi4KCAowcOdKPkRHS3JhuO9wGoGTlS34tWdXV1eHy5cvC82vXruHUqVOIiYlBamoqFi5ciN/+9rfo2bMnMjIysHTpUiQnJ9OgXeJ3tmMDOY7DhP6JEHEcpOKA/f8f9PyarI4dO4YxY8YIz61tTTNmzEB+fj7+53/+B/X19ZgzZw5qampwzz33YNeuXVAoFP4KmRBeUwHKWgW8K0nlx2BCA8c6eY8+rVYLtVoNjUYDlYr+oEj7YYyBMQhTxBDPefL5DNgGdkICHcdxwupp127Vw2i2IDUmDAop3ajxBapgE9IO9pwvw1dnbqK20eTvUDotKlkR4oUfrlbhVp0eg1Oi0C06jKY27gBUsiLECyXVDbhUXoc6PV+SomTle5SsCPGCMAd7U5ISpjamLuw+Q8mKEC/Y9rOy/UoFK9+hZEWIF5qX4uI/QtaZFqhk5TuUrAjxgrlp+eWmXEULnXYAuhtIiBesy25ZS1bZ6TFoNJoRG0lLovkKJStCvGC22JesMuMj/BhNaKBkRYgXZt/THSYLg4SG2nQYSlaEeEEk4iCzSVQV2kbUG8zoEiGDShE6c9F3JGpgJ6QdHL5ahc9PlqCoSufvUDotKlkR4iGLhWHXuTKIRRzG9I6HTCISui7Q3UDfoZIVIR4yWRgulNXifGnzykliIVn5K6rOj5IVIR6y7fhpbWCn4Ta+R8mKEA9Ze6+LOE6YeM9aDezkc1n6FSUrQjxkFpbhat7W3IPdHxGFBkpWhHioeWWb5o8PVQN9j+4GEuIhU1PvdduSVa+ESMSEy5CkVvopqs6PkhUhHhLmsrIpWaXEhCElJsxfIYUESlaEeCghUoFfju5Bc1d1sIBuszKbzVi6dCkyMjKgVCrRo0cPvP7663THhfiVSMRBIRVDKWtexUbbaMSNah2q6w1+jKxzC+iS1apVq7Bx40Zs2bIF/fr1w7FjxzBz5kyo1WosWLDA3+ERIjhXosUPV6swKEWN+/sk+DucTimgk9WhQ4cwefJkTJo0CQCQnp6O7du348iRI36OjISyMk0jzpZo0CVChiGp0QBs7wb6MbBOLqCrgaNGjcLevXtx8eJFAMDp06fx3XffYeLEiS7P0ev10Gq1dg9C2lO1zoAfSzS4VlkvbKOZQn0voEtWixcvhlarRZ8+fSAWi2E2m/HGG29g2rRpLs9ZuXIlVqxY0YFRklBjvRsoETdPEUM92H0voEtWf/3rX7Ft2zZ8/PHHOHHiBLZs2YK3334bW7ZscXnOkiVLoNFohEdxcXEHRkxCgTVZWdcK5L+37vNHRKEhoEtWL7/8MhYvXownn3wSADBgwAAUFhZi5cqVmDFjhtNz5HI55HKaB5v4TvPKNs3JiqqBvhfQJSudTgeRyD5EsVgMi4X+fRH/EUpWItuSFSUrXwvoktVDDz2EN954A6mpqejXrx9OnjyJ1atXY9asWf4OjYQwk9mxZJWgUuCenrGIUtKUxr4S0Mnqvffew9KlS/H888+joqICycnJmDt3LpYtW+bv0EgIszD71ZgBIC5SjjhahsunOObF7YurV6+ie/fuvoin3Wm1WqjVamg0GqhUKn+HQzoBg8kCvckMiUhk14udeM6Tz6dXbVaZmZkYM2YMPvroIzQ2NnoVJCHBSiYRIVIhtUtUjUYzyrWNuFWn92NknZtXyerEiRMYOHAgFi1ahMTERMydO5d6lZOQVlLTgI8LirDnfLm/Q+m0vEpWgwcPxrp161BaWopNmzbh5s2buOeee9C/f3+sXr0alZWV7R0nIQHjXKkG+y9U4EZ187JbYrob6HN31HVBIpFgypQp+PTTT7Fq1SpcvnwZL730ElJSUjB9+nTcvHmzveIkJGBcv6XDyaIa3KprnmFB6LpAM4X6zB0lq2PHjuH5559HUlISVq9ejZdeeglXrlzBnj17UFpaismTJ7dXnIQEDGGmUM52uA3/lXKV73jVdWH16tXYvHkzLly4gAcffBBbt27Fgw8+KHTgzMjIQH5+PtLT09szVkICgrOuC9SD3fe8SlYbN27ErFmzkJeXh6SkJKfHxMfH409/+tMdBUdIIDKZHZOVtRpIC0b4jlfJas+ePUhNTXUYCsMYQ3FxMVJTUyGTyVyO3yMkmDXPwe44kJkKVr7jVZtVjx49cOvWLYftt2/fRkZGxh0HRUggczaQOVwuwYiMGAxJi/JTVJ2fVyUrV53e6+rqoFAo7iggQgKds5JVuFyCUZmx/gopJHiUrBYtWgSAn2hs2bJlCAtrXnrIbDajoKAAgwcPbtcACQk0j2R1hdHMECEP6KG1nY5HP+2TJ08C4EtWZ8+ehUwmE/bJZDIMGjQIL730UvtGSEiAiVQ4zqxgsTDUNBhhYQyxETSg2Rc8Slb79u0DAMycORPr1q2jgcGENNEZzdhy6Do4Dlg4tpe/w+mUvCrHbt68ub3jICRoHLrM31wakhYNhZQfzGx7N5AxJszJTtqP28lqypQpyM/Ph0qlwpQpU1o9dseOHXccGCGB6lhhNcwWhv7d1DbJqjk5WRggplzV7txOVmq1WvhvoVarfRYQIYGMMda8uo2TTqEA34tdDMpW7c3tZGVb9aNqIAlVtj3UnXUKtR4jpTn52p1XnUIbGhqg0zVPj1FYWIi1a9fim2++abfACAlEJttkxTmODQSoF7uveJWsJk+ejK1btwIAampqMHz4cLzzzjuYPHkyNm7c2K4BEhJIbAcq2yYojuNsZl6gbOULXs8Ueu+99wIA/va3vyExMRGFhYXYunUr3n333XYNkJBAYrLpvd7yjt+Q1GgMS4+2S2Kk/XjVdUGn0yEyMhIA8M0332DKlCkQiUT42c9+hsLCwnYNkJBAYnYy44LVfb3iOjqckOL1ghGff/45iouLsXv3bowbNw4AUFFRQR1FSacWqZDgmZFpeGxYN3+HEnK8SlbLli3DSy+9hPT0dIwYMQIjR44EwJeysrKy2jVAQgKJRCxCbIQc8ZGOA/br9CZodEaYzLRiuC94lax+/vOfo6ioCMeOHcOuXbuE7Q888ADWrFnTbsEBQElJCZ5++ml06dIFSqUSAwYMwLFjx9r1PQhpD9sLirDp+2u4rTO0fTDxmNfDxhMTE5GYmGi3bfjw4XcckK3q6mrcfffdGDNmDL7++mvExcXh0qVLiI6Obtf3IcRdGp0R529qEamQoH9X+87RHE3A51NeJav6+nq8+eab2Lt3LyoqKmCx2Bd7r1692i7BrVq1CikpKXadUGlyP+JP1ToDfrhahXiV3CFZ0dTGvuVVspo9ezYOHDiAZ555BklJST4btPnll19i/PjxeOyxx3DgwAF07doVzz//PJ577jmX5+j1euj1zaviarVan8RGQpPQdcHJ3zwtGuFbXiWrr7/+Gl999RXuvvvu9o7HztWrV7Fx40YsWrQIv/nNb3D06FEsWLCg1fndV65ciRUrVvg0LhK6nK1sY2XdZKH2dZ/wqoE9OjoaMTEx7R2LA4vFgiFDhuB3v/sdsrKyMGfOHDz33HP44IMPXJ6zZMkSaDQa4VFcXOzzOEnocLayjRVHqzL7lFfJ6vXXX8eyZcvsxgf6QlJSEvr27Wu37a677kJRUZHLc+RyOVQqld2DkPbSesmKkpUveVUNfOedd3DlyhUkJCQgPT0dUqn9NK8nTpxol+DuvvtuXLhwwW7bxYsXkZaW1i6vT4inTE4Wi7DqmRCBRLXc6bTHDixmgBM130IkbfIqWeXm5rZzGM69+OKLGDVqFH73u9/h8ccfx5EjR/CHP/wBf/jDHzrk/QlpydlcVlbZ6W42jdy+Cvz4d0CuBrKmAbLw9gyx0+KYq3W1AsTOnTuxZMkSXLp0CRkZGVi0aFGrdwNb0mq1UKvV0Gg0VCUkd6xeb4K20Qi5RIyYcFnbJ7TEGFDwIdBQzT/vlg30HNu+QQYRTz6fXiermpoa/O1vf8OVK1fw8ssvIyYmBidOnEBCQgK6du3qVeC+QMmKdBS9yQyTmUEmEUEqdtEcrL0JHM9vfi4LB0b9KmSrg558Pr2qBp45cwZjx46FWq3G9evX8dxzzyEmJgY7duxAUVGRMNcVIaFk149luFpZj//qm+DQYVRQdYn/GtsTqL4GGOqB+ltABM3Y0Bav7gYuWrQIeXl5uHTpkt0KzA8++CAOHjzYbsEREmiKqnQ4XngbpTUNDvvc6rqgLeW/xmQAkUn897Wl7R1mp+RVsjp69Cjmzp3rsL1r164oKyu746AICVSXK2tx8OItXK+qd9gnFpKVi5MZA2pv8t9HJgOqZP577U0fRNr5eFUNlMvlToexXLx4EXFxVJwlnZd19hdnw22sNwhdjg1srAGMjYBIDETENzey11e0f6CdkFclq4cffhivvfYajEYjAL74W1RUhFdeeQWPPvpouwZISCAxN42lkThZGNBaDXR5z6q+iv8a1oVPWOGx/HNdFU3V4AavktU777yDuro6xMXFoaGhATk5OcjMzERkZCTeeOON9o6RkIBhLVmJWh3I7OJknTVZxcBiYagwKcEAvrRl9O1okM7Aq2qgWq3Gnj178P333+P06dOoq6vDkCFDMHZs6PYXIaHBZC1ZiRz/z7dZDWy4zX9VxuDApUqcKqrBz41SpCiNfCKjzqGt8jhZWSwW5OfnY8eOHbh+/To4jkNGRgYSExPBGPPZdDGEBALrnT4nuQpJaiVMFoa4SLnzk3XN1cBz5zUAAFlkF8BUBuhuA1Gpvgi50/CoGsgYw8MPP4zZs2ejpKQEAwYMQL9+/VBYWIi8vDw88sgjvoqTkIBgnXXBWcmqb7IK4/slIjM+wvnJOr5kZZJHCWMM1dFNN6T0NO9aWzwqWeXn5+PgwYPYu3cvxowZY7fv22+/RW5uLrZu3Yrp06e3a5CEBIqc3nFoNFjQJcLDoTYmPd8BFEC9RA3GdBCLOMgjooFKAI2a9g+2k/GoZLV9+3b85je/cUhUAHD//fdj8eLF2LZtW7sFR0igiY9UILVLGMLljv/nLRYGg8kCo7PVbRqbSk4SOfSMn5XBbGEo0klgZqx5P3HJo2R15swZTJgwweX+iRMn4vTp03ccFCHBqODabby/7zL+fanScae1mqdQo8FoFjbvuaqH0WShkpUbPKoG3r59GwkJCS73JyQkoLq6+o6DIiRQnSvVwGxhyIyPQJjM/uPT6rTGjTX81xbJSi+O4Nuv9LX8ic5a7gkAD0tWZrMZEonr/CYWi2Eyme44KEIC1eErVdj7UwVqGx3/zkVN2crsrIOntZonV0EhESOtSxgAwCAOg4lxALMAhlqfxd0ZeFSyYowhLy8PcrnzW7O2q8oQ0hlZ+1A56xQqaq0Hu001MD02HOmx4fjrsWKUVDfAIA4H0MiXrhQuZmsgniUrVyvK2KI7gaQzM7UyU6hQDXTWJ9TaJqVonrNJLuErNgZxGPhkVdeeoXY6HiUr28VGCQlFFmvJqpUFI5z2YG9sLllZySViAIBepOQ3GChZtcbr5eMJCTWMsVZLVi4XObWYmxORXIVdP97E1Vv1MDd1MNWLwwAzKFm1gZIVIW6yLTA5W91GrZSiZ0IEElQK+x16LT+rgkgCyMKhN2mgN1rQN1mFJLUCibVxQNklqga2gZIVIW4y2fRJcJasUmLCkBIT5niiUAVUARwHg4l/nYzYcPRKiATKYoAyUMmqDZSsCHGTRCTCI1ldYbIwp9VAl/TN3RYAwNhU/RMWlbDOtqCnrgutoWRFiJvEIg7psa1P48IY42t8tsmsxZ1A63Aco9mCoiod5AYJEgAqWbUhqLrLvvnmm+A4DgsXLvR3KIQ4uFpZh7X/uoS/HCu232EtMcntk9VNTSP+fuIGDhUb+P3GRsBMnapdCZpkdfToUXz44YcYOHCgv0MhIarRaMaPJRpcLHdeXRO5Wt1GSFaRAABDU7IKlzV1XWBSvvEdoNJVK4IiWdXV1WHatGn44x//iOjoaH+HQ0KUttGIPefLsf+C8wUehGTVsp+V0GbFJ6tElQJJagUiFHyCMjIA8qY5sChZuRQUyWrevHmYNGmSW9Mm6/V6aLVauwch7cF6M1DsYrAx56oHe4tq4JQh3fDk8FSoFPxUMUaTxaaRnZKVKwHfwP7JJ5/gxIkTOHr0qFvHr1y5EitWrPBxVCQUWbsuOFnYht/urFOoycC3RQFCycrKejfQaLYAMipZtSWgS1bFxcV44YUXsG3bNruVn1uzZMkSaDQa4VFcXNz2SYS4QShZiZ1/bJwOt7EmH7EUkNhPACBreh2ThTUnMkpWLgV0yer48eOoqKjAkCFDhG1msxkHDx7E+vXrodfrIRaL7c6Ry+UuZ4Ug5E40l6ycF62svRXs2teFbgtqgONwq06PT4/dgEopQe7grgAAg8kCJg0DBwhTHxNHAZ2sHnjgAZw9e9Zu28yZM9GnTx+88sorDomKEF+yVu9cdQiVS8TIiA2HQmrzd9nyTqDJgkajGXKJCHKJCPf1ioVEJAIT6fhkRW1WLgV0soqMjET//v3ttoWHh6NLly4O2wnxNVMrMy4AgDpMitysrvYbWyQrax8rqUQEiViEoWkx/P4qqga2JaCTFSGBJDlKif8emGRfcmqLi2Qla9lKLzSwUzXQlaBLVvv37/d3CCREqRRSobuB2xyqgfbjAiu0jdCbLIhXKCAH+GXkaS52p+gnQkg7qdebsP7bS1j/7aXmjQ6DmJuqgU3J6h9nbuJvx2+g2iAFOBHfOm+k0pUzlKwIcdOtOj0ulNWiQtvodD/H8TMqGM2seR52V21WTclK2lQdNFoAyJqml6GqoFOUrAhx0+WKOvzz7E2cueF8jT/bRSQsDIDZCBgb+A1NyUohFSNRrUBUGF+dtC5Dz3cMpV7srQm6NitC/MXa2VPsogu7fbJiEFtLVWIJIOE7Nffvqkb/rs3zsAslKzNramQvpzuCLlDJihA3tTb/OtDcKRRoSmy2YwJddCR1PuSGqoHOULIixE3mph7sEhd36mynOmYMDo3rzkiaSlYmC2uuBlKycoqqgYS4ydQ0HbHERTWQ4zhwHJ+oLIw5NK4DwO5zZbhR3YB7MmPROzHSRcmKpjd2hpIVIW6yVgOdLRZh1S2av6PHcXCarOr1JmgbjEL7V8/4CHQJl6FrtBLQU8mqNZSsCHFTW21WAPDzod2anzhJVkIPdgn/Gt3jItA9rmmnhtqsWkPJihA3ZaVEoUdcOJLUSvdOcNJmZWi5so0t264LjLlslA9VlKwIcZPTNQFbY7teYBNTU8lK0pSsGgxmaBqMkElEiFE0lawsJsCkB6TuzeEWKuhuICHt6M8/FGLD/suorNY2dwhVNPerau7BzpeaLpbXYvuRInx/+Zb9BH1UFXRAyYoQNxVV6XC1sg6NRrPLY4wmC/RGCyyNNfwGqcJuhlDrAqfWWUKbuy40TUNKdwRdomRFiJu+/U85vjhViqp6g8tjhHnYdTYzhNqICZchJlwmtFlZk5axaTYG6mvlGrVZEeImd+4GWpMVa6jhN7RIVlOHp9o9t7ZdGYWSFSUrV6hkRYib3OlnJSSyRuclK1fHWzucCusH6qka2BIlK0LcZPagZAW9NVlFtfqaMolND3aAxge2gqqBhLjJWvpptWRlHYrToAHEsCtZVdcb8NnJEkQoJHh8WAp/vMhm1gWAqoGtoGRFiBssFiasbuO0Q2eT6DAZ9GoL5NW1DslKb7JA02CE7Upd4XIJRmTECCUsWuzUNUpWhLjBZLNwaWslq9G94/kOnf9uqta10scK4CfjG5UZ2/wClKxcomRFiBtEHPBffRNgsrBW26wAAA3V/FepskUfK/spjZ2yVgONjYDZxE/cRwBQsiLELRKxyG6Gz1bpbvNfw2LsNhtdjAusrjfAaLYgJlwGiVQJiMSAxcwvHCF28z1DAN0NJKQd/XC1CjsLfkRJTQOgbJmsHKuBAPDxkSJsKyhCnd7ED16mudidCuhktXLlSmRnZyMyMhLx8fHIzc3FhQsX/B0WCUGNRjOu36pHaU1Dq8cZzRYw3W0+MYV1cdgHOJas7OZhB+iOoAsBnawOHDiAefPm4YcffsCePXtgNBoxbtw41NfTL5F0rGod3+3g6x/LWj1OLOKgMGr4aY1bVAOlYhGiw6SIkNu3vtitcANQI7sLAd1mtWvXLrvn+fn5iI+Px/Hjx3Hffff5KSoSikxCe1PrjesSjoPSpIGFcQ7VwJYr21hZX1PoxU7JyqmATlYtaTR8r+CYmBiXx+j1euj1euG5Vqv1eVyk87OWelwtFmEltTTAbDHAAjmgjHLrta3VQoOZxge2JqCrgbYsFgsWLlyIu+++G/3793d53MqVK6FWq4VHSkpKB0ZJOiujmyUrhZ6/E2iQRPLzU7nBOpjZRIOZWxU0yWrevHn48ccf8cknn7R63JIlS6DRaIRHcXFxB0VIOrPmudNb/8jI9LcAADpZrMO+f50vx0c/FOJKpX31jqqB7gmKauD8+fOxc+dOHDx4EN26dWv1WLlcDrlc3uoxhHjK3WpguLEKYTIxzJHxDvuqdQZU1uqbk1KTzPgIxEbIERvR9HdrXWCikZowbAV0smKM4Ve/+hU+++wz7N+/HxkZGf4OiYQo63CbtqqBiSItErtFAX37OOzTm/iEJ29ROuuX3KLR3Tpnu6GeerHbCOifwrx58/Dxxx/jiy++QGRkJMrK+NvGarUaSqWbK4wQ0g5SY8KQ0zsOMWEy1wdZzICuiv8+wrFkZTC5V5WENIxPUGYTv0JOmOsbSqEkoJPVxo0bAQCjR4+2275582bk5eV1fEAkZCWoFEhQtbHaTP0tPmFJ5E4n3TO4aPdqNJrRYDBDIuYQqZDyvdgVUfzrNWooWTUJ6GTFGGv7IEIChfYGdAYTTlRF4MbRYjzZYgpjVyWr08U1OHSlCgO6qjG2bwK/Ua5qTlYEQIAnK0ICRWWtHgazBdFhUoTJXHxsNDcAADcRi2qd0W6XyWwRZhqVtRhuI8zDbu1nBTSXzChZCYKm6wIh/nToyi389WgxrlS00vdJUwKxiEOtPME+8YBvoI8OkyJcLnZIVsLYQJs5syhZOaKSFSFusHY3kLi6G9hQDTRqIBKLUSeLh9nCYLYwYaI+hVSMvLud382WCstx2Zasmu4I6qn7ghWVrAhxg7V3ucuJ86quAADEUSkwi/g7hi1LV65IWy50ClDJyglKVoS4wdDWcJumZCWK7SnMJGpwM1k1z7pgUw1URvNfG7WA2ejkrNBDyYoQNxiEDp1iJzvrgZpC/vsumZA23e0z2FTrLlfU4qMfCvHvS5UOp0tbLscF8H2tpE1dJawzj4Y4arMixA16kxmAY+9zAED5Ob5/lSoJCO+CmDAdGmVmu0M0DSZU1uoRE+7YqTRCLsHglCiE285zxTVNMWMsBRpuA5EJ7Xo9wYiSFSFtYIy57n1uMQMlx/nvEwcAAB7PdpzpQ2/kk5dS6lgyUyulGNPHscc7wroA2tLmXvEhjpIVIW2wMOC+XnHQGy1QtEw2paeAhhp+WpeEAS5fo6EpWTmc3xprz3WqBgKgZEWIo4ZqoK4SMDUAYjnEEfEYkhLNV81s1ZYDV7/lv0+7G5C4HjdoTVZKmfNk1WAwo9FoRqRCInQSFeZwp5IVAEpWhPAsZuDmab5KV3/Lcb9UAai6AVEp/J26+kqguIAfbBydDnQdIhz6w9UqXCyvxeCUKAzsFgWAT0aA82ogAGw5fB0NBjOeGZnWPFWMbbJizDFZhhhKViS0mU1A2Rmg6HDz/FGciJ81QRYOGHUwasvRWFsLqe4CFFWX7c+PSgH65dolkgajGVV1BmgbTMK2Oj3/fbjcebJSSERC6UqgjAZEEr7rQkN1yA9opmRFQpO1JGWbpOQRQOpIIKF/c7cBAEXlGuw9ehbpotsYF2fkuypIw4DYXvyxLSbks5aeGmwSj0IqhlxqdljZRjhHJka1zohGo033BZEYiIgDtDeB2jJKVv4OgJAOZbEAFeeA69/xDeNAc5JKGux0orsGE1Avj0d9bDrQv/WZagEgTOaYrKa2mIGhJWvDu13JCgAiEvlkVVcGJPRt8707M0pWJDQwBty6BFw70NwmJQsH0ka5TFJWQuO41L2Pi7Vk1Wgwt3FkM2tnU2t/LkFkIv+1ttzt1+qsKFmRzq/+FnBxN1BTxD+XKoCUnwFdh7Z6B89K15R0XLU3tWS946czmNo4splCylclGwwthuiokvmv2hK+6iryoOtDJ0PJinReJgNQ+D1QfARgFr701C2bT1TSNmb9tKFrahwPc9HtoKVIBb8El7bRBIuF4dSNGvxYokG/ZDWGpkU7PcfallWnbzEOMDwOkCoBYwNQexNQt10N7awoWZHOqfIicHlPc+N5bE8gc6zbC4/aspasXE6610KkXIIwmRiRCikMZgtu1epRVWewGyvYUqJagYHd1EiOarG2AMcBUalA5QW+ZEjJipBOolEDXNrDt08B/FQrPf+LT1Zeqjd4VrISiTjMzekhPK+s41cI7xLhusrZLToM3aLDnO+MTuOTVdVlvo0tRFGyIp2DSQ/cOAoU/cD3S+JEQOoIvme5mysjuzIsLQbVOgO6RHi+HqXeZMatWgMAIEntftXTTmwvPgFrSvhk7GQxilBAyYoEt0ZNU8/zE3y7DsB31Ow5nu+j1A76Jqu8Prfg6m1YGENUmFRoy3LFZLagqt6AqDCp/VQ08ki++ldTzM/wEKKlK0pWJLjoa/kPreYGoCnmh71YV0EKiwHS7wXi7/L70BSLheHzUyUorNIBAO5KajvhbT9ShFt1BjyS1RXpseH2OxMH8td94xjQbXhILnwaFJPvvf/++0hPT4dCocCIESNw5MgRf4dEOkqjFrh5BvjpH8APG4FD64HzX/Bj+Ooq+EQVnQb0nQxkP8d3nGzHRFWmaUTxbZ1j/6c2iEQcUmLCIOI4dI1SYkiq87uAtqzVTGsbl52EfnwJy1AP3AjNv/+AT89/+ctfsGjRInzwwQcYMWIE1q5di/Hjx+PChQuIj3cyBxDhP8D6Wn6xAbORv20vUfC3wGXhgFjm95KHSyYDf9fr9lWg+prj9Cgcx4/bU6fwVSN1N/5D7CMF16pwtbIe9/SMRXa6Z8NdstNjkJUS1TyLQhsSVHJcKKtF8W2d43uJxED3HOCnnXzve1UyP4A6hHAswFcSHTFiBLKzs7F+/XoAgMViQUpKCn71q19h8eLFbZ6v1WqhVquh0WigUnnf9hBwzCbAqOP/0zbc5js+6qr47xuq+f2uiCWALIJPXLIIPpGB8UnNYgYsJpuvJsBi5BOgRA5IlHwfJYmST352jzCAE/MJhRM1PZq+B5qqa6z5q8XcnFTryvnxb9pSfrsVxwGRSXzpyZqgJJ43dHtDozNiy+HrMFsYnv5ZGuIiffu+NToDNn9/HSKOw7SfpTbPvmDFGHDuM/7OoEjMd2pN6Mf3xQrSzqKefD4DumRlMBhw/PhxLFmyRNgmEokwduxYHD582Ok5er0een1zMVqrdXMpo8JDfOOlK3Y5nfl3u8Xc9iICnIgvcYil/PcmPZ/czEY+kTXUNI+NCzQKNdClBxCdwfcx8qAD550wmCzYfqQIJguD2WJBg8ECC2PoGq1EbCvdDtpLVJgMGbHhuHarHh8XFCFSIcGUId2gVjY1zHMccNfDAPcPoOI/fGfX4iP8domiqcTc4h+EP0vQskhg0BPt9nIBnaxu3boFs9mMhAT7+acTEhLwn//8x+k5K1euxIoVKzx/M32d83mMAhknAmRh/FQiYV34hzKGb2hWqJ3/tzUbAUMdf72Gev5harT5IxfzJS+R7UMMgOMTnqmRfxgbmh46/qupkf/eYoZQSmM2X4GmDw7X/L01ocoi+NJBZCKg6srH74cPmVjE4Xa9wW5bXKQc4/slguugeMb2TcDO06W4qWlEjc4Ii6VFxUcsAfrmAglXgNKTgKaIrzpbfx+BRGFo+xgPBHSy8saSJUuwaNEi4blWq0VKiuOc2A66Dm3uOGj3h2nzvafb7/i1XOzjRHy1S6Lw/EMtlvLJzbrUU0cIkonjRBzw86HdIBFzEIs4yMViqJSSDktUAD/s5onsFGgajKjTmxChcPIR5TggNpN/MMb/wzE2AGYD/8/BWtVm7i0F5jOi9k0vAZ2sYmNjIRaLUV5uP+K8vLwciYmJTs+Ry+WQy71oWwjvwj9I+wuCRAUAHMffwfM3juMQFSZDVJgbVU+O46e4kUf4PjA/C+iuCzKZDEOHDsXevXuFbRaLBXv37sXIkSP9GBkhpKMFdMkKABYtWoQZM2Zg2LBhGD58ONauXYv6+nrMnDnT36ERQjpQwCerJ554ApWVlVi2bBnKysowePBg7Nq1y6HRnRDSuQV8P6s71Wn7WRHSCXjy+QzoNitCCLGiZEUICQqUrAghQSHgG9jvlLVJzu1hN4SQDmP9XLrTdN7pk1VtbS0AuNeLnRDiF7W1tVCrW58BtdPfDbRYLCgtLUVkZGSHDpvwlnV4UHFxcae6e9lZrwvovNfWEdfFGENtbS2Sk5MhErXeKtXpS1YikQjdugXfiiAqlapT/eFbddbrAjrvtfn6utoqUVlRAzshJChQsiKEBAVKVgFGLpdj+fLl3s0cEcA663UBnffaAu26On0DOyGkc6CSFSEkKFCyIoQEBUpWhJCgQMmKEBIUKFkFgDfeeAOjRo1CWFgYoqKi3DqHMYZly5YhKSkJSqUSY8eOxaVLl3wbqIdu376NadOmQaVSISoqCs8++yzq6upaPWf06NHgOM7u8Ytf/KKDInbN01XBP/30U/Tp0wcKhQIDBgzAP//5zw6K1DOeXFd+fr7D70ah6Jhl0gBKVgHBYDDgsccewy9/+Uu3z3nrrbfw7rvv4oMPPkBBQQHCw8Mxfvx4NDY2+jBSz0ybNg3nzp3Dnj17sHPnThw8eBBz5sxp87znnnsON2/eFB5vvfVWB0TrmnVV8OXLl+PEiRMYNGgQxo8fj4qKCqfHHzp0CFOnTsWzzz6LkydPIjc3F7m5ufjxxx87OPLWeXpdAN+b3fZ3U1hY2HEBMxIwNm/ezNRqdZvHWSwWlpiYyH7/+98L22pqaphcLmfbt2/3YYTuO3/+PAPAjh49Kmz7+uuvGcdxrKSkxOV5OTk57IUXXuiACN03fPhwNm/ePOG52WxmycnJbOXKlU6Pf/zxx9mkSZPsto0YMYLNnTvXp3F6ytPrcvfv01eoZBWErl27hrKyMowdO1bYplarMWLECJcrVXe0w4cPIyoqCsOGDRO2jR07FiKRCAUFBa2eu23bNsTGxqJ///5YsmQJdDqdr8N1yboquO3Puq1VwQ8fPmx3PACMHz8+YH43gHfXBQB1dXVIS0tDSkoKJk+ejHPnWlnFvJ11+oHMnVFZWRkAOF2p2rrP38rKyhAfH2+3TSKRICYmptUYn3rqKaSlpSE5ORlnzpzBK6+8ggsXLmDHjh2+Dtkpb1YFLysrC+jfDeDddfXu3RubNm3CwIEDodFo8Pbbb2PUqFE4d+5ch0wWQCUrH1m8eLFDY2TLh6s/ikDm6+uaM2cOxo8fjwEDBmDatGnYunUrPvvsM1y5cqUdr4J4Y+TIkZg+fToGDx6MnJwc7NixA3Fxcfjwww875P2pZOUjv/71r5GXl9fqMd27d/fqta2rUZeXlyMpKUnYXl5ejsGDB3v1mu5y97oSExMdGmpNJhNu377tcjVtZ0aMGAEAuHz5Mnr06OFxvHfKm1XBExMTPTreH7y5rpakUimysrJw+fJlX4TogJKVj8TFxSEuLs4nr52RkYHExETs3btXSE5arRYFBQUe3VH0hrvXNXLkSNTU1OD48eMYOnQoAODbb7+FxWIREpA7Tp06BQB2Sbkj2a4KnpubC6B5VfD58+c7PWfkyJHYu3cvFi5cKGzbs2dPQK0i7s11tWQ2m3H27Fk8+OCDPozUht+a9omgsLCQnTx5kq1YsYJFRESwkydPspMnT7La2lrhmN69e7MdO3YIz998800WFRXFvvjiC3bmzBk2efJklpGRwRoaGvxxCU5NmDCBZWVlsYKCAvbdd9+xnj17sqlTpwr7b9y4wXr37s0KCgoYY4xdvnyZvfbaa+zYsWPs2rVr7IsvvmDdu3dn9913n78ugTHG2CeffMLkcjnLz89n58+fZ3PmzGFRUVGsrKyMMcbYM888wxYvXiwc//333zOJRMLefvtt9tNPP7Hly5czqVTKzp49669LcMrT61qxYgXbvXs3u3LlCjt+/Dh78sknmUKhYOfOneuQeClZBYAZM2YwAA6Pffv2CccAYJs3bxaeWywWtnTpUpaQkMDkcjl74IEH2IULFzo++FZUVVWxqVOnsoiICKZSqdjMmTPtEvC1a9fsrrOoqIjdd999LCYmhsnlcpaZmclefvllptFo/HQFzd577z2WmprKZDIZGz58OPvhhx+EfTk5OWzGjBl2x//1r39lvXr1YjKZjPXr14999dVXHRyxezy5roULFwrHJiQksAcffJCdOHGiw2KlKWIIIUGB7gYSQoICJStCSFCgZEUICQqUrAghQYGSFSEkKFCyIoQEBUpWhJCgQMmKEBIUKFmRTiU/P99uauhXX33VbnB3Xl6eMBaOBBdKVsQreXl5LudHnzdvHjiOs5udwRdJIj09HWvXrrXb9sQTT+DixYsuz1m3bh3y8/OF56NHj7YbcEwCFyUr4rWUlBR88sknaGhoELY1Njbi448/Rmpqql9iUiqVDpP+2VKr1W4vykECCyUr4rUhQ4YgJSXFbhbPHTt2IDU1FVlZWXf02s5KPLm5uUJpbfTo0SgsLMSLL74oTPoHOFYDW7It4eXl5eHAgQNYt26d8BrXrl1DZmYm3n77bbvzTp06BY7jOmzuJuKIkhW5I7NmzcLmzZuF55s2bcLMmTN9/r47duxAt27d8NprrwkrrXhq3bp1GDlypN1qOqmpqQ7XBACbN2/Gfffdh8zMzPa6BOIhSlbkjjz99NP47rvvUFhYiMLCQnz//fd4+umnff6+MTExEIvFiIyMRGJiolezcKrVashkMoSFhQmvIRaLkZeXhwsXLghr6BmNRnz88ceYNWtWe18G8QDNFEruSFxcHCZNmoT8/HwwxjBp0iTExsb6O6w7kpycjEmTJmHTpk0YPnw4/vGPf0Cv1+Oxxx7zd2ghjUpW5I7NmjUL+fn52LJlS7uVPkQiEVpOtWY0Gtvltd0xe/Zs4ebB5s2b8cQTTyAsLKzD3p84omRF7tiECRNgMBhgNBoxfvz4dnnNuLg4u3Yos9nssKKxTCaD2Wy+o/dx9RoPPvggwsPDsXHjRuzatYuqgAGAqoHkjonFYvz000/C965oNBphAQirLl26ICUlxeHY+++/H4sWLcJXX32FHj16YPXq1aipqbE7Jj09HQcPHsSTTz4JuVzuVfUzPT0dBQUFuH79OiIiIhATEwORSCS0XS1ZsgQ9e/YMqMUeQhWVrEi7UKlUUKlUrR6zf/9+ZGVl2T1WrFjh9NhZs2ZhxowZmD59OnJyctC9e3eMGTPG7pjXXnsN169fR48ePbxeSeill16CWCxG3759ERcXh6KiImHfs88+C4PB0CF3N0nbaA52Qlz497//jQceeADFxcUOKxeTjkfJipAW9Ho9KisrMWPGDCQmJmLbtm3+DomAqoGEONi+fTvS0tJQU1ODt956y9/hkCZUsiKEBAUqWRFCggIlK0JIUKBkRQgJCpSsCCFBgZIVISQoULIihAQFSlaEkKBAyYoQEhT+HxQ8clWR79UdAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T14:27:59.708227Z", + "iopub.status.busy": "2024-03-26T14:27:59.707893Z", + "iopub.status.idle": "2024-03-26T14:28:00.099437Z", + "shell.execute_reply": "2024-03-26T14:28:00.098535Z" + }, + "papermill": { + "duration": 0.411806, + "end_time": "2024-03-26T14:28:00.101584", + "exception": false, + "start_time": "2024-03-26T14:27:59.689778", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASsAAAEmCAYAAAA6OrZqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABAQUlEQVR4nO3deXxTZb4G8Odkb5o03Tfa0gIFZBOQZVhkUa4IXoRhRhZRKSA4V3RERJFxQNFB3AccvTgzd6CiLC6DeEevIFMRRpayI9uwlNIWSmmBNumSZjnn3D9Oc5o0S9M0aZL29/18QptzTpI3pXn6vu95z/syPM/zIISQECcJdgEIIcQbFFaEkLBAYUUICQsUVoSQsEBhRQgJCxRWhJCwQGFFCAkLFFaEkLAgC3YBAo3jOJSWlkKr1YJhmGAXhxBih+d5VFdXIzU1FRKJ57pTuw+r0tJSpKenB7sYhBAPSkpKkJaW5vGYdh9WWq0WgPDDiIqKCnJpCCH2DAYD0tPTxc+pJ+0+rGxNv6ioKAorQkKUN1001MFOCAkLFFaEkLBAYUUICQvtvs+qvWJZFhaLJdjFIKRZcrkcUqm01c9DYRWGampqcPXqVdC8iSQcMAyDtLQ0aDSaVj0PhVWYYVkWV69ehVqtRkJCAg10JSGN53lUVFTg6tWryM7OblUNi8IqzFgsFvA8j4SEBERERAS7OB2eheVQU2+FRiWDXEpdwK4kJCTgypUrsFgsrQor+umGKapRhYbqeivMDYFFXPPX7yqFFSGtYGE5AIC54SsJHAorQkhYoLAiIS0nJwdTpkzx63OOGTMGixYt8nhMZmYm1qxZ4/EY+7OxMWq5H0pGPKGwIm3Gm5AIJzwAmYSBhGEg66Cd66+88gr69+/fJq8V1J/w3r17MWnSJKSmpoJhGGzfvl3cZ7FYsHTpUvTt2xeRkZFITU3FY489htLS0uAVmBA7EoZBnEaJBK0SEjrhEXBBDava2lrceeed+PDDD5321dXV4dixY1i+fDmOHTuGbdu24fz583jwwQeDUNLQZ7Zybm/WJp2/no61eHlsS+Xk5GDPnj1Yu3YtGIYBwzAoKCjAvHnzkJWVhYiICPTo0QNr1651+fiVK1ciISEBUVFR+M1vfgOz2ezV69bW1uKxxx6DRqNBSkoK3n33XadjysvLMWnSJERERCArKwubNm1yOoZhGKxbtw4TJkxAREQEunTpgi+//BI8z6PWZMWZ85fAMAw+//xz3H333YiIiMDgwYNx4cIFHD58GIMGDYJGo8GECRNQUVHh9c9t/fr16N27N5RKJVJSUvDUU0+J+4qLizF58mRoNBpERUVh2rRpuHHjhrjfVuv55JNPkJmZCZ1OhxkzZqC6ulo8huM4vPXWW+jWrRuUSiUyMjKwatUqcf/SpUvRvXt3qNVqdOnSBcuXLxevnMjNzcXKlStx8uRJ8f80NzfX6/fWUkEdZzVhwgRMmDDB5T6dToddu3Y5bPvggw8wZMgQFBcXIyMjoy2KGDY+3H3J7b6s+EhMGdBJvP+XvQWwsK5Hv6fFROChQY2TFa7fVwijmXU67tn/6N6i8q1duxYXLlxAnz598OqrrwIAYmJikJaWhi+++AJxcXHYv38/FixYgJSUFEybNk18bF5eHlQqFX788UdcuXIFc+bMQVxcnMOHyp3nn38ee/bswddff43ExET87ne/w7FjxxyaLjk5OSgtLcXu3bshl8vx29/+FuXl5U7PtXz5crzxxhtYu3YtPvnkE8yYMQOnTp1CbKcs1DX8jF5++WWsWbMGGRkZmDt3Lh5++GFotVqsXbsWarUa06ZNw4oVK7Bu3bpmy75u3TosXrwYb7zxBiZMmAC9Xo99+/YBEELGFlR79uyB1WrFwoULMX36dPz444/icxQUFGD79u345ptvUFlZiWnTpuGNN94Qf3bLli3DX//6V/zxj3/EyJEjcf36dfz73/8WH6/VapGbm4vU1FScOnUK8+fPh1arxQsvvIDp06fj9OnT2LFjB/75z38CED63gRJWg0L1ej0YhkF0dLTbY0wmE0wmk3jfYDC0QclIc3Q6HRQKBdRqNZKTk8XtK1euFL/PysrCgQMH8PnnnzuElUKhwPr166FWq9G7d2+8+uqreP755/Haa695nAq3pqYGf/vb3/Dpp5/i3nvvBQB8/PHHDjNSXrhwAd999x0OHTqEwYMHAwD+9re/4Y477nB6voceegiPP/44AOC1117D999/j7ffW4PV7zbWBpcsWYLx48cDAJ555hnMnDkTeXl5GDFiBABg3rx5Xtc+/vCHP+C5557DM888I26zlTEvLw+nTp1CYWGhOBPuxo0b0bt3bxw+fFg8juM45ObmipPbPfroo8jLy8OqVatQXV2NtWvX4oMPPsDs2bMBAF27dsXIkSPF1/v9738vfp+ZmYklS5Zg69ateOGFFxAREQGNRgOZTObwfxooYRNW9fX1WLp0KWbOnOlxEr3Vq1c7fAA6ioVju7ndJ2nSnbJgVFe3xzbtepk7Iqs1xWrWhx9+iPXr16O4uBhGoxFms9mpw/bOO++EWq0W7w8bNgw1NTUoKSlB586d3T53QUEBzGYzhg4dKm6LjY1Fjx49xPvnzp2DTCbDXXfdJW7r2bOnyz+Iw4YNc7g/5Be/wLHjJxy29evXT/w+KSkJANC3b1+Hba5qbU2Vl5ejtLRUDNmmzp07h/T0dIcpu3v16oXo6GicO3dODKvMzEyHWThTUlLE1z937hxMJpPb1wCAzz77DO+//z4KCgpQU1MDq9UatEksw+IUhsViwbRp08DzfLPV52XLlkGv14u3kpKSNiplcClkEre3pmeqPB3b9JIRd8f5w9atW7FkyRLMmzcP33//PU6cOIE5c+Z43R8VdC5a0nJ54xAG28jtpts4rvk+P39dSmX/2k1fv7nXOHDgAGbNmoWJEyfim2++wfHjx/HSSy8F7f8n5MPKFlRFRUXYtWtXs6muVCrFKYxpKuPQolAowLKN/V/79u3D8OHD8eSTT2LAgAHo1q0bCgoKnB538uRJGI1G8f7Bgweh0WiaXQika9eukMvlyM/PF7dVVlbiwoUL4v2ePXvCarXi6NGj4rbz58+jqqrK6fkOHjzocP/QoXxkd+/psQy+0mq1yMzMRF5ensv9d9xxB0pKShz+GJ89exZVVVXo1auXV6+RnZ2NiIgIt6+xf/9+dO7cGS+99BIGDRqE7OxsFBUVORzT9P80kEK6GWgLqosXL2L37t2Ii4sLdpFIK2RmZiI/Px9XrlyBRqNBdnY2Nm7ciJ07dyIrKwuffPIJDh8+jKwsx6an2WzGvHnz8Pvf/x5XrlzByy+/jKeeeqrZpZs0Gg3mzZuH559/HnFxcUhMTMRLL73k8LgePXrg/vvvxxNPPIF169ZBJpNh0aJFLmsdX3zxBQYNGoSRI0di06ZNOHL4MN5+v/mOcl+98sor+M1vfoPExERMmDAB1dXV2LdvH55++mmMGzcOffv2xaxZs7BmzRpYrVY8+eSTGD16NAYNGuTV86tUKixduhQvvPACFAoFRowYgYqKCpw5cwbz5s1DdnY2iouLsXXrVgwePBjffvstvvrqK4fnyMzMRGFhIU6cOIG0tDRotVoolcpA/DiCW7OqqanBiRMncOLECQAQ33RxcTEsFgt+/etf48iRI9i0aRNYlkVZWRnKysrCp5lAHCxZsgRSqRS9evVCQkICxo8fj6lTp2L69OkYOnQobt26hSeffNLpcffeey+ys7MxatQoTJ8+HQ8++CBeeeUVr17z7bffxt13341JkyZh3LhxGDlypEP/FABs2LABqampGD16NKZOnYoFCxYgMTHR6blWrlyJrVu3ol+/fti4cSM2bPwUPXo6d8T7y+zZs7FmzRr893//N3r37o3//M//xMWLFwEIzbmvv/4aMTExGDVqFMaNG4cuXbrgs88+a9FrLF++HM899xxWrFiBO+64A9OnTxf7tB588EE8++yzeOqpp9C/f3/s378fy5cvd3j8r371K9x///0YO3YsEhISsGXLFv+8eVf4INq9ezcPoeXvcJs9ezZfWFjoch8Afvfu3V6/hl6v5wHwer0+cG+kDRmNRv7s2bO80WgMdlE6FAD8V1995bCt2mjmy/RG/laNiTdZWJ5lueAULsR5+p1tyeczqM3AMWPGeJzt0tM+QoKOES618edJB+JeSPdZEeJJcXGxx87ks2fPBnTwsEYpg0bp+0fI0zS/3333He6++26fn7s9orAiYSs1NVXs73S331881fJNVhZWlodCKoG8BTUsT2Xv1KmT230dFYUVCVsymQzdurkfDNtW6i0c6i0sIpWyFoVVKJQ9nFBYEeIjg9ECK8dBHONJfawBRb2ChPjIyvGwsLw4kJ2iKrAorAjxmRBPNJVV26CwIsRHtlaf7UJxagUGFoUVIT6yZZPtgmXKqsCisCLER01rVhRXgUVhRdpMqC0Y0dry2LrWVTIpotVyRCro5HogUViRsBJKF7FLwIBhAJmUgVIm7bAr3LQV+umGO54HrObg3FrQo+zrghG2dQNXrVqF1NRUcZbP/fv3o3///lCpVBg0aBC2b98OhmEcRoWfPn0aEyZMgEajQVJSEh599FHcvHnTbXmuXLnSoh99vFaJRK0K0mamqiH+QfXWcMdagH85r9jSJu5+DpApvDq0tQtGREVFiQuIGAwGTJo0CRMnTsTmzZtRVFTk1JyrqqrCPffcg8cffxx//OMfYTQasXTpUkybNg0//PCDy/IkJCT49GOwshzMLAcJw0All/r0HKR5FFakTbRmwYjIyEj8z//8DxQKIRg/+ugjMAyDv/71r1CpVOjVqxeuXbuG+fPni4/54IMPMGDAALz++uvitvXr1yM9PR0XLlxA9+7dXZbHF1aOR3W9FXKphMIqgCiswp1ULtRwgvXareTNghF9+/YVgwoQph3u168fVCqVuG3IkCEOjzl58iR2797tcmaDgoICdO/esqXEmuI4HlVGCxgAEQpbQNHZwECisAp3DON1UyzU2BaMePfddzFs2DBotVq8/fbbDnOmA0LNqqVqamowadIkvPnmm077UlJSfC6zDQ9eXBBWDSGsaFBoYFFYkTbjacEIG1cLRjTVo0cPfPrppzCZTOJ834cPH3Y4ZuDAgfj73/+OzMxMyGSuf81bs9iBLZgY8R8SaHQag7QZ+wUjbt68iezsbBw5cgQ7d+7EhQsXsHz5cqfQceXhhx8Gx3FYsGABzp07h507d+Kdd94B0DiafOHChbh9+zZmzpyJw4cPo6CgADt37sScOXPEgGpaHm+WyLJpHL3emFVUsQosCivSZnxdMKKpqKgo/OMf/8CJEyfQv39/vPTSS1ixYgUAiP1Yqamp2LdvH1iWxX333Ye+ffti0aJFiI6OFle3aVqe4uJiH94VA6patQ2Gb+cTnRsMBuh0Ouj1+naxhmB9fT0KCwuRlZXl0MHc0W3atAlz5syBXq/32wKhnpitHCrrzJBKGESp5KisM0PCMEjQBmYZqnDm6Xe2JZ9P6rMiYWnjxo3o0qULOnXqhJMnT4pjqNoiqASNf+NlEga6CDlNFRNgFFYkLJWVlWHFihUoKytDSkoKHnroIaxatarNXp9HQwOQASQSBioJja8KNAorEpZeeOEFvPDCC0F7faVMisQoCqi2FNQO9r1792LSpElITU0FwzDYvn27w36e57FixQqkpKQgIiIC48aNE1ekJSRUcDwPo5mF0ezbMAjinaCGVW1tLe688058+OGHLve/9dZbeP/99/HRRx8hPz8fkZGRGD9+POrr69u4pKGnnZ8XCSscz8NQb4Gh3hLsooQkf/2uBrUZOGHCBEyYMMHlPp7nsWbNGvz+97/H5MmTAQidqklJSdi+fTtmzJjRlkUNGVKp0PQwm81t2JlMmjJZWNRZWCjoesBm2ab1sf3u+ipk+6wKCwtRVlaGcePGidt0Oh2GDh2KAwcOuA0rk8kEk8kk3jcYDAEva1uSyWRQq9WoqKiAXC4XxwyRtmU0W1FrZmGRMoBSDkvDB9Jo5MWBqQTgOA4VFRVQq9VuryTwVsiGVVlZGQAgKSnJYXtSUpK4z5XVq1c7XMnf3jAMg5SUFBQWFqKoqCjYxemwzFZhYVNhpgUJquutAACDSkZh1YREIkFGRkarfy4hG1a+WrZsGRYvXizeNxgMSE9PD2KJ/E+hUCA7OzukZs3saE6WVOJ4eRWyk7QYnBGLzfnCH46Hh3aGogWrMncECoXCLy2AkA0r2xxDN27ccLhK/saNG05TiNhTKpXixa3tmUQioRHsQcQychg5KSQyBSLVETByQn+MQqmkPqwACdk/AVlZWUhOTkZeXp64zWAwID8/H8OGDQtiyQgB2IYzXFIJY7e6jXBmkARGUGtWNTU1uHTpkni/sLAQJ06cQGxsLDIyMrBo0SL84Q9/QHZ2NrKysrB8+XKkpqZiypQpwSs0IRBmBwWEsGIYBvf3SYaEYSCnRSMCJqhhdeTIEYwdO1a8b+trmj17NnJzc/HCCy+gtrYWCxYsQFVVFUaOHIkdO3ZQ84cEX0MFStpQrbojJfwvkg91NOsCIT7ieR48L1wbSHxDsy4Q0gaEJbyE7wtv1sLCcsiIVVMHe4BQA5sQP9h1tgzf/nxdHG9F/I9qVoT44ODlW7hZY0L/9Gikxaghaahi0dnAwKGaFSE+uFZpxMUbNagxCTUpCqvAo7AixAesbehCQ0jZ+tht24n/UVgR4gP7cVb2X6liFTgUVoT4wDaCXdZwzZvtIl2qWQUOhRUhPmAbVmO2XZ9rq1lRn1Xg0NlAQnzANmSSrWY1ODMW9RYW8bQUV8BQWBHiA5ZzrFl1S9QEsTQdA4UVIT54fGQXWDkeMrrUps1QWBHiA4mEgcIuqMoN9ag1s4jTKBClkgexZO0XdbAT4gcHLt/C9uPXUHyrLthFabeoZkVIC3Ecjx1nyiCVMBjbIxEKmUQcukBnAwOHalaEtJCV43G+rBpnSxtXTpKKYRWsUrV/FFaEtJD9wE9bBztdbhN4FFaEtJBt9LqEYcSJ92zNwHY+l2VQUVgR0kIsa7susHFb4wj2YJSoY6CwIqSFGle2afz4UDMw8OhsICEtZG0YvW5fs+qepEVspAIpuogglar9o7AipIXEuazsalbpsWqkx6qDVaQOgcKKkBZK0qrwX2O60txVbSyk+6xYlsXy5cuRlZWFiIgIdO3aFa+99hqdcSFBJZEwUMmliFA0rmJjqLfgamUdKmvNQSxZ+xbSNas333wT69atw8cff4zevXvjyJEjmDNnDnQ6HX77298Gu3iEiM5cM+Dg5Vu4M12He3omBbs47VJIh9X+/fsxefJkPPDAAwCAzMxMbNmyBYcOHQpyyUhHVqavx6lresRpFBiYEQPA/mxgEAvWzoV0M3D48OHIy8vDhQsXAAAnT57ETz/9hAkTJrh9jMlkgsFgcLgR4k+VdWacvqZHYUWtuI1mCg28kK5ZvfjiizAYDOjZsyekUilYlsWqVaswa9Yst49ZvXo1Vq5c2YalJB2N7WygTNo4RQyNYA+8kK5Zff7559i0aRM2b96MY8eO4eOPP8Y777yDjz/+2O1jli1bBr1eL95KSkrasMSkI7CFlW2tQOF7275glKhjCOma1fPPP48XX3wRM2bMAAD07dsXRUVFWL16NWbPnu3yMUqlEkolzYNNAqdxZZvGsKJmYOCFdM2qrq4OEoljEaVSKTiO/nyR4BFrVhL7mhWFVaCFdM1q0qRJWLVqFTIyMtC7d28cP34c7733HubOnRvsopEOzMo616ySolQYmR2P6Aia0jhQQjqs/vSnP2H58uV48sknUV5ejtTUVDzxxBNYsWJFsItGOjCOd1yNGQAStEok0DJcAcXwPpy+uHz5Mrp06RKI8vidwWCATqeDXq9HVFRUsItD2gGzlYPJykImkTiMYict15LPp099Vt26dcPYsWPx6aefor6+3qdCEhKuFDIJtCq5Q1DVW1jcMNTjZo0piCVr33wKq2PHjqFfv35YvHgxkpOT8cQTT9CoctKhXasyYnN+MXadvRHsorRbPoVV//79sXbtWpSWlmL9+vW4fv06Ro4ciT59+uC9995DRUWFv8tJSMg4U6rHj+fLcbWycdktKZ0NDLhWDV2QyWSYOnUqvvjiC7z55pu4dOkSlixZgvT0dDz22GO4fv26v8pJSMi4crMOx4urcLOmcYYFcegCzRQaMK0KqyNHjuDJJ59ESkoK3nvvPSxZsgQFBQXYtWsXSktLMXnyZH+Vk5CQIc4UythfbiN8pawKHJ+GLrz33nvYsGEDzp8/j4kTJ2Ljxo2YOHGiOIAzKysLubm5yMzM9GdZCQkJroYu0Aj2wPMprNatW4e5c+ciJycHKSkpLo9JTEzE3/72t1YVjpBQZGWdw8rWDKQFIwLHp7DatWsXMjIynC6F4XkeJSUlyMjIgEKhcHv9HiHhrHEOducLmaliFTg+9Vl17doVN2/edNp++/ZtZGVltbpQhIQyVxcyRyplGJoVi4Gdo4NUqvbPp5qVu0HvNTU1UKlUrSoQIaHOVc0qUinD8G7xwSpSh9CisFq8eDEAYaKxFStWQK1uXHqIZVnk5+ejf//+fi0gIaHmlwM6wcLy0ChD+tLadqdFP+3jx48DEGpWp06dgkKhEPcpFArceeedWLJkiX9LSEiI0aqcZ1bgOB5VRgs4nke8hi5oDoQWhdXu3bsBAHPmzMHatWvpwmBCGtRZWHy8/woYBlg0rnuwi9Mu+VSP3bBhg7/LQUjY2H9JOLk0sHMMVHLhYmb7s4E8z4tzshP/8Tqspk6ditzcXERFRWHq1Kkej922bVurC0ZIqDpSVAmW49EnTWcXVo3hxPGAlLLK77wOK51OJ/610Ol0ASsQIaGM5/nG1W1cDAoFhFHsUlBa+ZvXYWXf9KNmIOmo7EeouxoUajtGTnPy+Z1Pg0KNRiPq6hqnxygqKsKaNWvw/fff+61ghIQiq31YMc7XBgI0ij1QfAqryZMnY+PGjQCAqqoqDBkyBO+++y4mT56MdevW+bWAhIQS+wuV7QOKYRi7mRcorQLB55lC7777bgDAl19+ieTkZBQVFWHjxo14//33/VpAQkKJ1W70etMzfgMzYjAoM8YhxIj/+DR0oa6uDlqtFgDw/fffY+rUqZBIJPjFL36BoqIivxaQkFDCuphxwWZU94S2Lk6H4vOCEdu3b0dJSQl27tyJ++67DwBQXl5OA0VJu6ZVyfDosM54aFBasIvS4fgUVitWrMCSJUuQmZmJoUOHYtiwYQCEWtaAAQP8WkBCQolMKkG8RolErfMF+zUmK/R1FlhZWjE8EHwKq1//+tcoLi7GkSNHsGPHDnH7vffeiz/+8Y9+KxwAXLt2DY888gji4uIQERGBvn374siRI359DUL8YUt+MdbvK8TtOnPzB5MW8/my8eTkZCQnJztsGzJkSKsLZK+yshIjRozA2LFj8d133yEhIQEXL15ETEyMX1+HEG/p6yw4e90ArUqGPp0cB0czNAFfQPkUVrW1tXjjjTeQl5eH8vJycJxjtffy5ct+Kdybb76J9PR0h0GoNLkfCabKOjMOXr6FxCilU1jR1MaB5VNYPf7449izZw8effRRpKSkBOyizf/93//F+PHj8dBDD2HPnj3o1KkTnnzyScyfP9/tY0wmE0ymxlVxDQZDQMpGOiZx6IKL33laNCKwfAqr7777Dt9++y1GjBjh7/I4uHz5MtatW4fFixfjd7/7HQ4fPozf/va3Hud3X716NVauXBnQcpGOy9XKNja2TRz1rweETx3sMTExiI2N9XdZnHAch4EDB+L111/HgAEDsGDBAsyfPx8fffSR28csW7YMer1evJWUlAS8nKTjcLWyjQ1DqzIHlE9h9dprr2HFihUO1wcGQkpKCnr16uWw7Y477kBxcbHbxyiVSkRFRTncCPEXzzUrCqtA8qkZ+O6776KgoABJSUnIzMyEXO44zeuxY8f8UrgRI0bg/PnzDtsuXLiAzp07++X5CWkpq4vFImyykzRI1ildTnvshGMBRtJ4CpE0y6ewmjJlip+L4dqzzz6L4cOH4/XXX8e0adNw6NAh/OUvf8Ff/vKXNnl9QppyNZeVzeBML7tGbl8GTv8dUOqAAbMARaQ/i9huMby7dbVCxDfffINly5bh4sWLyMrKwuLFiz2eDWzKYDBAp9NBr9dTk5C0Wq3JCkO9BUqZFLGRiuYf0BTPA/l/BoyVwv20wUD2OP8WMoy05PPpc1hVVVXhyy+/REFBAZ5//nnExsbi2LFjSEpKQqdOnXwqeCBQWJG2YrKysLI8FDIJ5FI33cGG68DR3Mb7ikhg+NMdtjnYks+nT83An3/+GePGjYNOp8OVK1cwf/58xMbGYtu2bSguLhbnuiKkI9lxugyXK2rxH72SnAaMim5dFL7GZwOVhYC5Fqi9CWhoxobm+HQ2cPHixcjJycHFixcdVmCeOHEi9u7d67fCERJqim/V4WjRbZRWGZ32eTV0wVAqfI3NArQpwvfVpf4uZrvkU1gdPnwYTzzxhNP2Tp06oaysrNWFIiRUXaqoxt4LN3HlVq3TPqkYVm4ezPNA9XXhe20qEJUqfG+4HoCStj8+NQOVSqXLy1guXLiAhASqzpL2yzb7i6vLbWwnCN1eG1hfBVjqAYkU0CQ2drLXlvu/oO2QTzWrBx98EK+++iosFgsAofpbXFyMpUuX4le/+pVfC0hIKGEbrqWRuVgY0NYMdHvOqvaW8FUdJwRWZLxwv+4WTdXgBZ/C6t1330VNTQ0SEhJgNBoxevRodOvWDVqtFqtWrfJ3GQkJGbaalcTjhcxuHlxnC6tYcByPcmsEeECobVkCezVIe+BTM1Cn02HXrl3Yt28fTp48iZqaGgwcOBDjxnXc8SKkY7DaalYS57/zzTYDjbeFrxGx2HOxAieKq/BrixzpERYhyGhwqEctDiuO45Cbm4tt27bhypUrYBgGWVlZSE5OBs/zAZsuhpBQYDvT5yKrkKKLgJXjkaBVun5wXWMz8MxZPQBAoY0DrGVA3W0gOiMQRW43WtQM5HkeDz74IB5//HFcu3YNffv2Re/evVFUVIScnBz88pe/DFQ5CQkJtlkXXNWseqVGYXzvZHRL1Lh+cJ1Qs7Iqo8VrDHUxDSekTDTvWnNaVLPKzc3F3r17kZeXh7Fjxzrs++GHHzBlyhRs3LgRjz32mF8LSUioGN0jAfVmDnGaFl5qYzUJA0AB1Mp04Pk6SCUMlJoYoAJAvd7/hW1nWlSz2rJlC373u985BRUA3HPPPXjxxRexadMmvxWOkFCTqFUhI06NSKXz33mO42G2crC4Wt2mvqHmJFPCxAuzMrAcj+I6GVieb9xP3GpRWP3888+4//773e6fMGECTp482epCERKO8gtv48Pdl/CvixXOO23NPJUORgsrbt512QSLlaOalRda1Ay8ffs2kpKS3O5PSkpCZWVlqwtFSKg6U6oHy/HolqiBWuH48fE4rXF9lfC1SViZpBqh/8pULTzQVc89AdDCmhXLspDJ3OebVCqF1WptdaEICVUHCm4h71w5quudf88lDWnFuhrgaWvmKaOgkknROU4NADBL1bDyDMBzgLk6YOVuD1pUs+J5Hjk5OVAqXZ+atV9VhpD2yDaGytWgUImnEex2zcDM+Ehkxkfi8yMluFZphFkaCaBeqF2p3MzWQFoWVu5WlLFHZwJJe2b1MFOo2Ax0NSbU1ielapyzSSkTGjZmqRpCWNX4s6jtTovCyn6xUUI6Is5Ws/KwYITLEez1jTUrG6VMCgAwSSKEDWYKK098Xj6ekI6G53mPNSu3i5xybGMQKaOw4/R1XL5ZC7ZhgKlJqgZYUFg1g8KKEC/ZV5hcrW6ji5AjO0mDpCiV4w6TQZhVQSIDFJEwWfUwWTj0So1Cik6F5OoEoOwiNQObQWFFiJesdmMSXIVVeqwa6bFq5weKTcAogGFgtgrPkxUfie5JWqAsFigD1ayaQWFFiJdkEgl+OaATrBzvshnolqlx2AIAWBqaf+KiErbZFkw0dMETCitCvCSVMMiM9zyNC8/zQovPPsyanAm0XY5jYTkU36qD0ixDEkA1q2aE1XDZN954AwzDYNGiRcEuCiFOLlfUYM0/L+KzIyWOO2w1JqVjWF3X1+Pvx65if4lZ2G+pB1gaVO1O2ITV4cOH8ec//xn9+vULdlFIB1VvYXH6mh4XbrhurkncrW4jhpUWAGBuCKtIRcPQBV4udL4DVLvyICzCqqamBrNmzcJf//pXxMTEBLs4pIMy1Fuw6+wN/Hje9QIPYlg1HWcl9lkJYZUcpUKKTgWNSggoCw9A2TAHFoWVW2ERVgsXLsQDDzzg1bTJJpMJBoPB4UaIP9hOBkrdXGzMuBvB3qQZOHVgGmYMyUCUSpgqxmLl7DrZKazcCfkO9q1bt+LYsWM4fPiwV8evXr0aK1euDHCpSEdkG7rgYmEbYburQaFWs9AXBYg1Kxvb2UALywEKqlk1J6RrViUlJXjmmWewadMmh5WfPVm2bBn0er14Kykpaf5BhHhBrFlJXX9sXF5uYwsfqRyQOU4AoGh4HivHNwYZhZVbIV2zOnr0KMrLyzFw4EBxG8uy2Lt3Lz744AOYTCZIpVKHxyiVSrezQhDSGo01K9dVK9toBYf+dXHYgg5gGNysMeGLI1cRFSHDlP6dAABmKwdergYDiFMfE2chHVb33nsvTp065bBtzpw56NmzJ5YuXeoUVIQEkq15525AqFImRVZ8JFRyu9/LpmcCrRzqLSyUMgmUMglGdY+HTCIBL6kTwor6rNwK6bDSarXo06ePw7bIyEjExcU5bSck0KweZlwAAJ1ajikDOjlubBJWtjFWcpkEMqkEd3WOFfbfomZgc0I6rAgJJanREfjPfimONafmuAkrRdNeerGDnZqB7oRdWP3444/BLgLpoKJUcnG4gdecmoGO1wWWG+phsnJIVKmgBIRl5GkudpfoJ0KIn9SarPjgh4v44IeLjRudLmJuaAY2hNU/fr6OL49eRaVZDjASoXfeQrUrVyisCPHSzRoTzpdVo9xQ73I/wwgzKlhYvnEednd9Vg1hJW9oDlo4AIqG6WWoKegShRUhXrpUXoP/O3UdP191vcaf/SISHA+AtQAWo7ChIaxUcimSdSpEq4XmpG0ZemFgKI1i9yTs+qwICRbbYE+pmyHsjmHFQ2qrVUllgEwY1Nynkw59OjXOwy7WrFi+oZP9Bp0RdINqVoR4ydP860DjoFCgIdjsrwl0M5DU9SU31Ax0hcKKEC+xDSPYZW7O1NlPdczzcOpcd0XWULOycnxjM5DCyiVqBhLiJWvDdMQyN81AhmHAMEJQcTzv1LkOADvPlOFqpREju8WjR7LWTc2Kpjd2hcKKEC/ZmoGuFouwSYsRzugxDFyGVa3JCoPRIvZ/ZSdqEBepQKeYCMBENStPKKwI8VJzfVYA8Ou70hrvuAgrcQS7THiOLgkadElo2KmnPitPKKwI8dKA9Gh0TYhEii7Cuwe46LMyN13Zxp790AWed9sp31FRWBHiJZdrAnpiv15gA2tDzUrWEFZGMwu90QKFTIJYVUPNirMCVhMg924Ot46CzgYS4kefHCzCf/94CRWVhsYBoarGcVWNI9iFWtOFG9XYcqgY+y7ddJygj5qCTiisCPFS8a06XK6oQb2FdXuMxcrBZOHA1VcJG+QqhxlCbQuc2mYJbRy60DANKZ0RdIvCihAv/fDvG/j6RClu1ZrdHiPOw15nN0OondhIBWIjFWKflS20LA2zMdBYK/eoz4oQL3lzNtAWVryxStjQJKxmDslwuG/ru7KINSsKK3eoZkWIl7wZZyUGWb3rmpW7420DTsX1A03UDGyKwooQL7EtqFnBZAuraI/PqZDZjWAH6PpAD6gZSIiXbLUfjzUr26U4Rj0ghUPNqrLWjK+OX4NGJcO0QenC8RK7WRcAagZ6QGFFiBc4jhdXt3E5oLNBjFoBk46DsrLaKaxMVg56owX2K3VFKmUYmhUr1rBosVP3KKwI8YLVbuFSTzWrMT0ShQGd/2po1nkYYwUIk/EN7xbf+AQUVm5RWBHiBQkD/EevJFg53mOfFQDAWCl8lUc0GWPlOKWxS7ZmoKUeYK3CxH0EAIUVIV6RSSUOM3x6VHdb+KqOddhscXNdYGWtGRaWQ2ykAjJ5BCCRAhwrLBwh9fI1OwA6G0iIHx28fAvf5J/GtSojENE0rJybgQCw+VAxNuUXo8ZkFS5eprnYXQrpsFq9ejUGDx4MrVaLxMRETJkyBefPnw92sUgHVG9hceVmLUqrjB6Ps7Ac+LrbQjCp45z2Ac41K4d52AE6I+hGSIfVnj17sHDhQhw8eBC7du2CxWLBfffdh9pa+k8kbauyThh28N3pMo/HSSUMVBa9MK1xk2agXCpBjFoOjdKx98VhhRuAOtndCOk+qx07djjcz83NRWJiIo4ePYpRo0YFqVSkI7KK/U2eO9dlDIMIqx4czzg1A5uubGNje05xFDuFlUshHVZN6fXCqODY2Fi3x5hMJphMJvG+wWAIeLlI+2er9bhbLMJGzhnBcmZwUAIR0V49t61ZaGbp+kBPQroZaI/jOCxatAgjRoxAnz593B63evVq6HQ68Zaent6GpSTtlcXLmpXKJJwJNMu0wvxUXrBdzGyli5k9CpuwWrhwIU6fPo2tW7d6PG7ZsmXQ6/XiraSkpI1KSNqzxrnTPX9kFKabAIA6RbzTvn+evYFPDxahoMKxeUfNQO+ERTPwqaeewjfffIO9e/ciLS3N47FKpRJKpdLjMYS0lLfNwEjLLagVUrDaRKd9lXVmVFSbGkOpQbdEDeI1SsRrGn5vbQtM1FMXhr2QDiue5/H000/jq6++wo8//oisrKxgF4l0ULbLbZprBiZLDEhOiwZ69XTaZ7IKgadsUjvrndqk0902Z7u5lkax2wnpn8LChQuxefNmfP3119BqtSgrE04b63Q6RER4ucIIIX6QEavG6B4JiFUr3B/EsUDdLeF7jXPNymz1rikJuVoIKNYqrJCjdn9CqSMJ6bBat24dAGDMmDEO2zds2ICcnJy2LxDpsJKiVEiKama1mdqbQmDJlC4n3TO76feqt7AwmlnIpAy0Krkwil0VLTxfvZ7CqkFIhxXP880fREioMFxFndmKY7c0uHq4BDOaTGHsrmZ1sqQK+wtuoW8nHcb1ShI2KqMaw4oACPGwIiRUVFSbYGY5xKjlUCvcfGz0VwEA1xGPyjqLwy4ry4kzjSqaXG4jzsNuG2cFNNbMKKxEYTN0gZBg2l9wE58fLkFBuYexT/prkEoYVCuTHIMHQgd9jFqOSKXUKazEawPt5syisHJGNStCvGAbbiBzdzbQWAnU6yGRSlGjSATL8WA5XpyoTyWXImeE67PZcnE5LvuaVcMZQRMNX7ChmhUhXrCNLnc7cd6tAgCANDodrEQ4Y9i0duWOvOlCpwDVrFygsCLEC+bmLrdpCCtJfLY4k6jZy7BqnHXBrhkYESN8rTcArMXFozoeCitCvGAWB3RKXeysBaqKhO/jukHecLbPbNesu1RejU8PFuFfFyucHi5vuhwXIIy1kjcMlbDNPNrBUZ8VIV4wWVkAzqPPAQA3zgjjq6JSgMg4xKrrUK9gHQ7RG62oqDYhNtJ5UKlGKUP/9GhE2s9zxTRMMWMpBYy3AW2SX99POKKwIqQZPM+7H33OscC1o8L3yX0BANMGO8/0YbII4RUhd66Z6SLkGNvTecQ71HGAobRxVHwHR2FFSDM4HhjVPQEmCwdV07ApPQEYq4RpXZL6un0OY0NYOT3eE9vIdWoGAqCwIsSZsRKoqQCsRkCqhFSTiIHpMULTzF71DeDyD8L3nUcAMvfXDdrCKkLhOqyMZhb1FhZalUwcJCrO4U41KwAUVoQIOBa4flJo0tXedN4vVwFRaUB0unCmrrYCKMkXLjaOyQQ6DRQPPXj5Fi7cqEb/9Gj0S4sGIIQR4LoZCAAfH7gCo5nFo8M6N04VYx9WPO8clh0MhRXp2FgrUPYzUHygcf4oRiLMmqCIBCx1sBhuoL66GvK681DduuT4+Oh0oPcUhyAxWljcqjHDYLSK22pMwveRStdhpZJJxNqVKCIGkMiEoQvGyg5/QTOFFemYbDUp+5BSaoCMYUBSn8ZhAwCKb+iRd/gUMiW3cV+CRRiqIFcD8d2FY5tMyGerPRntgkcll0IpZ51WthEfo5Ciss6Ceovd8AWJFNAkAIbrQHUZhVWwC0BIm+I4oPwMcOUnoWMcaAyplP4uJ7ozWoFaZSJq4zOBPp5nqgUAtcI5rGY2mYGhKVvHu0PNCgA0yUJY1ZQBSb2afe32jMKKdAw8D9y8CBTuaeyTUkQCnYe7DSkbsXNc7t3HxVazqjezzRzZyDbY1DaeS6RNFr5W3/D6udorCivS/tXeBC7sBKqKhftyFZD+C6DTXR7P4NnUNYSOu/6mpmxn/OrM1maObKSSC01Jo7nJJTpRqcJXwzWh6SppwdCHdobCirRfVjNQtA8oOQTwnFB7ShssBJW8mVk/7dQ1dI6r3Qw7aEqrEpbgMtRbwXE8TlytwulrevRO1eGuzjEuH2Pry6oxNbkOMDIBkEcAFiNQfR3QNd8Mba8orEj7VHEBuLSrsfM8PhvoNs7rhUft2WpWbifda0KrlEGtkEKrksPMcrhZbcKtGrPDtYJNJetU6JemQ2p0k7UFGAaIzgAqzgs1QworQtqJej1wcZfQPwUIU61k/4cQVj6qNbesZiWRMHhidFfxfkWNsEJ4nMZ9kzMtRo20GLXrnTGdhbC6dUnoY+ugKKxI+2A1AVcPA8UHhXFJjATIGCqMLPdyZWR3BnWORWWdGXGalq9HabKyuFltBgCk6LxvejqI7y4EsP6aEMYuFqPoCCisSHir1zeMPD8m9OsAwkDN7PHCGCU/6JUa5fNj8y/fBsfziFbLxb4sd6wsh1u1ZkSr5Y5T0Si1QvOvqkSY4aGD1q4orEh4MVULH1r9VUBfIlz2YlsFSR0LZN4NJN4R9EtTOI7H9hPXUHSrDgBwR0rzgbflUDFu1pjxywGdkBkf6bgzuZ/wvq8eAdKGdMiFT8Ni8r0PP/wQmZmZUKlUGDp0KA4dOhTsIpG2Um8Arv8MnPsHcHAdsP8D4OzXwjV8NeVCUMV0BnpNBgbPFwZO+jGoyvT1KLld5zz+qRkSCYP0WDUkDINO0REYmOH6LKA9WzPT1sflIKm3UMMy1wJXO+bvf8jH82effYbFixfjo48+wtChQ7FmzRqMHz8e58+fR2KiizmAiPABNlULiw2wFuG0vUwlnAJXRAJSRdBrHm5ZzcJZr9uXgcpC5+lRGEa4bk+XLjSNdGnChzhA8gtv4XJFLUZmx2NwZssudxmcGYsB6dGNsyg0IylKifNl1Si5Xef8WhIp0GU0cO4bYfR9VKpwAXUHwvAhvpLo0KFDMXjwYHzwwQcAAI7jkJ6ejqeffhovvvhis483GAzQ6XTQ6/WIivK97yHksFbAUif8pTXeFgY+1t0SvjdWCvvdkcoAhUYILoVGCDLwQqhxLMBZ7b5aAc4iBKBMCcgihDFKsggh/BxuaoCRCoHCSBpuDd8DDc01vvErxzaGas0N4fo3Q6mw3YZhAG2KUHuyBZSs5R3dvtDXWfDxgStgOR6P/KIzErSBfd2qOjM27LsCCcNg1i8yGmdfsOF54MxXwplBiVQY1JrUWxiLFaaDRVvy+QzpmpXZbMbRo0exbNkycZtEIsG4ceNw4MABl48xmUwwmRqr0QaDl0sZFe0XOi/dcch0PrjbObb5RQQYiVDjkMqF760mIdxYixBkxqrGa+NCjUoHxHUFYrKEMUYtGMDZGmYrhy2HimHleLAcB6OZA8fz6BQTgXgPww78JVqtQFZ8JApv1mJzfjG0KhmmDkyDLqKhY55hgDseBJh/AOX/Fga7lhwStstUDTXmJn8gglmDVmiBO6f77elCOqxu3rwJlmWRlOQ4/3RSUhL+/e9/u3zM6tWrsXLlypa/mKnG9TxGoYyRAAq1MJWIOk64RcQKHc0qneu/tqwFMNcI79dcK9ys9Xa/5FKh5iWxv0kBMELgWeuFm8XYcKsTvlrrhe85FmItjbf7CjR8cJjG722BqtAItQNtMhDVSSh/ED5kUgmD27Vmh20JWiXG904G00blGdcrCd+cLMV1fT2q6izguCYNH6kM6DUFSCoASo8D+mKh6Wz7/wglKnPzx7RASIeVL5YtW4bFixeL9w0GA9LTnefEdtLprsaBgw6/mHbft3R7q5/LzT5GIjS7ZKqWf6ilciHcbEs9tYUwmThOwgC/visNMikDqYSBUipFVISszYIKEC67mT44HXqjBTUmKzQqFx9RhgHiuwk3nhf+4FiMAGsW/jjYmtq8d0uBBYzEv/ES0mEVHx8PqVSKGzccrzi/ceMGkpOTXT5GqVRCqfShbyEyTrgR/wuDoAIAhhHO4AUbwzCIVisQrfai6ckwwhQ3Sk3gCxZkIT10QaFQ4K677kJeXp64jeM45OXlYdiwYUEsGSGkrYV0zQoAFi9ejNmzZ2PQoEEYMmQI1qxZg9raWsyZMyfYRSOEtKGQD6vp06ejoqICK1asQFlZGfr3748dO3Y4dboTQtq3kB9n1VrtdpwVIe1ASz6fId1nRQghNhRWhJCwQGFFCAkLId/B3lq2LjmvL7shhLQZ2+fSm67zdh9W1dXVAODdKHZCSFBUV1dDp/M8A2q7PxvIcRxKS0uh1Wrb9LIJX9kuDyopKWlXZy/b6/sC2u97a4v3xfM8qqurkZqaConEc69Uu69ZSSQSpKWF34ogUVFR7eoX36a9vi+g/b63QL+v5mpUNtTBTggJCxRWhJCwQGEVYpRKJV5++WXfZo4IYe31fQHt972F2vtq9x3shJD2gWpWhJCwQGFFCAkLFFaEkLBAYUUICQsUViFg1apVGD58ONRqNaKjo716DM/zWLFiBVJSUhAREYFx48bh4sWLgS1oC92+fRuzZs1CVFQUoqOjMW/ePNTU1Hh8zJgxY8AwjMPtN7/5TRuV2L2Wrgr+xRdfoGfPnlCpVOjbty/+7//+r41K2jIteV+5ublO/zcqVdsskwZQWIUEs9mMhx56CP/1X//l9WPeeustvP/++/joo4+Qn5+PyMhIjB8/HvX19QEsacvMmjULZ86cwa5du/DNN99g7969WLBgQbOPmz9/Pq5fvy7e3nrrrTYorXu2VcFffvllHDt2DHfeeSfGjx+P8vJyl8fv378fM2fOxLx583D8+HFMmTIFU6ZMwenTp9u45J619H0Bwmh2+/+boqKitiswT0LGhg0beJ1O1+xxHMfxycnJ/Ntvvy1uq6qq4pVKJb9ly5YAltB7Z8+e5QHwhw8fFrd99913PMMw/LVr19w+bvTo0fwzzzzTBiX03pAhQ/iFCxeK91mW5VNTU/nVq1e7PH7atGn8Aw884LBt6NCh/BNPPBHQcrZUS9+Xt7+fgUI1qzBUWFiIsrIyjBs3Ttym0+kwdOhQtytVt7UDBw4gOjoagwYNEreNGzcOEokE+fn5Hh+7adMmxMfHo0+fPli2bBnq6uoCXVy3bKuC2/+sm1sV/MCBAw7HA8D48eND5v8G8O19AUBNTQ06d+6M9PR0TJ48GWfOeFjF3M/a/YXM7VFZWRkAuFyp2rYv2MrKypCYmOiwTSaTITY21mMZH374YXTu3Bmpqan4+eefsXTpUpw/fx7btm0LdJFd8mVV8LKyspD+vwF8e189evTA+vXr0a9fP+j1erzzzjsYPnw4zpw50yaTBVDNKkBefPFFp87Ipjd3vxShLNDva8GCBRg/fjz69u2LWbNmYePGjfjqq69QUFDgx3dBfDFs2DA89thj6N+/P0aPHo1t27YhISEBf/7zn9vk9almFSDPPfcccnJyPB7TpUsXn57bthr1jRs3kJKSIm6/ceMG+vfv79Nzesvb95WcnOzUUWu1WnH79m23q2m7MnToUADApUuX0LVr1xaXt7V8WRU8OTm5RccHgy/vqym5XI4BAwbg0qVLgSiiEwqrAElISEBCQkJAnjsrKwvJycnIy8sTw8lgMCA/P79FZxR94e37GjZsGKqqqnD06FHcddddAIAffvgBHMeJAeSNEydOAIBDKLcl+1XBp0yZAqBxVfCnnnrK5WOGDRuGvLw8LFq0SNy2a9eukFpF3Jf31RTLsjh16hQmTpwYwJLaCVrXPhEVFRXxx48f51euXMlrNBr++PHj/PHjx/nq6mrxmB49evDbtm0T77/xxht8dHQ0//XXX/M///wzP3nyZD4rK4s3Go3BeAsu3X///fyAAQP4/Px8/qeffuKzs7P5mTNnivuvXr3K9+jRg8/Pz+d5nucvXbrEv/rqq/yRI0f4wsJC/uuvv+a7dOnCjxo1Klhvged5nt+6dSuvVCr53Nxc/uzZs/yCBQv46OhovqysjOd5nn/00Uf5F198UTx+3759vEwm49955x3+3Llz/Msvv8zL5XL+1KlTwXoLLrX0fa1cuZLfuXMnX1BQwB89epSfMWMGr1Kp+DNnzrRJeSmsQsDs2bN5AE633bt3i8cA4Dds2CDe5ziOX758OZ+UlMQrlUr+3nvv5c+fP9/2hffg1q1b/MyZM3mNRsNHRUXxc+bMcQjgwsJCh/dZXFzMjxo1io+NjeWVSiXfrVs3/vnnn+f1en2Q3kGjP/3pT3xGRgavUCj4IUOG8AcPHhT3jR49mp89e7bD8Z9//jnfvXt3XqFQ8L179+a//fbbNi6xd1ryvhYtWiQem5SUxE+cOJE/duxYm5WVpoghhIQFOhtICAkLFFaEkLBAYUUICQsUVoSQsEBhRQgJCxRWhJCwQGFFCAkLFFakXcnNzXWYbfWVV15xuF4yJydHvLyEhBcKK+KTnJwct1MOL1y4EAzDOFzwHIiQyMzMxJo1axy2TZ8+HRcuXHD7mLVr1yI3N1e8P2bMGIdr+EjoorAiPktPT8fWrVthNBrFbfX19di8eTMyMjKCUqaIiAinebTs6XQ6r+e5J6GFwor4bODAgUhPT3eYGG/btm3IyMjAgAEDWvXcrmo8U6ZMEWtrY8aMQVFREZ599llxHi3AuRnYlH0NLycnB3v27MHatWvF5ygsLES3bt3wzjvvODzuxIkTYBimzaZDIc4orEirzJ07Fxs2bBDvr1+/HnPmzAn4627btg1paWl49dVXxcULWmrt2rUYNmyYwwIVGRkZTu8JADZs2IBRo0ahW7du/noLpIUorEirPPLII/jpp59QVFSEoqIi7Nu3D4888kjAXzc2NhZSqRRarRbJyck+TWyn0+mgUCigVqvF55BKpcjJycH58+fFZaksFgs2b96MuXPn+vttkBagyfdIqyQkJOCBBx5Abm4ueJ7HAw88gPj4+GAXq1VSU1PxwAMPYP369RgyZAj+8Y9/wGQy4aGHHgp20To0qlmRVps7dy5yc3Px8ccf+632IZFI0HT2IovF4pfn9sbjjz8unjzYsGEDpk+fDrVa3WavT5xRWJFWu//++2E2m2GxWDB+/Hi/PGdCQoJDPxTLsk6LhCoUCrAs26rXcfccEydORGRkJNatW4cdO3ZQEzAEUDOQtJpUKsW5c+fE793R6/XinOo2cXFxSE9Pdzr2nnvuweLFi/Htt9+ia9eueO+991BVVeVwTGZmJvbu3YsZM2ZAqVT61PzMzMxEfn4+rly5Ao1Gg9jYWEgkErHvatmyZcjOzg6p+dM7KqpZEb+IiopCVFSUx2N+/PFHDBgwwOG2cuVKl8fOnTsXs2fPxmOPPYbRo0ejS5cuGDt2rMMxr776Kq5cuYKuXbv6vDjHkiVLIJVK0atXLyQkJKC4uFjcN2/ePJjN5jY5u0maR9MaE+LGv/71L9x7770oKSlxWgyUtD0KK0KaMJlMqKiowOzZs5GcnIxNmzYFu0gE1AwkxMmWLVvQuXNnVFVV4a233gp2cUgDqlkRQsIC1awIIWGBwooQEhYorAghYYHCihASFiisCCFhgcKKEBIWKKwIIWGBwooQEhYorAghYeH/AdbkM/r69dXCAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T14:28:00.140629Z", + "iopub.status.busy": "2024-03-26T14:28:00.139829Z", + "iopub.status.idle": "2024-03-26T14:28:00.301596Z", + "shell.execute_reply": "2024-03-26T14:28:00.300513Z" + }, + "papermill": { + "duration": 0.183853, + "end_time": "2024-03-26T14:28:00.303929", + "exception": false, + "start_time": "2024-03-26T14:28:00.120076", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAErCAYAAAChCiRcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA04UlEQVR4nO3deVgT59o/8C9hCUsIGEAQCwaLAm1RWlAKPxEXULF6ULRaS0UpR6s9dAO12p5KfX1bao+41qNXFxda11qlvtQtRQSqKIJLRREVcQVERFk1hPD8/vBkjilbQCAkc3+uq1eZZ56Z3APJ18kszxgwxhgIIUTPCbRdACGEdAUKO0IIL1DYEUJ4gcKOEMILFHaEEF6gsCOE8AKFHSGEFyjsCCG8QGFHCOEFCjtCCC/oXNitW7cOUqkUpqam8PX1RVZWVrN9v/vuOwQEBKBHjx7o0aMHgoKCWuxPCNFfOhV2O3fuRExMDOLi4nD69GkMHDgQo0ePRmlpaZP9jx49imnTpiE1NRWZmZlwcnLCqFGjcOfOnS6unBCibQa6NBCAr68vBg0ahG+++QYA0NDQACcnJ7z33ntYuHBhq8srlUr06NED33zzDSIiIjq7XEJIN2Kk7QI0VVdXh5ycHCxatIhrEwgECAoKQmZmpkbrqK2thUKhgEQiabaPXC6HXC7nphsaGlBeXg4bGxsYGBi0fwMIIR2OMYaqqio4OjpCIGj5i6rOhF1ZWRmUSiXs7e3V2u3t7XHp0iWN1vHxxx/D0dERQUFBzfaJj4/HkiVLnqlWQkjXunXrFp577rkW++hM2D2rr776Cjt27MDRo0dhamrabL9FixYhJiaGm66oqICzszMKCwthaWnZFaVqhUKhQGpqKoYPHw5jY2Ntl0OeEV/+nlVVVXBxcdHos6kzYWdrawtDQ0PcvXtXrf3u3btwcHBocdnly5fjq6++wu+//44BAwa02FcoFEIoFDZql0gkEIvFbS9cRygUCpibm8PGxkavPxx8wZe/p2rbNDnEpDNnY01MTODt7Y2UlBSuraGhASkpKfDz82t2ua+//hpLly7FwYMH4ePj0xWlEkK6IZ3ZswOAmJgYzJgxAz4+Phg8eDBWrVqFmpoaREZGAgAiIiLQu3dvxMfHAwCWLVuGxYsXY9u2bZBKpSgpKQEAiEQiiEQirW0HIaTr6VTYTZ06Fffu3cPixYtRUlICLy8vHDx4kDtpcfPmTbUzMuvXr0ddXR0mT56stp64uDh8/vnnXVk6IUTLdCrsACA6OhrR0dFNzjt69Kja9PXr1zu/IEKITtCZY3aEEPIsKOwIIbygc19jCSGN1dbWql1cX/1IjuPnC9DDNhsiM/VLqdzd3WFubt7VJWodhR1Pafrh4OsHQ9dcunQJ3t7ejdq/bqJvTk4OXnnllc4vqpuhsOMpTT8cfP1g6Bp3d3fk5ORw0/nFDxHz83mseN0Tbr2sG/XlIwo7ntL0w8HXD4auMTc3V/tHSXDjPoQZj+Dx0kB49bHRYmXdB4UdT9GHQ/cVltWgRl7f5LyCezXc/42Mmv+YWwiN4GJr0Sn1dTcUdoTooMKyGgxffrTVfrG7z7faJ3XeMF4EHoUdITpItUe3aqoXXHs2vvWx5pEcyUczMW6YHyzMGg9sAQBXS6vx4c6zze4d6hsKO0J0mGtPEV7qbdWoXaFQoMQOeKVPD70e9aQtKOwI0UFy5WMITO+gsDIfAtPGe3b19fUoqi9CXnles8fsCiurITC9A7nyMYDGgalvKOwI0UFFNTdg4bIWn7TysLx/H/x3i/MtXICiGi94w77FfvqAwo4QHeRo0Qc1he9h9VQvPN/EMbv6+noc++MY/t+Q/9fsnl1BaTU+2HkWjsP7dHa53QKFHSE6SGhoiobHveEidsMLNk0fsys0KoSHxKPZY3YNjyvQ8PgehIbNP6ZAn9BAAIQQXqCwI4TwAoUdIYQXKOwIIbxAYUcI4QUKO0IIL9ClJzzR0ggZAI2SQfQfhR0PaDpCBkCjZBD9RWHHA62NkAHQKBlE/1HY8UhzI2QANEoG0X90goIQwgsUdoQQXqCwI4TwAoUdIYQXKOwIIbxAYUcI4QUKO0IIL1DYEUJ4QefCbt26dZBKpTA1NYWvry+yslp+4sjPP/8Md3d3mJqawtPTE/v37++iSgkh3YlOhd3OnTsRExODuLg4nD59GgMHDsTo0aNRWlraZP/jx49j2rRpiIqKwpkzZzBhwgRMmDABubm5XVw5IUTbdOp2sRUrVmDWrFmIjIwEAGzYsAG//fYbNm7ciIULFzbqv3r1aowZMwbz588HACxduhQymQzffPMNNmzY0KW1a1NrzxgF6DmjRP/pTNjV1dUhJycHixYt4toEAgGCgoKQmZnZ5DKZmZmIiYlRaxs9ejSSkpKafR25XA65XM5NV1ZWAnhy76hCoXiGLdCeW5XXNHrGKKDZc0ZvVXpigELSQdWR9qh69OQ9eu5mOerrGw/MUPNYjux7gO21e7AwbWZgh/8M61VfX6+z7+221K0zYVdWVgalUgl7e/WH+drb2+PSpUtNLlNSUtJk/5KSkmZfJz4+HkuWLGnUfvjwYZibm7ejcu0rrKpDTeF7mO6qhINZ+9dT8gj48aohCg1vY39B04cOSNfIvGsAwBCf/nqxhV5G+PHqmVbXdSrzD9x4hveFNtXW1mrcV2fCrqssWrRIbW+wsrISTk5OGDVqFMRisRYra78LRZVYnXsCEwNexYuOTW+DQqGATCZDcHBws6OeXCiqxJbcExjewnpI13i1pg6eeaXoa2cBM2PDRvMvl1Rgwd48fD3RA/0dmj/kYCE0hNRGd8cmVH3z0oTOhJ2trS0MDQ1x9+5dtfa7d+/CwcGhyWUcHBza1B8AhEIhhMLGu/3GxsY6O/SR6hickZFRq9vQ0na2ZT2kc9lbGyPcz6XVfv0drODVx6YLKtKOtrwPdSbsTExM4O3tjZSUFEyYMAEA0NDQgJSUFERHRze5jJ+fH1JSUvDhhx9ybTKZDH5+fl1QcffxSKEEAOTeqWi2T82jJ8d4HG48aHHwTkJ0lc6EHQDExMRgxowZ8PHxweDBg7Fq1SrU1NRwZ2cjIiLQu3dvxMfHAwA++OADBAYGIiEhAa+99hp27NiB7OxsfPvtt9rcjC5X8J+QWrintSHXjfDj1VOtrs9CqFNvG0IA6FjYTZ06Fffu3cPixYtRUlICLy8vHDx4kDsJcfPmTQgE/7100N/fH9u2bcM///lPfPLJJ+jXrx+SkpLw0ksvaWsTtGLUi0++tj/fU9Tk8R0AyC+uQOzu80iY7Am3Xi0d46EH7hDdZMAYY9ouojurrKyElZUVKioqdPYERVNqa2vVzmLnFz9EzM/nseJ1T7j1suba3d3ddfYsNJ+dvXEfE9afQNLcV/X6mF1bPp86tWdHOs6lS5fg7e3dqP3NLerTOTk5eOWVV7qoKkI6D4UdT7m7uyMnJ4ebrn4kx2+pmXhtuB9ET52gcHd310Z5hHQ4CjueMjc3V9tjUygUeFBWCr/BPnRZCdFLOjUQACGEtBeFHSGEFyjsCCG8QGFHCOEFCjtCCC9Q2BFCeIHCjhDCCxR2hBBeoLAjhPAChR0hhBco7AghvEBhRwjhBQo7QggvUNgRQniBwo4QwgsUdoQQXqCwI4TwAoUdIYQXKOwIIbxAYUcI4QUKO0IIL1DYEUJ4gcKOEMILFHaEEF6gsCOE8AKFHSGEFyjsCCG8QGFHCOEFCjtCCC/oTNiVl5cjPDwcYrEY1tbWiIqKQnV1dYv933vvPbi5ucHMzAzOzs54//33UVFR0YVVE9L1ysvLMXnUENxcPQ2TRw1BeXm5tkvqFoy0XYCmwsPDUVxcDJlMBoVCgcjISMyePRvbtm1rsn9RURGKioqwfPlyvPDCC7hx4wbmzJmDoqIi7N69u4urJ6RrODg44O7du9x0weVLsLGxgb29PUpKSrRYWTfAdMDFixcZAHbq1Cmu7cCBA8zAwIDduXNH4/Xs2rWLmZiYMIVCofEyFRUVDACrqKhoU826pq6ujiUlJbG6ujptl0Layd7engFo9j97e3ttl9jh2vL5bPOe3bVr19C3b9+OS1sNZGZmwtraGj4+PlxbUFAQBAIBTp48iYkTJ2q0noqKCojFYhgZNb/Zcrkccrmcm66srAQAKBQKKBSKdm5B96faNn3eRn12+/ZttT26pty9excHDx6En58fzM3Nu6iyztWW92ubw87V1RWBgYGIiorC5MmTYWpq2tZVtFlJSQl69uyp1mZkZASJRKLxrnlZWRmWLl2K2bNnt9gvPj4eS5YsadR++PBhvXmDtEQmk2m7BNIOs2bN0qhfSEgIEhIS8Pzzz3dyRV2jtrZW474GjDHWlpWfPXsWmzZtwvbt21FXV4epU6ciKioKgwcPbnOhCxcuxLJly1rsk5eXhz179mDLli3Iz89Xm9ezZ08sWbIEc+fObXEdlZWVCA4OhkQiwb59+2BsbNxs36b27JycnFBWVgaxWKzBVukmhUIBmUyG4ODgFn8/pHsSiUSoq6sDAJiYmHA//3Xa2NgY9+7d05t/uCsrK2Fra8t9a2tJm/fsvLy8sHr1aiQkJGDfvn3YvHkzhgwZgv79++Ptt9/G9OnTYWdnp9G6YmNjMXPmzBb79O3bFw4ODigtLVVrr6+vR3l5ORwcHFpcvqqqCmPGjIGlpSX27t3b6gdZKBRCKBQ2ajc2NuZFCPBlO/WNgYEB9/OoUaPw8ccf4/bt23juueewbNkyJCcnAwAEAgGsrKy0VWaHa9N79VkPED5+/JitWLGCCYVCZmBgwIRCIZs+fTorKip61lVzVCcosrOzubZDhw61eoKioqKCvfrqqywwMJDV1NS067XpBAXRBZ6entyJiPLyciaTyVhMTAyTyWSsvLycm+fp6antUjtUWz6f7b7OLjs7G++++y569eqFFStWYN68eSgoKIBMJkNRURFCQ0Pbu+pGPDw8MGbMGMyaNQtZWVk4duwYoqOj8cYbb8DR0REAcOfOHbi7uyMrKwvAk93bUaNGoaamBj/88AMqKytRUlKCkpISKJXKDquNkO7ghRde4H6WSCQIDg7GihUruMM3TfXjmzZ/jV2xYgU2bdqE/Px8jB07FomJiRg7diwEgie56eLigs2bN0MqlXZooVu3bkV0dDRGjhwJgUCASZMmYc2aNdx8hUKB/Px87oDl6dOncfLkSQBPTqo8rbCwsMPrI0SbND3hoC8nJtqjzWG3fv16vP3225g5cyZ69erVZJ+ePXvihx9+eObiniaRSJq9gBgApFIp2FPnWoYNG6Y2TYg+CwwMxJdffqlRP75qc9jJZDI4Oztze3IqjDHcunULzs7OMDExwYwZMzqsSEIIeVZtPmb3/PPPo6ysrFF7eXk5XFxcOqQoQkjbpKWldWg/fdTmsGvuq2F1dXWXXGBMCGnsypUr3M9/vXTq6c/l0/34RuOvsTExMQCeXM+zePFitYsSlUolTp48CS8vrw4vkBDSujNnzgAADA0N0bNnT9y6dYubZ2dnh6KiIiiVSq4fH2kcdqpfEmMM58+fh4mJCTfPxMQEAwcOxLx58zq+QkJIq1TDnSmVSrWgA6A23dKwaPpO47BLTU0FAERGRmL16tV6fesUIbrG0dFRo/vEVdel8lGbj9lt2rSJgo6QbkbTkX807aePNNqzCwsLw+bNmyEWixEWFtZi3z179nRIYYQQzV28eLFD++kjjcLOysqKu9FYn24iJkRf3Lhxo0P76SONwm7Tpk1N/kwI6R6evjrCzMwMjx49anJaX4Z2ag+deQYFIaR53t7e+P333wEAw4cPx+jRo3HlyhX069cPhw4dwv79+7l+fKVR2L388stq42W15PTp089UECGk7Z4eQ3L//v1cuAHqY91pOtakPtIo7CZMmNDJZRBCnkVLg9g+fddTa4Pd6jONwi4uLq6z6yCEPIPevXtzPwsEAjQ0NDQ5/XQ/vtGZh2QTQprn7+8PIyMjWFlZwcnJSW2es7MzrKysYGRkBH9/fy1VqH0a7dlJJBJcvnwZtra26NGjR4vH7+jp44R0vePHj6O+vh4VFRUICAhATEwMd4JCJpNxz6A4fvw4hg0bpt1itUSjsFu5ciUsLS25nzU9WUEI6RrFxcUAgJ9++gmffPIJF24A0KdPH/z000946623uH58pFHYPT0QZ2tPAyOEdD3VqOEHDhzAnTt31Obdvn2bOzvb3OjifNDmY3aGhoaNHmsIAPfv34ehoWGHFEUIaZuAgACIxWJs3bq10ZiTjDFs27YNYrEYAQEBWqpQ+zps8E65XK427BMhpOsolUpUVVUBaPwZVU1XVVXx+sl6Gt9BoXqSl4GBAb7//nuIRCJunlKpRHp6Otzd3Tu+QkJIq9auXcuFmqmpaaPbxWpra8EYw9q1axEbG6utMrVK47BbuXIlgCf/SmzYsEHtK6uJiQmkUik2bNjQ8RUSQlqVkZEBABg8eDD++OMPpKWl4cCBAwgJCUFgYCD8/f2RnZ2NjIwMCrvWFBYWAnhy392ePXvQo0ePTiuKENI2NTU1AIAhQ4bA2NgYgYGBqKmpQWBgIIyNjTFkyBBkZ2dz/fiozcfsUlNTKegI6WZ8fHwAPBmVSC6XIy0tDenp6UhLS4NcLseWLVvU+vFRm0c9efvtt1ucv3HjxnYXQwhpn6CgIHz11Vd48OABzM3NudvDVqxYoXa7WFBQkDbL1Ko2h92DBw/UphUKBXJzc/Hw4UOMGDGiwwojhGhu2LBhEIvFqKysVLsvFgA3LRaLeXv3BNCOsNu7d2+jtoaGBsydOxfPP/98hxRFCGm71u5s4vudTx0yEIBAIEBMTAx3xpYQ0rWOHj2KioqKFvtUVFTg6NGjXVNQN9Rho54UFBSgvr6+o1ZHCGmDgwcPcj9XV1dDJpMhJiYGMplM7VmxT/fjmzZ/jY2JiVGbZoyhuLgYv/32m9o9tISQrvPrr78CeHICwsLCotGlJyNGjMCRI0fw66+/4l//+peWq9WONofdmTNn1KYFAgHs7OyQkJDQ6plaQkjnUCgUAJ7co97UCQrV0GuqfnzU5rBLTU3tjDoIIc/A09MT169fx5kzZxAaGooFCxbg0aNHOHHiBL7++mucPXuW68dXOjNScXl5OcLDwyEWi2FtbY2oqCi1YxEtYYwhJCQEBgYGSEpK6txCCdGCbdu2cT+npKRg6NChmDZtGoYOHYqUlJQm+/GNzoRdeHg4Lly4wI26mp6ejtmzZ2u07KpVq3h/2p3oN5FIhEGDBgGA2iAAT08PGjRIbQAPvtGJsMvLy8PBgwfx/fffw9fXF0OGDMHatWuxY8cOFBUVtbjs2bNnkZCQQHd2EL2XlZXFBd5fDRo0CFlZWV1cUfeiEw/JzszMhLW1tdp9fUFBQRAIBDh58iQmTpzY5HK1tbV48803sW7dOo0fISeXyyGXy7npyspKAE8O7OrzwV3VtunzNvLBsWPHUF1djYiICJw/fx6enp5ITEyESCTSy79tW7apw8Lu9u3b+J//+R98++23HbVKTklJCXr27KnWZmRkBIlEgpKSkmaX++ijj+Dv74/Q0FCNXys+Ph5Llixp1H748GGYm5trXrSOkslk2i6BdIC///3v3M/p6elarKRz1dbWaty3w8Lu/v37+OGHH9oUdgsXLsSyZcta7JOXl9euevbt24cjR440ulSmNYsWLVK7lrCyshJOTk4YNWoUxGJxu2rRBQqFAjKZDMHBwTA2NtZ2OeQZ8eXvqfrmpQmtfo2NjY1t9QE+ffv2hYODQ6PnXtTX16O8vLzZr6dHjhxBQUEBrK2t1donTZqEgICAZm+bEQqFEAqFjdqNjY31+k2jwpft5At9/3u2Zdu0GnZ2dnaws7NrtZ+fnx8ePnyInJwceHt7A3gSZg0NDfD19W1ymYULF6rtygNPrjFauXIlxo8f/+zFE0J0ik6coPDw8MCYMWMwa9YsbNiwAQqFAtHR0XjjjTfg6OgIALhz5w5GjhyJxMREDB48GA4ODk3u9Tk7O8PFxaWrN4EQomUah11YWFiL8x8+fPistbRo69atiI6OxsiRIyEQCDBp0iTuIUDAk2MU+fn5bTpgSQjhD43DzsrKqtX5ERERz1xQcyQSSYtXf0ul0mYf86jS2nxCiP7SOOw2bdrUmXUQQkin0ok7KAgh5FlpvGen6fBNdFsWIaQ70jjsNm/ejD59+uDll1+mY1+EEJ2jcdjNnTsX27dvR2FhISIjI/HWW29BIpF0Zm2EENJhND5mt27dOhQXF2PBggX4v//7Pzg5OWHKlCk4dOgQ7ekRQrq9Np2gEAqFmDZtGmQyGS5evIgXX3wR7777LqRSqcYDaRJCiDa0+2ysQCCAgYEBGGNQKpUdWRMhhHS4NoWdXC7H9u3bERwcjP79++P8+fP45ptvcPPmTV6PgEoI6f40PkHx7rvvYseOHXBycsLbb7+N7du3w9bWtjNrI4SQDqNx2G3YsAHOzs7o27cv0tLSkJaW1mS/PXv2dFhxhBDSUTQOu4iICHpoDSFEZ7XpomJCCNFVdG8sIYQXKOwIIbxAYUeInlEqlUhLS0N6ejrS0tLoOtj/oLAjRI/s2bMHrq6uCA4OxooVKxAcHAxXV1e6SgIUdoTojT179mDy5Mnw9PRERkYGtm/fjoyMDHh6emLy5Mm8DzwKO0L0gFKpRGxsLMaNG4ekpCT4+vrCzMwMvr6+SEpKwrhx4zBv3jxef6WlsCNED2RkZOD69ev45JNPIBCof6wFAgEWLVqEwsJCZGRkaKlC7aOwI0QPFBcXAwBeeumlJuer2lX9+IjCjhA90KtXLwBAbm5uk/NV7ap+fERhR4geCAgIgFQqxZdffomGhga1eQ0NDYiPj4eLiwsCAgK0VKH2UdgRogcMDQ2RkJCA5ORkTJgwASdOnMCjR49w4sQJTJgwAcnJyVi+fDkMDQ21XarWaHxvLCGkewsLC8Pu3bsRGxuLoUOHcu0uLi7YvXs3wsLCtFid9lHYEaJHwsLCEBoaitTUVBw4cAAhISEYPnw4r/foVCjsCNEzhoaGCAwMRE1NDQIDAyno/oOO2RFCeIHCjhDCCxR2hBBeoLAjhPAChR0hhBd0JuzKy8sRHh4OsVgMa2trREVFobq6utXlMjMzMWLECFhYWEAsFmPo0KF49OhRF1RMCOlOdCbswsPDceHCBchkMiQnJyM9PR2zZ89ucZnMzEyMGTMGo0aNQlZWFk6dOoXo6OhGo0IQQvSfTlxnl5eXh4MHD+LUqVPw8fEBAKxduxZjx47F8uXL4ejo2ORyH330Ed5//30sXLiQa3Nzc+uSmgkh3YtOhF1mZiasra25oAOAoKAgCAQCnDx5EhMnTmy0TGlpKU6ePInw8HD4+/ujoKAA7u7u+OKLLzBkyJBmX0sul0Mul3PTlZWVAACFQgGFQtGBW9W9qLZNn7eRT/jy92zL9ulE2JWUlKBnz55qbUZGRpBIJCgpKWlymWvXrgEAPv/8cyxfvhxeXl5ITEzEyJEjkZubi379+jW5XHx8PJYsWdKo/fDhwzA3N3/GLen+ZDKZtksgHUjf/561tbUa99Vq2C1cuBDLli1rsU9eXl671q0a5uadd95BZGQkAODll19GSkoKNm7ciPj4+CaXW7RoEWJiYrjpyspKODk5YdSoURCLxe2qRRcoFArIZDIEBwfD2NhY2+WQZ8SXv6fqm5cmtBp2sbGxmDlzZot9+vbtCwcHB5SWlqq119fXo7y8HA4ODk0upxqk8IUXXlBr9/DwwM2bN5t9PaFQCKFQ2Kjd2NhYr980KnzZTn2mVCpx/PhxpKenw8LCQq8HAmjLe1WrYWdnZwc7O7tW+/n5+eHhw4fIycmBt7c3AODIkSNoaGiAr69vk8tIpVI4OjoiPz9frf3y5csICQl59uIJ6Yb27NmD2NhYXL9+HQCwYsUKSKVSJCQk8H6IJ524BsPDwwNjxozBrFmzkJWVhWPHjiE6OhpvvPEGdyb2zp07cHd3R1ZWFgDAwMAA8+fPx5o1a7B7925cvXoVn332GS5duoSoqChtbg4hnYIepdgKpiPu37/Ppk2bxkQiEROLxSwyMpJVVVVx8wsLCxkAlpqaqrZcfHw8e+6555i5uTnz8/NjGRkZbXrdiooKBoBVVFR0xGZ0W3V1dSwpKYnV1dVpuxTSDvX19UwqlbLx48czpVKp9vdUKpVs/PjxzMXFhdXX12u71A7Vls+nTpyNBQCJRIJt27Y1O18qlYIx1qh94cKFatfZEaKPVI9S3L59OwQCgdrzYVWPUvT390dGRgaGDRumvUK1SCe+xhJCWkaPUmwdhR0heuDpRykqlUqkpaUhPT0daWlpUCqV9ChFAAasqe9+hFNZWQkrKytUVFTo/XV2+/fvx9ixY+nSEx2kVCrh6uoKW1tblJaWql1e5ezsjJ49e+L+/fu4cuWKXl2G0pbPJ+3ZEaIHDA0N8frrryM7Oxu3b99Wm3f79m1kZ2dj8uTJehV0bUVhR4geUCqV2Lx5MwA0uije1NQUALBlyxa1Exd8Q2FHiB44evQo7t27Bw8Pj0YX6tva2sLd3R2lpaU4evSodgrsBijsCNEDqhDLy8vDwIED1S4qHjhwIC5duqTWj48o7AjRA6qBL/z8/JCUlARfX1+YmZnB19cXSUlJePXVV9X68RGFHSF6QCKRAABqamqanK9qV/XjIwo7QvSAavSfP//8E6GhoThx4gQePXqEEydOIDQ0FOfPn1frx0c6c7sYIaR5vXv3BvBkAIyUlBQkJydz88zNzWFgYADGGNePjyjsCNEDAQEBkEqlsLW1RVlZGTfEEwDY29vDxsYG9+/fR0BAgPaK1DIKO0L0gKGhIRISEjB58mS89tpr+Oijj3DlyhX069cPMpkMv/32G3bv3s3ri4op7AjRE2FhYdi9ezdiY2PVvsa6uLhg9+7dvB+8k8KOED0SFhaGcePGYe3atThy5AhGjBiB9957DyYmJtouTevobCwhemTPnj1wc3PDvHnzsH//fsybNw9ubm40SjEo7AjRGzQse8so7AjRA0qlErGxsRg3blyTd1CMGzcO8+bNo4EACCG6TTUs+yeffAKBQP1jrRqWvbCwEBkZGVqqUPso7AjRAzQse+so7AjRA08Py94UGpadwo4QvaC6g+LLL79sNLJJQ0MD4uPj4eLiwus7KCjsCNEDqjsokpOTMWHCBLWBACZMmIDk5GQsX76c7qAghOi+p++gGDp0KNdOd1A8QWFHiB4JCwtDaGgoUlNTceDAAYSEhGD48OG83qNTobAjRM8YGhoiMDAQNTU1CAwMpKD7DzpmRwjhBQo7QggvUNgRQniBwo4QwgsUdoQQXqCwI4Twgs6EXXl5OcLDwyEWi2FtbY2oqChUV1e3uExJSQmmT58OBwcHWFhY4JVXXsEvv/zSRRUTQroTnQm78PBwXLhwATKZDMnJyUhPT8fs2bNbXCYiIgL5+fnYt28fzp8/j7CwMEyZMgVnzpzpoqoJ6XpKpRJpaWlIT09HWloar8ewU8N0wMWLFxkAdurUKa7twIEDzMDAgN25c6fZ5SwsLFhiYqJam0QiYd99953Gr11RUcEAsIqKirYXrkPq6upYUlISq6ur03Yp5Bn88ssvrE+fPgwA91+fPn3YL7/8ou3SOkVbPp86cQdFZmYmrK2t4ePjw7UFBQVBIBDg5MmTmDhxYpPL+fv7Y+fOnXjttddgbW2NXbt24fHjxxg2bFizryWXyyGXy7npyspKAIBCoYBCoeiYDeqGVNumz9uo7/bu3YupU6fCzMxMrb20tBSTJk3Czp07m/2s6Kq2vF91IuxKSkrQs2dPtTYjIyNIJBKUlJQ0u9yuXbswdepU2NjYwMjICObm5ti7dy9cXV2bXSY+Ph5Llixp1H748GGYm5u3fyN0hEwm03YJpB2USiVmzZoFAKivr1ebp5qePXs2jIyM9Or2sdraWo37ajXsFi5ciGXLlrXYJy8vr93r/+yzz/Dw4UP8/vvvsLW1RVJSEqZMmcI9hKQpixYtQkxMDDddWVkJJycnjBo1CmKxuN21dHcKhQIymQzBwcEwNjbWdjmkjY4cOaL2LeRpqumKigpYWFhgxIgRXV5fZ1Ftsya0GnaxsbGYOXNmi3369u0LBwcHlJaWqrXX19ejvLwcDg4OTS5XUFCAb775Brm5uXjxxRcBAAMHDkRGRgbWrVuHDRs2NLmcUCiEUChs1G5sbMyLEODLduqb9PR07mcTExN8+OGHcHFxQWFhIVatWoW6ujqu3+jRo7VVZodry3tVq2FnZ2cHOzu7Vvv5+fnh4cOHyMnJgbe3N4An/5I1NDTA19e3yWVUu7d/ffiIoaFho5FcCdF1165dA/Dk/V1VVQUDAwPs378fUVFRWLp0KczMzNDQ0MD14yOduPTEw8MDY8aMwaxZs5CVlYVjx44hOjoab7zxBhwdHQEAd+7cgbu7O7KysgAA7u7ucHV1xTvvvIOsrCwUFBQgISEBMpkMEyZM0OLWENLxLl++DACwt7eHkZH6PoyRkRH3DUjVj490IuwAYOvWrXB3d8fIkSMxduxYDBkyBN9++y03X6FQID8/n9ujMzY2xv79+2FnZ4fx48djwIABSExMxJYtWzB27FhtbQYhncLAwAAAUFRUhNDQULVh2UNDQ1FUVKTWj4904mwsAEgkEmzbtq3Z+VKpFIwxtbZ+/frRHROEFwYNGoScnBwAQEpKCpKTk7l5T19FMGjQoC6vrbvQmT07QkjzVqxY0ey8p3cCWuqn7yjsCNEDZmZmCA0NBQA8evRIbZ5qOjQ0tNEFx3xCYUeInkhKSuIC769CQ0ORlJTUtQV1MxR2hOiRpKQk1NbWYs6cOfDy8sKcOXNQW1vL+6ADdOgEBSFEM2ZmZlizZg3279+PsWPH0kXi/0F7doQQXqCwI4TwAoUdIYQX6JhdK1TXKLVldAVdpFAoUFtbi8rKSjrGowf48vdUfS7/ekNBUyjsWlFVVQUAcHJy0nIlhJDmVFVVwcrKqsU+BkyTSOSxhoYGFBUVwdLSUq/vK1SN23fr1i29HrePL/jy92SMoaqqCo6Ojo1GOPor2rNrhUAgwHPPPaftMrqMWCzW6w8H3/Dh79naHp0KnaAghPAChR0hhBco7AiAJ8PRx8XFNTkkPdE99PdsjE5QEEJ4gfbsCCG8QGFHCOEFCjtCCC9Q2HWwmTNndvjTy4YNG4YPP/ywxT5SqRSrVq3q0NclRJ9Q2LVAk5AhuuXzzz+Hl5eXtstoUnd7v3W3ep4VhR0heqSurk7bJXRbFHbNmDlzJtLS0rB69WoYGBjAwMAABQUFiIqKgouLC8zMzODm5obVq1c3ufySJUtgZ2cHsViMOXPmaPwmrKmpQUREBEQiEXr16oWEhIRGfUpLSzF+/HiYmZnBxcUFW7dubdTHwMAA69evR0hICMzMzNC3b1/s3r2bm3/9+nUYGBhg165dCAgIgJmZGQYNGoTLly/j1KlT8PHxgUgkQkhICO7du6fhbw3YuHEjXnzxRQiFQvTq1QvR0dHcvJs3byI0NBQikQhisRhTpkzB3bt3ufmqva4ff/wRUqkUVlZWeOONN7jBGIAn9yp//fXXcHV1hVAohLOzM7744gtu/scff4z+/fvD3Nwcffv2xWeffQaFQgEA2Lx5M5YsWYJz585xf9PNmzdrvG2dqb3vN9Vhky+++AKOjo5wc3MDABw/fhxeXl4wNTWFj48PkpKSYGBggLNnz3LL5ubmIiQkBCKRCPb29pg+fTrKysqaref69etd9evoHIw06eHDh8zPz4/NmjWLFRcXs+LiYvb48WO2ePFidurUKXbt2jX2008/MXNzc7Zz505uuRkzZjCRSMSmTp3KcnNzWXJyMrOzs2OffPKJRq87d+5c5uzszH7//Xf2559/snHjxjFLS0v2wQcfcH1CQkLYwIEDWWZmJsvOzmb+/v7MzMyMrVy5kusDgNnY2LDvvvuO5efns3/+85/M0NCQXbx4kTHGWGFhIQPA3N3d2cGDB9nFixfZq6++yry9vdmwYcPYH3/8wU6fPs1cXV3ZnDlzNKr93//+NzM1NWWrVq1i+fn5LCsri6tJqVQyLy8vNmTIEJadnc1OnDjBvL29WWBgILd8XFwcE4lELCwsjJ0/f56lp6czBwcHtd/dggULWI8ePdjmzZvZ1atXWUZGBvvuu++4+UuXLmXHjh1jhYWFbN++fcze3p4tW7aMMcZYbW0ti42NZS+++CL3N62trdVo2zrbs77fpk+fznJzc1lubi6rqKhgEomEvfXWW+zChQts//79rH///gwAO3PmDGOMsQcPHjA7Ozu2aNEilpeXx06fPs2Cg4PZ8OHDm62nvr5eG7+aDkNh14LAwEC1kGnKP/7xDzZp0iRuesaMGUwikbCamhqubf369UwkEjGlUtniuqqqqpiJiQnbtWsX13b//n1mZmbG1ZGfn88AsKysLK5PXl4eA9Ao7P4aUr6+vmzu3LmMsf+G3ffff8/N3759OwPAUlJSuLb4+Hjm5ubWYt0qjo6O7NNPP21y3uHDh5mhoSG7efMm13bhwgW1bYmLi2Pm5uassrKS6zN//nzm6+vLGGOssrKSCYVCtXBrzb/+9S/m7e3NTcfFxbGBAwdqvHxXau/7zd7ensnlcq5t/fr1zMbGhj169Ihr++6779TCbunSpWzUqFFq67516xYDwPLz8zWuR5fQqCdttG7dOmzcuBE3b97Eo0ePUFdX1+iA98CBA9Wewu7n54fq6mrcunULffr0aXbdBQUFqKurg6+vL9cmkUi4ryYAkJeXByMjI3h7e3Nt7u7usLa2brQ+Pz+/RtNPf40BgAEDBnA/29vbAwA8PT3V2kpLS5utWaW0tBRFRUUYOXJkk/Pz8vLg5OSkNi7gCy+8AGtra+Tl5XFPqpdKpbC0tOT69OrVi3v9vLw8yOXyZl8DAHbu3Ik1a9agoKAA1dXVqK+v1+lRPzR5v3l6esLExISbzs/Px4ABA2Bqasq1DR48WG2Zc+fOITU1FSKRqNFrFhQUoH///h27Id0AHbNrgx07dmDevHmIiorC4cOHcfbsWURGRur0QeGnR7FVjdf317aGhoZW19NRD1/+66i6T79+a6+RmZmJ8PBwjB07FsnJyThz5gw+/fRTnf37aPp+s7CwaPO6q6urMX78eJw9e1btvytXrmDo0KEdtQndCoVdC0xMTKBUKrnpY8eOwd/fH++++y5efvlluLq6oqCgoNFy586dU3sq+4kTJyASiVod7fj555+HsbExTp48ybU9ePAAly9f5qbd3d1RX1+PnJwcri0/Px8PHz5stL4TJ040mvbw8GixhvaytLSEVCpFSkpKk/M9PDxw69Yt3Lp1i2u7ePEiHj58iBdeeEGj1+jXrx/MzMyafY3jx4+jT58++PTTT+Hj44N+/frhxo0ban3++jftTtr7fvsrNzc3nD9/HnK5nGs7deqUWp9XXnkFFy5cgFQqhaurq9p/qvDszr+r9qCwa4FUKsXJkydx/fp1lJWVoV+/fsjOzsahQ4dw+fJlfPbZZ43eRMCT0/9RUVG4ePEi9u/fj7i4OERHR7c6kqpIJEJUVBTmz5+PI0eOIDc3FzNnzlRbzs3NDWPGjME777yDkydPIicnB3//+9+b3Ov5+eefsXHjRly+fBlxcXHIyspSOzva0T7//HMkJCRgzZo1uHLlCk6fPo21a9cCAIKCguDp6Ynw8HCcPn0aWVlZiIiIQGBgIHx8fDRav6mpKT7++GMsWLAAiYmJKCgowIkTJ/DDDz8AeBKGN2/exI4dO1BQUIA1a9Zg7969auuQSqUoLCzE2bNnUVZWphYI2tbe99tfvfnmm2hoaMDs2bORl5eHQ4cOYfny5QD+u/f+j3/8A+Xl5Zg2bRpOnTqFgoICHDp0CJGRkVzA/bUeTfbwuzVtHzTszvLz89mrr77KzMzMGAB26dIlNnPmTGZlZcWsra3Z3Llz2cKFC9UOeM+YMYOFhoayxYsXMxsbGyYSidisWbPY48ePNXrNqqoq9tZbbzFzc3Nmb2/Pvv7660YHiouLi9lrr73GhEIhc3Z2ZomJiaxPnz6NTlCsW7eOBQcHM6FQyKRSqdpZPNUJCtUBa8YYS01NZQDYgwcPuLZNmzYxKysrjX9nGzZsYG5ubszY2Jj16tWLvffee9y8GzdusL/97W/MwsKCWVpastdff52VlJRw85s6ebBy5UrWp08fblqpVLL//d//ZX369GHGxsbM2dmZffnll9z8+fPnc7/3qVOnspUrV6rV//jxYzZp0iRmbW3NALBNmzZpvG2d7Vneb3917NgxNmDAAGZiYsK8vb3Ztm3buHWqXL58mU2cOJFZW1szMzMz5u7uzj788EPW0NDQZD2FhYWd/BvoXDTEk54yMDDA3r17O/zWNaKbtm7disjISFRUVHTY8VVdQ2djCdFDiYmJ6Nu3L3r37o1z587h448/xpQpU3gbdACFXZe6efNmiwfjL168CGdn5y6sqG2aukxB5cCBAwgICOjCakhLSkpKsHjxYpSUlKBXr154/fXX1e404SP6GtuF6uvrW7zlRiqVwsio+/77c/Xq1Wbn9e7dm9d7DaT7o7AjhPACXXpCCOEFCjtCCC9Q2BFCeIHCjhDCCxR2hBBeoLAj3cbMmTO5UXGNjY1hb2+P4OBgbNy4sU33ZW7evLnJIa86W2c8bIl0HAo70q2MGTMGxcXFuH79Og4cOIDhw4fjgw8+wLhx41BfX6/t8ogu0+aNuYQ8rbmb2lNSUhgAboTihIQE9tJLLzFzc3P23HPPsblz57KqqirG2H8HM3j6v7i4OMYYY4mJiczb25uJRCJmb2/Ppk2bxu7evcu9Tnl5OXvzzTeZra0tMzU1Za6urmzjxo3c/Js3b7LXX3+dWVlZsR49erC//e1v3M3xcXFxjV43NTW1U35PpH1oz450eyNGjMDAgQOxZ88eAIBAIMCaNWtw4cIFbNmyBUeOHMGCBQsAAP7+/li1ahXEYjGKi4tRXFyMefPmAQAUCgWWLl2Kc+fOISkpCdevX8fMmTO51/nss89w8eJFHDhwAHl5eVi/fj1sbW25ZUePHg1LS0tkZGTg2LFjEIlEGDNmDOrq6jBv3jxMmTKF2zMtLi6Gv79/1/6iSMu0nbaEqDS3Z8cYY1OnTmUeHh5Nzvv555+ZjY0NN63psFSnTp1iALi9wvHjx7PIyMgm+/7444/Mzc2NG/6IMcbkcjkzMzNjhw4darV+on20Z0d0AmOMG3jy999/x8iRI9G7d29YWlpi+vTpuH//Pmpra1tcR05ODsaPHw9nZ2dYWloiMDAQwJMBGgBg7ty52LFjB7y8vLBgwQIcP36cW/bcuXO4evUqLC0tIRKJIBKJIJFI8PjxY41GDybaR2FHdEJeXh5cXFxw/fp1jBs3DgMGDMAvv/yCnJwcrFu3DkDLD4iuqanB6NGjIRaLsXXrVpw6dYobxVi1XEhICG7cuIGPPvqIe3iQ6itwdXU1vL29Gz2z4fLly3jzzTc7eetJR+i+Q2wQ8h9HjhzB+fPn8dFHHyEnJwcNDQ1ISEjghqvftWuXWv+mnp1w6dIl3L9/H1999RX3LJDs7OxGr2VnZ4cZM2ZgxowZCAgIwPz587F8+XK88sor2LlzJ3r27Nns08r07ZkN+ob27Ei3IpfLUVJSgjt37uD06dP48ssvERoainHjxiEiIgKurq5QKBRYu3Ytrl27hh9//BEbNmxQW4dUKkV1dTVSUlJQVlaG2tpaODs7w8TEhFtu3759WLp0qdpyixcvxq+//oqrV6/iwoULSE5O5h5QFB4eDltbW4SGhiIjIwOFhYU4evQo3n//fdy+fZt73T///BP5+fkoKyuDQqHoml8a0Yy2DxoSojJjxgzusg0jIyNmZ2fHgoKC2MaNG9UeML5ixQrWq1cvZmZmxkaPHs0SExMbPTtjzpw5zMbGRu3Sk23btjGpVMqEQiHz8/Nj+/bta/TgaA8PD2ZmZsYkEgkLDQ1l165d49ZZXFzMIiIimK2tLRMKhaxv375s1qxZrKKigjHGWGlpKQsODmYikYguPemGaDw7Qggv0NdYQggvUNgRQniBwo4QwgsUdoQQXqCwI4TwAoUdIYQXKOwIIbxAYUcI4QUKO0IIL1DYEUJ4gcKOEMIL/x9J9t2UncLY3QAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T14:28:00.343388Z", + "iopub.status.busy": "2024-03-26T14:28:00.342962Z", + "iopub.status.idle": "2024-03-26T14:28:00.566566Z", + "shell.execute_reply": "2024-03-26T14:28:00.565652Z" + }, + "papermill": { + "duration": 0.245783, + "end_time": "2024-03-26T14:28:00.568624", + "exception": false, + "start_time": "2024-03-26T14:28:00.322841", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUEAAAEmCAYAAAD8/yLTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABGfElEQVR4nO3deVxTV9oH8F8IJGFHqyagIKgoLigWJcWxtU7xRUtHmLdvXdoqWqp2s1bccAHq0sK4Impl7FStbRV1ptWOWqaUqp0qYkVcWVSKBZewiKyyJuf9A7lyJcEEQwLk+X4+9wOc+9ybcwl5OPfec88RMMYYCCHERJkZuwKEEGJMlAQJISaNkiAhxKRREiSEmDRKgoQQk0ZJkBBi0igJEkJMGiVBQohJMzd2BToqlUqFO3fuwNbWFgKBwNjVIYQ0wRhDeXk5nJycYGbWcluPkmAr3blzB87OzsauBiGkBXl5eejVq1eLMZQEW8nW1hZAwy/Zzs7OyLUhhDRVVlYGZ2dn7nPaEkqCrdR4CmxnZ0dJkJB2SptLVXRjhBBi0igJEkJMGiVBQohJo2uCBsYYQ319PZRKpbGrQkiHZmFhAaFQ+NT7oSRoQLW1tbh79y4ePHhg7KoQ0uEJBAL06tULNjY2T7UfSoIGolKpkJOTA6FQCCcnJ4hEIupkTUgrMcZQWFiIW7duwd3d/alahJQEDaS2thYqlQrOzs6wsrJq9X5q6pW4U1KN7rZi2Ijp7SOmq3v37rh58ybq6uqeKgnSjREDe9IjPE+SW/wA5dV1+L2wQk81IqRj0teZFCXBDqZOSfNiEaJPlAQJISaNkiBpEzNmzEBQUJBe9/niiy/io48+ajHG1dUVMTExen1dYhzavN/6QEmQPJGh/hiJ4Xz88cfw8vIydjXaBaMnwW3btsHV1RUSiQRyuRxnz55tMf7gwYPw8PCARCKBp6cnjh07xlvPGENERAQcHR1haWkJPz8/XL9+vdl+jh49CrlcDktLS3Tp0kXvrRZCCFBXV2fsKjyRUZPg/v37ERoaisjISJw/fx7Dhg2Dv78/CgoK1MafPn0aU6dORUhICNLS0hAUFISgoCBcuXKFi1m7di1iY2MRFxeHlJQUWFtbw9/fH9XV1VzMv/71L0ybNg0zZ87ExYsXcerUKbz++uttfryPY4zhQW29TktNnRLVDxddt226MKbdDZYZM2bg5MmT2Lx5MwQCAQQCAbKzsxESEgI3NzdYWlpiwIAB2Lx5s9rtV65cie7du8POzg7vvPMOamtrtXrdyspKTJ8+HTY2NnB0dMSGDRuaxRQUFOAvf/kLLC0t4ebmhm+++aZZjEAgwPbt2zFhwgRYWlqiT58++Oc//8mtv3nzJgQCAQ4cOIDnn38elpaWGDlyJK5du4bffvsNI0aMgI2NDSZMmIDCwkKt6g4AO3fuxODBgyEWi+Ho6IgPPviAW5ebm4vAwEDY2NjAzs4OkyZNQn5+Pre+sZX21VdfwdXVFfb29pgyZQrKy8u5GJVKhbVr16Jfv34Qi8VwcXHBJ598wq1fsmQJ+vfvDysrK/Tp0wfh4eFcQtq9ezdWrlyJixcvcu/p7t27n3hMmZmZGD16NCQSCQYNGoSffvoJAoEAhw4d4v0u9+/fjzFjxkAikeCbb77BvXv3MHXqVPTs2RNWVlbw9PTEvn37ePvW5v1uK0btaLZx40bMmjULM2fOBADExcXh6NGj2LlzJ8LCwprFb968GePHj8eiRYsAAKtXr0ZiYiK2bt2KuLg4MMYQExODFStWIDAwEACwZ88eSKVSHDp0CFOmTEF9fT3mzZuHdevWISQkhNv3oEGDDHDEfFV1SgyK+I/BXxcA0lf5w0r05Ld/8+bNuHbtGoYMGYJVq1YBALp06YJevXrh4MGDeOaZZ3D69GnMnj0bjo6OmDRpErdtUlISJBIJTpw4gZs3b2LmzJl45plneB9WTRYtWoSTJ0/i8OHD6NGjB5YtW4bz58/zTuFmzJiBO3fu4Pjx47CwsMCHH36o9h9oeHg4oqOjsXnzZnz11VeYMmUKLl++jIEDB3IxkZGRiImJgYuLC9566y28/vrrsLW1xebNm2FlZYVJkyYhIiIC27dvf2Ldt2/fjtDQUERHR2PChAkoLS3FqVOnADQkr8YEePLkSdTX1+P999/H5MmTceLECW4f2dnZOHToEI4cOYL79+9j0qRJiI6O5n53S5cuxeeff45NmzZh9OjRuHv3LjIzM7ntbW1tsXv3bjg5OeHy5cuYNWsWbG1tsXjxYkyePBlXrlxBQkICfvrpJwCAvb19i8ekVCoRFBQEFxcXpKSkoLy8HAsWLFAbGxYWhg0bNmD48OGQSCSorq6Gt7c3lixZAjs7Oxw9ehTTpk1D37594ePjA0C797vNMCOpqalhQqGQfffdd7zy6dOns4kTJ6rdxtnZmW3atIlXFhERwYYOHcoYYyw7O5sBYGlpabyYF154gX344YeMMcZSUlIYALZz507m5eXFZDIZGz9+PLt8+bJO9S8tLWUAWGlpqVbxVVVVLD09nVVVVXFllTV1rPeSI0ZZKmvqtD7WMWPGsHnz5rUY8/7777NXX32V+zk4OJh17dqVVVZWcmXbt29nNjY2TKlUtriv8vJyJhKJ2IEDB7iye/fuMUtLS64eWVlZDAA7e/YsF5ORkcEA8P5GALB33nmHt3+5XM7effddxhhjOTk5DAD7xz/+wa3ft28fA8CSkpK4sqioKDZgwIAW693IycmJLV++XO26H3/8kQmFQpabm8uVXb16lXcskZGRzMrKipWVlXExixYtYnK5nDHGWFlZGROLxezzzz/Xqj6MMbZu3Trm7e3N/RwZGcmGDRum9fY//PADMzc3Z3fv3uXKEhMTGQDuM9z4u4yJiXni/gICAtiCBQsYY9q93+qo+0w10uXzabSWYFFREZRKJaRSKa9cKpXy/qM1pVAo1MYrFApufWOZppjff/8dQMMpx8aNG+Hq6ooNGzbgxRdfxLVr19C1a1e1r11TU4Oamhru57KyMm0PVSNLCyHSV/nrtE2mohz1ShUAYEjPlv97P+m1n8a2bduwc+dO5ObmoqqqCrW1tc3+aw8bNoz3dIyvry8qKiqQl5eH3r17a9x3dnY2amtrIZfLubKuXbtiwIAB3M8ZGRkwNzeHt7c3V+bh4QEHB4dm+/P19W3284ULF3hlQ4cO5b5v/Pvx9PTklWm6TNNUQUEB7ty5g5deeknt+oyMDDg7O/OmZhg0aBAcHByQkZGBkSNHAmi4y910VGRHR0fu9TMyMlBTU6PxNYCGS02xsbHIzs5GRUUF6uvrn2rw36ysLDg7O0Mmk3Flja24x40YMYL3s1KpxKeffooDBw7g9u3bqK2tRU1NDfe3oc373ZaMfmPE0FSqhgSyfPlyvPrqq/D29sauXbsgEAhw8OBBjdtFRUXB3t6eW/Qxv4hAIICVyFynRWIh5BZdt226PE1v+/j4eCxcuBAhISH48ccfceHCBcycOVPr633tkYWFBfd94+/m8bLGv52WWFpa6r0+j7/+k14jOTkZb7zxBl5++WUcOXIEaWlpWL58ucHeH2tra97P69atw+bNm7FkyRIcP34cFy5cgL+/f7v5ezFaEuzWrRuEQiHvgjAA5Ofn8/7bNCWTyVqMb/zaUoyjoyMA/jVAsViMPn36IDc3V2N9ly5ditLSUm7Jy8vT5jA7BZFIxBv669SpUxg1ahTee+89DB8+HP369UN2dnaz7S5evIiqqiru5zNnzsDGxuaJ/0D69u0LCwsLpKSkcGX379/HtWvXuJ89PDxQX1+P1NRUriwrKwslJSXN9nfmzJlmPze9HqhPtra2cHV1RVJSktr1AwcORF5eHu/vJz09HSUlJVpfl3Z3d4elpaXG1zh9+jR69+6N5cuXY8SIEXB3d8cff/zBi3n8PX2SAQMGIC8vj/fZ+u2337Ta9tSpUwgMDMSbb76JYcOGoU+fPrz3Upv3uy0ZLQmKRCJ4e3vz3kiVSoWkpKRmpy+NfH19m73xiYmJXLybmxtkMhkvpqysDCkpKVyMt7c3xGIxsrKyuJi6ujrcvHmzxVM0sVjMzSdizHlFjDHujKurK1JSUnDz5k0UFRXB3d0d586dw3/+8x9cu3YN4eHhaj8QtbW1CAkJQXp6Oo4dO4bIyEh88MEHT3x+2sbGBiEhIVi0aBF+/vlnXLlyBTNmzOBtN2DAAIwfPx5z5sxBSkoKUlNT8fbbb6ttJR08eBA7d+7EtWvXEBkZibNnz/Lu1urbxx9/jA0bNiA2NhbXr1/H+fPnsWXLFgCAn58fPD098cYbb+D8+fM4e/Yspk+fjjFjxjQ7jdREIpFgyZIlWLx4Mfbs2YPs7GycOXMGX3zxBYCGJJmbm4v4+HhkZ2cjNjYW3333HW8frq6uyMnJwYULF1BUVMS71KPOuHHj0LdvXwQHB+PSpUs4deoUVqxYAeDJz/C6u7sjMTERp0+fRkZGBubMmcNLptq8323qiVcN21B8fDwTi8Vs9+7dLD09nc2ePZs5ODgwhULBGGNs2rRpLCwsjIs/deoUMzc3Z+vXr2cZGRksMjKSWVhY8G5qREdHMwcHB3b48GF26dIlFhgYyNzc3HgXT+fNm8d69uzJ/vOf/7DMzEwWEhLCevTowYqLi7Wuuz5ujLRG+p1SdjHvPruYd/+p9qOLrKws9txzzzFLS0sGgGVmZrIZM2Ywe3t75uDgwN59910WFhbGu9AeHBzMAgMDWUREBHvmmWeYjY0NmzVrFquurtbqNcvLy9mbb77JrKysmFQqZWvXrm12g+bu3bssICCAicVi5uLiwvbs2cN69+7d7MbItm3b2Lhx45hYLGaurq5s//793PrGi/lNb6YdP36cAWD379/nynbt2sXs7e21/p3FxcWxAQMGMAsLC+bo6Mjmzp3Lrfvjjz/YxIkTmbW1NbO1tWWvvfYa9zfPmPqbFps2bWK9e/fmflYqlWzNmjWsd+/ezMLCgrm4uLBPP/2UW79o0SLu9z558mS2adMmXv2rq6vZq6++yhwcHBgAtmvXriceU0ZGBvvTn/7ERCIR8/DwYP/+978ZAJaQkMAYU/+7ZKzhJkdgYCCzsbFhPXr0YCtWrGDTp09ngYGBXIw27/fj9HVjxKhJkDHGtmzZwlxcXJhIJGI+Pj7szJkz3LoxY8aw4OBgXvyBAwdY//79mUgkYoMHD2ZHjx7lrVepVCw8PJxJpVImFovZSy+9xLKysngxtbW1bMGCBaxHjx7M1taW+fn5sStXruhUb2MlwatGSIIdGZrcvST69euvvzIA7MaNG0Z5fX0lQQFjWvaaJTxlZWWwt7dHaWmpVqfG1dXVyMnJgZubGyQSSatfN/1uGXd3eGgvh1bvx1QIBAJ899139ESQHnz33XewsbGBu7s7bty4gXnz5qFLly749ddfjVKflj5Tunw+Te7uMDG+3Nxc2NjYaFxaukHVHrRU9//+97/Grl6rfPPNNxqPafDgwQCA8vJyvP/++/Dw8MCMGTMwcuRIHD582Mg1f3o0NDExOCcnp2b99B5fry9tcaLTUt179uyp99czhIkTJ/L66TXV2F1n+vTpmD59uiGrZRCUBDuYzjAribm5Ofr162fsarRaR667Jra2trzO2aaETocJISaNkiAhxKRREuxg6FY+IfpFSZAQYtIoCXYwneHGCCHtCSVB0uYen/yo6WjEhBgbdZEhBnf37l106dLF2NUgBAAlQWIEmoZKMyTGGJRKJczN6SNg6uh02JgYA2ordVoEdQ+4RddteYuOT1KUl5fjjTfegLW1NRwdHbFp06ZWT8WpbnKeb7/9FmPHjoWVlRWGDRuG5ORk3ja//vorNxGSs7MzPvzwQ1RWVnLrv/rqK4wYMQK2traQyWR4/fXXeSNBnzhxAgKBAD/88AM3nJqxnnkl7Qv9GzSmugfAp7o9Iuahr9dedgcQWT857qHQ0FCcOnUK33//PaRSKSIiIvQ6Ec7y5cuxfv16uLu7Y/ny5Zg6dSpu3LgBc3NzZGdnY/z48VizZg127tyJwsJCfPDBB/jggw+wa9cuAA1jQq5evRoDBgxAQUEBQkNDMWPGjGZTsoaFhWH9+vXo06cPnZITAJQEiRbKy8vx5ZdfYu/evdy8Frt27dLrM74LFy5EQEAAgIZpOgcPHowbN27Aw8MDUVFReOONN7hWp7u7O2JjYzFmzBhs374dEokEb731FrevPn36IDY2FiNHjkRFRQVsbGy4datWrcK4ceP0Vm/S8VESNCYLq4YWmQ4yFeWoeziUludTTLQEC6snxzz0+++/o66ujjexjr29vV4nwmk60VHjFAgFBQXw8PDAxYsXcenSJd68wowxqFQq5OTkYODAgUhNTcXHH3+Mixcv4v79+9x8HLm5ubxh67UdvZmYDkqCxiQQ6HRKCgDMQglm9nDCHx23bc/UTXTUmMgqKiowZ84cfPjhh822c3FxQWVlJfz9/eHv749vvvkG3bt3R25urtrJfB6fBIgQSoLkifr06QMLCwv89ttvcHFxAQCUlpbi2rVreOGFF9r89Z999lmkp6drHL3l8uXLuHfvHqKjo7lJnM6dO9fm9SKdA90dJk9ka2uL4OBgLFq0CMePH8fVq1cREhICMzOzp5q6U1tLlizB6dOn8cEHH+DChQu4fv06Dh8+zE2W5OLiApFIhC1btuD333/H999/j9WrV7d5vUjnQEmQaGXjxo3w9fXFK6+8Aj8/P/zpT3/CwIEDn2qqAG0NHToUJ0+exLVr1/D8889j+PDhiIiI4G7MdO/eHbt378bBgwcxaNAgREdHY/369W1eL9I50BwjrWSsOUYy7pZxN0aMOcdIZWUlevbsiQ0bNiAkJMRo9SCmS19zjNA1wQ6mMQEaWlpaGjIzM+Hj44PS0lKsWrUKABAYGGiU+hCiL5QEidbWr1+PrKwsiEQieHt747///S8yMjIwYcIEjdtUVFQYsIaE6I6SINHK8OHDkZqa2qy8qqqqxYmHCGnv2sWNkW3btsHV1RUSiQRyuRxnz55tMf7gwYPw8PCARCKBp6dns0ejGGOIiIiAo6MjLC0t4efnh+vXr/NiXF1dIRAIeEt0dLTej62zs7S0RL9+/TQuhLR3Rk+C+/fvR2hoKCIjI3H+/HkMGzYM/v7+vIffmzp9+jSmTp2KkJAQpKWlISgoCEFBQbhy5QoXs3btWsTGxiIuLg4pKSmwtraGv78/qqureftatWoV7t69yy1z585t02MF2mYKSEJMkd4+S8zIfHx82Pvvv8/9rFQqmZOTE4uKilIbP2nSJBYQEMArk8vlbM6cOYwxxlQqFZPJZGzdunXc+pKSEiYWi9m+ffu4st69e7NNmza1ut6lpaUMACstLdUqvr6+nqWnp7OioqJWvyZjjF3Mu88thJiykpISlp6ezmpra5ut0+XzadRrgrW1tUhNTcXSpUu5MjMzM/j5+TUbSqlRcnIyQkNDeWX+/v7c0Ew5OTlQKBTw8/Pj1tvb20MulyM5ORlTpkzhyqOjo7F69Wq4uLjg9ddfx/z58zWOL1dTU4Oamhru57KyMp2OVSgUwsHBgWvhWllZtaqjMat/9BjY4y1bQkyFSqVCYWEhrKysnnpMSKMmwaKiIiiVSkilUl65VCpFZmam2m0UCoXaeIVCwa1vLNMUAwAffvghnn32WXTt2hWnT5/G0qVLcffuXWzcuFHt60ZFRWHlypW6HeBjGgcT1XSqr42C+1Xc96Iqy6eqDyEdmZmZGVxcXJ76qSWTvTvctDU5dOhQiEQizJkzB1FRURCLxc3ily5dytumrKyMe05VWwKBAI6OjujRowfq6upaVe+3vz3BfZ+04MVW7YOQzkAkEsHM7Olvaxg1CXbr1g1CoRD5+fm88vz8fI1DsMtkshbjG7/m5+dzQzI1/tzSAKByuRz19fW4efOm2iGixGKx2uTYGkKhEEKhsFXb3i5Xct8b4pE1Qjo7o94dbux0m5SUxJWpVCokJSXB19dX7Ta+vr68eABITEzk4t3c3CCTyXgxZWVlSElJ0bhPALhw4QLMzMzQo0ePpzkkQkgHY/TT4dDQUAQHB2PEiBHw8fFBTEwMKisrMXPmTADA9OnT0bNnT0RFRQEA5s2bhzFjxmDDhg0ICAhAfHw8zp07hx07dgBoOOX86KOPsGbNGri7u8PNzQ3h4eFwcnJCUFAQgIabKykpKRg7dixsbW2RnJyM+fPn480336Qh1wkxMUZPgpMnT0ZhYSEiIiKgUCjg5eWFhIQE7sZGbm4u77x/1KhR2Lt3L1asWIFly5bB3d0dhw4dwpAhQ7iYxYsXo7KyErNnz0ZJSQlGjx6NhIQE7vRRLBYjPj4eH3/8MWpqauDm5ob58+c3u+tMCOn8aBSZVtJ1FBl9cQ07yn1/MzrAYK9LSEeiy+fT6E+MEEKIMVESJISYNEqChBCTRkmQEGLSKAkSQkwaJUFCiEl7qn6CFRUV3ATZjQzZXYQQQp6Wzi3BnJwcBAQEwNraGvb29ujSpQu6dOkCBwcHetqCENLh6NwSfPPNN8EYw86dOyGVSg0y+TYhhLQVnZPgxYsXkZqaqnakFUII6Wh0Ph0eOXIk8vLy2qIuhBBicDq3BP/xj3/gnXfewe3btzFkyBBYWFjw1g8dOlRvlSOEkLamcxIsLCxEdnY2N9QV0DB8FWMMAoEASqWyha0JIaR90TkJvvXWWxg+fDj27dtHN0YIIR2ezknwjz/+wPfff08TaxNCOgWdb4z8+c9/xsWLF9uiLoQQYnA6twT/8pe/YP78+bh8+TI8PT2b3RiZOHGi3ipHCCFtTeck+M477wAAVq1a1Wwd3RghhHQ0OifBx58VJoSQjkyna4J1dXUwNzfHlStX2qo+hBBiUDolQQsLC7i4uNApLyGk09D57vDy5cuxbNkyFBcXt0V9CCHEoHROglu3bsUvv/wCJycnDBgwAM8++yxvaY1t27bB1dUVEokEcrkcZ8+ebTH+4MGD8PDwgEQigaenJ44dO8ZbzxhDREQEHB0dYWlpCT8/P1y/fl3tvmpqauDl5QWBQIALFy60qv6EkI5L5xsjQUFBeq3A/v37ERoairi4OMjlcsTExMDf3x9ZWVno0aNHs/jTp09j6tSpiIqKwiuvvIK9e/ciKCgI58+f5yZgX7t2LWJjY/Hll1/Czc0N4eHh8Pf3R3p6OjcBe6PFixfDycmJ+j4SYqqYkfn4+LD333+f+1mpVDInJycWFRWlNn7SpEksICCAVyaXy9mcOXMYY4ypVComk8nYunXruPUlJSVMLBazffv28bY7duwY8/DwYFevXmUAWFpamtb1Li0tZQBYaWmp1tvoQ+8lR7iFEKKeLp/PVs8xkpqaiq+//hpff/010tLSWrWP2tpapKamws/PjyszMzODn58fkpOT1W6TnJzMiwcAf39/Lj4nJwcKhYIXY29vD7lczttnfn4+Zs2aha+++gpWVlZPrGtNTQ3Kysp4CyGk49P5dLigoABTpkzBiRMn4ODgAAAoKSnB2LFjER8fj+7du2u9r6KiIiiVSkilUl65VCpFZmam2m0UCoXaeIVCwa1vLNMUwxjDjBkz8M4772DEiBG4efPmE+saFRWFlStXanVchJCOQ+eW4Ny5c1FeXo6rV6+iuLgYxcXFuHLlCsrKyvDhhx+2RR31bsuWLSgvL8fSpUu13mbp0qUoLS3lFhpYlpDOQeckmJCQgM8++wwDBw7kygYNGoRt27bhhx9+0Glf3bp1g1AoRH5+Pq88Pz8fMplM7TYymazF+MavLcX8/PPPSE5Ohlgshrm5OTcizogRIxAcHKz2dcViMezs7HgLIaTj0zkJqlSqZoMmAA0dqXV9pE4kEsHb2xtJSUm8/SclJcHX11ftNr6+vrx4AEhMTOTi3dzcIJPJeDFlZWVISUnhYmJjY3Hx4kVcuHABFy5c4LrY7N+/H5988olOx0AI6eB0vesyceJE9sILL7Dbt29zZbdu3WJjxoxhQUFBuu6OxcfHM7FYzHbv3s3S09PZ7NmzmYODA1MoFIwxxqZNm8bCwsK4+FOnTjFzc3O2fv16lpGRwSIjI5mFhQW7fPkyFxMdHc0cHBzY4cOH2aVLl1hgYCBzc3NjVVVVauuQk5NDd4cJ6UR0+XzqfGNk69atmDhxIlxdXeHs7AwAyMvLw5AhQ/D111/rnIQnT56MwsJCREREQKFQwMvLCwkJCdyNjdzcXJiZPWqwjho1Cnv37sWKFSuwbNkyuLu749ChQ1wfQaCh719lZSVmz56NkpISjB49GgkJCc36CBJCiIAxxnTdiDGGn376ibuDO3DgwGbdVjq7srIy2Nvbo7S01KDXB13DjnLf34wOMNjrEtKR6PL51LklCDSMGzhu3DiMGzeuVRUkhJD2olVJMCkpCUlJSSgoKGh2M2Tnzp16qRghhBiCzklw5cqVWLVqFUaMGAFHR0eabY4Q0qHpnATj4uKwe/duTJs2rS3qQwghBqVzP8Ha2lqMGjWqLepCCCEGp3MSfPvtt7F37962qAshhBiczqfD1dXV2LFjB3766ScMHTq02dMjGzdu1FvlCCGkremcBC9dugQvLy8AaDbhEt0kIYR0NDonwePHj7dFPchj6pQqJGUUQO7WFV2sRWpjlCqGczeLMczZARILoYFrSEjn0OpBVUnb2vrzDbzzdSr+L+60xpjPjt/A5B1nMHdf6wa1JYRQEmy3jl6+CwDILqzUGPPVmT8AAInp+RpjCCEtoyTYgVkI6e0j5GnRp4gQYtJ0ToK//PIL6uvrm5XX19fjl19+0UuliHZaMQAQIeQxOifBsWPHori4uFl5aWkpxo4dq5dKEYA6GxFiGDonQcaY2v6A9+7dg7W1tV4qRQghhqJ1P8H//d//BdDQIXrGjBkQi8XcOqVSiUuXLtEzxQZWWas0dhUI6fC0ToL29vYAGlqCtra2sLS05NaJRCI899xzmDVrlv5rSDQqraozdhUI6fC0ToK7du0CALi6umLhwoV06tvGml5xqFOqYCYQQGhGVwoJ0TedH5uLjIxsi3qQFsg/TYKdxBzHF75o7KoQ0unofGMkPz8f06ZNg5OTE8zNzSEUCnkL0b/iylrcvPfA2NUgpFPSuSU4Y8YM5ObmIjw8nIbXJ4R0eDonwV9//RX//e9/ueG0SNsQUE9BQgxC59NhZ2dnvT+psG3bNri6ukIikUAul+Ps2bMtxh88eBAeHh6QSCTw9PTEsWPHeOsZY4iIiICjoyMsLS3h5+eH69ev82ImTpwIFxcXSCQSODo6Ytq0abhz545ej0vfHlCXGEL0TuckGBMTg7CwMNy8eVMvFdi/fz9CQ0MRGRmJ8+fPY9iwYfD390dBQYHa+NOnT2Pq1KkICQlBWloagoKCEBQUxBvgde3atYiNjUVcXBxSUlJgbW0Nf39/VFdXczFjx47FgQMHkJWVhX/961/Izs7G//3f/+nlmNrK/2yixxIJ0TcB07FZ16VLFzx48AD19fWwsrJqNry+ukfqWiKXyzFy5Ehs3boVAKBSqeDs7Iy5c+ciLCysWfzkyZNRWVmJI0eOcGXPPfccvLy8EBcXB8YYnJycsGDBAixcuBBAwyN9UqkUu3fvxpQpU9TW4/vvv0dQUBBqamqaHZM6usxw3xrjY35BpqJc6/ib0QF6rwMhHZUun0+drwnGxMS0tl7N1NbWIjU1FUuXLuXKzMzM4Ofnh+TkZLXbJCcnIzQ0lFfm7++PQ4cOAQBycnKgUCjg5+fHrbe3t4dcLkdycrLaJFhcXIxvvvkGo0aN0pgAa2pqUFNTw/1cVlam9XESQtovnZNgcHCw3l68qKgISqUSUqmUVy6VSpGZmal2G4VCoTZeoVBw6xvLNMU0WrJkCbZu3YoHDx7gueee47UuHxcVFYWVK1dqd2CEkA6jVeMJZmdnY8WKFZg6dSp37e6HH37A1atX9Vq5trZo0SKkpaXhxx9/hFAoxPTp0zXe9Fm6dClKS0u5JS8vz8C1bT2liiFo2ym8902qsatCSLujcxI8efIkPD09kZKSgm+//RYVFRUAgIsXL+r8NEm3bt0gFAqRn88fHj4/Px8ymUztNjKZrMX4xq/a7LNbt27o378/xo0bh/j4eBw7dgxnzpxR+7pisRh2dna8paPIuFuGC3klOHZZAZWKxiAkpCmdk2BYWBjWrFmDxMREiESPZkH785//rDGBaCISieDt7Y2kpCSuTKVSISkpCb6+vmq38fX15cUDQGJiIhfv5uYGmUzGiykrK0NKSorGfTa+LgDedb/OQtWkdZt+l65lEtKUztcEL1++jL179zYr79GjB4qKinSuQGhoKIKDgzFixAj4+PggJiYGlZWVmDlzJgBg+vTp6NmzJ6KiogAA8+bNw5gxY7BhwwYEBAQgPj4e586dw44dOwA0DPX10UcfYc2aNXB3d4ebmxvCw8Ph5OSEoKAgAEBKSgp+++03jB49Gl26dEF2djbCw8PRt2/fFhNlR1VTr+K+33c2F5/81dOItSGkfdE5CTo4OODu3btwc3PjlaelpaFnz546V2Dy5MkoLCxEREQEFAoFvLy8kJCQwN3YyM3NhZnZowbrqFGjsHfvXqxYsQLLli2Du7s7Dh06hCFDhnAxixcvRmVlJWbPno2SkhKMHj0aCQkJkEgkAAArKyt8++23iIyMRGVlJRwdHTF+/HisWLGCN05iZ5HRpPVXXt18agRCTJnO/QQXLlyIlJQUHDx4EP3798f58+eRn5+P6dOnY/r06SYzykxb9xOcsPm/vOT1JC31E4z56Rpifmp4YuaVoY7Y+vqzT10/QtozXT6fOl8T/PTTT+Hh4QFnZ2dUVFRg0KBBeOGFFzBq1CisWLGi1ZUmhkFzMxHCp/PpsEgkwueff47w8HBcuXIFFRUVGD58ONzd3duifkTPVJQFCeHROQk2cnFxgYuLiz7rQgxASV1kCOHRKgmGhoZi9erVsLa2bvbI2uM2btyol4qZurYaSItyICF8WiXBtLQ01NXVcd9rQgOstn80YTshfFolwePHj6v9nnQ8dE2QEL5WPTtMOpameY9SICF8WrUEGyde18a3337b6sqQR9rqygJdEySET6uWoL29PbfY2dkhKSkJ586d49anpqYiKSmJm6CdtC9NEypdEySET6uWYOPE60DDGHyTJk1CXFwcN8WmUqnEe++916FGVjFVdE2QED6drwnu3LkTCxcu5M0xLBQKERoaip07d+q1ckT/VKonxxBiSnROgvX19WpHfc7MzOSGoyJPr+2uCVJLkJCmdH5iZObMmQgJCUF2djZ8fHwANAxNFR0dzQ1/RdovyoGE8OmcBNevXw+ZTIYNGzbg7t27AABHR0csWrQICxYs0HsFiX5RS5AQPp2ToJmZGRYvXozFixdzM67RDRH9E7TRg3OUBAnha/UACgAlv7Zwo6ACEgv99mFvmveUlAMJ4WlVEvznP/+JAwcOIDc3F7W1tbx158+f10vFTFHJg1r4bTwJAPDs2TZ9LqmfICF8Ojc5YmNjMXPmTEilUqSlpcHHxwfPPPMMfv/9d0yYMKEt6mgy8oqr2vw1KAcSwqdzEvzss8+wY8cObNmyBSKRCIsXL0ZiYiI+/PBDlJaWtkUdTRJ1kSHEMHROgrm5uRg1ahQAwNLSEuXl5QCAadOmYd++ffqtHdE7yoGE8OmcBGUyGYqLiwE0jC7dONdwTk4OXW/qAKglSAifzknwz3/+M77//nsADR2n58+fj3HjxmHy5Mn461//qvcKEv2iJEgIn85JcMeOHVi+fDkA4P3338fOnTsxcOBArFq1Ctu3b29VJbZt2wZXV1dIJBLI5XKcPXu2xfiDBw/Cw8MDEokEnp6eOHbsGG89YwwRERFwdHSEpaUl/Pz8cP36dW79zZs3ERISAjc3N1haWqJv376IjIxsdqfbmGh4fUIMQ6ckWF9fjzVr1kChUHBlU6ZMQWxsLObOnQuRSKRzBfbv34/Q0FBERkbi/PnzGDZsGPz9/VFQUKA2/vTp05g6dSpCQkKQlpaGoKAgBAUF4cqVK1zM2rVrERsbi7i4OKSkpMDa2hr+/v6orq4G8Og557///e+4evUqNm3ahLi4OCxbtkzn+uuTIWYnUFEWJIRH58nXbWxscOXKFbi6uuqlAnK5HCNHjsTWrVsBACqVCs7Ozpg7dy7CwsKaxU+ePBmVlZU4cuQIV/bcc8/By8sLcXFxYIzByckJCxYswMKFCwEApaWlkEql2L17N6ZMmaK2HuvWrcP27dvx+++/a1Xvtph8/crtUryy5VcAwLBe9rh4S/u77S1Nvr4p8Ro2JzW0hF2fscKJRWOfrqKEtHNtOvn6Sy+9hJMnT7a6ck3V1tYiNTUVfn5+jypkZgY/Pz8kJyer3SY5OZkXDwD+/v5cfE5ODhQKBS/G3t4ecrlc4z6BhkTZtWtXjetrampQVlbGWzoiaggSwqfzEyMTJkxAWFgYLl++DG9vb1hbW/PWT5w4Uet9FRUVQalUQiqV8sqlUqna4boAQKFQqI1vPEVv/NpSzONu3LiBLVu2YP369RrrGhUVhZUrV7Z8QPrURufGNO8wIXw6J8H33nsPgPr5hQUCAZRK5dPXyoBu376N8ePH47XXXsOsWbM0xi1dupQ353JZWRmcnZ0NUcWn1jTt0d1hQvh0ToL6HDi1W7duEAqFyM/P55Xn5+dDJpOp3UYmk7UY3/g1Pz8fjo6OvBgvLy/ednfu3MHYsWMxatQo7Nixo8W6isViiMVirY6rPaMkSAifUafcFIlE8Pb2RlJSElemUqmQlJQEX19ftdv4+vry4gEgMTGRi3dzc4NMJuPFlJWVISUlhbfP27dv48UXX4S3tzd27doFM7NOPPtok8RHZ8OE8GndEqyqqkJSUhJeeeUVAA2nhzU1Ndx6oVCI1atXQyKR6FSB0NBQBAcHY8SIEfDx8UFMTAwqKyu5UaqnT5+Onj17IioqCgAwb948jBkzBhs2bEBAQADi4+Nx7tw5riUnEAjw0UcfYc2aNXB3d4ebmxvCw8Ph5OSEoKAgAI8SYO/evbF+/XoUFhZy9dHUAjW0NusnSFmQEB6tk+CXX36Jo0ePcklw69atGDx4MCwtLQE09L1zcnLC/PnzdarA5MmTUVhYiIiICCgUCnh5eSEhIYG7sZGbm8trpY0aNQp79+7FihUrsGzZMri7u+PQoUMYMmQIF7N48WJUVlZi9uzZKCkpwejRo5GQkMAl6MTERNy4cQM3btxAr169ePXpjI/+NT0iZSc8PkKehtb9BJ9//nksXrwYf/nLXwAAtra2uHjxIvr06QMA+Prrr7Ft27YWu6F0Jm3dT9DL2QEX8kq03ralfoIbfszClp9vaBVLSGfQJv0Eb9y4AU9PT+5niUTCa6H5+PggPT29FdUlbe3xf3O6JFdCOjutT4dLSkp41wCbXkcDGm5oNF1Pnk5bPkL36/VCeDk7tN0LENKBaN0S7NWrF+/53MddunSp2fU10j4w0HVAQjTROgm+/PLLiIiI4AYhaKqqqgorV65EQABdayKEdCxanw4vW7YMBw4cwIABA/DBBx+gf//+AICsrCxs3boV9fX1Rh+FpaO5UVCBwvIa+PZ9ptk6fZ4NP35NUGCI4WoI6SC0ToJSqRSnT5/Gu+++i7CwMK4riUAgwLhx4/DZZ581e16XtKxxZrkNrw1DpqIMfgPp90eIoen02JybmxsSEhJQXFyMGzcaulz069evxdFXyJMtOHgRAHAxr20mqqIrgoRo1qp5h7t27QofHx9918XkXSsoN8jrlFXVaVyXV/wAJ64V4jXvXpBYCA1SH0KMqRM/MNu+1dY3H4ii5IHm5PQ0Hr8m+PdfNA8c67fxJMIPXcGWn69rjCGkM6EkaCTX8g3T6tNVzcPkfDr7npFrQohhUBI0Aer6CW4/kd3yNnQhkZgISoJGYuxx/f6WoH7kbkJMTatujJCn87eEzCe2xPTal49adYRoRC1BI3hSAgQ655BehLRHlARNAKVTQjSjJEgIMWmUBNspfV4TbM2p9YW8EgRuO4XUP4r1Vg9C2iNKgkSji3kleHW7aYwUTkwXJUETQPdYCNGMkiAhxKRREjQB1BAkRDNKgu1UZU19m7+GkuYgJoSSYHulzwSl6ZrgnZIqvb0GIR2V0ZPgtm3b4OrqColEArlcjrNnz7YYf/DgQXh4eEAikcDT0xPHjh3jrWeMISIiAo6OjrC0tISfnx+uX+cPC/XJJ59g1KhRsLKygoODg74PSS9uGyBBUUuQECMnwf379yM0NBSRkZE4f/48hg0bBn9/fxQUFKiNP336NKZOnYqQkBCkpaUhKCgIQUFBvFnw1q5di9jYWMTFxSElJQXW1tbw9/fnTRBVW1uL1157De+++26bH2NrPahV6m1fmmaboxRIiJGT4MaNGzFr1izMnDkTgwYNQlxcHKysrLBz50618Zs3b8b48eOxaNEiDBw4EKtXr8azzz6LrVu3AmhoBcbExGDFihUIDAzE0KFDsWfPHty5cweHDh3i9rNy5UrMnz+fN5m8KTL2SDaEtAdGS4K1tbVITU2Fn5/fo8qYmcHPzw/Jyeo76CYnJ/PiAcDf35+Lz8nJgUKh4MXY29tDLpdr3Ke2ampqUFZWxls6Ck25jgZpIMSISbCoqAhKpbLZDHVSqRQKhULtNgqFosX4xq+67FNbUVFRsLe35xZnZ+dW7efSrZKnqocmrUloj59y1yubD/lPSGdn9BsjHcXSpUtRWlrKLXl5ea3az8Stp/Rcs9Zb9e903s//u/20kWpCiPEYLQl269YNQqEQ+fn5vPL8/HzIZDK128hkshbjG7/qsk9ticVi2NnZ8ZaO7twf93k/X7rVNlN+EqI3FYXA/Zt63aXRkqBIJIK3tzeSkpK4MpVKhaSkJPj6+qrdxtfXlxcPAImJiVy8m5sbZDIZL6asrAwpKSka92kK6Nof6XCq7gN/JAO/fQEcWwTsfgVY2wdY3w9IWKrXlzLq8PqhoaEIDg7GiBEj4OPjg5iYGFRWVmLmzJkAgOnTp6Nnz56IiooCAMybNw9jxozBhg0bEBAQgPj4eJw7dw47duwA0DD81EcffYQ1a9bA3d0dbm5uCA8Ph5OTE4KCgrjXzc3NRXFxMXJzc6FUKnHhwgUADRPJ29jYGPR3QIhJq6kACrOAgnSgMLPha0EGUH5XwwYCoO6BXqtg1CQ4efJkFBYWIiIiAgqFAl5eXkhISOBubOTm5sLM7FFjddSoUdi7dy9WrFiBZcuWwd3dHYcOHcKQIUO4mMWLF6OyshKzZ89GSUkJRo8ejYSEBEgkEi4mIiICX375Jffz8OHDAQDHjx/Hiy++2MZHbXjUDiRGV1cNFF3jJ7qCdKAkV/M29s5Aj4ENS/eHX7v1B0RWeq2agNG5UquUlZXB3t4epaWlOl0fdF9+DHVK/f/Kc6Je1jgQa8ThK9iT/IfadTejA7jvXcOOPjGGkBYp64B72UBhxqNEV5AJFGcDTEPvAxvpw2Q3COju8fDrAEDS+uvuunw+abY5AzPGvxz6N0f0TqVsuEHBa9llNrT2VHXqt7Hs0pDgegx8lOx6DASsuhq06o+jJGhg7e0pjbziB3Du2vLpBWNMv1OAko6DMaDsdpNW3cMWXmEWUK/h+XaRTfPT2B6DAJseQDv8O6IkaECh+y/AGGMWaHp2GACWfXcZX4XIW9xeqWIwF7a/P16iR4wBFQXNT2MLM4EaDU9HmUsaTlubJroeHg3X8tphstOEkqABfZt229hVaKa0quHUpaVLw7VKFcyF1K++03hQ3OQ0NvNR0qvSMKmWmXnDDYmmp7A9BgJdXAEzoUGr3hYoCZqAls7AG/9f17fQRK2rZ4BIv3UiBlBT/qj7SUGTa3cVGh4hFZgBXfs0T3Zd+wLmnfcPgJKgqXt42lLfwh3rWnqmuH2rq2q4IdF4va5xKW2p+4nLoyTXeBrbrT9gYWm4ercTlAQNRGXEAUy1eeU6leZEV0dJsH1Q1gH3bjRJdA9bdvdzWuh+IuMnusbuJ2Jbw9a9HaMkaCDttTXVeDqsbKElSEnQwBq7nzRNdoWZQNH1FrqfdG1yCuvxqM+dkbufdASUBA3EmEnwSb1y8suq8dnxGxrX19ZTEmwTjAGlefzrdYWN3U+q1W8jsm1yGtvkdNa6e4e6I9ueUBI0kLZOJIy17jMgEADTvkjBtfwKjTHttRXbYTAGVOQ3b9kVZAK15eq3MbdsOG1tmui6ewD2vSjZ6RklQQMx7iml5qagAGgxAQJok8f8Oq0HxfzrdY1dUaruq483s2i4IdHDg5/sOkn3k46AkqCBtNdTyvO5JU+MoWuCalSXNel+kvGok3FFvvp4gVlDV5MeTbqfdB8IPNMXEFoYtu6Eh5KggXTkRNJeE7hB1FU9THZNEl1BRsO1PE0cXJrcpHjYsuvWH7CQaN6GGA0lQQOpMWIiedrHlU3immB97cPuJ43X6x6e0hbnQOPlBFtH9aOfiGlMyo6EkqCBdOTranWdqSWoUjYktscH8bx3A1DVq9/G6hk1o594NIyKQjo8SoIGYsxTyqdtCXbIBK5SPex+ksEfFKDouubuJ2I7DaOfdDds3YlBURI0kA59TVCpfHKQsTAGlCseG/3kYV+7Wg13vc0tG1pyvNFPBgJ2TtT9xARREjQQo7YEn3KA/br6dtISrLzXfKingnSgukR9vFDUZPSTJo+OObgCZjQqDmlASbCtnf0cyL8KmbInfM0Yrqt6oQh2ePTAWvtn8Bsj1WWPjVj8cKksUB8vEDZ0NWk2+kkf6n5CnoiSYFvLOgZk/4yBAPY9HI2omNngOuuFa6peuMZ6cd8Xo23mMn76a4JtlARrHwBFWY+17DKAsluat3Hoze9+0sMDeMadup+QVqMk2NZGvg04euHO9TTU3k2Hi6AAXQUVkAsyITfL5IUWMTtcV/XCNdaTlyRLYNwRP546CdbXAveuNz+NvX8TmrufODV/RrYbdT8h+kdJsK15BAAeAYhn1xD7x3VIUIO+gjvoL7iF/ma34C64hf6CW3AxK0Q3QRm6CdPhi3TeLgqZ/WOtxp64xnqhDNolhKe9oqf19UxlfcOwTo8P4nnvBsA03Fyx6qZmqCcPwNLhKWtNiHbaRRLctm0b1q1bB4VCgWHDhmHLli3w8fHRGH/w4EGEh4fj5s2bcHd3x9/+9je8/PLL3HrGGCIjI/H555+jpKQEf/rTn7B9+3a4u7tzMcXFxZg7dy7+/e9/w8zMDK+++io2b97cJpOvV9UqEZt0HQBQDTGuMjdcZW5Ak9xiiWr0e5gc3c1ucUmyl6AI3QWl6C4sxZ9wlbfffOaAa6qGxIjzhY+GUZLY67X+tY93kVGpGgbsfHwQz6JrgLJG/U7E9s1bdt0HUvcTYnRGT4L79+9HaGgo4uLiIJfLERMTA39/f2RlZaFHjx7N4k+fPo2pU6ciKioKr7zyCvbu3YugoCCcP3+em4R97dq1iI2NxZdffgk3NzeEh4fD398f6enp3CTsb7zxBu7evYvExETU1dVh5syZmD17Nvbu3av3Y2ycx6MlVZDgMuuDy6wPLzlaowr9BLcfthpvc0myp+AepIISSIUleB5XgCMJjzaydWrSBcQDzpXmsIEFKqDrpNUMUtxHr3ungdM/NhkBJQuoq1S/iYVVk7uxTbqg2DpS9xPSLhl98nW5XI6RI0di69atAACVSgVnZ2fMnTsXYWFhzeInT56MyspKHDlyhCt77rnn4OXlhbi4ODDG4OTkhAULFmDhwoUAgNLSUkilUuzevRtTpkxBRkYGBg0ahN9++w0jRowAACQkJODll1/GrVu34OTk9MR6azO5M2MMRy7dxdx9aTr/Xp7EBg/gLrjNtRpD+tdAUJgJlN/RuM1t9szDa469cJ31xHVVw9dKWKIrytDf7Bb6C/IwQHAL/c3y0F9wC/aCB+p3JhQ1XKN7fPQTh97U/YQYXYeZfL22thapqalYunQpV2ZmZgY/Pz8kJyer3SY5ORmhoaG8Mn9/fxw6dAgAkJOTA4VCAT8/P269vb095HI5kpOTMWXKFCQnJ8PBwYFLgADg5+cHMzMzpKSk4K9//Wuz162pqUFNzaNTvbIyDdMQNiEQCLCthcFKn0YFrJDG3JGmbDjFP3jPFgIhYG1XARdVHnorcxsWVS561f+BHriPnoJ76Cm8hxdxkbevMmYFOw3Jrp6Z4Q+BI+5YuOKmsDdumrngD6EL7pg5QVUjBPLQsAB47AdC2oRnT3use22Y3vZn1CRYVFQEpVIJqVTKK5dKpcjMzFS7jUKhUBuvUCi49Y1lLcU8fqptbm6Orl27cjGPi4qKwsqVK7U8skfmj+uPOV+l6rydrrLyHw3OmYpeAHoBGMWV2aHi4XXG29zNmP5mt9BdUAo7wQOomAC5rAeus17IYr1wTeWMa6wXfmeOqIUF0OxJMw0tRELamL2lfvt+Gv2aYEexdOlSXgu0rKwMzs7OT9zOf7AMifNfgMjcDEUVtXDpagUGhlv3q/BTej4qa+pRXafCqH7PwNzMDBfy7uNafgWsREKUVtVBaCZAd1sxulqJIBAArt2skVv8AF69HGArsUBu8QN0txXD0qLlATgthAKIzM1wPb8CAx3tUFFTD9bDGgkZ2aguvg3hM25QWVhhSE97eIrMcev+A0gshFCqGMqrNQwsQIgRdKok2K1bNwiFQuTn8weizM/Ph0wmU7uNTCZrMb7xa35+PhwdHXkxXl5eXExBAf/pg/r6ehQXF2t8XbFYDLFYrP3BNeEubejn1/sZa66sh60Ez7o0H4UkYKhjszJ9Gv7Ya473GQxgcLM4mT11PiamwahXsEUiEby9vZGUlMSVqVQqJCUlwdfXV+02vr6+vHgASExM5OLd3Nwgk8l4MWVlZUhJSeFifH19UVJSgtTUR6epP//8M1QqFeRyud6OjxDSATAji4+PZ2KxmO3evZulp6ez2bNnMwcHB6ZQKBhjjE2bNo2FhYVx8adOnWLm5uZs/fr1LCMjg0VGRjILCwt2+fJlLiY6Opo5ODiww4cPs0uXLrHAwEDm5ubGqqqquJjx48ez4cOHs5SUFPbrr78yd3d3NnXqVK3rXVpaygCw0tJSPfwWCCH6pMvn0+hJkDHGtmzZwlxcXJhIJGI+Pj7szJkz3LoxY8aw4OBgXvyBAwdY//79mUgkYoMHD2ZHjx7lrVepVCw8PJxJpVImFovZSy+9xLKysngx9+7dY1OnTmU2NjbMzs6OzZw5k5WXl2tdZ0qChLRfunw+jd5PsKPSpR8SIcSwdPl8Uq9WQohJoyRICDFplAQJISaNOku3UuOlVG0enyOEGFbj51KbWx6UBFupvLzhMTVtnhohhBhHeXk57O1bHlqO7g63kkqlwp07d2BrawtBBx4iqvHxv7y8vE5zl5uOqeNoq+NijKG8vBxOTk4we8KoRtQSbCUzMzP06tXL2NXQGzs7u0714QLomDqStjiuJ7UAG9GNEUKISaMkSAgxaZQETZxYLEZkZGSrR8hpj+iYOo72cFx0Y4QQYtKoJUgIMWmUBAkhJo2SICHEpFESJISYNEqCndC2bdvg6uoKiUQCuVyOs2fPthh/8OBBeHh4QCKRwNPTE8eOHeOtZ4whIiICjo6OsLS0hJ+fH65fv96Wh9CMLsf0+eef4/nnn0eXLl3QpUsX+Pn5NYufMWMGBAIBbxk/fnxbHwaPLse0e/fuZvWVSPjzwHS09+nFF19sdkwCgQABAQFcjEHep7Ya2ZUYR3x8PBOJRGznzp3s6tWrbNasWczBwYHl5+erjT916hQTCoVs7dq1LD09na1YsULtdAX29vbs0KFD7OLFi2zixInNpitoT8f0+uuvs23btrG0tDSWkZHBZsyYwezt7dmtW7e4mODgYDZ+/Hh29+5dbikuLjbI8TCm+zHt2rWL2dnZ8erbOAVFo472Pt27d493PFeuXGFCoZDt2rWLizHE+0RJsJPx8fFh77//PvezUqlkTk5OLCoqSm38pEmTWEBAAK9MLpezOXPmMMYapiqQyWRs3bp13PqSkhImFovZvn372uAImtP1mB5XX1/PbG1t2ZdffsmVBQcHs8DAQH1XVWu6HtOuXbuYvb29xv11hvdp06ZNzNbWllVUVHBlhnif6HS4E6mtrUVqair8/Py4MjMzM/j5+SE5OVntNsnJybx4APD39+fic3JyoFAoeDH29vaQy+Ua96lPrTmmxz148AB1dXXo2rUrr/zEiRPo0aMHBgwYgHfffRf37t3Ta901ae0xVVRUoHfv3nB2dkZgYCCuXr3KresM79MXX3yBKVOmwNramlfe1u8TJcFOpKioCEqlElKplFculUqhUCjUbqNQKFqMb/yqyz71qTXH9LglS5bAycmJ9wEdP3489uzZg6SkJPztb3/DyZMnMWHCBCiVSr3WX53WHNOAAQOwc+dOHD58GF9//TVUKhVGjRqFW7duAej479PZs2dx5coVvP3227xyQ7xPNIoM6dSio6MRHx+PEydO8G4kTJkyhfve09MTQ4cORd++fXHixAm89NJLxqhqi3x9fXlzcY8aNQoDBw7E3//+d6xevdqINdOPL774Ap6envDx8eGVG+J9opZgJ9KtWzcIhULk5+fzyvPz8yGTydRuI5PJWoxv/KrLPvWpNcfUaP369YiOjsaPP/6IoUOHthjbp08fdOvWDTdu3HjqOj/J0xxTIwsLCwwfPpyrb0d+nyorKxEfH4+QkJAnvk5bvE+UBDsRkUgEb29vJCUlcWUqlQpJSUm8VkRTvr6+vHgASExM5OLd3Nwgk8l4MWVlZUhJSdG4T31qzTEBwNq1a7F69WokJCRgxIgRT3ydW7du4d69e3B0dNRLvVvS2mNqSqlU4vLly1x9O+r7BDR00aqpqcGbb775xNdpk/epTW+7EIOLj49nYrGY7d69m6Wnp7PZs2czBwcHrjvFtGnTWFhYGBd/6tQpZm5uztavX88yMjJYZGSk2i4yDg4O7PDhw+zSpUssMDDQ4F0vdDmm6OhoJhKJ2D//+U9e14ry8nLGGGPl5eVs4cKFLDk5meXk5LCffvqJPfvss8zd3Z1VV1e3y2NauXIl+89//sOys7NZamoqmzJlCpNIJOzq1au84+5I71Oj0aNHs8mTJzcrN9T7REmwE9qyZQtzcXFhIpGI+fj4sDNnznDrxowZw4KDg3nxBw4cYP3792cikYgNHjyYHT16lLdepVKx8PBwJpVKmVgsZi+99BLLysoyxKFwdDmm3r17MwDNlsjISMYYYw8ePGD/8z//w7p3784sLCxY79692axZs5r1u2tPx/TRRx9xsVKplL388svs/PnzvP11tPeJMcYyMzMZAPbjjz8225eh3icaSosQYtLomiAhxKRREiSEmDRKgoQQk0ZJkBBi0igJEkJMGiVBQohJoyRICDFplAQJISaNkiDpNNQNxW6MYfNJx0JDaZFOZfz48di1axevTCwWq42tq6uDhYUFr6y2thYikUjn123tdsT4qCVIOhWxWAyZTMZbunTpAgAQCATYvn07Jk6cCGtra3zyySf4+OOP4eXlhX/84x9wc3PjxhzMzc1FYGAgbGxsYGdnh0mTJvGGidK0Hel4KAkSk/Lxxx/jr3/9Ky5fvoy33noLAHDjxg3861//wrfffosLFy5ApVIhMDAQxcXFOHnyJBITE/H7779j8uTJvH09vh3pmOh0mHQqR44cgY2NDa9s2bJlWLZsGQDg9ddfx8yZM3nra2trsWfPHnTv3h1Aw3iKly9fRk5ODpydnQEAe/bsweDBg/Hbb79h5MiRarcjHRMlQdKpjB07Ftu3b+eVNZ1gSd0Aq7179+YlsoyMDDg7O3MJEAAGDRoEBwcHZGRkcEnw8e1Ix0RJkHQq1tbW6NevX4vrtSnT9rVIx0fXBAl5zMCBA5GXl4e8vDyuLD09HSUlJRg0aJARa0baArUESadSU1PTbIpHc3NzdOvWTet9+Pn5wdPTE2+88QZiYmJQX1+P9957D2PGjNFqvhLSsVBLkHQqCQkJcHR05C2jR4/WaR8CgQCHDx9Gly5d8MILL8DPzw99+vTB/v3726jWxJhoeH1CiEmjliAhxKRREiSEmDRKgoQQk0ZJkBBi0igJEkJMGiVBQohJoyRICDFplAQJISaNkiAhxKRREiSEmDRKgoQQk0ZJkBBi0v4f+5ugG+DdV/wAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.018695, + "end_time": "2024-03-26T14:28:00.606488", + "exception": false, + "start_time": "2024-03-26T14:28:00.587793", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4197.889978, + "end_time": "2024-03-26T14:28:03.348884", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/tab_ddpm_concat/42/mlu-eval.ipynb", + "output_path": "eval/insurance/tab_ddpm_concat/42/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/insurance/tab_ddpm_concat/42", + "path_prefix": "../../../../", + "random_seed": 42, + "single_model": "tab_ddpm_concat" + }, + "start_time": "2024-03-26T13:18:05.458906", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/insurance/tab_ddpm_concat/model.pt b/insurance/tab_ddpm_concat/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..edcd1ce96df7819b61be1f08a431c85c3b778646 --- /dev/null +++ b/insurance/tab_ddpm_concat/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cf82040a7b54dfb74f04184f6eb7357bc2d17bbdaedf7da723911d055c58a32 +size 34848313 diff --git a/insurance/tab_ddpm_concat/params.json b/insurance/tab_ddpm_concat/params.json new file mode 100644 index 0000000000000000000000000000000000000000..53073ab4d0a04c8ef979de5b5cf092e6c1215f05 --- /dev/null +++ b/insurance/tab_ddpm_concat/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 1, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.6896836352825375, "head_final_mul": "identity", "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "loss_balancer_beta": 0.7520229775744602, "loss_balancer_r": 0.9706519501751338, "mse_mag_multiply": true, "tf_pma_low_exp_2": 6, "dataset_size_exp_2": 11, "batch_size_exp_2": 2, "epochs": 100, "lr_mul": 0.08030439779404704, "n_warmup_steps": 85, "Optim": "diffgrad", "fixed_role_model": "tab_ddpm_concat", "mse_mag_target": 0.13044551835398707, "d_model_exp_2": 8, "attn_activation": "sigmoid", "tf_d_inner_exp_2": 8, "tf_n_layers_enc": 5, "tf_n_head_exp_2": 7, "tf_activation": "leakyhardsigmoid", "tf_activation_final": "leakyhardtanh", "tf_num_inds_exp_2": 6, "ada_d_hid_exp_2": 8, "ada_n_layers": 8, "ada_activation": "relu6", "ada_activation_final": "leakyhardtanh", "head_d_hid_exp_2": 8, "head_n_layers": 8, "head_n_head_exp_2": 5, "head_activation": "relu6", "head_activation_final": "softsign", "mse_mag": true, "models": ["tab_ddpm_concat"], "max_seconds": 3600} \ No newline at end of file diff --git a/insurance/tvae/eval.csv b/insurance/tvae/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..6e3d552a9cbe53d983c0eb5a022cfcbf4b22dcb7 --- /dev/null +++ b/insurance/tvae/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tvae,0.012883516520669219,0.004672739247980644,0.0003646601323998766,3.7268638610839844,0.005140321329236031,0.3903414011001587,0.009947109967470169,1.8891797992637294e-07,1.3032469749450684,0.013064631260931492,0.1989498883485794,0.019096076488494873,0.16100887954235077,0.00037080643232911825,5.030110836029053 diff --git a/insurance/tvae/history.csv b/insurance/tvae/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..a83eea40c845f9a931be8e9ab96551c144fdac67 --- /dev/null +++ b/insurance/tvae/history.csv @@ -0,0 +1,17 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.033494596945432324,0.9514266119047663,0.00740568548471212,0.4561071928011047,0.0,0.0,0.0,0.0,0.0338612375757657,900,113,139.86616706848145,1.2377536908715172,0.15540685229831272,0.13048605609322544,0.0035450709047209887,0.4135572405852655,1.1992556082892254e-05,0.0,0.0,0.0,0.0,0.0,0.0035450709047209887,450,57,43.52694535255432,0.7636306202202513,0.09672654522789849,0.06890298123879914 +1,0.005948302344315582,0.6644698490732288,0.00017848719000122755,0.07072600515352355,0.0,0.0,0.0,0.0,0.006019683801569044,900,113,139.209575176239,1.2319431431525576,0.15467730575137667,0.09152691003627482,0.003438505803383628,0.09945504239179795,4.135730159976308e-05,0.0,0.0,0.0,0.0,0.0,0.003438505803383628,450,57,45.21612048149109,0.7932652716051068,0.10048026773664687,0.08053733325939168 +2,0.004148594846030594,0.3845783284646484,0.00010671993065773853,0.04789311033363144,0.0,0.0,0.0,0.0,0.004195070721357802,900,113,139.53700613975525,1.234840762298719,0.15504111793306138,0.09683596715331078,0.0023919830305708777,0.2817465887051783,3.5287871541890406e-06,0.0,0.0,0.0,0.0,0.0,0.0023919830305708777,450,57,43.74946212768555,0.7675344232927289,0.09722102695041232,0.06890195867929019 +3,0.0031747633912083176,0.401544842024328,3.1176737821745636e-05,0.034890410486194824,0.0,0.0,0.0,0.0,0.0032127828725949967,900,113,138.64260721206665,1.2269257275404128,0.15404734134674072,0.09449338388785852,0.0037739401170983912,0.09890887374428546,0.00012161373627072356,0.0,0.0,0.0,0.0,0.0,0.0037739401170983912,450,57,45.92029070854187,0.8056191352375767,0.10204509046342637,0.08018968007608987 +4,0.0021943279545909413,0.27095252091449784,4.216843699032259e-05,0.02423756867647171,0.0,0.0,0.0,0.0,0.002222463636571774,900,113,144.14387917518616,1.2756095502228864,0.16015986575020685,0.09544809428002454,0.001543269329316293,0.09598122937022212,7.372837712272447e-06,0.0,0.0,0.0,0.0,0.0,0.001543269329316293,450,57,44.021047830581665,0.7722990847470468,0.09782455073462593,0.07492631013717568 +5,0.0009152512939504555,0.12106465140603044,1.8304799068708634e-06,0.017721550791627832,0.0,0.0,0.0,0.0,0.0009255373460796869,900,113,139.43437671661377,1.2339325373151662,0.15492708524068197,0.09658713597573537,0.001039498461003758,0.191177892819592,4.677953496311381e-07,0.0,0.0,0.0,0.0,0.0,0.001039498461003758,450,57,44.20726180076599,0.7755659965046665,0.09823835955725777,0.07235986795766573 +6,0.0010385865857501307,0.14289225078986598,9.29667037661573e-06,0.01798489984124899,0.0,0.0,0.0,0.0,0.001049564996954157,900,113,143.65014576911926,1.2712402280453032,0.15961127307679918,0.09901201174454352,0.00183226286931636,0.27243784420758393,1.1850463488066116e-06,0.0,0.0,0.0,0.0,0.0,0.00183226286931636,450,57,43.90792679786682,0.7703145052257337,0.09757317066192626,0.06361736614551199 +7,0.0011654625320160347,0.24595644928550064,3.943663092020979e-06,0.01612356576240725,0.0,0.0,0.0,0.0,0.0011781512379012485,900,113,143.4735608100891,1.269677529292824,0.15941506756676568,0.096535653403375,0.001652421035234713,0.22786178245158029,1.7284394548257756e-05,0.0,0.0,0.0,0.0,0.0,0.001652421035234713,450,57,44.54355502128601,0.7814658775664213,0.09898567782508003,0.06960312473963488 +8,0.0005234555136606408,0.057786288244384934,2.6204717372690923e-07,0.01177795834839344,0.0,0.0,0.0,0.0,0.0005286935975244786,900,113,140.09226059913635,1.2397545185764278,0.15565806733237372,0.0975041702076529,0.0010801255712673688,0.21078088797052652,7.320336708706144e-08,0.0,0.0,0.0,0.0,0.0,0.0010801255712673688,450,57,43.883389711380005,0.7698840300242106,0.09751864380306667,0.06990287764193981 +9,0.00042697534642583275,0.0412824616791532,1.6301317682804454e-07,0.00998444479993648,0.0,0.0,0.0,0.0,0.00043120299362474015,900,113,139.74913692474365,1.236718025882687,0.15527681880527072,0.09550961650973927,0.0011431862615669767,0.3449243648920886,8.091652926604335e-07,0.0,0.0,0.0,0.0,0.0,0.0011431862615669767,450,57,43.695363998413086,0.7665853333054927,0.09710080888536242,0.07124646358281832 +10,0.00040735370030031644,0.04582493726108598,1.9942591136076234e-07,0.010237777637731697,0.0,0.0,0.0,0.0,0.00041172872336270907,900,113,139.79106163978577,1.237089041060051,0.15532340182198417,0.09870326426704373,0.0012087840589164343,0.3767059982758602,1.1969978613053132e-05,0.0,0.0,0.0,0.0,0.0,0.0012087840589164343,450,57,43.84554171562195,0.7692200300986307,0.09743453714582655,0.06792440776827566 +11,0.00033184414415801357,0.03488898599820476,2.7725949718404836e-07,0.009320654780086543,0.0,0.0,0.0,0.0,0.0003353726631192128,900,113,141.7481198310852,1.254408140098099,0.15749791092342802,0.10097343308142329,0.0007768379625890197,0.39109475741188193,2.166238427795613e-07,0.0,0.0,0.0,0.0,0.0,0.0007768379625890197,450,57,45.42530655860901,0.7969352027826142,0.1009451256857978,0.06935281714980017 +12,0.00027347822246762615,0.04173435084108528,4.555697823765888e-08,0.008184957193831603,0.0,0.0,0.0,0.0,0.0002763797078417459,900,113,146.54646396636963,1.2968713625342445,0.16282940440707735,0.09951070125783439,0.000984192539084082,0.30668928338613405,9.389742739488819e-08,0.0,0.0,0.0,0.0,0.0,0.000984192539084082,450,57,44.021021604537964,0.772298624641017,0.0978244924545288,0.07037516528119643 +13,0.0002533764886902645,0.024310367316020987,1.0157818183775696e-07,0.008018655985490316,0.0,0.0,0.0,0.0,0.0002561400921856855,900,113,139.47040557861328,1.2342513768018875,0.1549671173095703,0.09854261146907785,0.0008298082016861169,0.47053791778261367,1.1205340189409292e-07,0.0,0.0,0.0,0.0,0.0,0.0008298082016861169,450,57,43.776551723480225,0.7680096793593022,0.09728122605217827,0.07100844480865226 +14,0.0002696858313962972,0.03735343955874807,6.127645766854687e-08,0.007558931692813834,0.0,0.0,0.0,0.0,0.0002726498826490974,900,113,139.74957585334778,1.2367219102066176,0.15527730650371976,0.10230876032296007,0.001009796441164023,0.39624184120929296,1.2299184930792467e-07,0.0,0.0,0.0,0.0,0.0,0.001009796441164023,450,57,44.01455473899841,0.7721851708596212,0.0978101216422187,0.06780717113022611 +15,0.00041215540432151305,0.051850774108070846,1.0660230064669582e-07,0.009949268912896514,0.0,0.0,0.0,0.0,0.0004164005695525298,900,113,139.9835147857666,1.2387921662457222,0.15553723865085178,0.10046833142044798,0.0008542559402445072,0.42860802830497374,1.2556846767389603e-07,0.0,0.0,0.0,0.0,0.0,0.0008542559402445072,450,57,44.051859617233276,0.7728396424076014,0.0978930213716295,0.07098463948881417 diff --git a/insurance/tvae/mlu-eval.ipynb b/insurance/tvae/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..8db16a483d3d4d3db362a8a8ab40c92749c360b3 --- /dev/null +++ b/insurance/tvae/mlu-eval.ipynb @@ -0,0 +1,2403 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:35.464856Z", + "iopub.status.busy": "2024-03-26T10:02:35.464320Z", + "iopub.status.idle": "2024-03-26T10:02:35.496480Z", + "shell.execute_reply": "2024-03-26T10:02:35.495612Z" + }, + "papermill": { + "duration": 0.046425, + "end_time": "2024-03-26T10:02:35.498358", + "exception": false, + "start_time": "2024-03-26T10:02:35.451933", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:35.523319Z", + "iopub.status.busy": "2024-03-26T10:02:35.522989Z", + "iopub.status.idle": "2024-03-26T10:02:35.529530Z", + "shell.execute_reply": "2024-03-26T10:02:35.528741Z" + }, + "papermill": { + "duration": 0.021243, + "end_time": "2024-03-26T10:02:35.531391", + "exception": false, + "start_time": "2024-03-26T10:02:35.510148", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:35.554714Z", + "iopub.status.busy": "2024-03-26T10:02:35.554449Z", + "iopub.status.idle": "2024-03-26T10:02:35.558226Z", + "shell.execute_reply": "2024-03-26T10:02:35.557436Z" + }, + "papermill": { + "duration": 0.017707, + "end_time": "2024-03-26T10:02:35.560103", + "exception": false, + "start_time": "2024-03-26T10:02:35.542396", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:35.583239Z", + "iopub.status.busy": "2024-03-26T10:02:35.582794Z", + "iopub.status.idle": "2024-03-26T10:02:35.586916Z", + "shell.execute_reply": "2024-03-26T10:02:35.586063Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017912, + "end_time": "2024-03-26T10:02:35.588899", + "exception": false, + "start_time": "2024-03-26T10:02:35.570987", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:35.611960Z", + "iopub.status.busy": "2024-03-26T10:02:35.611704Z", + "iopub.status.idle": "2024-03-26T10:02:35.617375Z", + "shell.execute_reply": "2024-03-26T10:02:35.616621Z" + }, + "papermill": { + "duration": 0.019488, + "end_time": "2024-03-26T10:02:35.619216", + "exception": false, + "start_time": "2024-03-26T10:02:35.599728", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "6906337c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:35.644394Z", + "iopub.status.busy": "2024-03-26T10:02:35.644133Z", + "iopub.status.idle": "2024-03-26T10:02:35.648834Z", + "shell.execute_reply": "2024-03-26T10:02:35.648018Z" + }, + "papermill": { + "duration": 0.019499, + "end_time": "2024-03-26T10:02:35.650700", + "exception": false, + "start_time": "2024-03-26T10:02:35.631201", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"insurance\"\n", + "dataset_name = \"insurance\"\n", + "single_model = \"tvae\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 4\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/insurance/tvae/4\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011166, + "end_time": "2024-03-26T10:02:35.672907", + "exception": false, + "start_time": "2024-03-26T10:02:35.661741", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:35.696249Z", + "iopub.status.busy": "2024-03-26T10:02:35.695956Z", + "iopub.status.idle": "2024-03-26T10:02:35.705143Z", + "shell.execute_reply": "2024-03-26T10:02:35.704377Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.02318, + "end_time": "2024-03-26T10:02:35.707127", + "exception": false, + "start_time": "2024-03-26T10:02:35.683947", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/insurance/tvae/4\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:35.731039Z", + "iopub.status.busy": "2024-03-26T10:02:35.730800Z", + "iopub.status.idle": "2024-03-26T10:02:37.647550Z", + "shell.execute_reply": "2024-03-26T10:02:37.646580Z" + }, + "papermill": { + "duration": 1.931008, + "end_time": "2024-03-26T10:02:37.649568", + "exception": false, + "start_time": "2024-03-26T10:02:35.718560", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:37.675958Z", + "iopub.status.busy": "2024-03-26T10:02:37.675118Z", + "iopub.status.idle": "2024-03-26T10:02:37.688380Z", + "shell.execute_reply": "2024-03-26T10:02:37.687696Z" + }, + "papermill": { + "duration": 0.028603, + "end_time": "2024-03-26T10:02:37.690482", + "exception": false, + "start_time": "2024-03-26T10:02:37.661879", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:37.714501Z", + "iopub.status.busy": "2024-03-26T10:02:37.714201Z", + "iopub.status.idle": "2024-03-26T10:02:37.721553Z", + "shell.execute_reply": "2024-03-26T10:02:37.720705Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021542, + "end_time": "2024-03-26T10:02:37.723382", + "exception": false, + "start_time": "2024-03-26T10:02:37.701840", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:37.747079Z", + "iopub.status.busy": "2024-03-26T10:02:37.746819Z", + "iopub.status.idle": "2024-03-26T10:02:37.838795Z", + "shell.execute_reply": "2024-03-26T10:02:37.838092Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.106291, + "end_time": "2024-03-26T10:02:37.840857", + "exception": false, + "start_time": "2024-03-26T10:02:37.734566", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:37.867301Z", + "iopub.status.busy": "2024-03-26T10:02:37.867020Z", + "iopub.status.idle": "2024-03-26T10:02:42.484361Z", + "shell.execute_reply": "2024-03-26T10:02:42.483570Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.633761, + "end_time": "2024-03-26T10:02:42.486688", + "exception": false, + "start_time": "2024-03-26T10:02:37.852927", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-26 10:02:40.086319: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-26 10:02:40.086378: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-26 10:02:40.087931: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:42.512029Z", + "iopub.status.busy": "2024-03-26T10:02:42.510906Z", + "iopub.status.idle": "2024-03-26T10:02:42.517655Z", + "shell.execute_reply": "2024-03-26T10:02:42.516958Z" + }, + "papermill": { + "duration": 0.021083, + "end_time": "2024-03-26T10:02:42.519473", + "exception": false, + "start_time": "2024-03-26T10:02:42.498390", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:42.545212Z", + "iopub.status.busy": "2024-03-26T10:02:42.544938Z", + "iopub.status.idle": "2024-03-26T10:02:50.875571Z", + "shell.execute_reply": "2024-03-26T10:02:50.874425Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.346408, + "end_time": "2024-03-26T10:02:50.878142", + "exception": false, + "start_time": "2024-03-26T10:02:42.531734", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (6) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-26T10:02:50.905472Z", + "iopub.status.busy": "2024-03-26T10:02:50.905123Z", + "iopub.status.idle": "2024-03-26T10:02:50.913038Z", + "shell.execute_reply": "2024-03-26T10:02:50.912150Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.02381, + "end_time": "2024-03-26T10:02:50.914882", + "exception": false, + "start_time": "2024-03-26T10:02:50.891072", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 36,\n", + " 'realtabformer': (19, 551, Embedding(551, 800), True),\n", + " 'lct_gan': 29,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:50.940106Z", + "iopub.status.busy": "2024-03-26T10:02:50.939431Z", + "iopub.status.idle": "2024-03-26T10:02:50.944208Z", + "shell.execute_reply": "2024-03-26T10:02:50.943335Z" + }, + "papermill": { + "duration": 0.019486, + "end_time": "2024-03-26T10:02:50.946171", + "exception": false, + "start_time": "2024-03-26T10:02:50.926685", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:50.971426Z", + "iopub.status.busy": "2024-03-26T10:02:50.970935Z", + "iopub.status.idle": "2024-03-26T10:02:51.033836Z", + "shell.execute_reply": "2024-03-26T10:02:51.032938Z" + }, + "papermill": { + "duration": 0.077733, + "end_time": "2024-03-26T10:02:51.035791", + "exception": false, + "start_time": "2024-03-26T10:02:50.958058", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/tvae/all inf False\n", + "../../../../ml-utility-loss/aug_test/insurance 0\n", + "Caching in ../../../../insurance/_cache_bs_test/tvae/all inf False\n", + "../../../../ml-utility-loss/bs_test/insurance 0\n", + "Caching in ../../../../insurance/_cache_synth_test/tvae/all inf False\n", + "../../../../ml-utility-loss/synthetics/insurance 600\n", + "600\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:51.062733Z", + "iopub.status.busy": "2024-03-26T10:02:51.062431Z", + "iopub.status.idle": "2024-03-26T10:02:51.382133Z", + "shell.execute_reply": "2024-03-26T10:02:51.381223Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.335631, + "end_time": "2024-03-26T10:02:51.384325", + "exception": false, + "start_time": "2024-03-26T10:02:51.048694", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.7,\n", + " 'head_final_mul': 'identity',\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'loss_balancer_beta': 0.79,\n", + " 'loss_balancer_r': 0.95,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 8,\n", + " 'epochs': 100,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'tvae',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.LeakyReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.ReLU,\n", + " 'ada_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 64,\n", + " 'head_activation': torch.nn.modules.activation.PReLU,\n", + " 'head_activation_final': torch.nn.modules.activation.Softsign,\n", + " 'models': ['tvae'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "# params = {\n", + "# **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "# }\n", + "params = getattr(PARAMS, dataset_name).BEST_DICT[gp][gp_multiply][single_model]\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).DEFAULTS,\n", + " **params,\n", + "}\n", + "if isinstance(params, (list, tuple)):\n", + " params = params[param_index]\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:51.409995Z", + "iopub.status.busy": "2024-03-26T10:02:51.409730Z", + "iopub.status.idle": "2024-03-26T10:02:51.514914Z", + "shell.execute_reply": "2024-03-26T10:02:51.513993Z" + }, + "papermill": { + "duration": 0.120333, + "end_time": "2024-03-26T10:02:51.516959", + "exception": false, + "start_time": "2024-03-26T10:02:51.396626", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_train/tvae/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/insurance [400, 0]\n", + "Caching in ../../../../insurance/_cache_aug_val/tvae/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/insurance [0, 200]\n", + "Caching in ../../../../insurance/_cache_bs_train/tvae/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/insurance [100, 0]\n", + "Caching in ../../../../insurance/_cache_bs_val/tvae/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/insurance [0, 50]\n", + "Caching in ../../../../insurance/_cache_synth/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/insurance [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-26T10:02:51.550753Z", + "iopub.status.busy": "2024-03-26T10:02:51.550377Z", + "iopub.status.idle": "2024-03-26T10:02:52.003759Z", + "shell.execute_reply": "2024-03-26T10:02:52.002782Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.474411, + "end_time": "2024-03-26T10:02:52.006072", + "exception": false, + "start_time": "2024-03-26T10:02:51.531661", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['tvae'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:52.039172Z", + "iopub.status.busy": "2024-03-26T10:02:52.038547Z", + "iopub.status.idle": "2024-03-26T10:02:52.042748Z", + "shell.execute_reply": "2024-03-26T10:02:52.041888Z" + }, + "papermill": { + "duration": 0.025052, + "end_time": "2024-03-26T10:02:52.044793", + "exception": false, + "start_time": "2024-03-26T10:02:52.019741", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:52.071448Z", + "iopub.status.busy": "2024-03-26T10:02:52.070514Z", + "iopub.status.idle": "2024-03-26T10:02:52.078065Z", + "shell.execute_reply": "2024-03-26T10:02:52.077217Z" + }, + "papermill": { + "duration": 0.022895, + "end_time": "2024-03-26T10:02:52.079973", + "exception": false, + "start_time": "2024-03-26T10:02:52.057078", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "9638537" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:52.105987Z", + "iopub.status.busy": "2024-03-26T10:02:52.105713Z", + "iopub.status.idle": "2024-03-26T10:02:52.194724Z", + "shell.execute_reply": "2024-03-26T10:02:52.193641Z" + }, + "papermill": { + "duration": 0.104517, + "end_time": "2024-03-26T10:02:52.196824", + "exception": false, + "start_time": "2024-03-26T10:02:52.092307", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1071, 36] --\n", + "├─Adapter: 1-1 [2, 1071, 36] --\n", + "│ └─Sequential: 2-1 [2, 1071, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1071, 1024] 37,888\n", + "│ │ │ └─ReLU: 4-2 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-4 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-6 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-8 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-10 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1071, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1071, 1024] 1,049,600\n", + "│ │ │ └─ReLU: 4-12 [2, 1071, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1071, 256] --\n", + "│ │ │ └─Linear: 4-13 [2, 1071, 256] 262,400\n", + "│ │ │ └─Softsign: 4-14 [2, 1071, 256] --\n", + "├─Adapter: 1-2 [2, 267, 36] (recursive)\n", + "│ └─Sequential: 2-2 [2, 267, 256] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-16 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-18 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-20 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-22 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-24 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 267, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 267, 1024] (recursive)\n", + "│ │ │ └─ReLU: 4-26 [2, 267, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 267, 256] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 267, 256] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 267, 256] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-3 [2, 16, 256] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-6 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-12 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-5 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-18 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-24 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-11 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 1071, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-30 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-36 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-17 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1071, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 1071, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 256] 8,192\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-42 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 1071, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 1071, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 1071, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 1071, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-48 [2, 1071, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 1071, 256] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 1071, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-23 [2, 1071, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 1071, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 256] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 1071, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 16, 1071] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 256] 65,792\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-54 [2, 16, 256] --\n", + "│ └─Encoder: 2-4 [2, 16, 256] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-60 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-66 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-31 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-72 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-78 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-37 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-14 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-40 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-84 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-41 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-90 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-15 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-42 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-43 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-44 [2, 267, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 16, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-16 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-45 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-46 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-96 [2, 32, 256] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-47 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 267, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 267, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-102 [2, 267, 256] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-17 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-48 [2, 267, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardsigmoid: 6-49 [2, 267, 512] --\n", + "│ │ │ │ │ └─Linear: 6-50 [2, 267, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-18 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-51 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-52 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 267, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 16, 4] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 16, 267] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 256] (recursive)\n", + "│ │ │ │ │ │ └─LeakyReLU: 7-108 [2, 16, 256] --\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 524,416\n", + "│ │ │ └─PReLU: 4-38 [2, 128] 1\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-40 [2, 128] 1\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-42 [2, 128] 1\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-44 [2, 128] 1\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-46 [2, 128] 1\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-48 [2, 128] 1\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-50 [2, 128] 1\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─PReLU: 4-52 [2, 128] 1\n", + "│ │ └─FeedForward: 3-25 [2, 1] --\n", + "│ │ │ └─Linear: 4-53 [2, 1] 129\n", + "│ │ │ └─Softsign: 4-54 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 9,638,537\n", + "Trainable params: 9,638,537\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 37.13\n", + "========================================================================================================================\n", + "Input size (MB): 0.39\n", + "Forward/backward pass size (MB): 307.09\n", + "Params size (MB): 38.55\n", + "Estimated Total Size (MB): 346.03\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:02:52.229131Z", + "iopub.status.busy": "2024-03-26T10:02:52.228819Z", + "iopub.status.idle": "2024-03-26T10:53:38.059080Z", + "shell.execute_reply": "2024-03-26T10:53:38.058161Z" + }, + "papermill": { + "duration": 3045.865391, + "end_time": "2024-03-26T10:53:38.077217", + "exception": false, + "start_time": "2024-03-26T10:02:52.211826", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 600]\n", + "Creating model of type \n", + "[*] Embedding False True\n", + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.033494596945432324, 'avg_role_model_std_loss': 0.9514266119047663, 'avg_role_model_mean_pred_loss': 0.00740568548471212, 'avg_role_model_g_mag_loss': 0.4561071928011047, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0338612375757657, 'n_size': 900, 'n_batch': 113, 'duration': 139.86616706848145, 'duration_batch': 1.2377536908715172, 'duration_size': 0.15540685229831272, 'avg_pred_std': 0.13048605609322544}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0035450709047209887, 'avg_role_model_std_loss': 0.4135572405852655, 'avg_role_model_mean_pred_loss': 1.1992556082892254e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0035450709047209887, 'n_size': 450, 'n_batch': 57, 'duration': 43.52694535255432, 'duration_batch': 0.7636306202202513, 'duration_size': 0.09672654522789849, 'avg_pred_std': 0.06890298123879914}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005948302344315582, 'avg_role_model_std_loss': 0.6644698490732288, 'avg_role_model_mean_pred_loss': 0.00017848719000122755, 'avg_role_model_g_mag_loss': 0.07072600515352355, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006019683801569044, 'n_size': 900, 'n_batch': 113, 'duration': 139.209575176239, 'duration_batch': 1.2319431431525576, 'duration_size': 0.15467730575137667, 'avg_pred_std': 0.09152691003627482}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.003438505803383628, 'avg_role_model_std_loss': 0.09945504239179795, 'avg_role_model_mean_pred_loss': 4.135730159976308e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003438505803383628, 'n_size': 450, 'n_batch': 57, 'duration': 45.21612048149109, 'duration_batch': 0.7932652716051068, 'duration_size': 0.10048026773664687, 'avg_pred_std': 0.08053733325939168}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004148594846030594, 'avg_role_model_std_loss': 0.3845783284646484, 'avg_role_model_mean_pred_loss': 0.00010671993065773853, 'avg_role_model_g_mag_loss': 0.04789311033363144, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004195070721357802, 'n_size': 900, 'n_batch': 113, 'duration': 139.53700613975525, 'duration_batch': 1.234840762298719, 'duration_size': 0.15504111793306138, 'avg_pred_std': 0.09683596715331078}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023919830305708777, 'avg_role_model_std_loss': 0.2817465887051783, 'avg_role_model_mean_pred_loss': 3.5287871541890406e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023919830305708777, 'n_size': 450, 'n_batch': 57, 'duration': 43.74946212768555, 'duration_batch': 0.7675344232927289, 'duration_size': 0.09722102695041232, 'avg_pred_std': 0.06890195867929019}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0031747633912083176, 'avg_role_model_std_loss': 0.401544842024328, 'avg_role_model_mean_pred_loss': 3.1176737821745636e-05, 'avg_role_model_g_mag_loss': 0.034890410486194824, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0032127828725949967, 'n_size': 900, 'n_batch': 113, 'duration': 138.64260721206665, 'duration_batch': 1.2269257275404128, 'duration_size': 0.15404734134674072, 'avg_pred_std': 0.09449338388785852}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0037739401170983912, 'avg_role_model_std_loss': 0.09890887374428546, 'avg_role_model_mean_pred_loss': 0.00012161373627072356, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0037739401170983912, 'n_size': 450, 'n_batch': 57, 'duration': 45.92029070854187, 'duration_batch': 0.8056191352375767, 'duration_size': 0.10204509046342637, 'avg_pred_std': 0.08018968007608987}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0021943279545909413, 'avg_role_model_std_loss': 0.27095252091449784, 'avg_role_model_mean_pred_loss': 4.216843699032259e-05, 'avg_role_model_g_mag_loss': 0.02423756867647171, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002222463636571774, 'n_size': 900, 'n_batch': 113, 'duration': 144.14387917518616, 'duration_batch': 1.2756095502228864, 'duration_size': 0.16015986575020685, 'avg_pred_std': 0.09544809428002454}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001543269329316293, 'avg_role_model_std_loss': 0.09598122937022212, 'avg_role_model_mean_pred_loss': 7.372837712272447e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001543269329316293, 'n_size': 450, 'n_batch': 57, 'duration': 44.021047830581665, 'duration_batch': 0.7722990847470468, 'duration_size': 0.09782455073462593, 'avg_pred_std': 0.07492631013717568}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0009152512939504555, 'avg_role_model_std_loss': 0.12106465140603044, 'avg_role_model_mean_pred_loss': 1.8304799068708634e-06, 'avg_role_model_g_mag_loss': 0.017721550791627832, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0009255373460796869, 'n_size': 900, 'n_batch': 113, 'duration': 139.43437671661377, 'duration_batch': 1.2339325373151662, 'duration_size': 0.15492708524068197, 'avg_pred_std': 0.09658713597573537}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001039498461003758, 'avg_role_model_std_loss': 0.191177892819592, 'avg_role_model_mean_pred_loss': 4.677953496311381e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001039498461003758, 'n_size': 450, 'n_batch': 57, 'duration': 44.20726180076599, 'duration_batch': 0.7755659965046665, 'duration_size': 0.09823835955725777, 'avg_pred_std': 0.07235986795766573}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0010385865857501307, 'avg_role_model_std_loss': 0.14289225078986598, 'avg_role_model_mean_pred_loss': 9.29667037661573e-06, 'avg_role_model_g_mag_loss': 0.01798489984124899, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001049564996954157, 'n_size': 900, 'n_batch': 113, 'duration': 143.65014576911926, 'duration_batch': 1.2712402280453032, 'duration_size': 0.15961127307679918, 'avg_pred_std': 0.09901201174454352}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00183226286931636, 'avg_role_model_std_loss': 0.27243784420758393, 'avg_role_model_mean_pred_loss': 1.1850463488066116e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00183226286931636, 'n_size': 450, 'n_batch': 57, 'duration': 43.90792679786682, 'duration_batch': 0.7703145052257337, 'duration_size': 0.09757317066192626, 'avg_pred_std': 0.06361736614551199}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0011654625320160347, 'avg_role_model_std_loss': 0.24595644928550064, 'avg_role_model_mean_pred_loss': 3.943663092020979e-06, 'avg_role_model_g_mag_loss': 0.01612356576240725, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011781512379012485, 'n_size': 900, 'n_batch': 113, 'duration': 143.4735608100891, 'duration_batch': 1.269677529292824, 'duration_size': 0.15941506756676568, 'avg_pred_std': 0.096535653403375}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001652421035234713, 'avg_role_model_std_loss': 0.22786178245158029, 'avg_role_model_mean_pred_loss': 1.7284394548257756e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001652421035234713, 'n_size': 450, 'n_batch': 57, 'duration': 44.54355502128601, 'duration_batch': 0.7814658775664213, 'duration_size': 0.09898567782508003, 'avg_pred_std': 0.06960312473963488}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0005234555136606408, 'avg_role_model_std_loss': 0.057786288244384934, 'avg_role_model_mean_pred_loss': 2.6204717372690923e-07, 'avg_role_model_g_mag_loss': 0.01177795834839344, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0005286935975244786, 'n_size': 900, 'n_batch': 113, 'duration': 140.09226059913635, 'duration_batch': 1.2397545185764278, 'duration_size': 0.15565806733237372, 'avg_pred_std': 0.0975041702076529}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0010801255712673688, 'avg_role_model_std_loss': 0.21078088797052652, 'avg_role_model_mean_pred_loss': 7.320336708706144e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010801255712673688, 'n_size': 450, 'n_batch': 57, 'duration': 43.883389711380005, 'duration_batch': 0.7698840300242106, 'duration_size': 0.09751864380306667, 'avg_pred_std': 0.06990287764193981}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00042697534642583275, 'avg_role_model_std_loss': 0.0412824616791532, 'avg_role_model_mean_pred_loss': 1.6301317682804454e-07, 'avg_role_model_g_mag_loss': 0.00998444479993648, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00043120299362474015, 'n_size': 900, 'n_batch': 113, 'duration': 139.74913692474365, 'duration_batch': 1.236718025882687, 'duration_size': 0.15527681880527072, 'avg_pred_std': 0.09550961650973927}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0011431862615669767, 'avg_role_model_std_loss': 0.3449243648920886, 'avg_role_model_mean_pred_loss': 8.091652926604335e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011431862615669767, 'n_size': 450, 'n_batch': 57, 'duration': 43.695363998413086, 'duration_batch': 0.7665853333054927, 'duration_size': 0.09710080888536242, 'avg_pred_std': 0.07124646358281832}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00040735370030031644, 'avg_role_model_std_loss': 0.04582493726108598, 'avg_role_model_mean_pred_loss': 1.9942591136076234e-07, 'avg_role_model_g_mag_loss': 0.010237777637731697, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00041172872336270907, 'n_size': 900, 'n_batch': 113, 'duration': 139.79106163978577, 'duration_batch': 1.237089041060051, 'duration_size': 0.15532340182198417, 'avg_pred_std': 0.09870326426704373}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0012087840589164343, 'avg_role_model_std_loss': 0.3767059982758602, 'avg_role_model_mean_pred_loss': 1.1969978613053132e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0012087840589164343, 'n_size': 450, 'n_batch': 57, 'duration': 43.84554171562195, 'duration_batch': 0.7692200300986307, 'duration_size': 0.09743453714582655, 'avg_pred_std': 0.06792440776827566}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00033184414415801357, 'avg_role_model_std_loss': 0.03488898599820476, 'avg_role_model_mean_pred_loss': 2.7725949718404836e-07, 'avg_role_model_g_mag_loss': 0.009320654780086543, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0003353726631192128, 'n_size': 900, 'n_batch': 113, 'duration': 141.7481198310852, 'duration_batch': 1.254408140098099, 'duration_size': 0.15749791092342802, 'avg_pred_std': 0.10097343308142329}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0007768379625890197, 'avg_role_model_std_loss': 0.39109475741188193, 'avg_role_model_mean_pred_loss': 2.166238427795613e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007768379625890197, 'n_size': 450, 'n_batch': 57, 'duration': 45.42530655860901, 'duration_batch': 0.7969352027826142, 'duration_size': 0.1009451256857978, 'avg_pred_std': 0.06935281714980017}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00027347822246762615, 'avg_role_model_std_loss': 0.04173435084108528, 'avg_role_model_mean_pred_loss': 4.555697823765888e-08, 'avg_role_model_g_mag_loss': 0.008184957193831603, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0002763797078417459, 'n_size': 900, 'n_batch': 113, 'duration': 146.54646396636963, 'duration_batch': 1.2968713625342445, 'duration_size': 0.16282940440707735, 'avg_pred_std': 0.09951070125783439}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.000984192539084082, 'avg_role_model_std_loss': 0.30668928338613405, 'avg_role_model_mean_pred_loss': 9.389742739488819e-08, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.000984192539084082, 'n_size': 450, 'n_batch': 57, 'duration': 44.021021604537964, 'duration_batch': 0.772298624641017, 'duration_size': 0.0978244924545288, 'avg_pred_std': 0.07037516528119643}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0002533764886902645, 'avg_role_model_std_loss': 0.024310367316020987, 'avg_role_model_mean_pred_loss': 1.0157818183775696e-07, 'avg_role_model_g_mag_loss': 0.008018655985490316, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0002561400921856855, 'n_size': 900, 'n_batch': 113, 'duration': 139.47040557861328, 'duration_batch': 1.2342513768018875, 'duration_size': 0.1549671173095703, 'avg_pred_std': 0.09854261146907785}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0008298082016861169, 'avg_role_model_std_loss': 0.47053791778261367, 'avg_role_model_mean_pred_loss': 1.1205340189409292e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008298082016861169, 'n_size': 450, 'n_batch': 57, 'duration': 43.776551723480225, 'duration_batch': 0.7680096793593022, 'duration_size': 0.09728122605217827, 'avg_pred_std': 0.07100844480865226}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0002696858313962972, 'avg_role_model_std_loss': 0.03735343955874807, 'avg_role_model_mean_pred_loss': 6.127645766854687e-08, 'avg_role_model_g_mag_loss': 0.007558931692813834, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0002726498826490974, 'n_size': 900, 'n_batch': 113, 'duration': 139.74957585334778, 'duration_batch': 1.2367219102066176, 'duration_size': 0.15527730650371976, 'avg_pred_std': 0.10230876032296007}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.001009796441164023, 'avg_role_model_std_loss': 0.39624184120929296, 'avg_role_model_mean_pred_loss': 1.2299184930792467e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001009796441164023, 'n_size': 450, 'n_batch': 57, 'duration': 44.01455473899841, 'duration_batch': 0.7721851708596212, 'duration_size': 0.0978101216422187, 'avg_pred_std': 0.06780717113022611}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00041215540432151305, 'avg_role_model_std_loss': 0.051850774108070846, 'avg_role_model_mean_pred_loss': 1.0660230064669582e-07, 'avg_role_model_g_mag_loss': 0.009949268912896514, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0004164005695525298, 'n_size': 900, 'n_batch': 113, 'duration': 139.9835147857666, 'duration_batch': 1.2387921662457222, 'duration_size': 0.15553723865085178, 'avg_pred_std': 0.10046833142044798}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0008542559402445072, 'avg_role_model_std_loss': 0.42860802830497374, 'avg_role_model_mean_pred_loss': 1.2556846767389603e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008542559402445072, 'n_size': 450, 'n_batch': 57, 'duration': 44.051859617233276, 'duration_batch': 0.7728396424076014, 'duration_size': 0.0978930213716295, 'avg_pred_std': 0.07098463948881417}\n", + "Stopped False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tvae', 'n_size': 600, 'n_batch': 75, 'role_model_metrics': {'avg_loss': 0.0003646601304838744, 'avg_g_mag_loss': 0.003987307718731851, 'avg_g_cos_loss': 0.010797024015676773, 'pred_duration': 1.3057825565338135, 'grad_duration': 3.728916645050049, 'total_duration': 5.034699201583862, 'pred_std': 0.16100887954235077, 'std_loss': 0.00037080643232911825, 'mean_pred_loss': 1.8891799413722765e-07, 'pred_rmse': 0.019096076488494873, 'pred_mae': 0.013064631260931492, 'pred_mape': 0.1989498883485794, 'grad_rmse': 0.009947109967470169, 'grad_mae': 0.005140321794897318, 'grad_mape': 0.3903414011001587}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0003646601304838744, 'avg_g_mag_loss': 0.003987307718731851, 'avg_g_cos_loss': 0.010797024015676773, 'avg_pred_duration': 1.3057825565338135, 'avg_grad_duration': 3.728916645050049, 'avg_total_duration': 5.034699201583862, 'avg_pred_std': 0.16100887954235077, 'avg_std_loss': 0.00037080643232911825, 'avg_mean_pred_loss': 1.8891799413722765e-07}, 'min_metrics': {'avg_loss': 0.0003646601304838744, 'avg_g_mag_loss': 0.003987307718731851, 'avg_g_cos_loss': 0.010797024015676773, 'pred_duration': 1.3057825565338135, 'grad_duration': 3.728916645050049, 'total_duration': 5.034699201583862, 'pred_std': 0.16100887954235077, 'std_loss': 0.00037080643232911825, 'mean_pred_loss': 1.8891799413722765e-07, 'pred_rmse': 0.019096076488494873, 'pred_mae': 0.013064631260931492, 'pred_mape': 0.1989498883485794, 'grad_rmse': 0.009947109967470169, 'grad_mae': 0.005140321794897318, 'grad_mape': 0.3903414011001587}, 'model_metrics': {'tvae': {'avg_loss': 0.0003646601304838744, 'avg_g_mag_loss': 0.003987307718731851, 'avg_g_cos_loss': 0.010797024015676773, 'pred_duration': 1.3057825565338135, 'grad_duration': 3.728916645050049, 'total_duration': 5.034699201583862, 'pred_std': 0.16100887954235077, 'std_loss': 0.00037080643232911825, 'mean_pred_loss': 1.8891799413722765e-07, 'pred_rmse': 0.019096076488494873, 'pred_mae': 0.013064631260931492, 'pred_mape': 0.1989498883485794, 'grad_rmse': 0.009947109967470169, 'grad_mae': 0.005140321794897318, 'grad_mape': 0.3903414011001587}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:53:38.112048Z", + "iopub.status.busy": "2024-03-26T10:53:38.111744Z", + "iopub.status.idle": "2024-03-26T10:53:38.116133Z", + "shell.execute_reply": "2024-03-26T10:53:38.115367Z" + }, + "papermill": { + "duration": 0.024087, + "end_time": "2024-03-26T10:53:38.117920", + "exception": false, + "start_time": "2024-03-26T10:53:38.093833", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:53:38.150564Z", + "iopub.status.busy": "2024-03-26T10:53:38.150277Z", + "iopub.status.idle": "2024-03-26T10:53:38.229933Z", + "shell.execute_reply": "2024-03-26T10:53:38.228948Z" + }, + "papermill": { + "duration": 0.098663, + "end_time": "2024-03-26T10:53:38.232248", + "exception": false, + "start_time": "2024-03-26T10:53:38.133585", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:53:38.267454Z", + "iopub.status.busy": "2024-03-26T10:53:38.267145Z", + "iopub.status.idle": "2024-03-26T10:53:38.548618Z", + "shell.execute_reply": "2024-03-26T10:53:38.547739Z" + }, + "papermill": { + "duration": 0.301827, + "end_time": "2024-03-26T10:53:38.551078", + "exception": false, + "start_time": "2024-03-26T10:53:38.249251", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAEUCAYAAAB+lmCLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/dUlEQVR4nO3de1xUdf4/8NdcmBnuV7m5CN4QV5FMY8JKMtmwSMN1I/mySX5N7bvqmqybsZtS7Xcby9zc1F9+bfertZuh7lp9F81dRK2UERQkNZVVw0vJoIiA3GaYmffvj2EODAw4M1wG5P18PM6DmXM+55zPTM6rc/mcz0dERATGGBsgxM6uAGOM2YNDizE2oHBoMcYGFA4txtiAwqHFGBtQOLQYYwMKhxZjbEDh0GKMDSgcWoyxAUXq7Ar0FaPRiOvXr8PT0xMikcjZ1WGMtUNEuHPnDkJDQyEWd3E8RQ7YtGkThYeHk1wup9jYWCooKOiy/K5du2jMmDEkl8tp/PjxtHfvXovlWVlZNGbMGHJzcyMfHx+aPn06HTt2zKJMeHg4AbCYVCqVzXW+du1ah/V54omn/jddu3aty9+y3UdaO3fuREZGBrZs2QKlUokNGzYgMTERpaWlCAwM7FA+Pz8fqampUKlUeOqpp7Bjxw4kJyejuLgY48ePBwBERkZi06ZNGDFiBBobG/Huu+/i8ccfx8WLFzFkyBBhW2+88QYWLlwovPf09LS53uay165dg5eXl70fmzHWy2praxEWFnb337XNhyotYmNjacmSJcJ7g8FAoaGhnR71pKSkUFJSksU8pVJJixcv7nQfNTU1BIAOHDggzAsPD6d3333X3up22GZNTY3D22CM9R5bf6N2XYjX6XQoKipCQkKCME8sFiMhIQFqtdrqOmq12qI8ACQmJnZaXqfTYevWrfD29kZMTIzFsrVr18Lf3x8TJ07EunXroNfrO62rVqtFbW2txcQYG/jsOj2srKyEwWBAUFCQxfygoCCcP3/e6joajcZqeY1GYzEvJycHc+fORUNDA0JCQpCbm4uAgABh+S9/+Uvcf//98PPzQ35+PjIzM1FeXo4//OEPVverUqnw+uuv2/PxGGMDQL+5ezht2jSUlJSgsrISH3zwAVJSUlBQUCBcJ8vIyBDKTpgwATKZDIsXL4ZKpYJcLu+wvczMTIt1zOfLjLGBza7QCggIgEQiQUVFhcX8iooKBAcHW10nODjYpvLu7u4YNWoURo0ahQcffBCjR4/Gn//8Z2RmZlrdrlKphF6vx+XLlzFmzJgOy+VyudUwY44xGAxobm52djXYAObi4gKJRNLt7dgVWjKZDJMmTUJeXh6Sk5MBmNo/5eXlYenSpVbXiYuLQ15eHl566SVhXm5uLuLi4rrcl9FohFar7XR5SUkJxGKx1TuWrOcQETQaDaqrq51dFXYP8PHxQXBwcLfaStp9epiRkYH09HRMnjwZsbGx2LBhA+rr6zF//nwAwLx58zB06FCoVCoAwPLlyxEfH4/169cjKSkJ2dnZOHHiBLZu3QoAqK+vx+9//3vMmjULISEhqKysxObNm/HDDz/gmWeeAWC6mF9QUIBp06bB09MTarUaK1aswM9//nP4+vo6/OHZ3ZkDKzAwEG5ubtwwlzmEiNDQ0IAbN24AAEJCQrq1Mbtt3LiRhg0bRjKZjGJjYy0agsbHx1N6erpF+V27dlFkZCTJZDIaN26cRePSxsZGmj17NoWGhpJMJqOQkBCaNWsWFRYWCmWKiopIqVSSt7c3KRQKGjt2LL355pvU1NRkc51tvZ1a3aAj9aVKOnap0uZt36v0ej2dPXuWKiv5u2A9o7Kyks6ePUt6vb7DMlt/oyKiwTGwRW1tLby9vVFTU9Nl49LDpTfw/LbjGBvihS+WP9KHNex/mpqaUFZWhoiICLi6ujq7Ouwe0NjYiMuXL2P48OFQKBQWy2z9jfID0+34u5su3lfVd349bbDhU0LWU3ri3xKHVjt+HjIAQFW9DoPkIJSxAYVDqx0/N1NoNRsId7Sdt7hnjDkHh1Y7rjIJXF1MbUlu1+ucXBs2UIlEInz22WfOrkaPeu2113Dfffc5uxocWtb4uZuOtm5xaLEBbPv27fDx8emx7a1cuRJ5eXk9tj1HcWhZ4W++rlXHocXufTqdbf/OPTw84O/v38u1uTsOLSt8W65rVTVwaLVFRGjQ6Z0y2XtTZP/+/Xj44Yfh4+MDf39/PPXUU7h06RIAYMqUKVi1apVF+Zs3b8LFxQVfffUVAKC8vBxJSUlwdXXF8OHDsWPHDkRERGDDhg0OfXenT5/GY489BldXV/j7+2PRokWoq6sTlh8+fBixsbFwd3eHj48PHnroIVy5cgUA8M033wgNq728vDBp0iScOHGiy/0dPnwY8+fPR01NDUQiEUQiEV577TUAQEREBH73u99h3rx58PLywqJFiwAAq1atQmRkJNzc3DBixAisXr3a4tGt9qeHzz//PJKTk/HOO+8gJCQE/v7+WLJkSa8/7tVvHpjuT/zdW+8gslaNzQb8eM0/nbLvs28kwk1m+z/X+vp6ZGRkYMKECairq8OaNWswe/ZslJSUIC0tDW+//TbWrl0r3ILfuXMnQkND8cgjprZ58+bNQ2VlJQ4fPgwXFxdkZGQIrbntVV9fj8TERMTFxeH48eO4ceMGXnjhBSxduhTbt2+HXq9HcnIyFi5ciE8++QQ6nQ6FhYVC3dLS0jBx4kS8//77kEgkKCkpgYuLS5f7nDJlCjZs2IA1a9agtLQUgOlIyeydd97BmjVrkJWVJczz9PTE9u3bERoaitOnT2PhwoXw9PTEyy+/3Ol+Dh06hJCQEBw6dAgXL17Es88+i/vuu8+is86exqFlhR+H1oA3Z84ci/f/+7//iyFDhuDs2bNISUnBSy+9hCNHjgghtWPHDqSmpkIkEuH8+fM4cOAAjh8/jsmTJwMA/vSnP2H06NEO1WXHjh1oamrCRx99BHd3dwDApk2bMHPmTLz11ltwcXFBTU0NnnrqKYwcORIAMHbsWGH9q1ev4te//jWioqIAwKZ6yGQyeHt7QyQSWe3M4LHHHsOvfvUri3mvvvqq8DoiIgIrV65EdnZ2l6Hl6+uLTZs2QSKRICoqCklJScjLy+PQ6mu+5gvxfE3LgquLBGffSHTavu1x4cIFrFmzBgUFBaisrITRaARgCoDx48fj8ccfx8cff4xHHnkEZWVlUKvV+J//+R8AQGlpKaRSKe6//35he6NGjXL4Oddz584hJiZGCCwAeOihh2A0GlFaWoqpU6fi+eefR2JiIn7yk58gISEBKSkpwvN5GRkZeOGFF/CXv/wFCQkJeOaZZ4Rwc5Q5jNvauXMn3nvvPVy6dAl1dXXQ6/V37Zp83LhxFj03hISE4PTp092q293wNS0rzKeHt/malgWRSAQ3mdQpk70tqWfOnImqqip88MEHKCgoQEFBAYDWi85paWn429/+hubmZuzYsQPR0dGIjo7u8e/MVtu2bYNarcaUKVOwc+dOREZG4tixYwBM15K+/fZbJCUl4eDBg/jxj3+MTz/9tFv7axuggKlTgrS0NDz55JPIycnByZMn8dvf/vauF+nbn6aKRCLhfxC9hUPLCm7yMLDdunULpaWlePXVVzF9+nSMHTsWt2/ftijz9NNPo6mpCfv378eOHTuQlpYmLBszZgz0ej1OnjwpzLt48WKHbdhq7Nix+Oabb1BfXy/MO3r0KMRisUVfcBMnTkRmZiby8/Mxfvx47NixQ1gWGRmJFStW4F//+hd++tOfYtu2bXfdr0wmg8FgsKmO+fn5CA8Px29/+1tMnjwZo0ePFm4E9DccWlYITR74+cMBydfXF/7+/ti6dSsuXryIgwcPWvRiC5iONJKTk7F69WqcO3cOqampwrKoqCgkJCRg0aJFKCwsxMmTJ7Fo0SK4uro69OxcWloaFAoF0tPTcebMGRw6dAjLli3Dc889h6CgIJSVlSEzMxNqtRpXrlzBv/71L1y4cAFjx45FY2Mjli5disOHD+PKlSs4evQojh8/bnHNqzMRERGoq6tDXl4eKisr0dDQ0GnZ0aNH4+rVq8jOzsalS5fw3nvvdftorrdwaFkhNHnga1oDklgsRnZ2NoqKijB+/HisWLEC69at61AuLS0N33zzDR555BEMGzbMYtlHH32EoKAgTJ06FbNnzxbupLXvmcAWbm5u+Oc//4mqqio88MAD+NnPfobp06dj06ZNwvLz589jzpw5iIyMxKJFi7BkyRIsXrwYEokEt27dwrx58xAZGYmUlBQ88cQTNo1/MGXKFLz44ot49tlnMWTIELz99tudlp01axZWrFiBpUuX4r777kN+fj5Wr15t92ftC9w1jRU1Dc2IeeNfAIDzv5sBhZ0Xge8V5q5prHUjMth8//33CAsLw4EDBzB9+nRnV2fA6urflK2/Ub57aIWXqxRSsQh6I+F2gw4h3tyX1GBz8OBB1NXVITo6GuXl5Xj55ZcRERGBqVOnOrtqgx6fHlohEom42cMg19zcjN/85jcYN24cZs+ejSFDhggNTT/++GN4eHhYncaNG9dndXziiSc6rcebb77ZZ/Xoa3yk1Qk/Nxlu3tFyA9NBKjExEYmJ1tukzZo1C0ql0uqyu7VU70l/+tOf0NjYaHWZn59fn9Wjr3FodcKP22qxTnh6esLT09PZ1cDQoUOdXQWn4NPDTph7MOXTQ8b6F4dCa/PmzYiIiIBCoYBSqURhYWGX5Xfv3o2oqCgoFApER0dj3759Fstfe+01REVFwd3dHb6+vkhISBBaMJtVVVUhLS0NXl5e8PHxwYIFCyyeku9p/NA0Y/2T3aG1c+dOZGRkICsrC8XFxYiJiUFiYmKnT8Dn5+cjNTUVCxYswMmTJ5GcnIzk5GScOXNGKBMZGYlNmzbh9OnTOHLkCCIiIvD444/j5s2bQpm0tDR8++23yM3NRU5ODr766iuhS43ewN3TMNZP2TtuWWxsLC1ZskR4bzAYKDQ0lFQqldXyKSkplJSUZDFPqVTS4sWLO92HefyzAwcOEBHR2bNnCQAdP35cKPPFF1+QSCSiH374waZ62zqmmtmH+WUUviqHFn90wqby96LGxkY6e/YsNTY2Orsq7B7R1b8pW3+jdh1p6XQ6FBUVISEhQZgnFouRkJAAtVptdR21Wm1RHjDdmemsvE6nw9atW+Ht7Y2YmBhhGz4+PhZPpickJEAsFnc4jTTTarWora21mOzB3dMw1j/ZFVqVlZUwGAwICgqymB8UFASNRmN1HY1GY1P5nJwceHh4QKFQ4N1330Vubi4CAgKEbQQGBlqUl0ql8PPz63S/KpUK3t7ewhQWFmbPR23z0DQ/f8jsdy8ObNFf9Ju7h9OmTUNJSQny8/MxY8YMpKSkONxTJABkZmaipqZGmK5du2bX+q1NHnq361jGektPD2wBmLpxFolEqK6u7tHt2sOu0AoICIBEIkFFRYXF/IqKCqu9IwJAcHCwTeXd3d0xatQoPPjgg/jzn/8MqVSKP//5z8I22geYXq9HVVVVp/uVy+Xw8vKymOzRtp2WwTgoHs9kbECwK7RkMhkmTZpkMYyQ0WhEXl4e4uLirK4TFxfXYdih3NzcTsu33a5WqxW2UV1djaKiImH5wYMHYTQaO22Z3F3mu4dEQDXfQTQhAnT1zpl4YIseG9hCq9Vi5cqVGDp0KNzd3aFUKnH48GFh3StXrmDmzJnw9fWFu7s7xo0bh3379uHy5cuYNm0aAFP3PyKRCM8//7xD30d32N0iPiMjA+np6Zg8eTJiY2OxYcMG1NfXY/78+QBMAwIMHToUKpUKALB8+XLEx8dj/fr1SEpKQnZ2Nk6cOIGtW7cCMHX6//vf/x6zZs1CSEgIKisrsXnzZvzwww945plnAJg6UZsxYwYWLlyILVu2oLm5GUuXLsXcuXMRGhraU9+FBReJGF4KKWqb9Kiq18HfQ94r+xlQmhuAN3vn+76r31wHZO53L9eCB7bofGCLpUuX4uzZs8jOzkZoaCg+/fRTzJgxA6dPn8bo0aOxZMkS6HQ6fPXVV3B3d8fZs2fh4eGBsLAw/P3vf8ecOXNQWloKLy8vuLr2fWcCdofWs88+i5s3b2LNmjXQaDS47777sH//fuFi+9WrVyEWtx7ATZkyBTt27MCrr76K3/zmNxg9ejQ+++wzjB8/HgAgkUhw/vx5fPjhh6isrIS/vz8eeOABfP311xYPn3788cdYunQppk+fDrFYjDlz5uC9997r7ufvkr+HXAgtNrDwwBbWB7a4evUqtm3bhqtXrwr/w1+5ciX279+Pbdu24c0338TVq1cxZ84cofvpESNGCOubn2kMDAzs8etlNuut9hj9jb3ttIiIfvr/jlL4qhzad+p6L9as/+rQpsZoJNLWOWcyGu2q+7///W+aO3cuDR8+nDw9Pcnd3Z0A0N69e4mI6MknnxTaCn733XcEgE6dOkVERJ999hlJpVIyGAwW2/T19aV3333Xpv0DoE8//ZSIiFasWEGPPvqoxfLq6moCQF9++SURET3//PMkl8vpqaeeog0bNtD1663/5rKyskgqldL06dNJpVLRxYsXbarDtm3byNvb22JeTk4OASB3d3eLSSqVUkpKChERffDBBySVSmnKlCm0Zs0a+uabb4T1Dx06RADo9u3bNtWhvT5vpzXYcF/x7YhEplM0Z0w8sEWPDGxRV1cHiUSCoqIilJSUCNO5c+fwxz/+EQDwwgsv4LvvvsNzzz2H06dPY/Lkydi4cWOPfdbu4tDqgl/LxfjbHFoDCg9sYWJtYIuJEyfCYDDgxo0bGDVqlMXU9jQyLCwML774Ivbs2YNf/epX+OCDD4RtArB5wIzewKHVBaGnBw6tAYUHtjCxNrBFZGQk0tLSMG/ePOzZswdlZWUoLCyESqXC3r17AQAvvfQS/vnPf6KsrAzFxcU4dOiQsL/w8HCIRCLk5OTg5s2bvdppQaccOjEdgBy5pvXBV5cofFUOLdtR3Is1678G8rOHubm5NHbsWJLL5TRhwgQ6fPiwxXUmIqJ9+/YRAJo6dWqH9a9fv05PPPEEyeVyCg8Ppx07dlBgYCBt2bLFpv2339epU6do2rRppFAoyM/PjxYuXEh37twhIiKNRkPJyckUEhJCMpmMwsPDac2aNWQwGEir1dLcuXMpLCyMZDIZhYaG0tKlS23+b/Liiy+Sv78/AaCsrCwiItLpdLRmzRqKiIggFxcXCgkJodmzZwvX9JYuXUojR44kuVxOQ4YMoeeee44qKyuFbb7xxhsUHBxMIpGI0tPTbaqHWU9c0+KBLbqwp/h7ZOz6Bg+PCsBfX+id9mD9GQ9s0YoHtugZPLBFL/Plh6YHLR7Yov/ia1pd4I4ABy8e2KL/4iOtLrTtnoaIHLoIywYmHtii/+LQ6oK/u+nRHZ3BiDqtHp6KvvsHyfovHtjCufj0sAuuMgkULqav6Hb94O2iZpDcq2F9oCf+LXFo3YX5aGswdgZoPtVpaGhwck3YvcL8b6k7p9F8engXfu4y/FDdOCgvxkskEvj4+Ai9G7i5ufF1PeYQIkJDQwNu3LgBHx8fSCQSh7fFoXUXg73Zg/nRju70IsuYmY+PT6cdd9qKQ+suBnuzB5FIhJCQEAQGBqK5efBe12Pd5+Li0q0jLDMOrbvgUXlMJBJJj/yDY6y7+EL8XXD3NIz1LxxadyEMcMGhxVi/wKF1F3ykxVj/wqF1F4P9Qjxj/Q2H1l3whXjG+heHQmvz5s2IiIiAQqGAUqlEYWFhl+V3796NqKgoKBQKREdHY9++fcKy5uZmrFq1CtHR0XB3d0doaCjmzZuH69evW2wjIiJCGL/NPK1du9aR6tvFHFp1Wj20eud1McsYM7E7tHbu3ImMjAxkZWWhuLgYMTExSExM7LTxYX5+PlJTU7FgwQKcPHkSycnJSE5OxpkzZwCYmvUXFxdj9erVKC4uxp49e1BaWopZs2Z12NYbb7yB8vJyYVq2bJm91bebl8IFErGpFfhgfv6QsX7Drr5SiSg2NpaWLFkivDcYDBQaGkoqlcpq+ZSUFEpKSrKYp1QqheGbrCksLCQAdOXKFWFeeHi4zcM3WeNId8tmk36XS+GrcujMD9UO758x1rVeGUJMp9OhqKgICQkJwjyxWIyEhASo1Wqr66jVaovygKmvos7KAxCG8m4/GOTatWvh7++PiRMnYt26ddDr9Z1uQ6vVora21mJyFF+MZ6z/sKtFfGVlJQwGgzCatFlQUBDOnz9vdR2NRmO1vEajsVq+qakJq1atQmpqqkU/0b/85S9x//33w8/PD/n5+cjMzER5eTn+8Ic/WN2OSqXC66+/bs/H65Svu+mJdA4txpyvXz3G09zcjJSUFBAR3n//fYtlbYeAmjBhAmQyGRYvXgyVSgW5XN5hW5mZmRbr1NbWIiwszKF6mbun4dBizPnsCq2AgABIJBJUVFRYzK+oqOj0ye3g4GCbypsD68qVKzh48OBdR8xRKpXQ6/W4fPmyxYCXZnK53GqYOYKbPTDWf9h1TUsmk2HSpEnIy8sT5hmNRuTl5SEuLs7qOnFxcRblASA3N9eivDmwLly4gAMHDsDf3/+udSkpKYFYLEZgYKA9H8Ehg717Gsb6E7tPDzMyMpCeno7JkycjNjYWGzZsQH19PebPnw8AmDdvHoYOHQqVSgUAWL58OeLj47F+/XokJSUhOzsbJ06cwNatWwGYAutnP/sZiouLkZOTA4PBIFzv8vPzg0wmg1qtRkFBAaZNmwZPT0+o1WqsWLECP//5z+Hr69tT30Wn+EI8Y/2II7cmN27cSMOGDSOZTEaxsbF07NgxYVl8fHyHUWd37dpFkZGRJJPJaNy4cbR3715hWVlZGQGwOh06dIiIiIqKikipVJK3tzcpFAoaO3Ysvfnmm9TU1GRznbvT5OH/Sn6g8FU59MyWfLvXZYzZhkeYbseREabN8i9W4j/+VIBRgR44kBHfSzVkbHCz9TfKzx7awJe7p2Gs3+DQsoH5mtbtBh2MxkFxYMpYv8WhZQPzkZaRgOpGfv6QMWfi0LKBi0QML4XpRmvVIBz/kLH+hEPLRq0NTPlIizFn4tCyUWto8ZEWY87EoWUjv5bnD7mveMaci0PLRv7c7IGxfoFDy0a+PCoPY/0Ch5aN+PlDxvoHDi0bcfc0jPUPHFo28vPg0GKsP+DQspGfG4cWY/0Bh5aN/NpciB8kHWMw1i9xaNnIv+X0UKc3ol7Hg7Yy5iwcWjZydZFALjV9XdxWizHn4dCykUgkEpo9cFstxpyHQ8sOrXcQ+flDxpyFQ8sOwvOHdXykxZizcGjZwc/NNNL07QYOLcachUPLDtzTA2PO51Bobd68GREREVAoFFAqlSgsLOyy/O7duxEVFQWFQoHo6Gjs27dPWNbc3IxVq1YhOjoa7u7uCA0Nxbx583D9+nWLbVRVVSEtLQ1eXl7w8fHBggULUFdX50j1HWZu9lDFp4eMOY3dobVz505kZGQgKysLxcXFiImJQWJiIm7cuGG1fH5+PlJTU7FgwQKcPHkSycnJSE5OxpkzZwAADQ0NKC4uxurVq1FcXIw9e/agtLQUs2bNsthOWloavv32W+Tm5iInJwdfffUVFi1a5MBHdpxfmwEuGGNOYu+AirGxsbRkyRLhvcFgoNDQUFKpVFbLp6SkUFJSksU8pVJJixcv7nQfhYWFBICuXLlCRERnz54lAHT8+HGhzBdffEEikYh++OEHq9toamqimpoaYbp27ZrDg7UK+zxdTuGrcih58xGHt8EYs87WwVrtOtLS6XQoKipCQkKCME8sFiMhIQFqtdrqOmq12qI8ACQmJnZaHgBqamogEong4+MjbMPHxweTJ08WyiQkJEAsFqOgoMDqNlQqFby9vYUpLCzM1o/ZKX9+aJoxp7MrtCorK2EwGBAUFGQxPygoCBqNxuo6Go3GrvJNTU1YtWoVUlNThVFmNRoNAgMDLcpJpVL4+fl1up3MzEzU1NQI07Vr12z6jF0Ruqfha1qMOY3U2RVoq7m5GSkpKSAivP/++93allwuh1wu76GamZhbxN/R6qHTGyGT8s1XxvqaXb+6gIAASCQSVFRUWMyvqKhAcHCw1XWCg4NtKm8OrCtXriA3N1c4yjJvo/2Ffr1ej6qqqk732xu8FC6QiEUA+GI8Y85iV2jJZDJMmjQJeXl5wjyj0Yi8vDzExcVZXScuLs6iPADk5uZalDcH1oULF3DgwAH4+/t32EZ1dTWKioqEeQcPHoTRaIRSqbTnI3SLWCyCb0sDU24Vz5iT2HuFPzs7m+RyOW3fvp3Onj1LixYtIh8fH9JoNERE9Nxzz9Err7wilD969ChJpVJ655136Ny5c5SVlUUuLi50+vRpIiLS6XQ0a9Ys+tGPfkQlJSVUXl4uTFqtVtjOjBkzaOLEiVRQUEBHjhyh0aNHU2pqqs31tvXOxN385A+HKXxVDn3975vd2g5jzJKtv1G7Q4uIaOPGjTRs2DCSyWQUGxtLx44dE5bFx8dTenq6Rfldu3ZRZGQkyWQyGjduHO3du1dYVlZWRgCsTocOHRLK3bp1i1JTU8nDw4O8vLxo/vz5dOfOHZvr3FOhlbIln8JX5dDnJdabWjDGHGPrb1RENDi64aytrYW3tzdqamosrpfZ6xcfF2HfaQ1em/ljPP/Q8B6sIWODm62/Ub79ZScelYcx5+LQspP5oekqvnvImFNwaNnJ3D0NH2kx5hwcWnby8+COABlzJg4tO/nzNS3GnIpDy07cPQ1jzsWhZafW0GqG0TgoWosw1q9waNnJ180UWgYjoaax2cm1YWzw4dCyk0wqhqfC1DkG9xXPWN/j0HKAP1/XYsxpOLQc4GseaZqbPTDW5zi0HMDNHhhzHg4tB7Q+f6h1ck0YG3w4tBwgPH9Yz3cPGetrHFoO8HM3P3/IR1qM9TUOLQeYj7S4yQNjfY9DywHc5IEx5+HQcoAvj3/ImNNwaDnAfKR1q16HQdJbNWP9BoeWA8xNHrR6Ixp0BifXhrHBxaHQ2rx5MyIiIqBQKKBUKlFYWNhl+d27dyMqKgoKhQLR0dHYt2+fxfI9e/bg8ccfh7+/P0QiEUpKSjps49FHH4VIJLKYXnzxRUeq321uMgnkLaNLcwNTxvqW3aG1c+dOZGRkICsrC8XFxYiJiUFiYmKHEaDN8vPzkZqaigULFuDkyZNITk5GcnIyzpw5I5Spr6/Hww8/jLfeeqvLfS9cuBDl5eXC9Pbbb9tb/R4hEol4gAvGnMXescliY2NpyZIlwnuDwUChoaGkUqmslk9JSaGkpCSLeUqlkhYvXtyhrHkMxJMnT3ZYFh8fT8uXL7e3uoKeGvfQ7Mk/fkXhq3Lo4LmKHtkeY4Odrb9Ru460dDodioqKkJCQIMwTi8VISEiAWq22uo5arbYoDwCJiYmdlu/Kxx9/jICAAIwfPx6ZmZloaGjotKxWq0Vtba3F1JP82lyMZ4z1Hak9hSsrK2EwGBAUFGQxPygoCOfPn7e6jkajsVpeo9HYVdH/+I//QHh4OEJDQ3Hq1CmsWrUKpaWl2LNnj9XyKpUKr7/+ul37sIfQVotDi7E+ZVdoOdOiRYuE19HR0QgJCcH06dNx6dIljBw5skP5zMxMZGRkCO9ra2sRFhbWY/Xx5SMtxpzCrtAKCAiARCJBRUWFxfyKigoEBwdbXSc4ONiu8rZSKpUAgIsXL1oNLblcDrlc3q19dMWfe3pgzCnsuqYlk8kwadIk5OXlCfOMRiPy8vIQFxdndZ24uDiL8gCQm5vbaXlbmZtFhISEdGs7juKeHhhzDrtPDzMyMpCeno7JkycjNjYWGzZsQH19PebPnw8AmDdvHoYOHQqVSgUAWL58OeLj47F+/XokJSUhOzsbJ06cwNatW4VtVlVV4erVq7h+/ToAoLS0FIDpKC04OBiXLl3Cjh078OSTT8Lf3x+nTp3CihUrMHXqVEyYMKHbX4IjuE8txpzEkVuTGzdupGHDhpFMJqPY2Fg6duyYsCw+Pp7S09Mtyu/atYsiIyNJJpPRuHHjaO/evRbLt23bRgA6TFlZWUREdPXqVZo6dSr5+fmRXC6nUaNG0a9//Wu7mi/0dJOHgu9uUfiqHIp/+2CPbI+xwc7W36iIaHA8PFdbWwtvb2/U1NTAy8ur29u7eKMOCX/4Ep4KKU6/ltgDNWRscLP1N8rPHjrIfCH+TpMezQajk2vD2ODBoeUgb1cXiEWm19xWi7G+w6HlILFYJIw2zW21GOs7HFrdwA9NM9b3OLS6gUOLsb7HodUNHFqM9T0OrW7gnh4Y63scWt3Azx8y1vc4tLrBT+iehp8/ZKyvcGh1Q2v3NHykxVhf4dDqBn+hpwe+psVYX+HQ6obWu4d8eshYX+HQ6gZ/j5ZrWg06GI2D4rlzxpyOQ6sbfNxcAAAGI6G2iY+2GOsLHFrdIJdK4Ck39aPIbbUY6xscWt3k58Gj8jDWlzi0uol7emCsb3FodZM/P3/IWJ/i0Oomfmiasb7FodVN5mtaHFqM9Q2HQmvz5s2IiIiAQqGAUqlEYWFhl+V3796NqKgoKBQKREdHY9++fRbL9+zZg8cffxz+/v4QiUTCmIZtNTU1YcmSJfD394eHhwfmzJnTYRBYZ/Bz49BirC/ZHVo7d+5ERkYGsrKyUFxcjJiYGCQmJuLGjRtWy+fn5yM1NRULFizAyZMnkZycjOTkZJw5c0YoU19fj4cffhhvvfVWp/tdsWIF/vGPf2D37t348ssvcf36dfz0pz+1t/o9jrunYayP2Ts2WWxsLC1ZskR4bzAYKDQ0lFQqldXyKSkplJSUZDFPqVTS4sWLO5QtKysjAHTy5EmL+dXV1eTi4kK7d+8W5p07d44AkFqttqnePT3uoVneOQ2Fr8qhp977uke3y9hgY+tv1K4jLZ1Oh6KiIiQkJAjzxGIxEhISoFarra6jVqstygNAYmJip+WtKSoqQnNzs8V2oqKiMGzYsE63o9VqUVtbazH1Bj9+aJqxPmVXaFVWVsJgMCAoKMhiflBQEDQajdV1NBqNXeU724ZMJoOPj4/N21GpVPD29hamsLAwm/dnDz837p6Gsb50z949zMzMRE1NjTBdu3atV/ZjvnvY1GxEg07fK/tgjLWS2lM4ICAAEomkw127iooKBAcHW10nODjYrvKdbUOn06G6utriaKur7cjlcsjlcpv34Sh3mQQyqRg6vRFV9Tq4yez6ShljdrLrSEsmk2HSpEnIy8sT5hmNRuTl5SEuLs7qOnFxcRblASA3N7fT8tZMmjQJLi4uFtspLS3F1atX7dpObxCJRNwqnrE+ZPdhQUZGBtLT0zF58mTExsZiw4YNqK+vx/z58wEA8+bNw9ChQ6FSqQAAy5cvR3x8PNavX4+kpCRkZ2fjxIkT2Lp1q7DNqqoqXL16FdevXwdgCiTAdIQVHBwMb29vLFiwABkZGfDz84OXlxeWLVuGuLg4PPjgg93+ErrL102G8pombvbAWF9w5Nbkxo0badiwYSSTySg2NpaOHTsmLIuPj6f09HSL8rt27aLIyEiSyWQ0btw42rt3r8Xybdu2EYAOU1ZWllCmsbGRfvGLX5Cvry+5ubnR7Nmzqby83OY691aTByKin//pGIWvyqG/nbjW49tmbLCw9TcqIqJB0eVmbW0tvL29UVNTAy8vrx7d9vLsk/i85DpeTRqLFx4Z0aPbZmywsPU3es/ePexL3Cqesb7DodUDhOcP6zi0GOttHFo9QOjpoYFDi7HexqHVA7jJA2N9h0OrB/hy9zSM9RkOrR5gHv/wVh0/f8hYb+PQ6gHmnh5qm/RoNhidXBvG7m0cWj3A29UFYpHp9W2+GM9Yr+LQ6gESsQg+fF2LsT7BodVDhFF5uK0WY72KQ6uHCKHFp4eM9SoOrR7CbbUY6xscWj3E1/z8IZ8eMtarOLR6CB9pMdY3OLR6CF/TYqxvcGj1EL57yFjf4NDqIX58eshYn+DQ6iEBHqZHecpu1eNw6Q0n14axexeHVg8ZE+SJuBH+0OmNmL/9OLZ8eQmDpCdrxvoUh1YPEYtF2P6fDyA1NgxEwNovzmN5dgkadQZnV42xe4pDobV582ZERERAoVBAqVSisLCwy/K7d+9GVFQUFAoFoqOjsW/fPovlRIQ1a9YgJCQErq6uSEhIwIULFyzKREREQCQSWUxr1651pPq9Ri6V4M3Z0fhd8nhIxSL83zfX8bMt+fj+doOzq8bYPcPu0Nq5cycyMjKQlZWF4uJixMTEIDExETduWL+Ok5+fj9TUVCxYsAAnT55EcnIykpOTcebMGaHM22+/jffeew9btmxBQUEB3N3dkZiYiKamJottvfHGGygvLxemZcuW2Vv9XicSifDcg+H4+AUl/N1l+PZ6LWZtOopj391ydtUYuzfYOzZZbGwsLVmyRHhvMBgoNDSUVCqV1fIpKSmUlJRkMU+pVNLixYuJiMhoNFJwcDCtW7dOWF5dXU1yuZw++eQTYV54eDi9++679lZX0JvjHnbm+9sN9OQfv6LwVTk0MnMvfZhfRkajsc/2z9hAYutv1K4jLZ1Oh6KiIiQkJAjzxGIxEhISoFarra6jVqstygNAYmKiUL6srAwajcaijLe3N5RKZYdtrl27Fv7+/pg4cSLWrVsHvV7faV21Wi1qa2stpr421McVf3txCmbFhEJvJKz5/Fu88vfT0Or5OhdjjrIrtCorK2EwGBAUFGQxPygoCBqNxuo6Go2my/Lmv3fb5i9/+UtkZ2fj0KFDWLx4Md588028/PLLndZVpVLB29tbmMLCwmz/oD3IVSbBH+feh8wnoiASATtPXEPq1mO4Udt095UZYx0MmLuHGRkZePTRRzFhwgS8+OKLWL9+PTZu3Ait1nq/7JmZmaipqRGma9eu9XGNW4lEIiyOH4ltzz8AL4UUxVerMXPTEZRcq3ZanRgbqOwKrYCAAEgkElRUVFjMr6ioQHBwsNV1goODuyxv/mvPNgFAqVRCr9fj8uXLVpfL5XJ4eXlZTM726JhAfL70YYwK9EBFrRYp/6PG34q+d3a1GBtQ7AotmUyGSZMmIS8vT5hnNBqRl5eHuLg4q+vExcVZlAeA3Nxcofzw4cMRHBxsUaa2thYFBQWdbhMASkpKIBaLERgYaM9HcLrhAe749BdTkDA2CDq9ESt3f4M3/nEWeh4QgzHb2HuFPzs7m+RyOW3fvp3Onj1LixYtIh8fH9JoNERE9Nxzz9Err7wilD969ChJpVJ655136Ny5c5SVlUUuLi50+vRpoczatWvJx8eHPv/8czp16hQ9/fTTNHz4cGpsbCQiovz8fHr33XeppKSELl26RH/9619pyJAhNG/ePJvr7Yy7h10xGIy0/l+lFL4qh8JX5dDTm47QP8+Uk97AdxfZ4GTrb9Tu0CIi2rhxIw0bNoxkMhnFxsbSsWPHhGXx8fGUnp5uUX7Xrl0UGRlJMpmMxo0bR3v37rVYbjQaafXq1RQUFERyuZymT59OpaWlwvKioiJSKpXk7e1NCoWCxo4dS2+++SY1NTXZXOf+FlpmX5y+TmNXfyGE1yNvHaQ/ff0d1TbqnF01xvqUrb9REdHgeECutrYW3t7eqKmp6RfXt9oqr2nER+or2FFwFTWNzQAAd5kEz0wOw/NTIhAR4G59xfpbgEEHeIX0YW0Z6x22/kY5tPqRRp0Be05+j21HL+PijToAgEgETI8KxPyHhmPKSH+IRCJArwPUG4Ev1wFkBObuAEYn3GXrjPVvHFrt2Bxa5d8AexYBMveWyaPd6/bv2732Gw64uHarrkSEry9UYtvRMhwqvSnMHxPkiVVRN/HopbUQV/67dQWJDHj2YyDy8W7tlzFnsvU3Ku3DOg0MDbeAm+cdX1/hA8QuBJQvAu4BDm1CJBJhauQQTI0cgks36/Bh/mUcPHEGi6v+Hx4rOAIAqHfxg3766/C+8i/g3D+AnWlAyl+AMTMcrztjAwAfabXXeBvQnAF09YCurmWqb/O+zWttu/eN1YC2xrQdqQKY+HMgbqnp6MtRRgNQtA104A2ItDUwQoSP9dOxTp+CBrEnZkYPQZb2D/C5vA8QuwApHwFRTzq+P8achE8P2+mTa1pGg+mo5+gG4PpJ0zyRGBg3G3hoORASY9/2rpcAOSuA68Wm9yEx0D+xHrk1P8K2o5dReLkKACCFHh96/wkPab8CiV0gemY7MPapnvpUjPUJDq12+vRCPBFw+WvgyAbgUpuGtSMfM4XX8HjTFfbONNUAB38PHP/AdKFd7gU89irwwAuAWCIUO/19DbYdLcM/Tl2H0aDHuy7/D7MkahhFUjQ+/QHc7/tp731GxnoYh1Y7Trt7WH4KyH8POLMHoJbeHULuAx5+CRg7yyKEQASc+Tvwz98AdS2PNY3/GZD4e8Cz80eaKmqb8NdjV5B9rAy/bf4jkiX50JMYe0a8gdik/+y8yQRj/QiHVjtOb/Jw+zKg3gwU/wXQN5rm+Y0ApiwDYv4DqPke2Pcr4LvDpmX+o4An3wFGTrN5F03NBnxefBV+B1bgJ82HoCcxXtIvQVPk0/jPh4cjbkRLkwnG+iEOrXacHlpm9ZVA4VbT1HjbNM8tANDWmhqKSuTA1JWm00ip3KFdkEGPio8XIfi7v8NAIqxoXoL/M07B2BAv/OdDEZgZEwqFi+TuG2KsD3FotdNvQstMV2866lJvAmpaus0ZlQA8uc50BNZdRiPwj2XAyb/CCDFeMf4Cu3RTAAABHjLMi4vAgoeHw13OrV5Y/8Ch1U6/Cy0zQzNwPgdwcQdG/6TrC/T2MhqBnOVA8UcgiHBo7Ov47XfjUV5j6oAwyEuOVTOikHzfUIjFfNrInItDq51+G1q9zWgE9mYARdsAiKCfuRF7JY9h/b/+jatVplGCYsJ8kDXzx7h/mG/v1aGhElB4O3zKy+59HFrtDNrQAkyhsW8lcOLPAETArPfQFJ2G/z1ahs0HL6K+ZWzG5PtCseqJKIR4d+8xJACmO6EVZ4DTfzPdETWfAss8ATc/09MCbv6m63kW783z/AF3f0DuDYgHTAe7rBs4tNoZ1KEFmELki5dNNwAAYMZaYOLPcUPrgnX/LMXfir8HEeDqIsF/PToSi6aOcOxifVUZcOZvprDqzuNQZhI58KPJQMQjwPBHgB89wEdr9ygOrXYGfWgBpuDa/wpQsMX0XiQGAscBYQ/gmvt4vHXGGzk/KACIMNTHFa88EYWnJoTcvZlE3Q3g28+A07uB79sM3CuRAZGJQPQzwKifAAYt0FBler6zvtL0t8H8t6rNvJZJa2UEJakCCIsFIqaaQiz0fkAq66lviDkRh1Y7HFotiICv3wGKPmw9ZWtDK/NFoX4k1NqRKDJGwiXsfrzy9CSMH+ptWbCpFji/1xRU3x1ubTgrEgPDp5qCKuopwNXH8arqtaCqKxBfPWp6wqDsa6C+3aDALm7AsAdbjsSmmhruSnrpjqjRAOi1gL7J1DxFrzVNBq3pvfcwwGNI7+x7EODQaodDy4ractOR0bWWqbzE9ONrQ09inKNhqA2YiPEPJsDbw9PUuv/f+00/XrOhk0xBNW424BkMvcGIeq0BtU3NqNPqUafV405TM+406YWpTmt6X9ekR22TabmpXOtrIwH3D/PBo2MCMS1yCMa6lEN0+WtTiF0+Yjoia0vmCYTHmYJMIjcFil5nGTTmeQZtm+Bpu6ztvKbW9c3B3BXPECB4AhAyofWvT3jP3hW+R3FotcOhZQO91vTYUUuQGa4WQFJX3mnxCtkwHHN/DIdlU3HJEIS6Jj3uaE0h1NjcOwPSBnnJ8WhkIKZFDcFDI/3gWXvRdARmDrGm6l7Zb0ci06mqVGb6K5IAd8oBWPk5KbxNAdY2zAIiOz8iJDL1GlJ3wzTVt/ytq7CcJxKbukJy9W2ZWl4L89osU/hYnkYTAdo7Lafn7U/NW17X37KcZzS02b6VfVjsu80ymYdNoc2h1Q6HloNqvsd3Jw/hzLFchDV8Cy/U44DxfnxueAhnKRxA1/8YFS5ieMhd4CGXwFPhAk+FFJ4KKTzkra+tvfdUuMBDLkWzwYivL1TicOkNHL14yyIMpWIRJkf4YtqYQEyLCsToIW4QVZwxhVh5ialuUpnpiEsqN11ja/tXqug4z1zWPEnkrcHUdplY2vGHqL0DVHxrCn7NN6a/N84BxuaOX4xUAQT+2BRiInFrGNVVAPU3geaGbv+n68DF3RQkRoMpiKzVqzeIpcD8/UDYA10W49Bqh0Ore4xGwv99cx35lyrhJpPCQ94SNoo2r+UubV6blrlIeq65QlOzAYVlVThcehOHS2/gu8p6i+Wh3go8GhWIaWMCMWWkf/9o7a/Xme6iak61hNkpQHPadCR1Ny7ugEdg6+QeCHgEma6bubcMndd423R02Xjb1J+b1fc1sHoEaN6Hm7/1Zihu/pbzxBIr+2iznw7zb7debvhFARAY1eXH5dBqh0Pr3nPlVj0Ol97EodIbUF+6Ba2+dexIiVgED7kUbjIJXGUSuMkkcHORCq+FeTIpXF3MryVwlUkhFYsgFosgFgFiUdu/IojFpp5lJeb3IrSUFUEmFUPhIoZcKoHCRQyFVAJ5y1+LJw6MRuB2malr74pvTWHgPqQlkNoElNyjZ74oo9HUOaU5XETi1jDqZtfg7RERtHojahqbUdugw536O2iqqURMVCTc3breV6+G1ubNm7Fu3TpoNBrExMRg48aNiI2N7bT87t27sXr1aly+fBmjR4/GW2+9hSefbO1dk4iQlZWFDz74ANXV1XjooYfw/vvvY/To0UKZqqoqLFu2DP/4xz8gFosxZ84c/PGPf4SHh23/YTm07m2NOgOOfXcLh0tv4FDpTaG1f38hk4ghl4ohd5FA3hJuipbXcqkELlKxUMZFYgpAmVQMF4npr1zS+to8ScUiiFrCUyI2BauoJUglbV6blosgankNmI67jKYhBGE0ml4bybTESK3vicj02ojWMGpqRm1jc8trPWrbzKtt1ENnZeDhL5Y/grEhXf/ueq2P+J07dyIjIwNbtmyBUqnEhg0bkJiYiNLSUqujPefn5yM1NRUqlQpPPfUUduzYgeTkZBQXF2P8+PEAgLfffhvvvfcePvzwQwwfPhyrV69GYmIizp49C4VCAQBIS0tDeXk5cnNz0dzcjPnz52PRokXYsWOHvR+B3YNcZRJMizJd23qNCDfuaHGnqRkNOkPLpBdeNwp/W+Y1m+fp0dhshMFoFH7I1PIDNlj5ERvNr8l0+qzVG6HVG6BtNqJJb0CzofV4QGcwQmcw4o5W78Rvqe+IRYCXqwu8XV3gpXDp0W3bfaSlVCrxwAMPYNOmTQAAo9GIsLAwLFu2DK+88kqH8s8++yzq6+uRk5MjzHvwwQdx3333YcuWLSAihIaG4le/+hVWrlwJAKipqUFQUBC2b9+OuXPn4ty5c/jxj3+M48ePY/LkyQCA/fv348knn8T333+P0NDQDvvVarXQarXC+9raWoSFhfGRFuszBiNBqzegqdmIpmYDtHrTX8vXpjBr1pv+6vRGNBuM0OpbX+talrWdr9MbhcA0GKk1QNuHKRGMxtbXBtPhlHCqK253RCYWweK9SNRaRiYRm0LIVSqEUWswSeHl2vreXSaxu++2XjnS0ul0KCoqQmZmpjBPLBYjISEBarXa6jpqtRoZGRkW8xITE/HZZ58BAMrKyqDRaJCQ0Dpun7e3N5RKJdRqNebOnQu1Wg0fHx8hsAAgISEBYrEYBQUFmD17dof9qlQqvP766/Z8PMZ6lEQsgptMCjdusN+j7Lq1U1lZCYPBgKCgIIv5QUFB0Gg0VtfRaDRdljf/vVuZ9qeeUqkUfn5+ne43MzMTNTU1wnTtWsfW34yxgacf3BPuHXK5HHI5P1jL2L3GriOtgIAASCQSVFRUWMyvqKhAcLD1gReCg4O7LG/+e7cyN25YPnOm1+tRVVXV6X4ZY/cmu0JLJpNh0qRJyMtrHRbLaDQiLy8PcXFxVteJi4uzKA8Aubm5Qvnhw4cjODjYokxtbS0KCgqEMnFxcaiurkZRUZFQ5uDBgzAajVAqlfZ8BMbYQEd2ys7OJrlcTtu3b6ezZ8/SokWLyMfHhzQaDRERPffcc/TKK68I5Y8ePUpSqZTeeecdOnfuHGVlZZGLiwudPn1aKLN27Vry8fGhzz//nE6dOkVPP/00DR8+nBobG4UyM2bMoIkTJ1JBQQEdOXKERo8eTampqTbXu6amhgBQTU2NvR+ZMdYHbP2N2h1aREQbN26kYcOGkUwmo9jYWDp27JiwLD4+ntLT0y3K79q1iyIjI0kmk9G4ceNo7969FsuNRiOtXr2agoKCSC6X0/Tp06m0tNSizK1btyg1NZU8PDzIy8uL5s+fT3fu3LG5zhxajPVvtv5G+TEexli/0Gst4gcqczbX1lrpDZMx5nTm3+bdjqMGTWjduXMHABAWFubkmjDGunLnzh14e3t3unzQnB4ajUZcv34dnp6ed328wPzIz7Vr1/hUsg3+XjrH34119nwvRIQ7d+4gNDQU4i5GYBo0R1pisRg/+tGP7FrHy8uL/wFawd9L5/i7sc7W76WrIywzHlCOMTagcGgxxgYUDi0r5HI5srKy+NnFdvh76Rx/N9b1xvcyaC7EM8buDXykxRgbUDi0GGMDCocWY2xA4dBijA0oHFqMsQGFQ6udzZs3IyIiAgqFAkqlEoWFhc6uktO99tprELWMyGKeoqK6Hi34XvTVV19h5syZCA0NhUgkEgZnMSMirFmzBiEhIXB1dUVCQgIuXLjgnMr2sbt9N88//3yHf0MzZsxwaF8cWm2Yx3TMyspCcXExYmJikJiY2KGr58Fo3LhxKC8vF6YjR444u0p9rr6+HjExMdi8ebPV5ebxO7ds2YKCggK4u7sjMTERTU1NfVzTvne37wYAZsyYYfFv6JNPPnFsZ73aq9cAExsbS0uWLBHeGwwGCg0NJZVK5cRaOV9WVhbFxMQ4uxr9CgD69NNPhfdGo5GCg4Np3bp1wrzq6mqSy+X0ySefOKGGztP+uyEiSk9Pp6effrpHts9HWi3MYzq2HX/xbmM6DiYXLlxAaGgoRowYgbS0NFy9etXZVepX7jZ+JwMOHz6MwMBAjBkzBv/1X/+FW7duObQdDq0WjozpOFgolUps374d+/fvx/vvv4+ysjI88sgjQh9lzLbxOwezGTNm4KOPPkJeXh7eeustfPnll3jiiSdgMBjs3tag6ZqGOe6JJ54QXk+YMAFKpRLh4eHYtWsXFixY4MSasYFi7ty5wuvo6GhMmDABI0eOxOHDhzF9+nS7tsVHWi0cGdNxsPLx8UFkZCQuXrzo7Kr0G7aM38lajRgxAgEBAQ79G+LQauHImI6DVV1dHS5duoSQkBBnV6XfsGX8Ttbq+++/x61btxz6N8Snh21kZGQgPT0dkydPRmxsLDZs2ID6+nrMnz/f2VVzqpUrV2LmzJkIDw/H9evXkZWVBYlEgtTUVGdXrU/V1dVZHBmUlZWhpKQEfn5+GDZsGF566SX893//N0aPHo3hw4dj9erVCA0NRXJysvMq3Ue6+m78/Pzw+uuvY86cOQgODsalS5fw8ssvY9SoUUhMTLR/Zz1yD/Ie0tWYjoPVs88+SyEhISSTyWjo0KH07LPP0sWLF51drT536NAhAtBhMo/zacv4nfeqrr6bhoYGevzxx2nIkCHk4uJC4eHhtHDhQmGAZ3txf1qMsQGFr2kxxgYUDi3G2IDCocUYG1A4tBhjAwqHFmNsQOHQYowNKBxajLEBhUOLMTagcGgxxgYUDi3G2IDCocUYG1D+P5pSXDWvVFBSAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:53:38.586255Z", + "iopub.status.busy": "2024-03-26T10:53:38.585967Z", + "iopub.status.idle": "2024-03-26T10:54:46.342506Z", + "shell.execute_reply": "2024-03-26T10:54:46.341460Z" + }, + "papermill": { + "duration": 67.776433, + "end_time": "2024-03-26T10:54:46.345102", + "exception": false, + "start_time": "2024-03-26T10:53:38.568669", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:54:46.382492Z", + "iopub.status.busy": "2024-03-26T10:54:46.381832Z", + "iopub.status.idle": "2024-03-26T10:54:46.402059Z", + "shell.execute_reply": "2024-03-26T10:54:46.401198Z" + }, + "papermill": { + "duration": 0.040945, + "end_time": "2024-03-26T10:54:46.404014", + "exception": false, + "start_time": "2024-03-26T10:54:46.363069", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tvae0.0128840.0046730.0003653.7268640.005140.3903410.0099471.889180e-071.3032470.0130650.198950.0190960.1610090.0003715.030111
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "tvae 0.012884 0.004673 0.000365 3.726864 0.00514 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "tvae 0.390341 0.009947 1.889180e-07 1.303247 0.013065 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "tvae 0.19895 0.019096 0.161009 0.000371 5.030111 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:54:46.439119Z", + "iopub.status.busy": "2024-03-26T10:54:46.438854Z", + "iopub.status.idle": "2024-03-26T10:54:46.887371Z", + "shell.execute_reply": "2024-03-26T10:54:46.886463Z" + }, + "papermill": { + "duration": 0.468433, + "end_time": "2024-03-26T10:54:46.889483", + "exception": false, + "start_time": "2024-03-26T10:54:46.421050", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:54:46.927267Z", + "iopub.status.busy": "2024-03-26T10:54:46.926741Z", + "iopub.status.idle": "2024-03-26T10:55:59.023347Z", + "shell.execute_reply": "2024-03-26T10:55:59.022545Z" + }, + "papermill": { + "duration": 72.117888, + "end_time": "2024-03-26T10:55:59.025782", + "exception": false, + "start_time": "2024-03-26T10:54:46.907894", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../insurance/_cache_aug_test/tvae/all inf False\n", + "Caching in ../../../../insurance/_cache_bs_test/tvae/all inf False\n", + "Caching in ../../../../insurance/_cache_synth_test/tvae/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:55:59.064540Z", + "iopub.status.busy": "2024-03-26T10:55:59.064204Z", + "iopub.status.idle": "2024-03-26T10:55:59.084370Z", + "shell.execute_reply": "2024-03-26T10:55:59.083679Z" + }, + "papermill": { + "duration": 0.040858, + "end_time": "2024-03-26T10:55:59.086401", + "exception": false, + "start_time": "2024-03-26T10:55:59.045543", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:55:59.120906Z", + "iopub.status.busy": "2024-03-26T10:55:59.120639Z", + "iopub.status.idle": "2024-03-26T10:55:59.125635Z", + "shell.execute_reply": "2024-03-26T10:55:59.124781Z" + }, + "papermill": { + "duration": 0.024585, + "end_time": "2024-03-26T10:55:59.127578", + "exception": false, + "start_time": "2024-03-26T10:55:59.102993", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tvae': 0.03756988301781045}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:55:59.162178Z", + "iopub.status.busy": "2024-03-26T10:55:59.161936Z", + "iopub.status.idle": "2024-03-26T10:55:59.511307Z", + "shell.execute_reply": "2024-03-26T10:55:59.510427Z" + }, + "papermill": { + "duration": 0.368913, + "end_time": "2024-03-26T10:55:59.513206", + "exception": false, + "start_time": "2024-03-26T10:55:59.144293", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8xUlEQVR4nO3deXhU5b0H8O85Z87smZnsCyQkEBYRWQWKiIBSF6xCaRWXVlARW6EulD6aPrcotl5oq1ZrLbftLYncqrgU9V4tokUBRVlkFzRsgYQsJJBkssx+znv/mAWGzCQzk0nOTPL7PM88T+bMmTO/k+WX933PeX8vxxhjIIQQBfFKB0AIIZSICCGKo0RECFEcJSJCiOIoERFCFEeJiBCiOEpEhBDFUSIihCiOEhEhRHGUiAghiqNEROLqiy++wFNPPYXm5malQyFJhBIRiasvvvgCK1eupEREokKJiBCiOEpEJG6eeuop/OIXvwAAFBUVgeM4cBwHo9GImTNndthflmUMGDAAP/zhDwPbnn32WVx11VVIT0+HTqfDhAkT8Pbbb4f8vH/84x+YMGECdDod0tLScMcdd6CqqqpnTo70KI7KgJB4OXjwIFavXo3XX38df/jDH5CRkQEAOHHiBJ5++mlUV1cjJycnsP+2bdswffp0vPXWW4FklJ+fj1tvvRUjR46Ey+XC+vXrsWvXLrz//vu4+eabA+995pln8Ktf/Qq33347pk+fjoaGBrz00kswGo3Yt28fLBZLr5476SZGSBz9/ve/ZwBYRUVFYFt5eTkDwF566aWgfR966CFmNBqZzWYLbLv4a8YYc7lcbNSoUezaa68NbDt16hQTBIE988wzQfseOnSIqVSqDttJ4qOuGelxw4YNw9ixY/HGG28EtkmShLfffhu33HILdDpdYPvFXzc1NcFqtWLatGnYu3dvYPuGDRsgyzJuv/12nDt3LvDIycnB0KFD8emnn/bOiZG4USkdAOkf5s+fj1/+8peorq7GgAEDsGXLFtTX12P+/PlB+73//vv4zW9+g/3798PpdAa2cxwX+PrYsWNgjGHo0KEhP0sUxZ45CdJjKBGRXjF//nyUlJTgrbfewqOPPoo333wTZrMZN954Y2Cfzz77DLfeeiuuueYa/PnPf0Zubi5EUURpaSlee+21wH6yLIPjOGzcuBGCIHT4LKPR2CvnROKHEhGJq4tbLhcrKirCpEmT8MYbb2Dp0qXYsGED5s6dC41GE9jnn//8J7RaLTZt2hS0vbS0NOhYQ4YMAWMMRUVFGDZsWM+cCOlVNEZE4spgMABAyBsa58+fjx07dmDt2rU4d+5ch26ZIAjgOA6SJAW2nTp1Cu+++27QfvPmzYMgCFi5ciXYJRd9GWM4f/58fE6G9Bq6fE/iavfu3Zg0aRJmz56NO+64A6Io4pZbboHBYMCZM2dQUFAAo9EIURRRV1cXNJ7zySef4LrrrsO0adNw1113ob6+Hi+//DJycnJw8ODBoKSzevVqlJSU4KqrrsLcuXORkpKCiooKvPPOO1i8eDGWL1+uxOmTWCl4xY70Ub/+9a/ZgAEDGM/zHS7lT506lQFgixYtCvnev//972zo0KFMo9GwESNGsNLSUvbkk0+yUL+q//znP9nVV1/NDAYDMxgMbMSIEWzJkiWsvLy8p06N9BBqERFCFEdjRIQQxVEiIoQojhIRIURxlIgIIYqjREQIURwlIkKI4hSf4lFdXY3HH38cGzduhM1mQ3FxMUpLS3HllVd2+V5ZllFTU4OUlJSwUwsIIcphjKG1tRV5eXng+fDtHkUTUVNTE6ZOnYqZM2di48aNyMzMxLFjx5CamhrR+2tqapCfn9/DURJCuquqqgoDBw4M+7qiNzQ+8cQT2L59Oz777LOY3m+1WmGxWFBVVQWTyRTn6Agh3dXS0oL8/Hw0NzfDbDaH3U/RRDRy5EjccMMNOHPmDLZu3YoBAwbgoYcewgMPPBByf6fTGVSjxn+SVquVEhEhCailpQVms7nLv1FFB6tPnjyJNWvWYOjQodi0aRN++tOf4uGHH8Yrr7wScv9Vq1bBbDYHHtQtI6RvULRFpFarceWVV+KLL74IbHv44Yexe/dufPnllx32pxYRIcklKVpEubm5GDlyZNC2yy67DJWVlSH312g0MJlMQQ9CSPJT9KrZ1KlTUV5eHrTt6NGjGDRoUNw+gzEGj8cTVGyLREYQBKhUKro1gvQ4RRPRY489hquuugr/+Z//idtvvx27du3CX//6V/z1r3+Ny/FdLhdqa2ths9nicrz+SK/XIzc3F2q1WulQSB+meD2i999/HyUlJTh27BiKioqwbNmysFfNLtVZ/1OWZRw7dgyCICAzMxNqtZr+s0eBMQaXy4WGhgZIkoShQ4d2ekMaIaFEOkak+J3V3/ve9/C9730v7sd1uVyQZRn5+fnQ6/VxP35/oNPpIIoiTp8+DZfLBa1Wq3RICU+WGT75th4emeG6y7IgCpS8I6F4Iupp9F+8e+j7F50TDW04VG0FAORZtBg90KJsQEmCfssIiaPiLCPG5HvvID7TZFc4muRBiYiQOOI4DsVpaqQ461DXTIkoUpSI+rnCwkK88MILSofRp2RVbcTlZ/8XqXWfQZJpbYpIUCIiJE4cbgmb936DhlPfQOCArNYjaLW7lA4rKVAi6gNcLvplTwQtdjdqKk+ivtUBjSiAZxJaG+uUDisp9MtE5PLIYR8eSY54X3cE+8ZixowZWLp0KZYuXQqz2YyMjAz86le/Cqx0WlhYiF//+te45557YDKZsHjxYgDA559/jmnTpkGn0yE/Px8PP/ww2tvbA8etr6/HLbfcAp1Oh6KiIrz66qsxxUdCa3G4YXCdg1olIM2gRmaKBjpPs9JhJYU+f/k+lJc/PR72taIMA+aOGxB4/tdtJ+CWQvfzB6bqcNuVFyoArN1eAbsreCrJY98dFlOMr7zyCu6//37s2rULX331FRYvXoyCgoLAzZ7PPvssVqxYgSeffBIAcOLECdx44434zW9+g7Vr16KhoSGQzEpLSwEACxcuRE1NDT799FOIooiHH34Y9fX1McVHOmpxeKDxtEKj4pGf6rt3jWtTNqgk0S8TUTLIz8/HH/7wB3Ach+HDh+PQoUP4wx/+EEhE1157LX7+858H9l+0aBHuvvtuPProowCAoUOH4o9//COmT5+ONWvWoLKyEhs3bsSuXbswceJEAMDf//53XHbZZb1+bn1Vu9MDracVaj0PmPKAlhrA3qR0WEmhXyaiJTOLw77GXzILZPE1Q8Lue+mMkfumFnUnrCDf+c53gqakTJkyBc8991xg8u6lNb0PHDiAgwcPBnW3GGOQZRkVFRU4evQoVCoVJkyYEHh9xIgRsFgscYu5v7O5JOilNoiCAJgHQrJWQ7K1gGbpda1fJiK1KvKhsZ7at7sMBkPQ87a2Njz44IN4+OGHO+xbUFCAo0eP9lZo/ZbTbodJdkIUDDjhMqO+ohFaixrjJnT93v6uXyaiZLBz586g5zt27MDQoUMhCELI/cePH48jR46guDh0a2/EiBHweDzYs2dPoGtWXl6O5ubmuMbdn3nszQAAQaOHaMwAAMjOVgUjSh798qpZMqisrMSyZctQXl6O119/HS+99BIeeeSRsPs//vjj+OKLL7B06VLs378fx44dw3vvvYelS5cCAIYPH44bb7wRDz74IHbu3Ik9e/Zg0aJF0Ol0vXVKfd6tl5kxvsACs9kCrTHFu9FlA2SqhdUVSkQJ6p577oHdbsekSZOwZMkSPPLII4HL9KGMHj0aW7duxdGjRzFt2jSMGzcOK1asQF5eXmCf0tJS5OXlYfr06Zg3bx4WL16MrKys3jidfkEl2aFRCVBpjNDpUsDAwS0xMGoVdYm6ZglKFEW88MILWLNmTYfXTp06FfI9EydOxEcffRT2mDk5OXj//feDtv34xz/uVpzkIm7f3DJRB42oglvQQy21w2VvhUZnUTS0REctIkLioN3pwa6jZ1DZaAPUBogCB4/Key+R29HexbsJJSJC4qDd5UFNw3k0tDoAUQeO48CJ3kJyLjsloq5Q1ywBbdmyRekQSJScbhmi7IDA84DovbUiM80CHVcPkTm7eDehRERIHDg9ElSSHSqBA0TvlcgrBuUAqhpA8CgcXeKjrhkhceDwtYhUPAeoffPMVL4a326HcoElCUpEhMSB0yNDlBwQeA4QvYmIqbSQGIPbSctZdYUSESFx4HS7oZKd3haRLxF9VW3HropGVNSdUzi6xEeJiJA4cDsdAJhvsNo7RiT4umiyi2pXd4UGqwmJg6sKUyDXWMALIsB75wOqNDo4ATAXdc26QomIkDgQmQdQCYD6wiKUgtrbMmIeunzfFUW7Zk899ZT3xq+LHiNGjFAyJEJiI/mSjXCh+pDKl5SYxwkou7J7wlO8RXT55Zfj3//+d+C5SqV4SIREbe/JOqSdb0d2bjr89QxUWu9XsiR5Z+AL9LsdjuLfGZVKhZycnN75MMYAyd07n3UxQexYzjGMdevW4bHHHkNNTQ00Gk1g+9y5c5GSkoL/+Z//6akoSTecrGuC0+pAap4YSERq3xQPmTFvi4kSUViKf2eOHTuGvLw8aLVaTJkyBatWrUJBQUHIfZ1OJ5zOC/3tlpaW6D5McgOfPdedcGMz7eeAKrKCobfddhsefvhh/O///i9uu+02AN7VNz744INOZ9YTZUlu7++l6qIxIqNWhDnFCD3vATxOQG0I9/Z+T9ExosmTJ6OsrAwffvgh1qxZg4qKCkybNg2traHrt6xatQpmsznwyM/PD7lfMtPpdLjrrrsCK28AwD/+8Q8UFBRgxowZygVGwpJlBtnjT0QXWrGpBjVGDsxAYbrBm4hIWIq2iG666abA16NHj8bkyZMxaNAgvPnmm7j//vs77F9SUoJly5YFnre0tESXjATR2zrpbYIY1e4PPPAAJk6ciOrqagwYMABlZWVYuHBhUDF9kjhckgxB9i5yqRI1wS+qNICz9cJgNglJ8a7ZxSwWC4YNG4bjx0OvO6bRaILGTaLGcRF3kZQ0btw4jBkzBuvWrcP111+Pw4cP44MPPlA6LBKGW5IhyG5wAHhRG/QaE9SQGQPvcYL+jYSXUHdWt7W14cSJE8jNzVU6FMUtWrQIZWVlKC0txaxZs/pkN7SvcHlkqJgLAs+BU134R8kYw8flTdhV0QiHne6u7oyiiWj58uXYunUrTp06hS+++ALf//73IQgC7rzzTiXDSgh33XUXzpw5g7/97W+47777lA6HdMItMQiy2zvhVbiQiDiOg+xLTG4XzcDvjKKJ6MyZM7jzzjsxfPhw3H777UhPT8eOHTuQmZmpZFgJwWw24wc/+AGMRiPmzp2rdDikE5kpGlw7zIKRuaYOXX/el4g8NN+sU4qOEa1fv17Jj0941dXVuPvuu7s3LkZ6nMBz0PMSIApBLSIA4EV/IqIWUWcSarCaeDU1NWHLli3YsmUL/vznPysdDomE//L8JVdIOV9xNImKo3WKElECGjduHJqamvDb3/4Ww4cPVzoc0oWqRhuctY1I5Z1IV13SIvI9l1x0+b4zlIgSULh1y0hiqmtxwHHeChgY0i/tmvlucJSpRdQpSkSEdJPbI0NgLgi8usNgdZopBaJehJanZac70+cTEaPyC91C37+uudwuaJgMgeM6DFZfUZAJNJsAfULdspdw+ux3RxS9g4Y2G1XH6w7/98///SQduX3jP977iC65c98/eK1E1Yck0mdbRIIgwGKxoL6+HgCg1+tprlYUGGOw2Wyor6+HxWKBIAhKh5Sw/OM/vEoE+Ev+t/PeRMQkmuLRmT6biAAE6hz5kxGJnsVi6b16UUnKf48Qr+p4v9eBOhvsJ88j1ewGXf8Mr08nIo7jkJubi6ysLLjd1DSOliiK1BKKgOyrRcRfOvMeACeowQDv7HvGIi6Q19/06UTkJwgC/UGRHnPtsFRwDjPEFGOH11Sid8xIlmUqF9sJ+q4Q0k0GXgLUKkCj6/Cav2KjLDNAclEiCqPPXjUjpNeEWMHDT1SpIHMCJMYAmYYHwqFEREg37T9Vj6omG9xcx1scRIGDzIm+FhElonAoERHSDR5JxrHq8zjTZAfjQ7SIBB4Sr4LM4O2akZCow0pIN7glBoH56lWrtR1e14oCTEYDUjwuSkSdoERESDe4PN561QIX+vK9WSdifGEm0OKhrlknqGtGSDe4JBkCc4MPNb3Dz7+dWkRhUSIipBv8SwmFnGfmR4moS5SICOkGl8fbIhI4zruGWQgffduInRXn0U4reYRFiYiQbnBH0CJyw3vVTHZTiygcGqwmpBsGpeuRNygFgpOFbRHxvmJpHjeViw2HEhEh3aBRCdAIoVfw8ON8iUiiFlFY1DUjpDsYAzy+BCOELh7nT0Syh1pE4VCLiJBuOHm2GerGNph0IkxddM2oRRQeJSJCuqGyvgmaJjsGAjCFGazmVd6WkuyhRBROwnTNVq9eDY7j8OijjyodCiER8y+cyIuasEXPLCkGmLQqqOHpzdCSSkK0iHbv3o2//OUvGD16tNKhEBIVt6+7xas6zjPzm1CUDbSaAT1VZwxH8RZRW1sb7r77bvztb39Damqq0uEQEhV/mVhODHNXNXDh/iKZWkThKJ6IlixZgptvvhmzZs3qcl+n04mWlpagByFKurCCR+iBagA0xSMCinbN1q9fj71792L37t0R7b9q1SqsXLmyh6MiJHL+S/JCiJn3fvtrbfCcbkSa2Y3BvRVYklGsRVRVVYVHHnkEr776KrTa8P3ri5WUlMBqtQYeVVVVPRwlIZ3zd82EELWI/CROBbfEvFM8aOXckBRrEe3Zswf19fUYP358YJskSdi2bRv+9Kc/wel0dlh5Q6PRQKPppAlMSC+bMcQEgTdBqzeE3UclquGCfyUPT9gbH/szxRLRddddh0OHDgVtu/feezFixAg8/vjjtPwPSQoWNQCtCHTSIvIvKSQxX91qSkQdKJaIUlJSMGrUqKBtBoMB6enpHbYTkrCkzqd3AP6VPFQXlhSCvndiSyKKXzUjJFk5PRLKq8+h1moPO+EVAFQ8501EDFQuNoyEuKHRb8uWLUqHQEjEHC4Zx2vOI9NpR64q/H1E/pU8JNlNa5uFkVCJiJBk4q9XLXDotEWkFQXodTroZVrJIxxKRITEyFuv2lc4v5MbGjNTNMgckg20yNQ1C4PGiAiJkdsjQ2Aub73qrq6EBe6upkQUCiUiQmLk9rWIvPWqu7i/jfclKuqahUSJiJAYOf0toi66Zh5JxifHmrDndGNgtj4JRomIkBi5PRIE2QOe62RNMwACz6HNw8ElMSqgHwYNVhMSo2EZGkh5KV22iDiOAydQudjOUCIiJEYGleyd3sHxAN/5nxLnG8z2UCIKibpmhMTq4tU7wpSJDfC1mGglj9AoERESo1P1jai12tEmdT1BW/AX0KcWUUgxJaKTJ0/GOw5Cks7JukacOm9Dk7PrWtSBRRZpJY+QYkpExcXFmDlzJv7xj3/A4XDEOyZCkoK/TKwgdl3Yz6jXw6AWoGJUtzqUmBLR3r17MXr0aCxbtgw5OTl48MEHsWvXrnjHRkhCk1ze8R6+s8L5PlcPz8XogRak63o6quQUUyIaO3YsXnzxRdTU1GDt2rWora3F1VdfjVGjRuH5559HQ0NDvOMkJOH4u1md1asO8E8BoSkeIXVrsFqlUmHevHl466238Nvf/hbHjx/H8uXLkZ+fj3vuuQe1tbXxipOQhMMiqFcdQHPNOtWtRPTVV1/hoYceQm5uLp5//nksX74cJ06cwMcff4yamhrMmTMnXnESknBYYAWPrhPRvuo27KtqwplzzT0cVXKK6YbG559/HqWlpSgvL8fs2bOxbt06zJ49GzzvzWtFRUUoKytDYWFhPGMlJKGwCJYS8nPIKsAtw+2mFlEoMSWiNWvW4L777sPChQuRm5sbcp+srCz8/e9/71ZwhCSyKYVGCPUp0Gq7HoEWVCIkAMzjW1Koqxsg+5mYEtHHH3+MgoKCQAvIjzGGqqoqFBQUQK1WY8GCBXEJkpBElKHlAL260xU8/AS1BhIA2b+SRyelZfujmMaIhgwZgnPnznXY3tjYiKKiom4HRUhSCKzg0XVSEVUiAA4SA9UkCiGmFhELs1plW1tbxKu2EpLM7C4J5xqaoXM4kdFVUTQAKkGAFLSkELlYVIlo2bJlALxlDVasWAG9/sL6TJIkYefOnRg7dmxcAyQkEVntbhyvOQ8zbMiIoJulVnGQeRUk5vau9kqCRJWI9u3bB8DbIjp06BDU6gs/ALVajTFjxmD58uXxjZCQBOSWfNUZhQjKxALQqASo1VqoeYlaRCFElYg+/fRTAN6loV988UWYTKYeCYqQROf0+OpVq7iIBp7z0/TIH5oNtDVQIgohpjGi0tLSeMdBSFJxezzgmQc8L0Y0WA3gorurqWt2qYgT0bx581BWVgaTyYR58+Z1uu+GDRu6HRghiczj8s28j2QFD79AIqIW0aUivnxvNpvB+W7CMpvNnT4itWbNGowePRomkwkmkwlTpkzBxo0boz8LQnqZx+lNRJygAoSu/5/bXB58frIZB840UyIKIeIW0cXdsXh1zQYOHIjVq1dj6NChYIzhlVdewZw5c7Bv3z5cfvnlcfkMQnqCx1eLiO+kaP7FeI5DkxMQXBIkjxtd13TsX2IaI7Lb7WCMBS7fnz59Gu+88w5GjhyJ66+/PuLj3HLLLUHPn3nmGaxZswY7duwImYicTieczgs1f1taWmIJn5BuG5ahhpyTApUxsgs2osBD5rx/bh63kxLRJWK6s3rOnDlYt24dAKC5uRmTJk3Cc889hzlz5mDNmjUxBSJJEtavX4/29nZMmTIl5D6rVq0K6gLm5+fH9FmEdJdZZEjVq5FiMES0P88BMu9fUogK6F8q5gqN06ZNAwC8/fbbyMnJwenTp7Fu3Tr88Y9/jOpYhw4dgtFohEajwU9+8pNAyyqUkpISWK3WwKOqqiqW8AnpPv9qHBEOVHvXNvMWR6O61R3F1DWz2WxISUkBAHz00UeYN28eeJ7Hd77zHZw+fTqqYw0fPhz79++H1WrF22+/jQULFmDr1q0hk5FGo4FGE+EVCkJ6UFVDM7StThjMAvRd7w4A4FXUIgon5uL57777LqqqqrBp06bAuFB9fX3UNzmq1WoUFxdjwoQJWLVqFcaMGYMXX3wxlrAI6TXlZxpwvKENVlfkoz2cyt8ioppEl4opEa1YsQLLly9HYWEhJk+eHBjT+eijjzBu3LhuBSTLctCANCGJSPZ4r5qpIikT66PV6qBR8eBkSkSXiqlr9sMf/hBXX301amtrMWbMmMD26667Dt///vcjPk5JSQluuukmFBQUoLW1Fa+99hq2bNmCTZs2xRIWIb1G9nWvVOrIhwq+O2ogwKUCNLrQQUyJCABycnKQk5MTtG3SpElRHaO+vj5QZN9sNmP06NHYtGkTvvvd78YaFiG9wr90dDQtIiqgH15Miai9vR2rV6/G5s2bUV9fD1mWg16PdCVYKiVLkpEss8AKHmJUiYiWFAonpkS0aNEibN26FT/+8Y+Rm5sbmPpBSH/gkmQIzJtMomkR7T3TBrHaiswMETld796vxJSINm7ciA8++ABTp06NdzyEJDynR4ZKdoLnAJU68qVbW90c1E4PzC66GHOpmBJRamoq0tLS4h0LIUlBJwoYm6cHZzcCEc41Ay4sTc3ohsYOYrp8/+tf/xorVqyAzWaLdzyEJDy1ikeGFkg3aCIvAQJA5UtasiwBstRT4SWlmFpEzz33HE6cOIHs7GwUFhZCFMWg1/fu3RuX4AhJWJKvexXFskCC6FtSSPYtKcTT1Fe/mBLR3Llz4xwGIcmjqdUOT3M7NCIPUzQtIlEFDzjf2mYuIIKlqvuLmBLRk08+Ge84CEkaZ85b0dbQhjSDGqYoxohEnoeNFyH5W0QkIKYxIsBb/uO///u/UVJSgsbGRgDeLll1dXXcgiMkEXmcdgC+6oxRdK80Ig9epfGWl6VpHkFiahEdPHgQs2bNgtlsxqlTp/DAAw8gLS0NGzZsQGVlZaBWESF9kb9eNR9l12pYdgqGDcsBbI1ULvYSMbWIli1bhoULF+LYsWNBK7vOnj0b27Zti1twhCQityu6MrFB6O7qkGJKRLt378aDDz7YYfuAAQNQV1fX7aAISWQXWkSxJCKabxZKTIlIo9GErBd99OhRZGZmdjsoQhKZ7PaOEfFR3FUNeJep3nm6Bd/UtVDX7BIxJaJbb70VTz/9NNxub1bnOA6VlZV4/PHH8YMf/CCuARKSaGT/mmZRjhExxnDOztDq8NBg9SViSkTPPfcc2trakJmZCbvdjunTp6O4uBgpKSl45pln4h0jIQllVLYaxVlGWMzRVSNVCTwkTvTO3qdpHkFiumpmNpvx8ccfY/v27Thw4ADa2towfvx4zJo1K97xEZJwsrQAjBpAH9kKHn6iwEHmVGAAZI+LlhS6SNSJSJZllJWVYcOGDTh16hQ4jkNRURFycnLAGKOSIKTv85WJhSq6MSKR5yHx3qtmHjclootF1TVjjOHWW2/FokWLUF1djSuuuAKXX345Tp8+jYULF0ZVJpaQZCTLDFUNjWi0uSBFMb0DAHieA3h/AX0qBXKxqFpEZWVl2LZtGzZv3oyZM2cGvfbJJ59g7ty5WLduHe655564BklIonBJMo5U1sPkbMWkCdHPFeP8SwpRTaIgUbWIXn/9dfzyl7/skIQA4Nprr8UTTzyBV199NW7BEZJoHG4JKtkJgecgRFMm1kclqqHiOch0H1GQqBLRwYMHceONN4Z9/aabbsKBAwe6HRQhicrhlqGSXVDxXNRjRABw6/hBmFiYBnPk1UP6hagSUWNjI7Kzs8O+np2djaampm4HRUii8reIvImoO3dW0+X7i0WViCRJgkoVflhJEAR4PJ5uB0VIonK63OCZByqBB8ToW0Q01yy0qAarGWNYuHBh2PXnaYVW0te5HO0A4C3lEeVVMwDYc6YNutoWZGfrkR7v4JJYVIlowYIFXe5DV8xIX+ZyeOu086IW4KOfmHDezpBid8NC/7SDRJWISktLeyoOQpLC4FQVPFlGiAZzTO8XfDP2GY0RBYm5QmM8rFq1ChMnTkRKSgqysrIwd+5clJeXKxkSIZ1KVcvINGpgMaXE9H5B5R0jkt0ugLF4hpbUFE1EW7duxZIlS7Bjxw58/PHHcLvduP7669He3q5kWISEF5jeEVvhe38NI5nJtKTQRWKa9BovH374YdDzsrIyZGVlYc+ePbjmmmsUioqQ8GoazkNrc8GQpkEMF++h8i2y6F1SyAUIiv4JJoyE+i5YrVYACLuKrNPpDLoyF6o4GyE96fCpWhjPtqI4V41YSgCKKhVcnACJwXcvkT7OESYnRbtmF5NlGY8++iimTp2KUaNGhdxn1apVMJvNgUd+fn4vR0n6O8nlHTZQaWNLIKLAg/EifLVA4hhZckuYRLRkyRJ8/fXXWL9+fdh9SkpKYLVaA4+qqqpejJD0d4wxyL6lhDQ6Y0zHGD3AjKnDclCcZaS7qy+SEF2zpUuX4v3338e2bdswcODAsPtpNJqwN1MS0tOcHhmC5E9EsV014/1TQ5ytlIguomgiYozhZz/7Gd555x1s2bIFRUVFSoZDSKdsLgmi7ICK5yBoujG2E5jmQV0zP0UT0ZIlS/Daa6/hvffeQ0pKSmApIrPZDJ0uhnk8hPQgm8sDUXJAFDhAjK5MrJ/V5sbJ6nYY7a0YSi2iAEXHiNasWQOr1YoZM2YgNzc38HjjjTeUDIuQkOxON1SyE2KsE14BSIyhtk1Gs81NXbOLKN41IyRZZOsZUrMM3nGeGBORWuWtWy3JDMzjAFV490qIwWpCkoFJ8HhX7xC1AB9b6Xu1wEPi1GAAPC4nxPiGmLQS5vI9IQnP7Z15DzH2gWpR4CD5iqP5l64mlIgIiVhtw3k02lxwcrHNMwO8qyJzvsqOlIguoERESIS+PlWL8rpWNLq6tyIZ71uqWqJEFECJiJAIue1tAAC1Ibqlpi8liBpwACS3PQ5R9Q00WE1IBBhjkBzeSdaaGIui+d00ZhAEMQ3d6OH1OZSICImA3S1B5W4HB0DXzUSk0ugAcICH7iPyo64ZIRFoc3iglmwQBQ6CNrZ5ZgH+ovsS1a32oxYRIRFodXogSjaIIg+oY5t573ekwQHhbCvSTCymmkZ9ESUiQiLQbndClO1QCyKg6V4iOmcHxHYXdKIdkOWYVgPpaygRERKBQSmAO8voXWC0Gzc0AoBK7bt8L8ve7hlPE7wpERESATPv8E7v0JoArnszxDRqEQ5OgCQzwOOMed5aX0JtQkIi4fTVR9d07x4iAFALAiRefVHdakKJiJAIHK88gyabC5I6DolI5Z34GmgREUpEhHTF6ZHw9YlKfFvXCikeLaKLSoFQIvKiRERIF6w2NzSeVogCB7Ux9FJX0RAFzts1kxndS+RDg9WEdKHZ7oZGaoNWFABt9+6qBoBcsw45o/IhNDqoReRDiYiQLjS2OaHxtEKrUcUlEQk8B2j08E7zoBn4AHXNCOlSU1MjeCZBrxXjctUMAKDyzXilGfgAKBER0qW2Ru/qMjpTRswlYi+1p8aO8rOtcDra43K8ZEeJiJBOONwSPG0NAABDWnbcjnvKytDY7oLTbovbMZMZjRER0gm1wOOmIhU8ohHqlKy4HVfwTfPwOCkRAZSICOkUz3OwsBbv9A5DRtyO618p1uOiRARQ14yQzjEGtHu7ZtDHLxGpNN6VYiUnDVYDlIgICcvhlvDh7iNoaG4B44W4tohEjXeiq0wtIgAKJ6Jt27bhlltuQV5eHjiOw7vvvqtkOIQEOV7fhnPVJ1HTbAeXkhu3K2YAoNJ6u2aSxwNI7rgdN1kpmoja29sxZswYvPzyy0qGQUgHjDEcONMMo6seGUYNYMqL6/HVai0YOG9NIrqXSNnB6ptuugk33XSTkiEQEtLhmhbUtzgx3l2HrHQNYB4Y1+NfWZgOvjYPgsfuu7s6TjdKJqmkumrmdDrhdF6Ym9PS0qJgNCSZMcZQ1+LA+TYXRg24MG3jREMbKhttOFhlhcbTgmKDE6LKAFgGxfXz1SoeUBu8SYhaRMmViFatWoWVK1cqHQZJYrLMcKS2BV+dakSTzQ21ig9KRAfPNOPUOe8A8jjdWeRptN7WkNgDi5D5j0nzzZIrEZWUlGDZsmWB5y0tLcjPz1cwIpJMzrU58fHhOrTXn4LFcQZpvARz5kBIroGBGwzzzDqkaEQUZegxuGILOBsHZF0W91isdjcqz7pgaGvDYGoRJVci0mg00Gg0SodBktDx+lZ8crAC+Wc/QbGrGgNSdchO0UKQqoFdB4CCq4ABEzB5cLr3DWePALbzgCACWSPjHo8kM1RYJeTaXZSIkGSJiJBYNLa7sGnfCVxe9x6yRAeGDEqHJvdyb9H68ycAWyNw4hOgZi8waKp3ZvyxTd4350/ukW6ZVuTh5nXwyAySsx3xuzEgOSmaiNra2nD8+PHA84qKCuzfvx9paWkoKChQMDLSl6SpZczlP4dL78aggfngR99+4ebEwdcCZw8BFdsAezPw7QcX3mjKBQqm9EhMOlGAR+W9qdHlaEV/X8dD0UT01VdfYebMmYHn/vGfBQsWoKysTKGoSJ8ieYCv/4kBqlYgLxMYcwegv6jcK88DuWOAzMuAM7uBhm+9K2ukFwNF1wBCz/yJcBwHwbdQo9tGiUjRRDRjxgwwxpQMgfRRbknGtvKzmOraDm1zJaBSA6PnByehi6nUQOFU76OXCLoUb6z2tl77zERFc81In7T75Hm0HvoXjn69G4zjgVE/AFJylA4riFrrnfjqdrQqHInyKBGRPqex3YXag/9Gdts3GGDRgxs5B0gtVDqsDtR6b4tIdtoAWVY4GmXRVTPS5xza9QnymvcgVS8ibez3gKwRSocU0rSRg6BqTIeKA+C2Ab4xo/6IWkSkT6k/eQDqk5vBARgw9npwAycoHVJYWrUIla9AGtz9uxwIJSLSd7TUoGHnWwAY1AUTkDJihtIRdU3tHSeCq38PWFMiIn2DvRnNu15Hi82BFn0BhkydB3Cc0lF1qtnmwqFzMirOtQPO/j1gTYmIJD+PEzj0FgxwIDNnIHRj58GsT/ypQJLMcLKFx7k2J+Do35UkaLCaJDfGgPKNQPs5iLoUFF+9AMVxWI21Nxi1KjgFIzwyg9vWDFHpgBRELSKS3Gr2gdUfATgeGDUvLktC9xaNSgDTeC/hO9qaFI5GWZSISPJqrQOOb4bV5san7pGoZulKRxQ1tcECAHC1W5UNRGGUiEhy8jiBw++CyW5848rCfjYMx+uT78qTxmgBALjbrd5uZj9FiYgkH8aAox8C9iY0Sjrs1k2FqOJx5aBUpSOLms6fiFzOfn0vESUiknzOfg2cPQIZHD7lp0AStBg90AKDJvmuvRj1OrgEPTwy85Yh6aeS7ydH+jdbI3DUW7TshGEcqqxp0KkFTCoKM6s+wV0xwAzhimEQW6q8FSHNA5QOSRHUIiLJQ/IAR94DJDfshgH4uL0IAHDVkHRoxeSscagVBYgpmd4ntvPKBqMgSkQkOTDmLd/aWgeIWhw0ToPTA2SbtBiVlzyX7EPS+6729eNERF0zkhzOfAXUHvRO2xg5B5NSCyEam1GYbgDPJ/ZUjq7sP89DXduCPP4sLEoHoxBqEZHEV3cIOP5vMDB4imYAaYPBcRzGF6QizaBWOrpuq3bpYbW7Ybee83Y/+yFKRCSxVe8Bvv0ATo+Er9yD8W7DAEhy37rfxmxOh5vXweZwAW1nlQ5HEdQ1IwlJdtrQcvhD2Cr3o8XuxlG+GCdSx4FvdqCq0YbCDIPSIcZNjkWHo5pMtDmrgZaafnnljBIRSSweFyoPb0fVgU/Bu20AOFRaJqImZQwGpOkxfVgmsk09sPyzgnLNWuxVZ8FurYSr+QzU+ROVDqnXUSIiCYG5bOCq9wDVXyGtrQ3Vbhs8Ggvsg69H8YBizEzTIyulbyUgP4NGBc40AMz6FdrOViBtFEv4WkrxRomIKMrjaMfXX34IU9PXGJzqHXg2WDIxbMgNSC0aD17VP35FLblFkM6IsDY3Iq21zru4Yz/SP37KJPEwhpaKr3B8x/tw2tvhAJCVNRTGodPAZY5AOt+/rqMMzrHghDEfPFcDnD9GiYiQHtdah5pd7+JM5QlIMoOkTUPh5FtgHDKq33VJ/IrSDSicNg3Ct+9775caNBXgk/Nu8VgkRCJ6+eWX8fvf/x51dXUYM2YMXnrpJUyaNEnpsBKStd2BMxXfYKRQDa6lGnBYUd/qQG0boE/PRVZeIczZRYApD1Druzwek9ywtjtQ0ypDLfIozkoJvFZndcCkU0GvjtOviccJ1/EtOH1wG861OiHxIlwFUzHp6u/CqEv80q49iec5IOsy4OSn3vrVdYeAvLFKh9VrFE9Eb7zxBpYtW4b/+q//wuTJk/HCCy/ghhtuQHl5ObKyspQOLyHYnC6cPvktzp04ADSUQyU70JpngknrLS7aanPC1u6Erb0F5yrLoVcLSDeoYcnIhSGzAJzGCHAC4HFAcrWjrcWKFmsz2lqtsLW3wSV5F/czppiAoYMBfQaYIRP/PmxHIyzIsJhQmKFHYboBOSZt9Hcyu9qB2gNA1S4cO12HFocH5w1DkDVuNq4aWpD0d0bHjaAC8iej8eBGuPf/C9mWgvBLZPcxHFN48fnJkydj4sSJ+NOf/gQAkGUZ+fn5+NnPfoYnnnii0/e2tLTAbDbDarXCZDL1Rrg9jzHAbUOb9TxOn66Ate4UpHPHoJIcAAAOgN5oQt6wscgouBzQp8PqkFFbX4+62kq01Z+G3nEWOnczAEAUOEwoSAXn6/Lsr2qG3S0FfSTPea/cWHQiBqZ6W1EeWcbX1VbY3TKcqhTYxDS4BR0g6pBuTkFRZgoGpBoBjgfjOHC84C3XyvEAOMguGxxtTbA1VMLoOgut4P38Oo8e2zEWUyZOQp5F1yvf0mRy+lwrjm78M1JcDcjKSEP6qFkwDRgBTpOSlN3WSP9GFW0RuVwu7NmzByUlJYFtPM9j1qxZ+PLLLzvs73Q64XQ6A89bWiJf+aC29gwqt70G4OK8y8AYwIFhkO+/PcDQ4nDjSE1L8L4X5ev8VB0G+P6I2pxufF1tDdqXYyzwbIBFi3zfH7fN5cHBM80djueXa9ZikEUEZAmc04OWais4eH9IWp0BKfkjkTN0PIxZQ4CLBnPNOsCcmoYRw0fA4ZZwoqENJ6sb0FR3Cjl8M7h8MyBLgKjFeVcLzrlFWCwWZKSmITsjHdmpKRDh8a4kYTsPtDdA1d6Asbp6uGxWNNtcaLbVwmpze+vmNAFSsw5I856Xwy3h4Jlm8BwXOG9ZvvA9GJSuR97AImDgRGRlXoZbwUEU+tdgdKQK0o2oGHMb2vevx9lzjTi75U3wHMCrNJAFNYbnpcJi8P7unW114NS5dnj/PXU0LDsFqXoR4Dg0tDlxsqE97OcOyTQiw+i9anm+3dVptcuiDAOyUjTAFbcD2vg0ABRNROfOnYMkScjOzg7anp2djW+//bbD/qtWrcLKlStj+iyPyw25rT7kawwA09oBle8/tMMNZuskyelcgMPl/drpAbtkKZigFOOUAJd3C+eWwFzhq/AxNwNk7x+ozmiCPjsD+vSByC68DKbswUHJJxytKODyPDMuzzNDkofA5vIA2gvrQ0wZ6IFOFAItpAs03sX+Lrlao3bbkdVWjyzbOchOG6ytrWhoboFerwIM3qRpb3fi/LkG8EwGBxlgDB5eA4/KAC4lB4Yhw5E3bDAA75wiSkHhcRyH6VcU4WT2Q6g8+BnY2a+hczZCdjsBtxOcnQPgSyhtDsht4ZMLMzoA5puL1+qE3NpJKV2dDYBvnK69i321BoDTAkyO6tw6o/gYUTRKSkqwbNmywPOWlhbk5+dH9N6MzGzwM+8HgOA/Qt/XKToR8FX4U3sYimyuoNd9T8BxgEGtCvxxqz0yhtg9HY7r/1qnFgCNd19RkjF0jKfDPv5ja0UB0GkBtRE8L2BMRGcWnsBzSNEGL1IT9cCzqANSBwGpg8ADSPU9LmaWGaZe4YHEWOB/s0bkwyQ80hWO4zAk24Ih370Fsvw9tDkccLY2Qfa4oNFygUxucLoxyB5qkqz3H59OLwK+Ok16l4QCmzvsZ+r1IqD27qvrcl8VoFZdWKU2DhRNRBkZGRAEAWfPBk/0O3v2LHJycjrsr9FooNHEdnVFp9djwODLItpXC6Djp4emBhDpkLoKQEaSl84JReA5mPX9eVWunsPzHEx6HaDvOJ5m9D0iYfA9IqH3PXqToq1ktVqNCRMmYPPmzYFtsixj8+bNmDJlioKREUJ6k+Jds2XLlmHBggW48sorMWnSJLzwwgtob2/Hvffeq3RohJBeongimj9/PhoaGrBixQrU1dVh7Nix+PDDDzsMYBNC+i7F7yPqjj55HxEhfUikf6N0JZUQojhKRIQQxVEiIoQoTvHB6u7wD29FM9WDENJ7/H+bXQ1FJ3Uiam1tBYCI764mhCijtbUVZnP4u3mT+qqZLMuoqalBSkpKUk0l8E9Nqaqq6jNX+/riOQF0Xt3FGENrayvy8vLAdzJXMqlbRDzPY+DAgUqHETOTydSnfrmBvnlOAJ1Xd3TWEvKjwWpCiOIoERFCFEeJSAEajQZPPvlkzJUEElFfPCeAzqu3JPVgNSGkb6AWESFEcZSICCGKo0RECFEcJSJCiOIoEfWCZ555BldddRX0ej0sFktE72GMYcWKFcjNzYVOp8OsWbNw7Nixng00So2Njbj77rthMplgsVhw//33o62tk9UfAMyYMQMcxwU9fvKTn/RSxKG9/PLLKCwshFarxeTJk7Fr165O93/rrbcwYsQIaLVaXHHFFfjXv/7VS5FGJ5rzKisr6/Bz0Wq1vRcsIz1uxYoV7Pnnn2fLli1jZrM5ovesXr2amc1m9u6777IDBw6wW2+9lRUVFTG73d6zwUbhxhtvZGPGjGE7duxgn332GSsuLmZ33nlnp++ZPn06e+CBB1htbW3gYbVaeynijtavX8/UajVbu3YtO3z4MHvggQeYxWJhZ8+eDbn/9u3bmSAI7He/+x07cuQI+4//+A8miiI7dOhQL0feuWjPq7S0lJlMpqCfS11dXa/FS4moF5WWlkaUiGRZZjk5Oez3v/99YFtzczPTaDTs9ddf78EII3fkyBEGgO3evTuwbePGjYzjOFZdXR32fdOnT2ePPPJIL0QYmUmTJrElS5YEnkuSxPLy8tiqVatC7n/77bezm2++OWjb5MmT2YMPPtijcUYr2vOK9Hezp1DXLAFVVFSgrq4Os2bNCmwzm82YPHlyyBVwlfDll1/CYrHgyiuvDGybNWsWeJ7Hzp07O33vq6++ioyMDIwaNQolJSWw2cIvOtmT/CsNX/x97mylYcB73hfvDwA33HBDwvxcgNjOCwDa2towaNAg5OfnY86cOTh8+HBvhAsgySe99lV1dXUAEHIFXP9rSqurq0NWVvCKbiqVCmlpaZ3GeNddd2HQoEHIy8vDwYMH8fjjj6O8vBwbNmzo6ZA7iHalYcB73on8cwFiO6/hw4dj7dq1GD16NKxWK5599llcddVVOHz4cK9MLKcWUYyeeOKJDoN7lz7C/dATWU+f1+LFi3HDDTfgiiuuwN13341169bhnXfewYkTJ+J4FiRaU6ZMwT333IOxY8di+vTp2LBhAzIzM/GXv/ylVz6fWkQx+vnPf46FCxd2us/gwYNjOrZ/lduzZ88iN/fCWvRnz57F2LFjYzpmpCI9r5ycHNTX1wdt93g8aGxsDLlKbziTJ08GABw/fhxDhgyJOt7uiHalYcD7s4lmfyXEcl6XEkUR48aNw/Hjx3sixA4oEcUoMzMTmZmZPXLsoqIi5OTkYPPmzYHE09LSgp07d+KnP/1pj3ymX6TnNWXKFDQ3N2PPnj2YMGECAOCTTz6BLMuB5BKJ/fv3A0BQwu0tF680PHfuXAAXVhpeunRpyPdMmTIFmzdvxqOPPhrY9vHHHyfUysSxnNelJEnCoUOHMHv27B6M9CKKDZP3I6dPn2b79u1jK1euZEajke3bt4/t27ePtba2BvYZPnw427BhQ+D56tWrmcViYe+99x47ePAgmzNnTkJevh83bhzbuXMn+/zzz9nQoUODLt+fOXOGDR8+nO3cuZMxxtjx48fZ008/zb766itWUVHB3nvvPTZ48GB2zTXXKHUKbP369Uyj0bCysjJ25MgRtnjxYmaxWAKXrn/84x+zJ554IrD/9u3bmUqlYs8++yz75ptv2JNPPpmwl++jOa+VK1eyTZs2sRMnTrA9e/awO+64g2m1Wnb48OFeiZcSUS9YsGABA9Dh8emnnwb2AcBKS0sDz2VZZr/61a9YdnY202g07LrrrmPl5eW9H3wnzp8/z+68805mNBqZyWRi9957b1ByraioCDrPyspKds0117C0tDSm0WhYcXEx+8UvfqHofUSMMfbSSy+xgoICplar2aRJk9iOHTsCr02fPp0tWLAgaP8333yTDRs2jKnVanb55ZezDz74oJcjjkw05/Xoo48G9s3OzmazZ89me/fu7bVYqQwIIURxdNWMEKI4SkSEEMVRIiKEKI4SESFEcZSICCGKo0RECFEcJSJCiOIoERFCFEeJiCSVsrKyoHK7Tz31VNBE4IULFwbmV5HkQYmIhLRw4cKw9aSXLFkCjuOCZun3RAIoLCzECy+8ELRt/vz5OHr0aNj3vPjiiygrKws8nzFjRtAEVZKYKBGRsPLz87F+/XrY7fbANofDgddeew0FBQWKxKTT6ToUZLuY2WyOeIECkjgoEZGwxo8fj/z8/KDqiRs2bEBBQQHGjRvXrWOHaqnMnTs30MqaMWMGTp8+jcceeyxQkA3o2DW71MUts4ULF2Lr1q148cUXA8eoqKhAcXExnn322aD37d+/HxzH9Vr9HRKMEhHp1H333YfS0tLA87Vr1+Lee+/t8c/dsGEDBg4ciKeffhq1tbWora2N+hgvvvgipkyZggceeCBwjIKCgg7nBAClpaW45pprUFxcHK9TIFGgREQ69aMf/Qiff/45Tp8+jdOnT2P79u340Y9+1OOfm5aWBkEQkJKSgpycnJgqIJrNZqjVauj1+sAxBEHAwoULUV5eHljny+1247XXXsN9990X79MgEaIKjaRTmZmZuPnmm1FWVgbGGG6++WZkZGQoHVa35OXl4eabb8batWsxadIk/N///R+cTiduu+02pUPrt6hFRLp03333oaysDK+88krcWg08z+PSUlhutzsux47EokWLAgPxpaWlmD9/PvR6fa99PglGiYh06cYbb4TL5YLb7cYNN9wQl2NmZmYGjftIkoSvv/46aB+1Wg1Jkrr1OeGOMXv2bBgMBqxZswYffvghdcsURl0z0iVBEPDNN98Evg7HarUGiuH7paenIz8/v8O+1157LZYtW4YPPvgAQ4YMwfPPP4/m5uagfQoLC7Ft2zbccccd0Gg0MXUJCwsLsXPnTpw6dQpGoxFpaWngeT4wVlRSUoKhQ4cmVPH7/ohaRCQiJpMJJpOp0322bNmCcePGBT1WrlwZct/77rsPCxYswD333IPp06dj8ODBmDlzZtA+Tz/9NE6dOoUhQ4bEvGLK8uXLIQgCRo4ciczMTFRWVgZeu//+++FyuXrlKiDpHNWsJv3WZ599huuuuw5VVVUdVkUlvYsSEel3nE4nGhoasGDBAuTk5ODVV19VOqR+j7pmpN95/fXXMWjQIDQ3N+N3v/ud0uEQUIuIEJIAqEVECFEcJSJCiOIoERFCFEeJiBCiOEpEhBDFUSIihCiOEhEhRHGUiAghivt/7aMXprusM0wAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:55:59.552002Z", + "iopub.status.busy": "2024-03-26T10:55:59.551749Z", + "iopub.status.idle": "2024-03-26T10:55:59.920014Z", + "shell.execute_reply": "2024-03-26T10:55:59.919141Z" + }, + "papermill": { + "duration": 0.390549, + "end_time": "2024-03-26T10:55:59.922017", + "exception": false, + "start_time": "2024-03-26T10:55:59.531468", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEoCAYAAAD8JfzQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA7NElEQVR4nO3deXxU5b348c+ZM1smy2QhKwQSCKDsoIUL1J2qYF3vrVapgnsV21KuV+X+Wlq8tWirXmnrxS4XqLdQqi3VV2uVUlSwqIjIpihLgBAgIYGQZTL7Oef3xyQDIftkyMnyfb9e83oxM2fOfCdhvnme5zzP91EMwzAQQggTWcwOQAghJBEJIUwniUgIYTpJREII00kiEkKYThKREMJ0koiEEKaTRCSEMJ0kIiGE6SQRCSFMZzU7gGPHjvH444/z5ptv4vV6KSoqYsWKFVx88cXtvlbXdY4fP05ycjKKonRDtEKIzjAMg7q6OvLy8rBYWm/3mJqITp8+zfTp07niiit48803yczMZP/+/aSlpXXo9cePHyc/P/88RymE6KrS0lIGDRrU6vOKmYten3jiCTZv3sx7770X0+trampITU2ltLSUlJSUOEcnhOiq2tpa8vPzqa6uxu12t3qcqYlo1KhRXHPNNRw9epSNGzcycOBAHn74Ye6///4Wjw8EAgQCgej9xg9ZU1MjiUiIHqi2tha3293ud9TUweqDBw+ybNkyhg8fzrp163jooYf49re/zW9/+9sWj1+yZAlutzt6k26ZEH2DqS0iu93OxRdfzPvvvx997Nvf/jZbt27lgw8+aHa8tIiE6F16RYsoNzeXUaNGNXnswgsv5MiRIy0e73A4SElJaXITQvR+pl41mz59Onv37m3y2L59+xgyZEjc3sMwDMLhMJqmxe2cojlVVbFarTKNQsTE1ET03e9+l2nTpvHjH/+YW2+9lY8++ohf/epX/OpXv4rL+YPBIGVlZXi93ricT7TN5XKRm5uL3W43OxTRy5g6RgTw17/+lYULF7J//34KCwtZsGBBq1fNztVW/1PXdfbv34+qqmRmZmK32+Wv9XliGAbBYJDKyko0TWP48OFtTl4T/UdHx4hMn1n91a9+la9+9atxP28wGETXdfLz83G5XHE/v2gqISEBm81GSUkJwWAQp9Npdkim0HWDt7+oIKwbXHVhFjZVEnJHmJ6Izjf5y9x95GcNxZUedh+rASAv1cm4QanmBtRLyP8cIeKoKCuJ8fmRGcRHT/tMjqb3kEQkRBwpikJRup3kQDnl1ZKIOqrPd82E6G5ZpW8y+sQnlPvGoumFqBa5SNIeaRH1QJdffjnz5883OwzRSf6QxoZPPqfy8OeoCmTV7aHOFzQ7rF5BEpEQcVLrC3H8yEEq6vw4bCoWQ6OuqtzssHqFfpmIgmG91VtY0zt8bKgDx3bW3Llz2bhxI0uXLkVRFBRFYdCgQSxbtqzJcdu3b8disVBSUgLA888/z9ixY0lMTCQ/P5+HH34Yj8fT5DX//Oc/ueSSS0hISCA/P59vf/vb1NfXdzpG0bJaf4jE4EnsVpX0RDuZyQ4SwtVmh9Ur9MsxohffOdDqc4UDErlp4sDo/V9tKiaktTznc1BaAl+7+EwFgOWbD+ELNl1K8t2vjOhUbEuXLmXfvn2MGTOGJ598EoDnnnuO1atX89BDD0WPW7VqFdOnT48uh7FYLPzsZz+jsLCQgwcP8vDDD/PYY4/xP//zPwAUFxdz7bXX8qMf/Yjly5dTWVnJI488wiOPPMKKFSs6FaNoWa0/jCNch8NqIT+tYe6a4mn7RQLopy2insztdmO323G5XOTk5JCTk8Ps2bPZvHlzdDGwruusWbOG2bNnR183f/58rrjiCgoKCrjyyiv50Y9+xCuvvBJ9fsmSJcyePZv58+czfPhwpk2bxs9+9jNefvll/H5/t3/Ovqg+EMYZrsNutUBKXuRB32lzg+ol+mWLaN4VRa0+d+4FjgcuHdbqseeuGLlnemFXwmrVhAkTuPDCC1m9ejVPPPEEGzdupKKigq997WvRY/7xj3+wZMkSvvjiC2prawmHw/j9frxeLy6Xi507d7Jr1y5WrVoVfY1hGOi6zqFDh7jwwgvPS+z9iTeo4dI82FQV3IPQao6heWuRlXft65ctIrvV0urNes6U/LaOPXf6fkvHxMvs2bNZvXo1AKtXr+baa68lIyMDgMOHD/PVr36VcePG8ac//Ylt27bx4osvApGlLgAej4cHH3yQHTt2RG87d+5k//79DBvWerIVHRfw+bDqAWyqQnHQzUeHqvjs8HGzw+oV+mWLqKez2+3NypbccccdfO9732Pbtm388Y9/5KWXXoo+t23bNnRd57nnnosuszi7WwYwadIk9uzZQ1FR661B0TVhXzUAqsOFLWkAAHqgzsSIeo9+2SLq6QoKCtiyZQuHDx/m5MmT6LpOQUEB06ZN495770XTNG644Ybo8UVFRYRCIX7+859z8OBB/u///q9JogJ4/PHHef/993nkkUfYsWMH+/fv5/XXX+eRRx7p7o/XZ91woZtJg1Nxu1NxJiVHHgx6QZdaWO2RRNQDPfroo6iqyqhRo8jMzIwOUs+ePZudO3dy8803k5CQED1+/PjxPP/88zzzzDOMGTOGVatWsWTJkibnHDduHBs3bmTfvn1ccsklTJw4kUWLFpGXl9etn60vs2o+HFYVqyOJhIRkDBRCmoEhraJ2mV6PqCvaqnXi9/s5dOgQhYWF/bYkRXfr9z/zox/D/vWQOZLgBTfxwe8WY9fqmXTzd3Ck98+NHnpFzWoh+or6QJiP9h3lSJUX7InYVIWwNTKXKOSXSaPtkUQkRBzUB8McrzxFZZ0fbAmRWfG2SKsw6JNE1B65aiZEHARCOjbdj2qxgC0RgMz0VBKUCmxGoJ1XC0lEQsRBIKxh1XxYVQVskQsJY4fkgPU4qGGTo+v5pGsmRBz4G1pEVosC9oZ1ZtaGAfuQLKFpjyQiIeIgENaxaf5IETRbJBEZVieaYRAKyHZW7ZFEJEQcBEIhrHog0iJqSEQfH/Px0aEqDpWfNDm6nk8SkRBxEAr4AaNhsDoyRqQ2dNH0oNSubo8MVgsRB9MKktGPp2JRbWBRAbA6EggARlC6Zu2RRCREHNiMMFhVsJ+ZUa7aIy0jIyyX79tjatfshz/8YbQcauPtggsuMDOkHqGnFc/vafH0SFpDslHPVB+yNiQlIxyA3ruSqluY3iIaPXo0//jHP6L3rVbTQ+oTgsEgdruU5OounxwsJ/1UPdm5GTQuR7Y6I//SNS2yAl+V/9utMX2w2mq1Rkui5uTkMGDAgPP3ZoYB4WD33zrx17Cl4vnFxcXce++9FBYWkpCQwMiRI1m6dGmz191000089dRT5OXlMXLkSADef/99JkyYgNPp5OKLL+a1115DURR27NgRfe2nn37KzJkzSUpKIjs7mzvvvJOTJ0+2Gs/hw4e7/Kvoaw6Wn6asxk9QsUUfszcs8dAN40yLSbTI9BS9f/9+8vLycDqdTJ06lSVLljB48OAWjw0EAgQCZ36htbW1nXszLQTvPdeVcGNzyb+DtWOtk5aK56elpTFo0CBeffVVMjIyeP/993nggQfIzc3l1ltvjb52w4YNpKSksH79eiDy87n++uuZNWsWq1evpqSkpFkXq7q6miuvvJL77ruP//7v/8bn8/H4449z66238vbbb7cYT2ZmZhx+KH2LFor8v7SeNUaU5LThTk7CZQlDOAD2RLPC6/FMTURTpkxh5cqVjBw5krKyMhYvXswll1zCp59+SnJycrPjlyxZwuLFi02ItPucWzy/0dmfu7CwkA8++IBXXnmlSSJKTEzkN7/5TbRL9tJLL6EoCr/+9a9xOp2MGjWKY8eOcf/990df84tf/IKJEyfy4x//OPrY8uXLyc/PZ9++fYwYMaLFeMQZum6ghxsTkSP6eFqinbRBAyBQF0lEolWmJqKZM2dG/z1u3DimTJnCkCFDeOWVV7j33nubHb9w4UIWLFgQvV9bW0t+fifqvKi2SOuku6m29o9px4svvsjy5cs5cuQIPp+PYDDIhAkTmhwzduzYJuNCe/fuZdy4cU1qA02ePLnJa3bu3Mk777xDUlJSs/csLi5mxIjObYfUHwU1HVWP1Aa32hxNn7Q6IolIumZtMr1rdrbU1FRGjBjBgQMt7zvmcDhwOBwtPtchitLhLlJPsmbNGh599FGee+45pk6dSnJyMj/96U/ZsmVLk+MSEzvf9Pd4PFx//fU888wzzZ7Lzc2NOeb+JKTpqHoIBbDYmhaEM1Q7umFgCQdQWn65oIclIo/HQ3FxMXfeeafZoZjq3OL5mzdvZtq0aTz88MPRx4qLi9s9z8iRI/nd735HIBCIJvCtW7c2OWbSpEn86U9/oqCgoNUrli0V8xdnBMM6ViOIalFQrGf+UBqGwfq9p0n2VTGhyEdCG+fo70y9avboo4+yceNGDh8+zPvvv8/NN9+MqqrcfvvtZoZlunOL5w8fPpyPP/6YdevWsW/fPr7//e83SygtueOOO9B1nQceeIDPP/+cdevW8eyzzwKgNGzKNm/ePKqqqrj99tvZunUrxcXFrFu3jrvvvjuafFoq5i/OCGkGqh6KLHhVzyQiRVHQGxJTKCgr8NtiaiI6evQot99+OyNHjuTWW28lIyODDz/8sN9flTm3eP4111zDLbfcwm233caUKVM4depUk9ZRa1JSUvjLX/7Cjh07mDBhAv/v//0/Fi1aBBAdN8rLy2Pz5s1omsbVV1/N2LFjmT9/PqmpqdGtiVor5i8iMpMdXDkilVG5Kc26/paGRBSW9WZtkuL5/cyqVau4++67qampabITSDz065/59t9BdSmMvhmyzqwO+MdfVpFYsZ1hF11F1oSZbZygb+po8fweNUYk4u/ll19m6NChDBw4kJ07d0bnCMU7CfV7jZfnz7lCqjQUR9OkOFqbJBH1ceXl5SxatIjy8nJyc3P52te+xlNPPWV2WH1KaZWXQFkVaZYAGdamV3Ubu2ZaUC7ft0USUR/32GOP8dhjj5kdRp9WXuvHf6oGEg0y1HMSUcMER11aRG2SRCREF4XCOqoRRLXYmw1Wp6ckY3PZcFpk+kNb+nwi6sVj8b1Of/1ZB0NBHIaOqjS9fA8wdnAmVKeAy/T15T1an/3p2GyRQUOvV6rjdZfGn3Xjz76/CDWM/0TmEZ0zc79x8FoLdXNUvUufbRGpqkpqaioVFRUAuFyu6CQ+EV+GYeD1eqmoqCA1NRVVVc0OqVs1jv9YrDawnPO33RJJRIYmSzza0mcTERBdLd6YjMT5lZqa2i9X6IeDjYmo+TrIneVefAdPkeYOMbK7A+tF+nQiUhSF3NxcsrKyCIWkaXw+2Wy2ftcSaqQ31CKynLvyHlBUOwZEVt8bRmThtWimTyeiRqqq9tsviTj/rhyRhuJ3Y0tuXkrFaouMGem6LuVi2yA/FSG6KNGigd0Kjuaz1RsrNuq6AVpQElEr+uxVMyG6TQs7eDSyWa3oiopmGKDL8EBrJBEJ0UU7DldQetpLSGk+bcGmKuiKraFFJImoNZKIhOiCsKaz/9gpjp72YVhaaBGpFjSLFd0g0jUTLZIOqxBdENIMVKOhXrW9eekTp00lJSmR5HBQElEbJBEJ0QXBcKRetaq0fPnenWBjUkEm1Iala9YG6ZoJ0QVBTUc1QlhaWt7RqPFxaRG1ShKREF3QuJVQi+vMGkkiapckIiG6IBiOtIhURYnsYdaCv39RxZZDp6j3Sd3q1kgiEqILQh1oEYWIXDXTQ9Iiao0MVgvRBUMyXOQNSUYNGK22iCwNxdLCISkX2xpJREJ0gcOq4lA1sKnNiqI1UhoSkSYtolZJ10yIrjAMCDckGLXlgnCNiUgPS4uoNdIiEqILDp6oxl7lISXBRko7XTNpEbVOEpEQXXCk4jSO0z4GASmtDFZbrJGWkh6WRNQaSURCdEHjxokWm6PVomepyYk4nVbshLsztF6lx4wRPf300yiKwvz5880ORYgOCzV0tyzW1rfYvqgwm9F5bjJdUp2xNT0iEW3dupVf/vKXjBs3zuxQhOiUxjKxiq2VWdVwZn6RLi2i1pieiDweD7Nnz+bXv/41aWlpbR4bCASora1tchPCTGd28Gh5oBqQJR4dYHoimjdvHtdddx0zZsxo99glS5bgdrujt/z8/G6IUIjWNV6SV1tYed9oR5mXj0uqOHjidHeF1euYmojWrFnDJ598wpIlSzp0/MKFC6mpqYneSktLz3OEQrStsWumtlCLqJGmWAlpRmSJRz/dDbc9pl01Ky0t5Tvf+Q7r16/H6Wz9l3g2h8OBw9FGE1iIbnb5sBRUSwpOV2Krx1htdoI07uQRbnXiY39mWiLatm0bFRUVTJo0KfqYpmls2rSJX/ziFwQCAdkCSPR4qXbAaYM2WkSNWwppRkPdaklEzZiWiK666ip2797d5LG7776bCy64gMcff1ySkOgdtLaXd0DjTh7WM1sK4eqe2HoR0xJRcnIyY8aMafJYYmIiGRkZzR4XoicKhDUOHztJSo2P3FYWvAJYLUokERlSLrY1pl81E6K38gd1Dhw/RelpH1hbn0fUuJOHpsveZq3pUUs83n33XbNDEKLDGutVqwqtlgCByE4eroQEXLrs5NGaHpWIhOhNIvWqGwrntzGhMTPZQeawbKjVpWvWCumaCRGjUFhHNYKRetXtXQmLzq6WRNQSSURCxCjU0CKK1KtuZ36bpSFRSdesRZKIhIhRoLFF1E7XLKzpvL3/NNtKqqKr9UVTkoiEiFEorKHqYSxKG3uaAapFwRNWCGqGFNBvhQxWCxGjEQMcaHnJ7baIFEVBUaVcbFskEQkRo0SrHlneoVjA0vZXSWkYzA5LImqRdM2EiNXZu3e0UiY2qqHFJDt5tEwSkRAxOlxRRVmND4/W/rpItbGAvrSIWiSJSIgYHSyv4vApL6cD7deijm6yKDt5tCimRHTw4MF4xyFEr9NYJla1tV9PK8nlItGuYjWkbnVLYkpERUVFXHHFFfzud7/D7/fHOyYhegUtGBnvsbRVOL/Bl0fmMm5QKhkJ5zuq3immRPTJJ58wbtw4FixYQE5ODg8++CAfffRRvGMTokdr7Ga1Va86qnEJiCzxaFFMiWjChAksXbqU48ePs3z5csrKyvjyl7/MmDFjeP7556msrIx3nEL0OEYH6lVHyVqzNnVpsNpqtXLLLbfw6quv8swzz3DgwAEeffRR8vPzueuuuygrK4tXnEL0OEZ0B4/2E9H2Yx62l57m6Mnq8xxV79SlRPTxxx/z8MMPk5uby/PPP8+jjz5KcXEx69ev5/jx49x4443xilOIHsfowFZCjfy6FX9IJxSSFlFLYppZ/fzzz7NixQr27t3LrFmzePnll5k1axYWSySvFRYWsnLlSgoKCuIZqxA9ytSCJNSKZJzO9kegVasNDTDCDVsKtTcBsp+JKREtW7aMe+65h7lz55Kbm9viMVlZWfzv//5vl4IToicb4FTAZW9zB49Gqt2BBuiNO3m0UVq2P4opEa1fv57BgwdHW0CNDMOgtLSUwYMHY7fbmTNnTlyCFKJHiu7g0X5SsVltBFHQjIbXSSJqIqYxomHDhnHy5Mlmj1dVVVFYWNjloITo6XxBjdLKak56Au0XRQOsqorWZEshcbaYWkRGK9vmejyeDu/aKkRvVuMLceD4Kdx4GdCB1o3dqqBbrGhGKLLbq2iiU4lowYIFQKS+yqJFi3C5zmwUp2kaW7ZsYcKECXENUIieKKQ1VGdUO1AmFnBYVex2J3aLJi2iFnQqEW3fvh2ItIh2796N3X7mL4Hdbmf8+PE8+uij8Y1QiB4oEG6oV21VOjTek5/uIn94NngqJRG1oFOJ6J133gEiW0MvXbqUlJSU8xKUED1dKBzGYoSxWGwdGqwGzppdLV2zc8U0RrRixYp4xyFErxIONqy878gOHo2iiUhaROfqcCK65ZZbWLlyJSkpKdxyyy1tHrt27doOnXPZsmUsW7aMw4cPAzB69GgWLVrEzJkzOxqWEKYIByKJSFGtoLb/NfIGw3xysJpkTzXjR0giOleHE5Hb7UZpmA3qdrvj8uaDBg3i6aefZvjw4RiGwW9/+1tuvPFGtm/fzujRo+PyHkKcD+GGWkSWNormn82iKJwOgBrU0MIh2q/p2L90OBGd3R2LV9fs+uuvb3L/qaeeYtmyZXz44YctJqJAIEAgcKbmb21tbVziEKKzRgywo+ckY03q2DipTbWgK5GvWzgUkER0jpgmNPp8Prxeb/R+SUkJL7zwAn//+99jDkTTNNasWUN9fT1Tp05t8ZglS5bgdrujt/z8/JjfT4iucNsM0lx2khMTO3S8RQHd0rilkBTQP1dMiejGG2/k5ZdfBqC6uprJkyfz3HPPceONN7Js2bJOnWv37t0kJSXhcDj45je/yZ///GdGjRrV4rELFy6kpqYmeistLY0lfCG6rnE3jg4OVEf2NosUR5O61c3FXKHxkksuAeCPf/wjOTk5lJSU8PLLL/Ozn/2sU+caOXIkO3bsYMuWLTz00EPMmTOHPXv2tHisw+EgJSWlyU0IM5RWVlNZF8BrdLyTZbFKi6g1MV2+93q9JCcnA/D3v/+dW265BYvFwr/8y79QUlLSqXPZ7XaKiooAuOiii9i6dStLly7ll7/8ZSyhCdEt9h6tJLHSQ0GWiqv9wwFQrI0tIqlJdK6Yi+e/9tprlJaWsm7dOq6++moAKioqutxK0XW9yYC0ED2RHo5cNbN2pExsA6czAYfVgqJLIjpXTC2iRYsWcccdd/Dd736Xq666Kjq4/Pe//52JEyd2+DwLFy5k5syZDB48mLq6OlavXs27777LunXrYglLiG6jN3SvrPYOTmYEvjJmEChp0PGX9BsxJaJ/+7d/48tf/jJlZWWMHz8++vhVV13FzTff3OHzVFRURGtbu91uxo0bx7p16/jKV74SS1hCdJvGraM70yKSAvqtiykRAeTk5JCTk9PkscmTJ3fqHFLBUfRGum5Ed/CwdSoRyZZCrYkpEdXX1/P000+zYcMGKioq0HW9yfOyE6zoy4KajmpEkklnWkSfHPVgO1ZD5gAbOe0f3q/ElIjuu+8+Nm7cyJ133klubm506YcQ/UEgrGPVA1gUsNo7vnVrXUjBHgjjDsrFmHPFlIjefPNN3njjDaZPnx7veITo8RJsKhPyXCi+JOjgWjM4szW1IRMam4kpEaWlpZGenh7vWIToFexWCwOcgMXR8RIggLUhaem6BroGFllx1iimeUT/9V//xaJFi5qsNxOiX9Eauled2I2jcSPGSAF9GbA+W0wtoueee47i4mKys7MpKCjAZrM1ef6TTz6JS3BC9ESn63yEq+tx2CykdKZFZLMSRmnY2ywIHdiqur+IKRHddNNNcQ5DiN7j6KkaPJUe0hPtpHRijMhmseC12NCkRdRMTInoBz/4QbzjEKLXCAd8QEN1xk6M8zhsFixWB6olALLMo4mYxoggUv7jN7/5DQsXLqSqqgqIdMmOHTsWt+CE6Ika61VbOtm1GpGdzLQROQzPSpa61eeIqUW0a9cuZsyYgdvt5vDhw9x///2kp6ezdu1ajhw5Eq1VJERfFAr6Ueh4mdgmZHZ1i2JqES1YsIC5c+eyf//+Jju7zpo1i02bNsUtOCF6ojMtolgSkaw3a0lMiWjr1q08+OCDzR4fOHAg5eXlXQ5KiJ5MD0XGiCydmFUNkW2qt5TU8nl5rXTNzhFT18zhcLRYuH7fvn1kZmZ2OSghejK9cU+zTo4RGYbBSZ+B4g/LYPU5YmoR3XDDDTz55JOEQpEfpqIoHDlyhMcff5x//dd/jWuAQvQ0Y7LtFGUlkeruXBFAq2pBU2yR1fuyzKOJmBLRc889h8fjITMzE5/Px2WXXUZRURHJyck89dRT8Y5RiB4lywmZSQ4SXR3bwaORTVXQFSsGoEsiaiKmrpnb7Wb9+vVs3ryZnTt34vF4mDRpEjNmzIh3fEL0PA1lYrF2bozIZrGgWSJXzcKhoOxtdpZOJyJd11m5ciVr167l8OHDKIpCYWEhOTk5GIYhJUFEn6brBscqq0j0BnGrjk4lE4tFAUtjAX0pBXK2TnXNDMPghhtu4L777uPYsWOMHTuW0aNHU1JSwty5cztVJlaI3iio6ew5UsHe8jqwdn6tmNK4pZDUJGqiUy2ilStXsmnTJjZs2MAVV1zR5Lm3336bm266iZdffpm77rorrkEK0VP4QxpWPYBqUVA7Uya2gdVmx2pR0GUeUROdahH9/ve/5z//8z+bJSGAK6+8kieeeIJVq1bFLTghehp/SMeqB7FalE6PEQHcMGkIXypIx93x6iH9QqcS0a5du7j22mtbfX7mzJns3Lmzy0EJ0VM1togiiagrM6vlqtnZOpWIqqqqyM7ObvX57OxsTp8+3eWghOipAsEQFiOMVbWArfMtIllr1rJOjRFpmobV2vpLVFUlHA53OSgheqqgvx4A1aJ0qkxso21HPSSU1ZKd7SIj3sH1Yp1KRIZhMHfuXByOln8BslW06OuC/kh5ZIvNCZbOzwc+5TNI9oVIle9KE51KRHPmzGn3GLliJvqyoWlWwllJ2BLdMb2+sW61IWNETXQqEa1YsSKub75kyRLWrl3LF198QUJCAtOmTeOZZ55h5MiRcX0fIeIlza5DkgOSk2N6vWqNjBHpoSAYBsgEYKALFRrjYePGjcybN48PP/yQ9evXEwqFuPrqq6mvrzczLCFaF13eEVvh+8YaRrqhR7YUEkCMa83i5a233mpyf+XKlWRlZbFt2zYuvfRSk6ISonXHK0/h9AZJTHcQw8V7rA2bLEa2FAqCaupXsMfoUT+FmpoagFY3bwwEAk0GxFuqiSTE+fTZ4TKSTtRRlGsnlspbNquVoKKiGTTMJXLFOcLeydSu2dl0XWf+/PlMnz6dMWPGtHjMkiVLcLvd0Vt+fn43Ryn6Oy0YGTawOmNLIDbVgmGx0VALJI6R9W49JhHNmzePTz/9lDVr1rR6zMKFC6mpqYneSktLuzFC0d8ZhoHesJWQIyEppnOMG+hm+ogcirKSZHb1WXpE1+yRRx7hr3/9K5s2bWLQoEGtHudwOFqdwyTE+RYI66haYyKK7aqZpXFpSKBOEtFZTE1EhmHwrW99iz//+c+8++67FBYWmhmOEG3yBjVsuh+rRUF1dGFsJ7rMQ7pmjUxNRPPmzWP16tW8/vrrJCcnR3cAcbvdJCTEsI5HiPPIGwxj0/zYVAVsnSsT26jGG+LgsXqSfHUMlxZRlKljRMuWLaOmpobLL7+c3Nzc6O0Pf/iDmWEJ0SJfIIRVD2CLdcEroBkGZR6dam9IumZnMb1rJkRvke0ySMtKjIzzxJiI7NZI3WpNNzDCkR1jRQ8ZrBaiN0hRw5HlHTYnWGIrfW9XLWiKHQMIBwPY4htir9VjLt8L0eOFIivvscU+UG1TFbSG4miNW1cLSURCdFhZ5SmqvEECSmzrzCCyGanSUNlREtEZkoiE6KBPD5ext7yOqmDXdiSzNGxVrUkiipJEJEQHhXweAOyJndtq+lyqzYECaCFfHKLqG2SwWogOMAwDzR9ZZO2IsShao5njh6Da0ulCD6/PkUQkRAf4QhrWUD0KkNDFRGR1JAAKhGUeUSPpmgnRAR5/GLvmxaYqqM7Y1plFNRbd16RudSNpEQnRAXWBMDbNi81mAXtsK+8b7an0o56oIz3FiKmmUV8kiUiIDqj3BbDpPuyqDRxdS0QnfWCrD5Jg84Gux7QbSF8jiUiIDhiSDKGspMi+fl2Y0AhgtTdcvtf1SPfMIgu8JREJ0QFuiz+yvMOZ0uWdNxx2G35FRdMNCAdiXrfWl0ibUIiOCDTUR3d0bQ4RgF1V0Sz2s+pWC0lEQnTAgSNHOe0NotnjkIiskYWv0RaRkEQkRHsCYY1Pi4/wRXkdWjxaRGeVApFEFCGJSIh21HhDOMJ12FQFe1LLW111hk1VIl0z3ZC5RA1ksFqIdlT7Qjg0D06bCs6uzaoGyHUnkDMmH7XKLy2iBpKIhGhHlSeAI1yH02GNSyJSLQo4XESWecgKfJCumRDtOn26Couh4XLa4nLVDABrw4pXWYEPSCISol2eqsjuMgkpA2IuEXuubcd97D1RR8BfH5fz9XaSiIRogz+kEfZUApCYnh238x6uMaiqDxLweeN2zt5MxoiEaINdtTCz0ErYloQ9OStu51UblnmEA5KIQBKREG2yWBRSjdrI8o7EAXE7b+NOseGgJCKQrpkQbTMMqI90zXDFLxFZHZGdYrWADFaDJCIhWuUPaby1dQ+V1bUYFjWuLSKbI7LQVZcWEWByItq0aRPXX389eXl5KIrCa6+9ZmY4QjRxoMLDyWMHOV7tQ0nOjdsVMwCrM9I108Jh0EJxO29vZWoiqq+vZ/z48bz44otmhiFEM4ZhsPNoNUnBCgYkOSAlL67nt9udGCiRmkQyl8jcweqZM2cyc+ZMM0MQokWfHa+lojbApFA5WRkOcA+K6/kvLsjAUpaHGvY1zK6O00TJXqpXXTULBAIEAmfW5tTW1poYjejNDMOgvNbPKU+QMQPPLNsorvRwpMrLrtIaHOFaihID2KyJkDokru9vt1rAnhhJQtIi6l2JaMmSJSxevNjsMEQvpusGe8pq+fhwFae9IexWS5NEtOtoNYdPRgaQJyacIM/hjLSGbOdhE7LGc8p6s96ViBYuXMiCBQui92tra8nPzzcxItGbnPQEWP9ZOfUVh0n1HyXdouHOHIQWHBSdYJjnTiDZYaNwgIuhh95F8SqQdWHcY6nxhThyIkiix8NQaRH1rkTkcDhwOBxmhyF6oQMVdby96xD5J96mKHiMgWkJZCc7UbVj8NFOGDwNBl7ElKEZkRec2APeU6DaIGtU3OPRdINDNRq5vqAkInpZIhIiFlX1QdZtL2Z0+etk2fwMG5KBI3d0pGj9qWLwVkHx23D8ExgyPbIyfv+6yIvzp5yXbpnTZiFkSSCsG2iBeuI3MaB3MjUReTweDhw4EL1/6NAhduzYQXp6OoMHDzYxMtGXpNt1brL8k6ArxJBB+VjG3XpmcuLQK+HEbji0CXzV8MUbZ16YkguDp56XmBJsKmFrZFJj0F9Hf9/Hw9RE9PHHH3PFFVdE7zeO/8yZM4eVK1eaFJXoU7QwfPonBlrrIC8Txn8dXGeVe7VYIHc8ZF4IR7dC5ReRnTUyiqDwUlDPz1dEURTUho0aQ15JRKYmossvvxzDMMwMQfRRIU1n094TTA9uxll9BKx2GHdb0yR0NqsdCqZHbt1ETUiOxOrzdNt79lSy1kz0SVsPnqJu99/Y9+lWDMUCY/4VknPMDqsJuzOy8DXkrzM5EvNJIhJ9TlV9kLJd/yDb8zkDU10oo26EtAKzw2rG7oq0iPSAF3Td5GjMJVfNRJ+z+6O3yaveRprLRvqEr0LWBWaH1KJLRg3BWpWBVQFCXmgYM+qPpEUk+pSKgzuxH9yAAgyccDXKoIvMDqlVTrsNa0OBNEL9uxyIJCLRd9Qep3LLq4CBffBFJF9wudkRtc8eGSci2L8HrCURib7BV031R7+n1uun1jWYYdNvAUUxO6o2VXuD7D6pc+hkPQT694C1JCLR+4UDsPtVEvGTmTOIhAm34Hb1/KVAmm5wsNbCSU8A/P27koQMVovezTBg75tQfxJbQjJFX55DURx2Y+0OSU4rATWJsG4Q8lZjMzsgE0mLSPRux7djVOwBxQJjbonLltDdxWFVMRyRS/h+z2mTozGXJCLRe9WVw4EN1HhDvBMaxTEjw+yIOs2emApAsL7G3EBMJolI9E7hAHz2GoYe4vNgFjuMERyo6H1XnhxJqQCE6msi3cx+ShKR6H0MA/a9Bb7TVGkJbE2Yjs1q4eIhaWZH1mkJjYkoGOjXc4kkEYne58SncGIPOgrvWKaiqU7GDUol0dH7rr0kuRIIqi7CuhEpQ9JP9b7fnOjfvFWwL1K0rDhxIqU16STYVSYXtrKqvocbO9CNOnYEttrSSEVI90CzQzKFtIhE76GFYc/roIXwJQ5kfX0hANOGZeC09c4ah06bii05M3LHe8rcYEwkiUj0DoYRKd9aVw42J7uSLiEQhuwUJ2Pyes8l+xa5Gq729eNEJF0z0Tsc/RjKdkWWbYy6kclpBdiSqinISMRi6dlLOdqz45QFe1kteZYTpJodjEmkRSR6vvLdcOAfGBiECy+H9KEoisKkwWmkJ9rNjq7LjgVd1PhC+GpORrqf/ZAkItGzHdsGX7xBIKzxcWgor1UORNP71nwbtzuDkCUBrz8InhNmh2MK6ZqJHkkPeKn97C28R3ZQ6wuxz1JEcdpELNV+Squ8FAxINDvEuMlJTWCfIxNP4BjUHu+XV84kEYmeJRzkyGebKd35DpaQF1A4kvoljiePZ2C6i8tGZJKdch62fzZRrtvJJ/YsfDVHCFYfxZ7/JbND6naSiESPYAS9KMe2wbGPSfd4OBbyEnak4ht6NUUDi7gi3UVWct9KQI0SHVaUlIEYNR/jOXGI9DFGj6+lFG+SiISpwv56Pv3gLVJOf8rQtMjAc2JqJiOGXUNa4SQs1v7xXzQ1txDtqI2a6irS68ojmzv2I/3jtyx6HsOg9tDHHPjwrwR89fiBrKzhJA2/BCXzAjIs/es6ytCcVIqT8rEox+HUfklEQpx3deUc/+g1jh4pRtMNNGc6BVOuJ2nYmH7XJWlUmJFIwSWXoH7x18h8qSHTwdI7Z4vHQhJRL1NT7+fooc8ZpR5DqT0G/hoq6vyUecCVkUtWXgHu7EJIyQO7q93zGVqImno/x+t07DYLRVnJ0efKa/ykJFhx2eP03yQcIHjgXUp2beJkXQDNYiM4eDqTv/wVkhJ6fmnX88liUSDrQjj4TqR+dfluyJtgdljdpkckohdffJGf/vSnlJeXM378eH7+858zefJks8PqMbyBICUHv+Bk8U6o3ItV91OXl0KKM1JctM4bwFsfwFtfy8kje3HZVTIS7aQOyCUxczCKIwkUFcJ+tGA9ntoaamuq8dTV4K33ENQim/slJafA8KHgGoCRmMk/PvNRRSoDUlMoGOCiICORnBRn52cyB+uhbCeUfsT+knJq/WFOJQ4ja+Ispg0f3OtnRseNaoX8KVTtepPQjr+RnTq49S2y+xjTE9Ef/vAHFixYwEsvvcSUKVN44YUXuOaaa9i7dy9ZWVlmh9f9DANCXjw1pygpOURN+WG0k/uxan6sgAK4klII5kyAwaPBlcEAv06oooLysiN4Kkrw+U/gPV1N6emD2A4e4qLBaSgNXZ7dpdX4QlqTt7QokSs3qdYQVJdCdSmarjPmRA2+kE6gPJlqWzpb1QSwJZDhTqYwM5mBaUmgWDAUBcWiRsq1KhZAQQ968XtO4608QlLwBE418v4ZWbl8wQSmfmkyeakJ3fqj7Q1KXKPYd/pdkoOV1K9/iYwxM0gZeAGKI7lPd1sVwzC3LNyUKVP40pe+xC9+8QsAdF0nPz+fb33rWzzxxBNNjg0EAgQCgej92tpa8vPzqampISUlpc33KSs7ypFNq4GzP66BYYCCwZCGv/ZgUOsPsed4bdNjz/ox5aclMLDhS+QJhPj0WE2TYxXDiN4bmOokPy3SRfIGw+w6Wt3sfI1y3U6GpNpA16gPhNl17Ez5UGdCIsn5o8gZPomkrGHQymCuP6RRXOnh4LFKTpcfJsdSzdUj3KBrYHOyYX8tJ0M2UlNTGZCWTvaADLLTkrERjuwk4T0F9ZWRm6eCoLeGam+Qam+IGl8oUjcHGJiawOD0yOfyhTR2Ha3GoijRz63rZ34GQzJc5A0qhEFfQs+8EA0Fm9q/BqM7yjAMNu4+hLZjDa5QFRD5Q2GxOtBVOyPz0khNjPzfO1Hn5/DJeiJ/npobkZ1MmssGikKlJ8DByvpW33dYZhIDkiJXLU/VB9usdlk4IJGsZAeMvRWcbX/vamtrcbvd7X5HTW0RBYNBtm3bxsKFC6OPWSwWZsyYwQcffNDs+CVLlrB48eKY3iscDKF7Klp8zgAMpw+sDX+h/SEMbxvbuyQEwR+M/DsQxjhnK5gmKSagQTDyiBLSMIKtV+EzQgbokS9oQlIKruwBuDIGkV1wISnZQ1tNPmdz2lRG57kZnedG04fhDYbBeWZ/iKmDwiTY1GgL6QxHZLO/c67W2EM+sjwVZHlPoge81NTVUVldi8tlhcRI0vTVBzh1shKLoaOgg2EQtjgIWxNRknNIHDaSvBFDgciaIklBrVMUhcvGFnIw+2GO7HoP48SnJASq0EMBCAVQfArQkFA8fnRP68nFSPKD0bAWry6AXtdGKd0EL9AwTlffzrHORFCcYOid+mxtMbVFdPz4cQYOHMj777/P1KlTo48/9thjbNy4kS1btjQ5vistIp/XS1V5CUDTL2HDv1MSbCQ1VPjzhw2qvcEmzzfcQVEg0W4lqeHLHQzrVPvCzc7b+O8Eu0qiI3JsWNOp9oebHdN4bqdNJTHBCfakXnXFRNMNPP4wmmFE/zY7bJZWEp7oDF038Pj9BOpOo4eDuJ0KzoZM7gmEqPW1tEg28pVOc9lIaKjTVB/UqPaGWn2fNJcNlz1yrDeocbqNY90uK0l2K7gHgdr2Jki9okXUWQ6HA4cjtqsrCS4XA4de2KFjnUBOB89rBzo6kmUFBvTy0jktUS0Kbld/3pXr/LFYFFJcCeBqPp6W1HDriMSGW0e4Gm7dydRW8oABA1BVlRMnmq44PnHiBDk5HU0FQojeztREZLfbueiii9iwYUP0MV3X2bBhQ5OumhCibzO9a7ZgwQLmzJnDxRdfzOTJk3nhhReor6/n7rvvNjs0IUQ3MT0R3XbbbVRWVrJo0SLKy8uZMGECb731FtnZ2WaHJoToJqbPI+qKjo7ICyHM0dHvqEzpEEKYThKREMJ0po8RdUVjr7K2to1Z0EII0zR+N9sbAerViaiurg6A/Px8kyMRQrSlrq4Ot7v12by9erBa13WOHz9OcnJyr1pK0Lg0pbS0tM8MsvfFzwTyubrKMAzq6urIy8vD0sZayV7dIrJYLAwaNMjsMGKWkpLSp/5zQ9/8TCCfqyvaagk1ksFqIYTpJBEJIUwnicgEDoeDH/zgBzFXEuiJ+uJnAvlc3aVXD1YLIfoGaREJIUwniUgIYTpJREII00kiEkKYThJRN3jqqaeYNm0aLpeL1NTUDr3GMAwWLVpEbm4uCQkJzJgxg/3795/fQDupqqqK2bNnk5KSQmpqKvfeey8eTxu7PwCXX345iqI0uX3zm9/spohb9uKLL1JQUIDT6WTKlCl89NFHbR7/6quvcsEFF+B0Ohk7dix/+9vfuinSzunM51q5cmWz34vT6ey+YA1x3i1atMh4/vnnjQULFhhut7tDr3n66acNt9ttvPbaa8bOnTuNG264wSgsLDR8Pt/5DbYTrr32WmP8+PHGhx9+aLz33ntGUVGRcfvtt7f5mssuu8y4//77jbKysuitpqammyJubs2aNYbdbjeWL19ufPbZZ8b9999vpKamGidOnGjx+M2bNxuqqho/+clPjD179hjf+973DJvNZuzevbubI29bZz/XihUrjJSUlCa/l/Ly8m6LVxJRN1qxYkWHEpGu60ZOTo7x05/+NPpYdXW14XA4jN///vfnMcKO27NnjwEYW7dujT725ptvGoqiGMeOHWv1dZdddpnxne98pxsi7JjJkycb8+bNi97XNM3Iy8szlixZ0uLxt956q3Hdddc1eWzKlCnGgw8+eF7j7KzOfq6O/t88X6Rr1gMdOnSI8vJyZsyYEX3M7XYzZcqUFjeeNMMHH3xAamoqF198cfSxGTNmYLFYmu1Hd65Vq1YxYMAAxowZw8KFC/F6W9908nxq3ODz7J9zWxt8QuRzn308wDXXXNNjfi8Q2+cC8Hg8DBkyhPz8fG688UY+++yz7ggX6OWLXvuq8vJygGZ1u7Ozs6PPma28vJysrKY7ulmtVtLT09uM8Y477mDIkCHk5eWxa9cuHn/8cfbu3cvatWvPd8jNnDx5Ek3TWvw5f/HFFy2+pry8vEf/XiC2zzVy5EiWL1/OuHHjqKmp4dlnn2XatGl89tln3bKwXFpEMXriiSeaDe6de2vtl96Tne/P9cADD3DNNdcwduxYZs+ezcsvv8yf//xniouL4/gpRGdNnTqVu+66iwkTJnDZZZexdu1aMjMz+eUvf9kt7y8tohj9+7//O3Pnzm3zmKFDh8Z07sbNJU+cOEFu7pm96E+cOMGECRNiOmdHdfRz5eTkUFFR0eTxcDhMVVVVpzbHnDJlCgAHDhxg2LBhnY63K2LZ4DMnJ6fHbwgaj41LbTYbEydO5MCBA+cjxGYkEcUoMzOTzMzM83LuwsJCcnJy2LBhQzTx1NbWsmXLFh566KHz8p6NOvq5pk6dSnV1Ndu2beOiiy4C4O2330bX9Why6YgdO3YANEm43eXsDT5vuukm4MwGn4888kiLr5k6dSobNmxg/vz50cfWr1/fozYEjeVznUvTNHbv3s2sWbPOY6RnMW2YvB8pKSkxtm/fbixevNhISkoytm/fbmzfvt2oq6uLHjNy5Ehj7dq10ftPP/20kZqaarz++uvGrl27jBtvvLFHXr6fOHGisWXLFuOf//ynMXz48CaX748ePWqMHDnS2LJli2EYhnHgwAHjySefND7++GPj0KFDxuuvv24MHTrUuPTSS836CMaaNWsMh8NhrFy50tizZ4/xwAMPGKmpqdFL13feeafxxBNPRI/fvHmzYbVajWeffdb4/PPPjR/84Ac99vJ9Zz7X4sWLjXXr1hnFxcXGtm3bjK9//euG0+k0Pvvss26JVxJRN5gzZ44BNLu988470WMAY8WKFdH7uq4b3//+943s7GzD4XAYV111lbF3797uD74Np06dMm6//XYjKSnJSElJMe6+++4myfXQoUNNPueRI0eMSy+91EhPTzccDodRVFRk/Md//Iep84gMwzB+/vOfG4MHDzbsdrsxefJk48MPP4w+d9lllxlz5sxpcvwrr7xijBgxwrDb7cbo0aONN954o5sj7pjOfK758+dHj83OzjZmzZplfPLJJ90Wq5QBEUKYTq6aCSFMJ4lICGE6SURCCNNJIhJCmE4SkRDCdJKIhBCmk0QkhDCdJCIhhOkkEYleZeXKlU3K7f7whz9sshB47ty50fVVoveQRCRaNHfu3FbrSc+bNw9FUZqs0j8fCaCgoIAXXnihyWO33XYb+/bta/U1S5cuZeXKldH7l19+eZMFqqJnkkQkWpWfn8+aNWvw+XzRx/x+P6tXr2bw4MGmxJSQkNCsINvZ3G53hzcoED2HJCLRqkmTJpGfn9+keuLatWsZPHgwEydO7NK5W2qp3HTTTdFW1uWXX05JSQnf/e53owXZoHnX7Fxnt8zmzp3Lxo0bWbp0afQchw4doqioiGeffbbJ63bs2IGiKN1Wf0c0JYlItOmee+5hxYoV0fvLly/n7rvvPu/vu3btWgYNGsSTTz5JWVkZZWVlnT7H0qVLmTp1Kvfff3/0HIMHD272mQBWrFjBpZdeSlFRUbw+gugESUSiTd/4xjf45z//SUlJCSUlJWzevJlvfOMb5/1909PTUVWV5ORkcnJyYqqA6Ha7sdvtuFyu6DlUVWXu3Lns3bs3us9XKBRi9erV3HPPPfH+GKKDpEKjaFNmZibXXXcdK1euxDAMrrvuOgYMGGB2WF2Sl5fHddddx/Lly5k8eTJ/+ctfCAQCfO1rXzM7tH5LWkSiXffccw8rV67kt7/9bdxaDRaLhXNLYYVCobicuyPuu+++6ED8ihUruO2223C5XN32/qIpSUSiXddeey3BYJBQKMQ111wTl3NmZmY2GffRNI1PP/20yTF2ux1N07r0Pq2dY9asWSQmJrJs2TLeeust6ZaZTLpmol2qqvL5559H/92ampqaaDH8RhkZGeTn5zc79sorr2TBggW88cYbDBs2jOeff57q6uomxxQUFLBp0ya+/vWv43A4YuoSFhQUsGXLFg4fPkxSUhLp6elYLJboWNHChQsZPnx4jyp+3x9Ji0h0SEpKCikpKW0e8+677zJx4sQmt8WLF7d47D333MOcOXO46667uOyyyxg6dChXXHFFk2OefPJJDh8+zLBhw2LeMeXRRx9FVVVGjRpFZmYmR44ciT537733EgwGu+UqoGib1KwW/dZ7773HVVddRWlpabNdUUX3kkQk+p1AIEBlZSVz5swhJyeHVatWmR1SvyddM9Hv/P73v2fIkCFUV1fzk5/8xOxwBNIiEkL0ANIiEkKYThKREMJ0koiEEKaTRCSEMJ0kIiGE6SQRCSFMJ4lICGE6SURCCNP9f4E3CJl0dMYEAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:55:59.960077Z", + "iopub.status.busy": "2024-03-26T10:55:59.959799Z", + "iopub.status.idle": "2024-03-26T10:56:00.173203Z", + "shell.execute_reply": "2024-03-26T10:56:00.172340Z" + }, + "papermill": { + "duration": 0.234595, + "end_time": "2024-03-26T10:56:00.175153", + "exception": false, + "start_time": "2024-03-26T10:55:59.940558", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAErCAYAAAChCiRcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2DklEQVR4nO3de1xUVb8/8M8wwHAHkZsXZFBU6PGOSnhAVFCRJLylKakgR8uii2gl/Urz8RRZatbR8KmTmqVpjyh5SAREUTRCRPIRRVTEVARR0eEmwzCzfn94Zj+ODDAjl7ns7/v14iWz99oz3+1mf2fttddeS8AYYyCEECNnousACCGkK1CyI4TwAiU7QggvULIjhPACJTtCCC9QsiOE8AIlO0IIL1CyI4TwAiU7QggvULIjhPCCwSW7LVu2QCwWw8LCAn5+fjh9+nSLZb/77jsEBgaiW7du6NatG0JCQlotTwgxXgaV7Pbu3Yu4uDisXr0aZ8+exdChQzF58mRUVlaqLZ+VlYW5c+fi2LFjyMnJgbu7OyZNmoSysrIujpwQomsCQxoIwM/PD6NGjcLmzZsBAAqFAu7u7njzzTexcuXKNreXy+Xo1q0bNm/ejAULFnR2uIQQPWKq6wA01djYiPz8fMTHx3PLTExMEBISgpycHI3eo76+HjKZDI6Oji2WkUqlkEql3GuFQoGqqip0794dAoHg2XeAENLhGGOoqalBz549YWLS+oWqwSS7e/fuQS6Xw9XVVWW5q6srLl26pNF7vP/+++jZsydCQkJaLJOQkIA1a9a0K1ZCSNe6efMmevfu3WoZg0l27fXZZ59hz549yMrKgoWFRYvl4uPjERcXx72WSCTo06cPSktLYWtr2xWh6oRMJsOxY8cwfvx4mJmZ6Toc0k58OZ41NTXw9PTU6Nw0mGTn5OQEoVCIO3fuqCy/c+cO3NzcWt12/fr1+Oyzz3DkyBEMGTKk1bIikQgikajZckdHR9jZ2WkfuIGQyWSwsrJC9+7djfrk4Au+HE/lvmnSxGQwd2PNzc3h6+uLzMxMbplCoUBmZib8/f1b3O7zzz/H2rVrcfjwYYwcObIrQiWE6CGDqdkBQFxcHBYuXIiRI0di9OjR2LRpE+rq6hAdHQ0AWLBgAXr16oWEhAQAwLp167Bq1Srs3r0bYrEYFRUVAAAbGxvY2NjobD8IIV3PoJLdnDlzcPfuXaxatQoVFRUYNmwYDh8+zN20uHHjhsodmcTERDQ2NmLWrFkq77N69Wp8/PHHXRk6IUTHDCrZAUBsbCxiY2PVrsvKylJ5ff369c4PiBBiEAymzY4QQtqDkh0hhBcM7jKWENJcfX29Suf62kdS/H6+BN2czsDGUrUrlbe3N6ysrLo6RJ2jZEeIEbh06RJ8fX2bLf9cTdn8/HyMGDGi84PSM5TseErTmgBfawGGxtvbG/n5+dzr4vKHiPvneWx8aTAG9nBoVpaPKNnxlKY1Ab7WAgyNlZWVynEy+es+RNmP4DNoKIZ5dNdhZPqDkh1PaVoT4GstwBCU3qtDnbRJ7bqSu3Xcv6amLZ/m1iJTeDpZd0p8+oaSHU+oOzHMXfv9+3eFBCK3RzB39YK5qz23/NoDGfBAwr3m08mhz0rv1WH8+qw2yy3fd77NMsdWjOPFMaVkxwOanhgAnRyGQvnFtWnOMHi5NH/0se6RFClZOZg6zh/Wls0HtgCAq5W1eGfvny3WDo0NJTseaOvEAOjkMFReLjYY1Mu+2XKZTIYKZ2CERzejHvVEG5TseKSlEwOgk8PQSOUNMLEoQ2l1MUwsmn+BNTU14XbTbRRVFbXYZldaXQsTizJI5Q0A1P9dGBNKdjzQ1okB0MlhaG7X/QVrz//GB21MlvfN4W9aXW/tCdyuGwZfuLZazhhQsuMBTU8MgE4OQ9HT2gN1pW/iqznD0E9N00RTUxNOnTyF/wj4jxa/vEoqa/H23j/Rc7xHZ4erFyjZ8UBbJwZAJ4ehEQktoGjoBU+7gXiuu/o2u1LTUvg4+rTYLKFokEDRcBciYcvTFBgTSnY80NaJAdDJQYwfjXpCCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBYNLdlu2bIFYLIaFhQX8/Pxw+nTrEyv885//hLe3NywsLDB48GAcOnSoiyIlhOgTg0p2e/fuRVxcHFavXo2zZ89i6NChmDx5MiorK9WW//333zF37lzExMSgoKAA06ZNw7Rp01BYWNjFkRNCdM2gkt3GjRuxePFiREdH47nnnsPWrVthZWWFbdu2qS3/1VdfITQ0FO+++y58fHywdu1ajBgxAps3b+7iyAkhumYwE+40NjYiPz8f8fHx3DITExOEhIQgJydH7TY5OTmIi4tTWTZ58mQkJye3+DlSqRRSqZR7XV1dDeDxhDQymawde6A7NY8e78+5G1VoampSW6auQYozdwGna3dhbSFSW+bq3ToAj2ciM9T/C2PR1jHly/HUJm6DSXb37t2DXC6Hq6vqfKWurq64dOmS2m0qKirUlq+oqGjxcxISErBmzZpmy9PT02FlZfUMketezh0BACH+368X2yhpih+vFrT5fnk5J/GXZYeERp6RZsfU+I9nfX29xmUNJtl1lfj4eJXaYHV1Ndzd3TFp0iTY2dnpMLJn93xdIwYXVaKvszUszYRqy1yukOC9A0X4fLoPBripn24RAKxFQoi7W3dWqERDbR1TvhxP5ZWXJgwm2Tk5OUEoFOLOnTsqy+/cuQM3Nze127i5uWlVHgBEIhFEoubVfjMzsxbnU9V3rg5miPT31KjsADd7DPPo3skRkfbS9Jga+/HU5pw0mBsU5ubm8PX1RWZmJrdMoVAgMzMT/v7+arfx9/dXKQ8AGRkZLZYnhBgvg6nZAUBcXBwWLlyIkSNHYvTo0di0aRPq6uoQHR0NAFiwYAF69eqFhIQEAMDbb7+NoKAgbNiwAS+88AL27NmDM2fO4Ntvv9XlbhBCdMCgkt2cOXNw9+5drFq1ChUVFRg2bBgOHz7M3YS4ceMGTEz+XVkdM2YMdu/ejQ8//BAffPAB+vfvj+TkZAwaNEhXu0AI0RGDSnYAEBsbi9jYWLXrsrKymi176aWX8NJLL3VyVIQQfWcwbXaEENIelOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOwIIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILBpPsqqqqEBkZCTs7Ozg4OCAmJga1tbWtln/zzTcxcOBAWFpaok+fPnjrrbcgkUi6MGpCul5tbS2WLZ6P29+/gWWL57d6nvCJwSS7yMhIXLhwARkZGUhJScGJEyewZMmSFsvfvn0bt2/fxvr161FYWIgdO3bg8OHDiImJ6cKoDYNcLseZnJOou3gcZ3JOQi6X6zok8oxGjx4NW1tbZGWkQnbvL2RlpMLW1hajR4/WdWi6xwzAxYsXGQCWl5fHLUtNTWUCgYCVlZVp/D6//PILMzc3ZzKZTONtJBIJA8AkEolWMRuKpKQkJhaLGQDuRywWs6SkJF2HRrQ0atQoleP49M+oUaN0HWKH0+b8NNVJhtVSTk4OHBwcMHLkSG5ZSEgITExMkJubi+nTp2v0PhKJBHZ2djA1bXm3pVIppFIp97q6uhoAIJPJIJPJnnEP9E99fT22bt2K+Ph4jBkzBgMHDcOJ4jsYO9AVtQ/uYtasWUhISMBrr70GKysrXYdL2lBbW4u8vLxWy+Tl5eHBgwewsbHpoqg6nzbnpNbJ7tq1a+jbt6+2m7VLRUUFXFxcVJaZmprC0dERFRUVGr3HvXv3sHbt2lYvfQEgISEBa9asabY8PT3dqE76K1euYOXKlQCAU6dOccvTrvy7zMqVKyEUCtG/f/+uDo9o6eOPP+Z+NzExgUKhUPva398ff//73yESibo6xE5RX1+vcVmtk52XlxeCgoIQExODWbNmwcLCQtu34KxcuRLr1q1rtUxRUdEzv79SdXU1XnjhBTz33HMqfxTqxMfHIy4uTmVbd3d3TJo0CXZ2du2ORV+kpaVxv3fr1g1Dhw7DXUktnO1tcO7cn3jw4AEAwNvbG5MnT9ZVmERDs2fP5n5/MtE9/bq4uBhisRjDhw/vstg6k/LKSxNaJ7uzZ89i+/btiIuLQ2xsLObMmYOYmJhnagBdvnw5oqKiWi3Tt29fuLm5obKyUmV5U1MTqqqq4Obm1ur2NTU1CA0Nha2tLQ4cOAAzM7NWy4tEIrXfemZmZm1ua0iUyczKygoSiQRZWce4dSYmJrCyskJ9fT0ePHhgVPttrBhjAB7//To6OqK8vJxb16NHD1RVVUEqlcLU1BSDBg0ymmOqzX5oneyGDRuGr776Chs2bMDBgwexY8cOBAQEYMCAAVi0aBHmz58PZ2dnjd7L2dlZo7L+/v54+PAh8vPz4evrCwA4evQoFAoF/Pz8WtyuuroakydPhkgkwsGDB9tVCzU2ubm5AB5fBri4uCAyMhL19fWwsrLCrl27uC+X3NxczJ8/X5ehEg04OTmhvLwcUqkUQ4cORXx8PC5fvowBAwbg0KFDOHz4MIDH55wxNcdopb13QxoaGtjGjRuZSCRiAoGAiUQiNn/+fHb79u32vrWK0NBQNnz4cJabm8tOnjzJ+vfvz+bOncutv3XrFhs4cCDLzc1ljD2+S+Pn58cGDx7Mrl69ysrLy7mfpqYmjT/XWO/GvvbaawwAEwqFzMPDQ+WunYeHBxMKhQwAe+2113QdKtHAsmXLWr0Tq/xZtmyZrkPtUNqcn8/cz+7MmTN4/fXX0aNHD2zcuBErVqxASUkJMjIycPv2bURERLQnBzeza9cueHt7Izg4GGFhYQgICMC3337LrZfJZCguLuYaLM+ePYvc3FycP38eXl5e6NGjB/dz8+bNDo3NEClv7MjlcjQ0NCAxMRHbtm1DYmIiGhoauL52mt4AIrrVu3fvDi1njLS+jN24cSO2b9+O4uJihIWFYefOnQgLC4OJyeO86enpiR07dkAsFndooI6Ojti9e3eL68ViMdduAQDjxo1TeU1UKe9uC4VCWFhYYOnSpdw6sVgMoVAIuVze7C440U/aNB3xldbJLjExEYsWLUJUVBR69OihtoyLiwu+//77dgdHOo+yYVcul6O+vh5jx47FvXv34OTkhKKiIq5mZywN2cbu7t27HVrOGGmd7DIyMtCnTx+uJqfEGMPNmzfRp08fmJubY+HChR0WJOl4fn5+2LJlC0xNTXH37t1mJ4GpqSmamppavQFE9Mf9+/c7tJwx0jrZ9evXD+Xl5c0ub6qqquDp6UnPVRoId3d3AI+78AgEAnh5eUEgEIAxhqtXr6KpqUmlHNFvT553AoEAw4cPh6WlJR49eoSCggKuSYfP56fWya6ldrDa2lrq2mFAlDU2ZYK7cuWKynrlcqrZGYbCwkLud8YYzp4922Y5vtE42SmfKhAIBFi1apVKXx25XI7c3FwMGzaswwMkneMf//gHgJa/vJTL//GPf+Cdd97pqrDIMzp37hz3u7OzM3x8fFTaYJXNFE+W4xuNk11BQQGAxyfB+fPnYW5uzq0zNzfH0KFDsWLFio6PkHSKp2ty7S1H9Ie6NliiRbI7duzx40TR0dH46quvjOo5UT56su3GyckJ48aNQ1VVFRwdHZGVlYV79+41K0f017Bhw3Dr1i2NyvGV1m1227dv74w4SBdTPhsLPH4+dt++fdzrPn36qC1H9Ne0adOQkpLCvR45ciT69euHkpISnDlzRqUcX2mU7GbMmIEdO3bAzs4OM2bMaLXs/v37OyQw0rkuXrzI/V5RUYF3330Xnp6eKC0txVdffaW2HNFfT49ld+bMGZUk92Q5vo7WrVGys7e3h0Ag4H4nhk/ZDGFqagq5XI4vvviCW2dqasr1s6PmCsOgvMtqYWGBhoaGZuuVy+lubBuevHSly1jjMGjQIPz+++9oamrClClT0K9fP26UjJKSEqSmpnLliP5TVkYaGhpavRurLMdHBjEsO+l4Tw6kkJaWxg3wmJ6ervJ0TEBAgE7iI9oJDw/HyZMnYWJiAktLS5w4cYJb5+HhwY1WHB4ersModUujZDd8+HCNvxFa6sxI9MuTT0a0NrItPUFhGJR3WRUKRbNRfW7cuMH1m6S7sW3g8x0cYxUYGMiNblJaWtpszgJPT08oFAoEBgbqMEqiqSefeX26o/iTr+nZ2DasXr26s+MgXUwoFGLDhg2YNWsWpkyZAnNzc5SUlKBfv35obGxEamoq9u3bB6FQqOtQiQaUz6r36tULFRUVKv0jTU1N4erqirKyMl4P2UVtdjw2Y8YMrFixAhs2bOBqdufPn4eJiQlWrFjRZjcjon88PT1x9epVbNmyBUePHsWECRPwxhtvICQkBGVlZboOT6c0SnaOjo64fPkynJyc0K1bt1bb76qqqjosONK59u/fjy+++KLZ8WSM4YsvvsDzzz9PCc9AKOcMOXnyJGbNmgVPT080NTXh+vXrmDVrFjdd5tMTV/GJRsnuyy+/hK2tLfc7n29fGwu5XI7o6GgALbfxREdHIyIigi5lDYByIN3AwED89ttv3PL09HRueXZ2dosD7vKBgNHY5a2qrq6Gvb09JBKJUXWwTU9P12g+2LS0NEyaNKkLIiLtIZfL4ejoiOrqari4uMDHxwd3796Fs7MzioqKUFlZCTs7O1RVVRnVl5c256fWE+4IhUK1VeH79+8b1X+isfvhhx+431944QVkZ2fj559/RnZ2Nl544QW15Yj+ksvlqK2tBfD4UvX48eO4ePEijh8/zp2vtbW1vB7YQetk11JFUCqVqgz7RPSbclwzDw8P7Nu3D7m5ufjxxx+Rm5uLffv2wcPDQ6Uc0W/ffPMNd5Pp6SkTlJUQhUKBb775pstj0xca3439+uuvATx+3OR//ud/YGNjw62Ty+U4ceIEvL29Oz5C0imU7a5VVVWwsbHhvvEPHTqE999/nxucldpnDcPly5cBPB6u69atW8jOzkZqaiqmTJmCwMBA9OrVC/fv3+fK8ZHGye7LL78E8Lhmt3XrVpVLVnNzc4jFYmzdurXjIySdYtCgQSgsLERNTU2zdXK5nFtOz8YahvLycgDAlClTIBKJEBQUhLq6OgQFBcHMzAyhoaHYtWsXV46PNE52paWlAIDx48dj//796NatW6cFRTpfZGQk9uzZo1E5ov/c3NwAAIcPH4ZUKkV2djZOnDgBa2trBAYGIi0tTaUcH2ndqVg5YjExbEVFRRqXmzp1aidHQ9pr4MCBAB4PyW5lZcW1323cuJEbBODJcnykddeTRYsWtbp+27Zt7QpI3xhr15OIiAgcPHiwzXIvvvgifv311y6IiLRHY2MjLC0tmw3q8CQTExM8evTIqG4kanN+al2ze3qYbplMhsLCQjx8+BATJkzQ9u2IjijnK+jWrRtsbGxURsro06cPampq8ODBA43mNSC6JxQKYWFhgfr6+hbLWFhY8Lp7mNbJ7sCBA82WKRQKLF26FP369euQoEjnE4lEAB5/WV25cqXZ3bvu3burlCP6LSsrq9VEBwD19fXIyspCcHBwF0WlX7TuZ6f2TUxMEBcXx92xJfqvb9++AB53NPXw8MCVK1cwaNAgXLlyBR4eHqirq1MpR/Tb4cOHud8lEgnWr1+PsLAwrF+/HhKJRG053mEd5LfffmNOTk4d9XZ6QyKRMABMIpHoOpQOlZaWxgC0+ZOWlqbrUIkG+vfvzwCwkJAQxhhjjY2NLDk5mTU2NjLGGJswYQIDwPr376/LMDucNuen1pexcXFxTydLlJeX47fffsPChQvbk3dJFwoODoadnR2qq6tV7tYB4F7b2dnx9pLH0MhkMgCPH9tUN/K0cjQiZTk+0voytqCgQOXnX//6FwBgw4YN2LRpU0fHRzqJUCjkJk9iLYx6sn37dl43aBuSwYMHA3h8fkZEROCPP/7Ao0eP8McffyAiIgJ//vmnSjle6uxqZke5f/8+mzdvHrO1tWX29vZs0aJFrKamRqNtFQoFCw0NZQDYgQMHtPpcY72MVUpKSmJ9+vRRuXT18PBgSUlJug6NaKGmpoY7fpaWlirH88nXmp4zhkKb87NDblB0hcjISFy4cAEZGRlISUnBiRMnsGTJEo223bRpEz3j2YIZM2bg2rVryMjIQFxcHDIyMlBSUkKDdhoYGxsbjBo1CgDw6NEjlXXK16NGjVJ5pp13uiD5ttvFixcZAJaXl8ctS01NZQKBgJWVlbW6bUFBAevVqxcrLy+nml0rnm7QJoZp1KhRam80jRo1StehdYpOvUGhCzk5OXBwcMDIkSO5ZSEhITAxMUFubi6mT5+udrv6+nrMmzcPW7Zs0fiZQKlUCqlUyr2urq4G8Lhh15gbd5X7Zsz7yAenTp1CbW0tFixYgPPnz2Pw4MHYuXMnbGxsjPLYarNPBpHsKioqms2KZGpqCkdHR1RUVLS43bJlyzBmzBhERERo/FkJCQlYs2ZNs+Xp6encsEfGLCMjQ9chkA7wn//5n9zvT06YbWza6kj9pA5Ldrdu3cLf//53bpZ5TaxcuRLr1q1rtYymD6w/7eDBgzh69CgKCgq02i4+Pl6le011dTXc3d0xadIko3o29mkymQwZGRmYOHEizMzMdB0OaSe+HE/llZcmOizZ3b9/H99//71WyW758uWIiopqtUzfvn3h5ubWbCj4pqYmVFVVtXh5evToUZSUlMDBwUFl+cyZMxEYGIisrCy124lEIrWPSJmZmRn1H40SX/aTL4z9eGqzbzq9jHV2doazs3Ob5fz9/fHw4UPk5+fD19cXwONkplAo4Ofnp3ablStXqlTlgcd9jL788kuEh4e3P3hCiEExiDY7Hx8fhIaGYvHixdi6dStkMhliY2Px8ssvo2fPngCAsrIyBAcHY+fOnRg9ejTc3NzU1vr69OkDT0/Prt4FQoiOGUw/u127dsHb2xvBwcEICwtDQECAyiWzTCZDcXGxVg2WhBD+0Lhm11Yn04cPH7Y3llY5Ojpi9+7dLa4Xi8Utznym1NZ6Qojx0jjZ2dvbt7l+wYIF7Q6IEEI6g8bJTvnQOCGEGCKDabMjhJD20Lhm19ZEO0rGNuEOIcQ4aJzsduzYAQ8PDwwfPpwa+gkhBkfjZLd06VL8/PPPKC0tRXR0NF555RU4Ojp2ZmyEENJhNG6z27JlC8rLy/Hee+/hf//3f+Hu7o7Zs2cjLS2NanqEEL2n1Q0KkUiEuXPnIiMjAxcvXsTf/vY3vP766xCLxaitre2sGAkhpN2e+W6siYkJBAIBGGOQy+UdGRMhhHQ4rZKdVCrFzz//jIkTJ2LAgAE4f/48Nm/ejBs3bvB7uGdCiN7T+AbF66+/jj179sDd3R2LFi3Czz//DCcnp86MjRBCOozGyW7r1q3o06cP+vbti+PHj+P48eNqy+3fv7/DgiOEkI6icbJbsGABzdBFCDFYWnUqJoQQQ0XPxhJCeIGSHSFGRi6X4/jx4zhx4gSOHz9OXcP+DyU7QozI/v374eXlhYkTJ2Ljxo2YOHEivLy86MYhKNkRYjT279+PWbNmYfDgwcjOzsbPP/+M7OxsDB48GLNmzeJ9wqNkR4gRkMvlWL58OaZOnYrk5GT4+fnB0tISfn5+SE5OxtSpU7FixQpeX9JSsiPECGRnZ+P69ev44IMPYGKielqbmJggPj4epaWlyM7O1lGEukfJjhAjUF5eDgAYNGiQ2vXK5cpyfETJjhAj0KNHDwBAYWGh2vXK5cpyfETJjhAjEBgYCLFYjE8//RQKhUJlnUKhQEJCAjw9PREYGKijCHWPkh0hRkAoFGLDhg1ISUnBtGnT8Mcff+DRo0f4448/MG3aNKSkpGD9+vUQCoW6DlVnNH5cjBCi32bMmIF9+/Zh+fLlGDt2LLfc09MT+/bta3Oie2NHyY4QIzJjxgxERETg2LFjSE1NxZQpUzB+/Hhe1+iUKNkRYmSEQiGCgoJQV1eHoKAgSnT/h9rsCCG8QMmOEMILlOwIIbxAyY4QwgsGk+yqqqoQGRkJOzs7ODg4ICYmRqO5anNycjBhwgRYW1vDzs4OY8eOxaNHj7ogYkKIPjGYZBcZGYkLFy4gIyMDKSkpOHHiBJYsWdLqNjk5OQgNDcWkSZNw+vRp5OXlITY2ttmD0nxHgz0SXmAG4OLFiwwAy8vL45alpqYygUDAysrKWtzOz8+Pffjhh+36bIlEwgAwiUTSrvfRV0lJSUwsFjMA3I9YLGZJSUm6Do20Q2NjI0tOTmaNjY26DqVTaXN+GkQ/u5ycHDg4OGDkyJHcspCQEJiYmCA3NxfTp09vtk1lZSVyc3MRGRmJMWPGoKSkBN7e3vjkk08QEBDQ4mdJpVJIpVLudXV1NQBAJpNBJpN14F7p3oEDB/Dyyy9jypQpCAsLw9WrV+Hl5YXr169j1qxZ2LNnj9r/W6L/lH+rxvY3+zRt9s8gkl1FRQVcXFxUlpmamsLR0REVFRVqt7l27RoA4OOPP8b69esxbNgw7Ny5E8HBwSgsLET//v3VbpeQkIA1a9Y0W56eng4rK6t27on+kMvlePPNN+Hq6orDhw9zD4+np6fDxMQErq6ueOutt2BqakqdUg1YRkaGrkPoVPX19RqX1WmyW7lyJdatW9dqmaKiomd6b+XJ++qrryI6OhoAMHz4cGRmZmLbtm1ISEhQu118fDzi4uK419XV1XB3d8ekSZNgZ2f3TLHoo+PHj6OyshIA4OLigtWrV8PKygr19fVYs2YN9yViZ2eHoKAgXYZKnoFMJkNGRgYmTpwIMzMzXYfTaZRXXprQabJbvnw5oqKiWi3Tt29fuLm5cSemUlNTE6qqquDm5qZ2O+W4Xc8995zKch8fH9y4caPFzxOJRBCJRM2Wm5mZGdUfjXIQRxcXF5SVlYExhkOHDiEyMhJLlixBr169UFlZifLycqPab74xtr/bp2mzbzq9Lens7Axvb+9Wf8zNzeHv74+HDx8iPz+f2/bo0aNQKBTw8/NT+95isRg9e/ZEcXGxyvLLly/Dw8OjU/fLEOTm5gIAFi1aBFNT1e88U1NT7ktIWY4YDrq7rp5B9MHw8fFBaGgoFi9ejNOnT+PUqVOIjY3Fyy+/jJ49ewIAysrK4O3tjdOnTwMABAIB3n33XXz99dfYt28frl69io8++giXLl1CTEyMLndHLzDGAAD5+flqB3ssKChQKUcMA02l2DKDSHYAsGvXLnh7eyM4OBhhYWEICAjAt99+y62XyWQoLi5WabB85513EB8fj2XLlmHo0KHIzMxERkYG+vXrp4td0CvKGzQZGRlqB3tUNmy3dCOH6B+aSrF1AkZf3a2qrq6Gvb09JBKJUd2gaGxshLW1NaytreHg4IC//vqLWycWi/HgwQPU1dWhrq4O5ubmOoyUaEIul8PLywuDBw9GcnIy5HI5Dh06hLCwMAiFQkybNg2FhYW4cuWKUd1d1+b8NJiaHelY5ubmWLZsGSQSCW7evKmy7saNG5BIJFi2bBklOgNBUym2jZIdjz3//PMAoLbN7sn1RP89OZWiuhsUNJWigXQqJh1PLpfjtddeAwCEhYWhb9++uHz5MgYMGIBr167h0KFDWLp0KSIiIozqssdYKbtabd68GVu3buWaJTZu3AgPDw+8+uqrKuV4qXOfXDN8xvps7JEjRxgAFhAQwORyucqzlHK5nAUEBDAA7MiRI7oOlWigqamJubi4MADM0tJS5Vln5WsXFxfW1NSk61A7lDbnJ13G8lRWVhYAYM2aNWrbeFavXq1Sjui/hoYGAI+feklMTMT27duRmJjINdwr1/MVXcYSlTYea2trjB8/XtchES1lZWWhuroaPj4+qK+vx9KlS7l1YrEY3t7euHTpErKyshAcHKzDSHWHanY8NW7cOADAG2+8obYT6ptvvqlSjug3ZQ18zpw5zdYxxrjlfK6pU82Op8aNGwd7e3tcunQJLi4uSExMhIWFBRoaGrB69Wpcv34d9vb2lOwMzMcff4zw8HD89NNPuHXrFnr37o3PP/9c7Ug+fEPJjseUfehqampULnssLS0BQO2ACEQ/jR07FgDQrVs37N+/H4wx3L9/H35+fti/fz9cXFzw4MEDrhwf0WUsT2VnZ+Pu3btISEiAq6uryjo3Nzd8+umnqKys5HUnVEOivMn04MEDTJ8+XeXxv+nTp+PBgwcq5fiIanY8pexcGhsbi3fffRfHjh1DamoqpkyZgvHjx6O+vh4ffPABrzuhGhLlEGgCgQCZmZlISUnh1llZWUEgEIAx1myoND7hb5rnOWXn0sLCQgiFQgQFBWHs2LEICgqCUChEYWGhSjmi35TH6dNPP21WU3d1dcUnn3yiUo6PKNnxVGBgIMRiMT799FO1j4slJCTA09MTgYGBOoqQaEN5PH///XdcvnwZGRkZiIuLQ0ZGBoqLi5GTk8P740nJjqeEQiE2bNiAlJQUtUM8paSkYP369fSomIF48njOnDkTIpEIo0aNgkgkwsyZM+l4AvS4WFuM9XExJXVTKXp6etJUigaKb8fT6KZSJJ1nxowZiIiIaHaDgtc1AAPHnhqi8ulmCr6iy1ii9gYFMTzKkYqHDBmiMlLxkCFDaKRiULIjxCjI5XIsX74cU6dORXJyMvz8/GBpaQk/Pz8kJydj6tSpWLFiBa8n36FkR4gRoJGK20bJjhAj8ORIxerQSMWU7AgxCk92EleHOolTsiPEKFAn8bZRsiPECFAn8bZRPztCjMSMGTOwb98+LF++XGUoJ09PT+zbtw8zZszQYXS6R8mOECNCncRbRsmOECOj7CReV1dHncSfQG12hBBeoGRHCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBYNJdlVVVYiMjISdnR0cHBwQExOD2traVrepqKjA/Pnz4ebmBmtra4wYMQJJSUldFDEhRJ8YTLKLjIzEhQsXkJGRgZSUFJw4cQJLlixpdZsFCxaguLgYBw8exPnz5zFjxgzMnj0bBQUFXRQ1IV1PLpfj+PHjOHHiBI4fP87rMexUdP4o8e138eJFBoDl5eVxy1JTU5lAIGBlZWUtbmdtbc127typsszR0ZF99913Gn+2sc9BodTY2MiSk5NZY2OjrkMh7ZCUlMQ8PDxU5qDw8PCgOSiYgcxBkZOTAwcHB4wcOZJbFhISAhMTE+Tm5mL69OlqtxszZgz27t2LF154AQ4ODvjll1/Q0NCAcePGtfhZUqkUUqmUe11dXQ0AkMlkkMlkHbNDekYulyMrKwsnTpyASCTCuHHjqNe9ATpw4ADmzJkDS0tLleWVlZWYOXMm9u7d2+K5Yqi0OScNItlVVFTAxcVFZZmpqSkcHR1RUVHR4na//PIL5syZg+7du8PU1BRWVlY4cOAAvLy8WtwmISEBa9asabY8PT0dVlZWz74TeionJwfff/897t27BwDYuHEjnJycEBMTA39/fx1HRzQll8uxePFiAIC3tzekUilqampga2sLkUiEgoICLFmyBKampkb1RVZfX69xWZ0mu5UrV2LdunWtlikqKnrm9//oo4/w8OFDHDlyBE5OTkhOTsbs2bORnZ2NwYMHq90mPj4ecXFx3Ovq6mq4u7tj0qRJsLOze+ZY9NGBAwfU/v/fu3cP69atM8qagLE6evQoqqurYWNjo7ZN2sbGBhKJBNbW1pgwYYIOIuwcyisvTeg02S1fvhxRUVGtlunbty/c3NxQWVmpsrypqQlVVVVwc3NTu11JSQk2b96MwsJC/O1vfwMADB06FNnZ2diyZQu2bt2qdjuRSASRSNRsuZmZGczMzDTYK8Mgl8vb/L+PiorCzJkzjaomYKxOnjwJAKitrYVAIEBkZCR8fX2Rn5+PXbt2cT0XTp48icmTJ+sy1A6lzTmp02Tn7OwMZ2fnNsv5+/vj4cOHyM/Ph6+vL4DH32QKhQJ+fn5qt1FWb5+efEQoFNI8mnh8Wf7o0SMAjxP8k+2UytePHj1Ceno6pkyZoqswiYaePH719fUQCoU4dOgQ3njjDXz33XdcO96T5fjGILqe+Pj4IDQ0FIsXL8bp06dx6tQpxMbG4uWXX0bPnj0BAGVlZfD29sbp06cBPG638PLywquvvorTp0+jpKQEGzZsQEZGBqZNm6bDvdEPGzdu5H6fOHGiyjyjEydOVFuO6K9Tp04BeFyBMDc3V1lnbm4OJycnlXJ8ZBDJDgB27doFb29vBAcHIywsDAEBAfj222+59TKZDMXFxVyNzszMDIcOHYKzszPCw8MxZMgQ7Ny5Ez/88APCwsJ0tRt648aNGwCAgQMH4tdff1WZZ/TXX39F//79VcoR/aZsu7p79y4iIiJUhmWPiIjgbkBp08ZlbAzibiwAODo6Yvfu3S2uF4vFYIypLOvfvz89MdECGxsbAEBNTY3a9XV1dSrliH7z8vLiZhDLzMxESkoKt+7JXgSt9UQwdgZTsyMdKyQkBABw+/ZthIeHq9QEwsPDcfv2bZVyRL/9+OOP3O9Pd9N68vWT5fhGwJ6uDhEV1dXVsLe3h0QiMaquJ5mZmRolsiNHjiA4OLgLIiLtNXr0aOTl5QEAgoOD4ebmhoqKCmRmZgIARo0axbVpGwttzk9Kdm0w1mQnl8vRs2fPZl16nuTi4oLbt29T1xMD8mTCe5IxJjpAu/OTLmN5SigUIjExEQKBoFm/QpFIBIFAgMTEREp0Bub06dOoqalBeHg4PDw8EB4ejpqaGqNMdNqiZMdjynlGe/ToobK8Z8+eNM+oAbOxsUFSUhK++uorJCUl0U2m/2Mwd2NJ56B5RglfULIjNM8o4QW6jCWE8AIlO0IIL1CyI4TwArXZtUHZDdHYnymUyWSor69HdXW1UQ1lxVd8OZ7K81KT7sKU7NqgfHbU3d1dx5EQQlpSU1MDe3v7VsvQExRtUCgUuH37NmxtbSEQCHQdTqdRjsh88+ZNo3pShK/4cjwZY6ipqUHPnj2bjV35NKrZtcHExAS9e/fWdRhdxs7OzqhPDr7hw/Fsq0anRDcoCCG8QMmOEMILlOwIgMcP/69evVrtZEPE8NDxbI5uUBBCeIFqdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnZGaNy4cXjnnXd0HQbRkr4dN32Lp70o2RFiRBobG3Udgv5ixKgsXLiQAVD56dWrF/vmm29Uyp09e5YJBAJ2/fp1xhhjGzZsYIMGDWJWVlasd+/ebOnSpaympkZlm+zsbBYQEMAsLCxY79692Ztvvslqa2u7bN+MmbrjdvXqVbZo0SImFouZhYUFGzBgANu0aVOz7SIiIth//dd/sR49ejCxWMwYY+zUqVNs6NChTCQSMV9fX3bgwAEGgBUUFHDbnj9/noWGhjJra2vm4uLCXnnlFXb37t0W4yktLe2q/45OQcnOyDx8+JD5+/uzxYsXs/LyclZeXs5WrFjBAgICVMotX75cZdmXX37Jjh49ykpLS1lmZiYbOHAgW7p0Kbf+6tWrzNramn355Zfs8uXL7NSpU2z48OEsKiqqy/bNmKk7bg0NDWzVqlUsLy+PXbt2jf3000/MysqK7d27l9tu4cKFzMbGhs2fP58VFhaywsJCJpFImKOjI3vllVfYhQsX2KFDh9iAAQNUkt2DBw+Ys7Mzi4+PZ0VFRezs2bNs4sSJbPz48S3G09TUpIv/mg5Dyc4IBQUFsbfffpt7XVBQwAQCAfvrr78YY4zJ5XLWq1cvlpiY2OJ7/POf/2Tdu3fnXsfExLAlS5aolMnOzmYmJibs0aNHHbsDPPX0cVPnjTfeYDNnzuReL1y4kLm6ujKpVMotS0xMZN27d1c5Lt99951Kslu7di2bNGmSynvfvHmTAWDFxcUax2NIqM2OB4YNGwYfHx/s3r0bAHD8+HFUVlbipZde4socOXIEwcHB6NWrF2xtbTF//nzcv38f9fX1AIBz585hx44dsLGx4X4mT54MhUKB0tJSnewXH2zZsgW+vr5wdnaGjY0Nvv32W9y4cUOlzODBg2Fubs69Li4uxpAhQ2BhYcEtGz16tMo2586dw7Fjx1SOp7e3NwCgpKSkE/dIdyjZ8URkZCSX7Hbv3o3Q0FB0794dAHD9+nVMnToVQ4YMQVJSEvLz87FlyxYA/27wrq2txauvvoo///yT+zl37hyuXLmCfv366WanjNyePXuwYsUKxMTEID09HX/++Seio6Ob3YSwtrbW+r1ra2sRHh6ucjz//PNPXLlyBWPHju2oXdArNJ6dETI3N4dcLldZNm/ePHz44YfIz8/Hvn37sHXrVm5dfn4+FAoFNmzYwA2A+Msvv6hsP2LECFy8eBFeXl6dvwM89fRxO3XqFMaMGYPXX3+dW6ZJrWvgwIH46aefIJVKuYEA8vLyVMqMGDECSUlJEIvFMDVVnwbU/R0ZMqrZGSGxWIzc3Fxcv34d9+7dg0KhgFgsxpgxYxATEwO5XI4XX3yRK+/l5QWZTIb//u//xrVr1/Djjz+qJEMAeP/99/H7778jNjaWqwH8+uuviI2N7erdM1pPH7f+/fvjzJkzSEtLw+XLl/HRRx81S1rqzJs3DwqFAkuWLEFRURHS0tKwfv16AOBG237jjTdQVVWFuXPnIi8vDyUlJUhLS0N0dDSX4NT9HRk0XTcako5XXFzMnn/+eWZpaanSZeCbb75hANiCBQuabbNx40bWo0cPZmlpySZPnsx27tzJALAHDx5wZU6fPs0mTpzIbGxsmLW1NRsyZAj75JNPumivjN/Tx+3SpUssKiqK2dvbMwcHB7Z06VK2cuVKNnToUG4bZdeTp506dYoNGTKEmZubM19fX7Z7927uPZUuX77Mpk+fzhwcHJilpSXz9vZm77zzDlMoFGrjMfSuJzTEEyE8sGvXLkRHR0MikcDS0lLX4egEtdkRYoR27tyJvn37olevXjh37hzef/99zJ49m7eJDqBkR4hRqqiowKpVq1BRUYEePXrgpZdewieffKLrsHSKLmMJIbxAd2MJIbxAyY4QwguU7AghvEDJjhDCC5TsCCG8QMmOEMILlOyI3oiKioJAIIBAIICZmRlcXV0xceJEbNu2TavnMnfs2AEHB4fOC7QFUVFRmDZtWpd/LtEMJTuiV0JDQ1FeXo7r168jNTUV48ePx9tvv42pU6eiqalJ1+ERQ6bbR3MJ+beWHmrPzMxkANh3333HGGt9voxjx441mzth9erVjDHGdu7cyXx9fZmNjQ1zdXVlc+fOZXfu3OE+p6qqis2bN485OTkxCwsL5uXlxbZt28atv3HjBnvppZeYvb0969atG3vxxRe5h+NXr17d7HOPHTvWKf9P5NlQzY7ovQkTJmDo0KHYv38/AMDExARff/01Lly4gB9++AFHjx7Fe++9BwAYM2YMNm3aBDs7O5SXl6O8vBwrVqwAAMhkMqxduxbnzp1DcnIyrl+/jqioKO5zPvroI1y8eBGpqakoKipCYmIinJycuG0nT54MW1tbZGdn49SpU7CxsUFoaCgaGxuxYsUKzJ49m6uZlpeXY8yYMV37H0Vap+tsS4hSSzU7xhibM2cO8/HxUbvu6fkytm/fzuzt7dv8vLy8PAaAqxWGh4ez6OhotWV//PFHNnDgQG74I8YYk0qlzNLSkqWlpbUZP9E9qtkRg8AY4waebGu+jJbk5+cjPDwcffr0ga2tLYKCggCAm9Nh6dKl2LNnD4YNG4b33nsPv//+O7ftuXPncPXqVdja2nJzNjg6OqKhocFo52wwNpTsiEEoKiqCp6enRvNlqFNXV4fJkyfDzs4Ou3btQl5eHg4cOKCy3ZQpU/DXX39h2bJluH37NoKDg7lL4NraWvj6+jabs+Hy5cuYN29eJ+896Qg0xBPRe0ePHsX58+exbNkyjebLUDd3wqVLl3D//n189tlncHd3BwCcOXOm2Wc5Oztj4cKFWLhwIQIDA/Huu+9i/fr1GDFiBPbu3QsXFxfY2dmpjdPY5mwwNlSzI3pFKpWioqICZWVlOHv2LD799FNERERg6tSpWLBggUbzZYjFYtTW1iIzMxP37t1DfX09+vTpA3Nzc267gwcPYu3atSrbrVq1Cr/++iuuXr2KCxcuICUlBT4+PgAez87m5OSEiIgIZGdno7S0FFlZWXjrrbdw69Yt7nP/9a9/obi4GPfu3YNMJuua/zSiGV03GhKitHDhQq7bhqmpKXN2dmYhISFs27ZtTC6Xc+U0mS/jtddeY927d1fperJ7924mFouZSCRi/v7+7ODBg80mjvbx8WGWlpbM0dGRRUREsGvXrnHvWV5ezhYsWMCcnJyYSCRiffv2ZYsXL2YSiYQxxlhlZSU3Rweo64neocE7CSG8QJexhBBeoGRHCOEFSnaEEF6gZEcI4QVKdoQQXqBkRwjhBUp2hBBeoGRHCOEFSnaEEF6gZEcI4QVKdoQQXvj/OQySBskOzuUAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T10:56:00.213892Z", + "iopub.status.busy": "2024-03-26T10:56:00.213613Z", + "iopub.status.idle": "2024-03-26T10:56:00.501992Z", + "shell.execute_reply": "2024-03-26T10:56:00.501187Z" + }, + "papermill": { + "duration": 0.310135, + "end_time": "2024-03-26T10:56:00.503802", + "exception": false, + "start_time": "2024-03-26T10:56:00.193667", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUIAAAEmCAYAAAAXyJnQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABstUlEQVR4nO2dZ3hU1daA36npDdIhkFCkSS8xWFCJBtSLKB8CcqWIgCCiRFRApOhVUIpYUGwUFQTxKnhRUYgUkUgJoRfpoSSBENLLtPP9mMxkJpkkM2HS9/s858mZc/bZZ53JzJq191p7LZkkSRICgUDQgJHXtAACgUBQ0whFKBAIGjxCEQoEggaPUIQCgaDBIxShQCBo8AhFKBAIGjxCEQoEggaPUIQCgaDBo6xpAeoqBoOBq1ev4uXlhUwmq2lxBAKBBZIkkZ2dTWhoKHJ5xfaeUISV5OrVq4SFhdW0GAKBoBwuXbpE06ZNK2wnFGEl8fLyAoxvtLe3dw1LIxAILMnKyiIsLMz8Pa0IoQgriWk47O3tLRShQFBLsXfaSjhLBAJBg0coQoFA0OARilAgEDR4xByhoFwkSUKn06HX62taFIGTUCgUKJVKEfZlgVCEgjLRaDQkJyeTl5dX06IInIy7uzshISGo1eqaFqVWIBShwCYGg4Hz58+jUCgIDQ1FrVYLC6IeIEkSGo2G69evc/78eVq3bm1XwHG9R6phPvroI6l58+aSi4uL1KtXL2nPnj1ltj169Kj0+OOPS82bN5cA6b333ivVxnSu5DZx4kRzmz59+pQ6P378eIfkzszMlAApMzOz3HazNhyR7l+4Tdp6PMWh/mua/Px86fjx41Jubm5NiyKoAnJzc6Xjx49L+fn5NS1KlWDv99NEjf4UrFu3jtjYWGbPns2BAwfo3LkzMTExXLt2zWb7vLw8WrRowfz58wkODrbZZt++fSQnJ5u3LVu2ADB48GCrdmPHjrVq9+677zr34YpIzSrk7PVcrmTkV0n/VY2wFuon4v9qTY2+G4sXL2bs2LGMHj2a9u3bs2zZMtzd3Vm+fLnN9j179mTBggUMHToUFxcXm20CAgIIDg42b5s2baJly5b06dPHqp27u7tVu6oKig70Nsp5LauwSvoXCAS3To0pQo1GQ0JCAtHR0cXCyOVER0cTHx/vtHt88803PP3006Xmt1avXo2/vz+3334706dPr9AhUFhYSFZWltVmDwGeRkV4PVsoQoGgtlJjzpK0tDT0ej1BQUFWx4OCgjh58qRT7rFhwwYyMjIYNWqU1fEnn3yS5s2bExoayuHDh3n11Vc5deoUP/zwQ5l9zZs3j7lz5zosg9kizC5w+FqBoLKMGjWKjIwMNmzYUNOi1Anqtdf4yy+/pH///oSGhlodHzdunHm/Y8eOhISE0LdvX86ePUvLli1t9jV9+nRiY2PNr02LuisiwKvIIswRFmF1ce+999KlSxeWLFlS06II6gg1pgj9/f1RKBSkpqZaHU9NTS3TEeIIFy9eZOvWreVaeSYiIyMBOHPmTJmK0MXFpcx5yfII9HIFxByhwHG0Wi0qlaqmxWgQ1NgcoVqtpnv37sTFxZmPGQwG4uLiiIqKuuX+V6xYQWBgIA8//HCFbQ8ePAhASEjILd+3JCaL8EauBr1Bcnr/1YkkSeRpdNW+SZL979uoUaPYsWMH77//PjKZDJlMRtOmTfnkk0+s2iUmJiKXy7l48SJgdNx17NgRDw8PwsLCmDhxIjk5OVbX7Nq1i7vvvhs3NzfCwsKYPHkyubm5dsmVnJzMww8/jJubGxEREaxZs4bw8HArq1Umk/HJJ58wYMAAPDw8eOutt9Dr9YwZM4aIiAjc3Nxo06YN77//vlXfer2e2NhYfH19ady4Ma+88opD75mghofGsbGxjBw5kh49etCrVy+WLFlCbm4uo0ePBmDEiBE0adKEefPmAUbnx/Hjx837V65c4eDBg3h6etKqVStzvwaDgRUrVjBy5EiUSutHPHv2LGvWrOGhhx6icePGHD58mClTpnDPPffQqVMnpz9jYw81MhnoDRLpuRqzYqyL5Gv1tJ/1W7Xf9/gbMbir7fuovv/++/zzzz/cfvvtvPHGGwAsWrSINWvWMGHCBHO71atXc+edd9K8eXPA6Kj74IMPiIiI4Ny5c0ycOJFXXnmFjz/+GDB+bvr168d//vMfli9fzvXr15k0aRKTJk1ixYoVFco1YsQI0tLS2L59OyqVitjYWJthYnPmzGH+/PksWbIEpVKJwWCgadOmrF+/nsaNG7N7927GjRtHSEgITzzxhPn5Vq5cyfLly2nXrh2LFi3ixx9/5P7777frPRPUsCIcMmQI169fZ9asWaSkpNClSxc2b95sdqAkJSVZxTtdvXqVrl27ml8vXLiQhQsX0qdPH7Zv324+vnXrVpKSknj66adL3VOtVrN161az0g0LC2PQoEHMnDmzSp5RqZDT2ENNWo6G69mFdVoR1gV8fHxQq9Xm8CiA4cOHs2jRIpKSkmjWrBkGg4G1a9da/c9ffPFF8354eDj/+c9/ePbZZ82KcN68eQwfPtzcrnXr1nzwwQf06dOHTz75BFdX1zJlOnnyJFu3bmXfvn306NEDgC+++ILWrVuXavvkk0+aDQETlk66iIgI4uPj+e6778yKcMmSJUyfPp3HH38cgGXLlvHbb9X/g1WXqXFnielX1RaWyg2MH1B7TP4HH3ywzHZhYWHs2LHDYTlvhQAvV9JyNFzLLqA9dTeJq5tKwfE3YmrkvrdCly5daNeuHWvWrGHatGns2LGDa9euWQXZb926lXnz5nHy5EmysrLQ6XQUFBSQl5eHu7s7hw4d4vDhw6xevdp8jSRJ5qWI7dq1K/P+p06dQqlU0q1bN/OxVq1a4efnV6qtSVFasnTpUpYvX05SUhL5+floNBq6dOkCQGZmJsnJyeZ5bgClUkmPHj3E8NgBalwRNgQCvFw4kQzX6ngsoUwms3uIWtsYPny4WRGuWbOGfv360bhxYwAuXLjAI488woQJE3jrrbdo1KgRu3btYsyYMWg0Gtzd3cnJyWH8+PFMnjy5VN/NmjVzmpweHh5Wr9euXcvUqVNZtGgRUVFReHl5sWDBAvbs2eO0ewqEIqwWAr1EUHV1olarS6UNe/LJJ5k5cyYJCQl8//33LFu2zHwuISEBg8HAokWLzFMx3333ndX13bp14/jx41Zz0fbSpk0bdDodiYmJdO/eHTBGKNy8ebPCa//66y969+7NxIkTzcfOnj1r3vfx8SEkJIQ9e/Zwzz33AKDT6UhISLCyQAXlIxYcVgMBQhFWK+Hh4ezZs4cLFy6QlpaGwWAgPDyc3r17M2bMGPR6PQMGDDC3b9WqFVqtlg8//JBz587x9ddfWylKgFdffZXdu3czadIkDh48yOnTp9m4cWOZ0zqWtG3blujoaMaNG8fevXtJTExk3LhxuLm5VZjRp3Xr1uzfv5/ffvuNf/75h9dff519+/ZZtXnhhReYP38+GzZs4OTJk0ycOJGMjAz73zCBUITVgbAIq5epU6eiUCho3749AQEBJCUlAcbh8aFDh3jsscdwc3Mzt+/cuTOLFy/mnXfe4fbbb2f16tXmSAUTnTp1YseOHfzzzz/cfffddO3alVmzZpUK1i+Lr776iqCgIO655x4ee+wxxo4di5eXV7lOFoDx48fz+OOPM2TIECIjI7lx44aVdQjw0ksv8dRTTzFy5Ejz8Pmxxx6zSy6BEZkkZlQrRVZWFj4+PmRmZlaYsGHT4atMWpNIz3A/1j/bu5okvDUKCgo4f/48ERERFX5ZBY5z+fJlwsLC2Lp1K3379q32+9f3/68j308Qc4TVgml1ibAIGy5//PEHOTk5dOzYkeTkZF555RXCw8PN83qCmkUowmrANEdY173GAtv8+eef9O/fv8zzOTk5aLVaZsyYwblz5/Dy8qJ3796sXr1aLKGrJQhFWA2Y5gjzNHpyC3V4uIi3vT7Ro0cP8zLNsoiJiSEmpvpjMAX2Ib6R1YCHixJ3tYI8jZ5r2YVECEVYr3Bzc6tUWI2g9iC8xtWE8BwLBLUXoQirieJ5QpGgVSCobQhFWE0Iz7FAUHsRirCaEJ5jgaD2IhRhNSGW2QkEtRehCKsJYRHWDWxljRYFkOo/Io6jmhBe47pJcnKyzbyBgvqFUITVRPHQWHiN6xLOKCR2q0iShF6vL1V2QuA8xNC4mjB5jW/katDpDTUsTSWRJNDkVv/mYF6Q7Oxshg8fjoeHByEhIbz33nvce++9Vun47cVyaHzhwgVkMhk//PAD9913H+7u7nTu3Jn4+Hirayoq8vT111/To0cPvLy8CA4O5sknn7SqX7J9+3ZkMhm//vor3bt3x8XFhV27djksu8B+avwnZunSpSxYsICUlBQ6d+7Mhx9+SK9evWy2PXbsGLNmzSIhIYGLFy/y3nvvlfpwz5kzp1Qh9jZt2lgVjS8oKOCll15i7dq1FBYWEhMTw8cff1yq2LwzaeShRi4Dg2RUhkHedTDjhzYP3rYv7ZRTmXEV1B4VtysiNjaWv/76i59++omgoCBmzZrFgQMHzOntb5XXXnuNhQsX0rp1a1577TWGDRvGmTNnUCqVdhV50mq1vPnmm7Rp04Zr164RGxvLqFGj+OWXX6zuM23aNBYuXEiLFi3E8LyKqVFFuG7dOmJjY1m2bBmRkZEsWbKEmJgYTp06RWBgYKn2eXl5tGjRgsGDBzNlypQy++3QoQNbt241vy45pJgyZQo///wz69evx8fHh0mTJvH444/z119/Oe/hSqCQy/D3dOFadiHXswvrpiKsA2RnZ7Nq1SrWrFljTm+1YsUKu/MG2sPUqVPNZWLnzp1Lhw4dOHPmDG3btrWryJNlUbEWLVrwwQcf0LNnT3JycvD09DSfe+ONN3jggQecJregbGpUES5evJixY8eaq3YtW7aMn3/+meXLlzNt2rRS7Xv27EnPnj0BbJ43oVQqy5zbyczM5Msvv2TNmjXmcocrVqygXbt2/P3339xxxx23+lhlEuBlVITG1SU+VXafKkPlbrTOauK+dnLu3Dm0Wq3VqMLHx4c2bdo4TRzLsq+mWtjXrl2jbdu2dhV5SkhIYM6cORw6dIibN29iMBinSpKSkmjfvr35OluFnARVQ40pQo1GQ0JCAtOnTzcfk8vlREdHl5pzcZTTp08TGhqKq6srUVFRzJs3z1xgJyEhAa1WS3R0tLl927ZtadasGfHx8VWqCAO9XDhGHfYcy2QODVHrK5aps0yp9k3KrKIiT7m5ueZMNKtXrzZn0I6JiUGj0Vi1L1nISVB11JgiTEtLQ6/Xl5qXCwoKsprPc5TIyEhWrlxJmzZtSE5OZu7cudx9990cPXoULy8vUlJSUKvV+Pr6lrpvSkpKmf0WFhZSWFiswLKyshyWzRxLmFVHFWEdoEWLFqhUKvbt22f+8cvMzOSff/6pliSoFRV5OnLkCDdu3GD+/PmEhYUBsH///iqXS1A+9c5r3L9/fwYPHkynTp2IiYnhl19+ISMjo1RVMkeZN28ePj4+5s30IXYE83rjHKEIqwovLy9GjhzJyy+/zLZt2zh27BhjxoxBLpdXWCjJGVRU5KlZs2ao1WpzoaiffvqJN998s8rlEpRPjSlCf39/FAoFqampVsdTU1OdGrvl6+vLbbfdxpkzZwBjXJhGoylV5aui+06fPp3MzEzzdunSJYdlERZh9bB48WKioqJ45JFHiI6O5s4776Rdu3bVUpujoiJPAQEBrFy5kvXr19O+fXvmz5/PwoULq1wuQfnU2NBYrVbTvXt34uLiGDhwIGCcZ4mLi7OrRKK95OTkcPbsWZ566ikAunfvjkqlIi4ujkGDBgFw6tQpkpKSiIqKKrMfFxcXXFxcbkkW8+oSYRFWKV5eXlbOitzcXObOncu4ceMqvPbChQtWry1rm4WHh1Oy1pmvr2+pYz179uT3338v8x7Dhg1j2LBhZd7n3nvvLdVnbSOrQItcJsOzniQZrtGniI2NZeTIkfTo0YNevXqxZMkScnNzzV7kESNG0KRJE3NpRY1Gw/Hjx837V65c4eDBg3h6eprnZKZOncq//vUvmjdvztWrV5k9ezYKhcL8wfPx8WHMmDHExsbSqFEjvL29ef7554mKiqpSRwlAoLfISVgdJCYmcvLkSXr16kVmZiZvvPEGAI8++mgNS1Y/yCnUcf/C7Xi7qvjlhbtxVSlqWqRbpkYV4ZAhQ7h+/TqzZs0iJSWFLl26sHnzZrMDJSkpCbm8ePR+9epVunbtan69cOFCFi5cSJ8+fdi+fTtgLJM4bNgwbty4QUBAAHfddRd///03AQEB5uvee+895HI5gwYNsgqormoCPItzEkqSVC1zVg2VhQsXcurUKfPI488//+TEiRMVFlkSVMyRy5mk5WhIy9HwY+IVhvVqVtMi3TKirnElcbRuKkC+Rk+7WZsBODznQbxda28Fs/pY9zY/P58rV66Ueb4h1R25lf/vl7vO8+Ym48gswt+DrbF9UMhr14+6qGtci3FTK/ByUZJdqON6dmGtVoT1EVFkyTkcv1ocOnY+LZffj6XQv2NIDUp069S78JnaToB33fIciwFD/eRW/q/Hk42KsEOo0dJatuNsnf+cCEVYzQR41g3PsWn1RF5eXg1LIqgKTP9XRwvMF+r0nE7NBuDtxzriopRz6HIm8eduOF3G6kQMjauZwKJkC9eyarfnWKFQ4Ovra04P5e7uLpw79QBJksjLy+PatWv4+vqiUDjm8T2dmoPOIOHtqqRTUx+e6BHG139fZNmOc/Ru6V9FUlc9QhFWM3XFIoTipKSWufIE9QNfX99KLVwwDYvbh3ojk8kYe3cLVu+5yM5/rnPsaiYdQutgMhGEIqx2TLGE1+vAHKFMJiMkJITAwEC0Wm1NiyNwEiqVymFL0ITJUWJSeM0au/Nwp1D+d+gqn+44xwfDupZ3ea1FKMJqpi5ZhCYUCkWlvziC+oXZIgwpDkkZf08L/nfoKpsOX+XlmDaENbI/bVptQThLqpnAOuY1FghMSJLEiavFQ2MTtzfx4e7W/hgk+PzPczUl3i0hFGE1EyDWGwvqKJfS88ku1KFWyGkZ4Gl1bkKflgB8t/8SN+rgZ1sowmrGlIorPVeDRldHizgJGiTHkzMBaB3kiVpprTqiWjamU1MfCrQGVu2+UAPS3RpCEVYzvm4qlEXLkW7k1r1fTkHDpdhRUnrJmkwm49kiq3BV/EVyC3XVKtutIhRhNSOXy0ReQkGdxJajxJKYDsGEN3YnM1/L2n2O5+usSYQirAGKi70LRSioOxw3O0psxwoq5DLG3tMCgC//PIe2DtXvFoqwBjAlaL0mFKGgjnAzV8PVTONqqLYhXmW2G9StKf6eLlzNLOB/h2qg4mElEYqwBhAWoaCuYRoWN2vkXm7WJFeVgtF3hgPw6Y5zdSYZg1CENUBAkedYZKoW1BXKc5SU5N93NMfTRcmp1Gy2naobyzOFIqwBhEUoqGtU5CixxMdNxZORxqzVy7bXjQDrW1KEOTk5ZGVlWW2CihFzhIK6xnEbK0rK4+k7I1ApZOy9kE7CxZtVKZpTcFgRnj9/nocffhgPDw98fHzw8/PDz88PX19f/Pz8qkLGeoewCAV1iQKtnjPXjfVc7FWEwT6uPNa1CWBM3FrbcVgR/vvf/+bmzZssX76cuLg4/vjjD/744w+2bdvGH3/84bAAS5cuJTw8HFdXVyIjI9m7d2+ZbY8dO8agQYMIDw9HJpOxZMmSUm3mzZtHz5498fLyIjAwkIEDB3Lq1CmrNvfeey8ymcxqe/bZZx2WvbIEWijCujKZLGi4/JOajd4g4eeuItjb/vom4+5piUwGW46ncuZadhVKeOs4nH3m0KFDJCQk0KZNm1u++bp164iNjWXZsmVERkayZMkSYmJiOHXqFIGBgaXa5+Xl0aJFCwYPHsyUKVNs9rljxw6ee+45evbsiU6nY8aMGTz44IMcP34cDw8Pc7uxY8eayzyCMfFodeFflIFGozeQma/F111dbfcWCBzFcljsSHLeVoGePNAuiN+Pp/LpjnMsGNy5qkS8ZRy2CHv27MmlS86JGl+8eDFjx45l9OjRtG/fnmXLluHu7s7y5cvLvPeCBQsYOnRomcXWN2/ezKhRo+jQoQOdO3dm5cqVJCUlkZCQYNXO3d2d4OBg82ZvJTpn4KpS4ONmDEEQw2NBbae4RonjSVefvde47G7DwSskZ+Y7VS5n4rAi/OKLL3jnnXdYtWoVCQkJHD582GqzF41GQ0JCAtHR0cXCyOVER0cTHx/vqFhlkplpXCjeqFEjq+OrV6/G39+f22+/nenTp1dYm6OwsNCpjqEA4TAR1BHMFqEdHuOSdGvmR6+IRmj1Est3nXe2aE7D4aHx9evXOXv2LKNHjzYfk8lk5oLler3ern7S0tLQ6/XmYu4mgoKCOHnypKNi2cRgMPDiiy9y5513cvvtt5uPP/nkkzRv3pzQ0FAOHz7Mq6++yqlTp/jhhx/K7GvevHnMnTvXKXKBcZ7wzLWcWmERimLzgrIwGCROJDvmMS7JhD4t2Xs+nTV7kph0X2t83GtfGVuHFeHTTz9N165d+fbbbwkKCqrVX6DnnnuOo0ePsmvXLqvj48aNM+937NiRkJAQ+vbty9mzZ2nZsqXNvqZPn05sbKz5dVZWFmFhYZWWrdgirNmg6v8mXGberyf5fER3ujYTXn+BNRfT88jV6FEr5bTw96j4Ahvc2yaAtsFenEzJ5ps9F3nuvtpXW9phRXjx4kV++umnWy6U7e/vj0KhIDU11ep4ampqpYrKlGTSpEls2rSJnTt30rRp03LbRkZGAnDmzJkyFaGLi0uZ85KVIbCWhND8fjyFtJxC9p5PF4pQUArTsLhtsBdKReXCjmUyGeP7tGDKukOs+Os8Y+6KwFVVu0o/OPxk999/P4cOHbrlG6vVarp3705cXJz5mMFgIC4ujqioqEr3K0kSkyZN4scff+SPP/4gIiKiwmsOHjwIQEhISKXv6yi1ZY4wPVcDgAjiqb/sv5DOU1/uqVQIiykZqz1L68rjkU6hNPF1Iy1Hw/cJl2+pr6rAYYvwX//6F1OmTOHIkSN07NixVIHoAQMG2N1XbGwsI0eOpEePHvTq1YslS5aQm5trnn8cMWIETZo0Yd68eYDRwXL8+HHz/pUrVzh48CCenp5mC/W5555jzZo1bNy4ES8vL1JSUgDw8fHBzc2Ns2fPsmbNGh566CEaN27M4cOHmTJlCvfccw+dOnVy9O2oNKZM1TVtEd7IMSpCg4hnrLes3XeJP0+n8fPhFF6ILjtzjC1uxVFiiUoh55m7I5j7v+N8/uc5hvVqhkJee6bVHFaEpsBjyxg8E444SwCGDBnC9evXmTVrFikpKXTp0oXNmzebHShJSUnI5cVG69WrV+natbhc4MKFC1m4cCF9+vRh+/btAHzyySeAMWjakhUrVjBq1CjUajVbt241K92wsDAGDRrEzJkz7ZbbGdQWizCtqL6E0IP1l4w8449dvtb+76aJ47foKLFkSM8wPog7zcUbefx6NJlHOoXecp/OwmFFaDA4N9nipEmTmDRpks1zJuVmIjw8vMKVGBWdDwsLY8eOHQ7JWBXUhjlCjc5AVoExpbrBIDRhfeVmnrEmdYGDijAtp5DUrEJkMmgTfOuK0F2tZERUOO/HnWbZjrM83DGk1jhbHZoj1Gq1KJVKjh49WlXyNBhMFmFmvtbhD6izMM0PAgg9WH8xWYRlfc4u38xj/4X0UsdNw+Lwxh54ujinBPrI3uG4quQcvZLFX2duOKVPZ+CQIlSpVDRr1syh4a/ANj5uKnMlsLQaKn9oeV8xR1h/ySiyCG0NjSVJYsTyvfzfsniSblgvKnDmsNhEIw81Q3sWpeiqRckYHPYav/baa8yYMYP09NK/IAL7kclkBHjW7DyhpUUokj/UTyRJIiO/7KHx+bRczl3PBeDSzRKK0EmOkpKMuSsChVzGrjNpHLmc6dS+K4vD9u5HH33EmTNnCA0NpXnz5laJDAAOHDjgNOHqOwFeLlzJyK+xeULLcqJiaFw/yS7UoS/65xZoS8/v/3k6zbyfWaQwTVSFRQgQ1sidf3UKYcPBqyzbeZalT3Zzav+VwWFFOHDgwCoQo2FS0wlaTaEzAJKIJKyXZOQWKzdbQ+M/T18vbptn0Vaj51xRDsIOTrYIwZiia8PBq/x6JJmLN3Jp3rhyq1achcOKcPbs2VUhR4OkphO0puUIZ0l952Ze8f+4sIQi1OoNxJ8tdlhYWoQnU7IwSODvqTZ/Tp1J+1Bv+twWwI5/rvPZznO89VhH+y7cvxxuHwSujmfCKY9Kp+pPSEjgm2++4ZtvviExMdGZMjUYioOqa2a98Q3hLKn3ZOSXbREmJmWQq9FbtC1WmsXDYp8qC3F5to9xOev6hMv2GQMJK2HTFPgiGrTO/c44bBFeu3aNoUOHsn37dnx9fQHIyMjgvvvuY+3atQQEBDhVwPpMTVuE1s6SGhFBUAUs3XYGpVzG+D4tzaEzUHqO0HJYDJBpMTSuKkeJJXe0aETnMF8OXcpg5e7zvBzTtuzGSXvg56nG/U5PgMr+TNn24LBF+Pzzz5Odnc2xY8dIT08nPT2do0ePkpWVxeTJk50qXH2npucI0yzjCMXYuF5wPbuQBb+dYt6vJynQ6rlp8T8uaRHuLHKUdG5qHGZaDo2rylFiiUwmY0KfFgB8HX+RnEKd7YZZyfDdU2DQQrsBcPdUp8visCLcvHkzH3/8Me3atTMfa9++PUuXLuXXX391qnD1nZq2CK2HxjUigsDJ5FooE4NF6AxYh89k5Gk4fDkDgIc7hRQdM7bVGyROJhsTNFSlRQjwQPtgWvh7kFWg49s9SaUb6AqNSjAnFQLbw8BPoAqG6g4rQoPBUCrRAhiDrZ29/K6+E+hdrAhrwiK7YeUsEZqwPqDVF38HJcnaE2ypCP86cwNJgtuCPLktyJiIwWQRnk/LJV+rx02lIKKSOQjtRSGXMe4eo1X45a7zaHQWOkSS4OeX4PI+o3Nk6Gpw8awSOSqVhuuFF17g6tWr5mNXrlxhypQp9O3b16nC1Qs0ubDvC5uTcI09jIpQZ7D+5a4O8jQ6q6GSCKiuH2j1xf9HgyRZeY21eskcU2iaH7y7dYC5eJhJEZqGxW1DvKolQ8xj3ZoQ6OVCSlYBGw9eKT6x/0tI/Bpkcvi/5dCoRZXJ4LAi/Oijj8jKyiI8PJyWLVvSsmVLIiIiyMrK4sMPP6wKGesueh2s6G/8Vfv7k1Kn1Uo5jTyMH8LqzlRtaQ2CyEdYX7C0CA0Ga4sQjFahJEnmQOq7W/vjW1RIzKwIq8FRYomLUsHTdxnzhn6685xxdHRxN/z6qrFB39nQKrqcHm4dh73GYWFhHDhwgK1bt5pri7Rr186qCJOgCIUSOj8JyYfg95kQ1B5a3GvVJMDThfRcDdezC2l764m57eZGrrUiFEPj+oHOYnrKIElWXmMwOkxSswq4kpGPWiEnMqKxecicU6hDqzdUi6OkJE9GNmPpH2c4cy2HXQcOcc+2EWDQQYfH4c4Xqvz+lUopIZPJeOCBB3jggQecLU/9I3K8UREeWgPrR8O4beAXbj4d6O3CqdRsrmVVr8PkRolED8JZUj8oOTQuOeVSoNWbrcEe4X64qRXm5B9gtAqr2yIE8HZVMfyO5qzYcYLQzc+A7joEdYRHP6oS50hJKqUI4+LiiIuL49q1a6UcJGXVJG6wyGTwyHtw/SRcPQBr/w1jfgO1cRLalHjhejVnoCk1NBYWYb3A0tlgkLAKnwGTIiyeHwSjw8LLVUl2gY7TqTmk5RQil0FbJ+QgdISnezenTfzLtNKdRuvih2roavP3pKpxeI5w7ty5PPjgg8TFxZGWlsbNmzetNoENVK4w5BvwCIDUI7Bxktl5ElDkOa5uizAtt4RFKBz+9YJCC0Wo1Rcn3jVZfVkFOvOyurtb+5vb+haV2Iw/a7QWWwR44qau3gJLgSdW8ph8JzpJzvu+08GvebXd22GLcNmyZaxcuZKnnnqqKuSpv/g0gSe+hlWPwLEfILQL3PlCjVmE6UUWoUxm1MlijrB+YGkRWjpKgrxduJSeT/zZG+Rq9DT2UFsNfX3d1Fwin/hzRiVZncNiAM7vhN9eA+Bt/XCWX2zKv1KyaRPsWI2VyuKwRajRaOjdu3dVyFL/aR4F/d817m+dA2e2EuhtXCp0LauavcZFQ6ZGRaETYo6wflCoKw6JMi2h9HJV4qE22jxbjhvL597V2h+5RWiMT5HnODEpA6heRwk3L8J3I0HSQ6ehJLcZBcCnO6svcavDivCZZ55hzZo1ThNg6dKlhIeH4+rqSmRkJHv37i2z7bFjxxg0aBDh4eHIZDKWLFlSqT4LCgp47rnnaNy4MZ6engwaNKhUfeUqo8fT0G0ESAb4/mnCDMlA9VuEpuzUptUtYo6wfmA5NDblm/RzV5vrCB+8lAHAXa38ra7zKRoa64p+EavNItTkwbrhkJ8OIV3gX0t49l5jRcqfDl7lSkZ+tYjhsCIsKChg8eLF9OnTh+eff57Y2FirzRHWrVtHbGwss2fP5sCBA3Tu3JmYmBiuXbtms31eXh4tWrRg/vz5ZRaBt6fPKVOm8L///Y/169ezY8cOrl69yuOPP+6Q7JVGJoOHFkLTnlCQSYc/J+BOAder3WtstBb8i4bmYmhcP7AcGpssQj93Fa4q66+6yVFiwmQRmqgWi1CS4KfnIeUIuPsbV46o3Ogc5ktUi8boDBJf/nm+6uWgEorw8OHDdOnSBblcztGjR0lMTDRvpkLp9rJ48WLGjh3L6NGjad++PcuWLcPd3b1Mz3PPnj1ZsGABQ4cOxcXFdo60ivrMzMzkyy+/ZPHixdx///10796dFStWsHv3bv7++2+H5K80ShfjfKFnMOr0UyxSfUJ2oZZ8TfXVgjFZC2aLsNruLKhKbA2NfdzVuKmKHR+3BXkS7GOdvcXXQhEGebuYfyCrlN0fwtHvQa6EJ74Cn6bmU8/ea0zRtXZfUqlYyKrAYWfJtm3bnHJjjUZDQkIC06dPNx+Ty+VER0cTHx9fZX0mJCSg1WqtAsDbtm1Ls2bNiI+P54477rDZd2FhIYWFxVZbVlZWpWQ04x0CQ75BWvkQ/dnHc4aNXM++n2aN3W+tXzuQJMn8JTEpQjFHWD/QWA2Niy1Cy+MlrUEo9hpDNQ2Lz8TB1qIkz/3mQ/idVqfvae1P+xBvjidn8VX8RSb3bV2l4lQ6MeutkpaWhl6vNxdzNxEUFERKSkqV9ZmSkoJarTbnUrT3vvPmzcPHx8e8hYWFVUpGK8J6IntoIQAvKdejOfHzrfdpB1kFOnPgrb+nyVkiNGF9wHKO8KZZEVpbhJZhMyYsh8ZVPixOPwffP22cJ+/6b+j5TKkmMpmM8UUpulbuvlDlo6UaU4R1jenTp5OZmWneLl265JyOu4/kV7dHkMskmm9/EdJOO6ffcjCtKvFyUZq/IMJZUj8otGER+ripcCn6P5uW1ZXEx01t3m8f4tw0+NYC5hgXFRRkQJMe8NCiMleOPNwxhKZ+bqTnalif4KTvWxnUmCL09/dHoVCU8tampqaW6QhxRp/BwcFoNBoyMjIcuq+Liwve3t5Wm7PYFPI8ewxtUelyYO2TUHCLw+4KMIfOeKrNadhFQHX9oCxniekHz7SsriTVYhFKEmycCNeOgWcQDPm63EzTSoWcsXcbrcLPdp5Dp6+6D2mNKUK1Wk337t2Ji4szHzMYDMTFxREVFVVlfXbv3h2VSmXV5tSpUyQlJVX6vrdKYx9PntO8QJY6ENL+gR/HV6lmMlmEjT3UyE2KUFiE9QJLZ4lpaOzrrqZbc18Uchn/172pzesCvIwWoZeLkuaNqmieetdiOL4R5Cqjs9A7tMJLnugRRiMPNZdv5vPzkeSqkYtKKMKdO3ei05VOqa3T6di5c6dDfcXGxvL555+zatUqTpw4wYQJE8jNzWX06NEAjBgxwsrxodFoOHjwIAcPHkSj0XDlyhUOHjzImTNn7O7Tx8eHMWPGEBsby7Zt20hISGD06NFERUWV6SipagI8XUjDh1VN/wMKFzj1C+x4p8ruZ7IIG3u6mEclQhHWDwot6pKYchH6uqt4pFMox9+I4fFuthVhywBPYh+4jXf+r5NVoLXT+Od3iHvTuP/QAmgWaddlbmoFI6PCAVi241yVTeE47DW+7777SE5OJjAw0Op4ZmYm9913H3q9/ZOaQ4YM4fr168yaNYuUlBS6dOnC5s2bzc6OpKQk5PJiXX316lW6du1qfr1w4UIWLlxInz592L59u119Arz33nvI5XIGDRpEYWEhMTExfPzxx46+FU7DlKn6gD4C/vU+bHgWdsyH4I7Q7hGn3684hrB4Et1WzVtB3aOwRNIFMDpLwJj3ryxkMlnVeWbTzsB/nwEk6D4aeox26PIRUc1ZtuMsJ5Kz2Hk6jT63Ob9AnMOKUJIkm+X9bty4gYeH45kiJk2axKRJk2yeMyk3E+Hh4Xb9IpTXJ4CrqytLly5l6dKlDslaVZhrl+QUQpdhxrRdez4xDpEbx0FgOdW9KkHx0NjFvKKgZAJPQd3E1g+aZWhMtVOQZZz3LsyEsDuKl5g6gJ+HmqG9wljx1wWWbT9bs4rQtPJCJpMxatQoq4BmvV7P4cOHxRrkSmKqb2zOQPPgm5B6FC78CWuHwdht4ObrtPulmYfGarO1IBRh/SBPU3raypSKv9oxGODHZyHtFHiFGIOmlZWT5Zm7W/B1/EXiz93g0KUMOof5OlVUu+cITfFzkiTh5eVlFVMXHBzMuHHj+Oabb5wqXEPBZBHeyNUYa0ooVDB4Jfg0M8Zc/fcZMDhv6GqyCBt5qM0rCm5WQ/S+oOopGW+nkMvwdq1U2tFbZ+cCOPUzKNQwZDV4BVV8TRk08XVjQGejc2XZDucnY7D7HVqxYgVgHJ5OnTq1UsNggW0ae6iRyYxlFNNzNUbF6OEPQ7+BL2PgzBbY9hb0neWU+5nCKvw9XcwWYZ5GT6FOX+48kqD2k1dCEfq4qWxOZVU5J3+B7W8b9x95D5p2v+Uux/dpyYaDV5Ak43fFmYWlHPYaz549WyhBJ6NUyGlcVMTJqsZxSGcYUFQQ689FcOxHp9zP5Cxp7KnGy1WJ6fOUKYbHdZ6Sc4Q1Mj94/RT8MM6432u8cfWIE2gT7MWuV+9n2VPdnV5dz2FFmJqaylNPPUVoaChKpRKFQmG1CSpHgGmesGQ1u06Doffzxv0NEyHl6C3dR2+QSC8aBjf2cEEul5mDaau7pKjA+ZQcGvtV9/xgfgZ8Oww02dD8Loh5y6ndh/q6ObU/Ew5PHowaNYqkpCRef/11QkJCasbsrocEeLlwIrmERWii7xyjAjy3zeiBG7cd3BtV6j438zRIknFVk1+RteDnruZmnrZUfQtB3aPk0NjXrRotQoMefhgL6WfBu6lxnltRgx5rB3BYEe7atYs///yTLl26VIE4DZfAIofJNVuKUKE0Frj+/D64ecG4YH3498bjDmIaFvu6qVAqjAMCcwiNsAjrNAaDZGNoXI0W4ba34fTvoHQ1zm97Oj/MpapweGgcFhYmFuhXASZFaNMiBKMFOHQNqNyNlmHcnErdx5SHsLFFvrniEBphEdZFlu04y7xfTlCgKx1Z4Fddc4THN8KfxkxK/OsDCO1afvtahsOKcMmSJUybNo0LFy5UgTgNl4CKFCFAUAcY+Ilxf/eHcHi9w/cxO0o8ii2F4hAaYRHWNSRJYsFvp/h05zku3yyd1r5anCWpx+HHCcb9qEnQeUjV39PJODy2GjJkCHl5ebRs2RJ3d3dUKus3Oj093WnCNSQCy3KWlKTDQEiONS5g/+l5CLjN6F22E1MMoWUGYl8RVF1n0egNxthTIMvG1EaVD43z0o1B/9pciOgD0XOr9n5VhMOKsKyCSYJbwy6L0MT9M411Hs5sgbXDjc4Tj9LJNm1xw2JViQk/8zI7MTSua1iuLc4pLL2qpEq9xgY9/HeMcd7at1mRc6SGgrdvEYelHjlyZFXI0eAp11lSErmCi/d9gPflGPwykzj54SBav7QFhariD31a0dC4keXQWKw3rrNY5h/MLazmdcZxc+HsH6B0M85fVzKSoTZQqXyEZ8+eZebMmQwbNsxcHe7XX3/l2LFjThWuIWGyCPM0enJt/LKD0Vpc8dd5Hl36F30+TGRw5iRyJFfaFhzkzOopdt0n3YazxDR8Esvs6h6FVorQ1jrjKlKER76Hv9437g9casyUVIdxWBHu2LGDjh07smfPHn744QdycnIAOHToELNnz3a6gA0FDxclHkWZgy2twtxCHT8mXmbk8r3cMS+Ouf87zqFLGSjkMpq07sr6Zq8D0ObCN+Tv+7rC+5hTcNmwCDNF+EydQ1MTQ+Pkw7CxKLvTnS/C7YOcf49qxuGh8bRp0/jPf/5DbGwsXl5e5uP3338/H330kVOFa2gEeLmQeyOP5Ix8zqflsCHxKluOp1rFhnUJ82Vgl1Ae6RyKv6cLGl0PVr5zhFHatah+iYXQ9tCk7HWdlklZTfgJi7DOoqluizD3hnFeWpcPLfs6bf17TeOwIjxy5Ahr1qwpdTwwMJC0tDSnCNVQCfRy5cKNPEat2IfGoj5DhL8Hj3YJ5dEuTYjwt17nrVbKCX10NlvWneYBRQK6NcNRTtgJnoEluwcgzZSL0MJZ4mMRPlNWvklB7cQyNX9OiRRcaqXcqnrdLaPXwfejIDMJ/CLg/74Eef1YVuuwIvT19SU5OZmIiAir44mJiTRp0sRpgjVEmvi5wQVjSIS/p5pHOoUysGsTOjf1KVc5PdAhhDFNXiPi6iRa5V6F70bAiJ9K5X4r1OnJLjB+WSzjCP2K9jU6AwVag83iPoLaRXJmPr8eSSEioPiHsaRF6Ofu5MwzW2bB+Z2g8oBh34Kbn/P6rmEcVoRDhw7l1VdfZf369chkMgwGA3/99RdTp05lxIgRVSFjg+GFvq0J83OjW3M/7mrlb14CVxEymYyXBvRg/Eex/Kh6He+kePhtOjy8yKrdzVzjHKBSLsPbtXjI5KFWoJTL0BkkbuZpcFNXzcJ2gfMY9PFurmYWEOpTXAWupNfY182J84OH1sLfRRndH1sGge2c13ctwGFnydtvv03btm0JCwsjJyeH9u3bc88999C7d29mzpxZKSGWLl1KeHg4rq6uREZGsnfv3nLbr1+/nrZt2+Lq6krHjh355ZdfrM7LZDKb24IFC8xtwsPDS52fP39+peR3FuH+HsQ+2IZ72wTarQRNdAj1oUe3SF7QTsKADPZ9AQmrrNqkWSRktSzQI5PJRFB1HeNqZoHVXyjtLHHa/OCVA/DTZOP+Pa9A+wHO6bcW4bAiVKvVfP7555w9e5ZNmzbxzTffcPLkSb7++utKpeFat24dsbGxzJ49mwMHDtC5c2diYmLMYTkl2b17N8OGDWPMmDEkJiYycOBABg4cyNGjxempkpOTrbbly5cjk8kYNMjau/XGG29YtXv++ecdlr828VLMbexV9mCRdrDxwC9T4VLxj4otR4kJEVRd9yk5NHaKIsy5Buv+DfpCuK0f3Du94mvqIJWua9ysWTMeeughnnjiCVq3rnz1q8WLFzN27FhGjx5N+/btWbZsGe7u7ixfvtxm+/fff59+/frx8ssv065dO9588026detm5bEODg622jZu3Mh9991HixYtrPry8vKyalfXE84Gerky8b5WLNU/yh/yO0CvgXVPQZaxHmzx8rrSQyZfkYGmzlN6jvAWh8Z6LXw3ErKuQOPW8PhnIK+0yqjV2DVHGBsby5tvvomHhwexsbHltl28eLHdN9doNCQkJFjVLpbL5URHRxMfH2/zmvj4+FIyxMTEsGHDBpvtU1NT+fnnn1m1alWpc/Pnz+fNN9+kWbNmPPnkk0yZMgWl0vZbUlhYSGFhcXxfVlZWRY9XI4y5K4I1e5KYlDGOnY2u459zFr57Ckb9bI4htFxVYkIEVdd9Sg+Nb1ERbp4OSbtB7WVcOeLqc2v91WLsUoSJiYlotVrzflk46qFKS0tDr9db1RwGCAoK4uTJkzavSUlJsdk+JSXFZvtVq1bh5eVlrsJnYvLkyXTr1o1GjRqxe/dupk+fTnJycpmKfN68ecydW/sXlLuqFEzr35bnv03kyezJbHafhfzyPvhlKmlKY4aQxh6lh8amDDRijrDu4tQ5wgNfw77PjfuDPjcm96jH2KUIt23bZnO/LrB8+XKGDx+Oq6ur1XFLq7JTp06o1WrGjx/PvHnzrEqVmpg+fbrVNVlZWYSFhVWd4LfAI51CWLn7AgkX4Yug1xl36RU48BW3hQQA3a1iCE2YQmjEHGHdJa/Ia+zlqiS7QEfrQM/KdXR5P/xc9Fm/7zVo099JEtZeanTA7+/vj0KhIDU11ep4amoqwcHBNq8JDg62u/2ff/7JqVOneOaZZyqUJTIyEp1OV2aeRRcXF7y9va222opMJuP1R9oD8PY/oST3eBWAR5OX0FN20uYcoY/ISVjnMQVUv/no7fw06U7ub2s7qL5cslOKnCMaaPsI3D3VyVLWTuyyCEsOK8vjhx9+sLutWq2me/fuxMXFMXDgQAAMBgNxcXFMmjTJ5jVRUVHExcXx4osvmo9t2bKFqKioUm2//PJLunfvTufOFefrO3jwIHK5nMDASnx4aiFdwnyJ6RDEb8dSWat6jCkdTqE89gMfq5dwUhYNNLNqLwq9131MieN93VV0aurreAe6QqNzLTsZAtoa4wXrqXOkJHY9pWUxd29vb+Li4ti/f7/5fEJCAnFxcfj4OD6ZGhsby+eff86qVas4ceIEEyZMIDc3l9GjRwMwYsQIK2fKCy+8wObNm1m0aBEnT55kzpw57N+/v5TizMrKYv369Tatwfj4eJYsWcKhQ4c4d+4cq1evZsqUKfz73//Gz6/+RMubKn5pDRI8+hGnZeEEyLLo9vck0FongBXhM7WfhIs3eWDxDv48fb3cdpVeVvfrK3B5L7j4GJ0jLl4VX1NPsMsiNBV3B3j11Vd54oknWLZsmTluUK/XM3HixEoNF4cMGcL169eZNWsWKSkpdOnShc2bN5sdIklJScgtfpV69+7NmjVrmDlzJjNmzKB169Zs2LCB22+/3arftWvXIkkSw4YNK3VPFxcX1q5dy5w5cygsLCQiIoIpU6ZU6BGvayiLgqb1BgnUHjyri2W9fAaN0g7Dpikw8GNjOTtEAae6wJbjqZy+lsNvx2w7Bk24qyuRHHX/ckhYCciMa4gbt6yUjHUVmeRgJaaAgAB27dpFmzZtrI6fOnWK3r17c+PGDacKWFvJysrCx8eHzMzMWjtfOP/XkyzbcZYxd0Xw0oO30X7Wb/SWH2W1yzvIJD30ewfueBaAE8lZ9H//T/w91eyf+UANSy6wxWs/HmH1niSe6NGU7/ZfLrPd1tg+tHLEUZL0N6x8BAxa6Dsb7q77BoGj30+HJwB0Op3N0JaTJ09iMBhsXCGoKUyr9PQGyRxDeEDRCR54w3jitxnGRfRYZ6kWVQprJ6aEGVp9+f8fd0eSZmRdNc4LGrTQfiDcZV+C3/qGwzb06NGjGTNmDGfPnqVXr14A7Nmzh/nz55vn9QS1A0XRlILOYChOv+XhgizqOUg5DIfXwfpRMG47fh5NitpK5BTq8HK1PwZt1+k0fjuWwmsPt8PVmWmfBFaY4gQtcxDawm5FqC0weohzr0FgB6upkoaGw4pw4cKFBAcHs2jRIpKTjUu3QkJCePnll3nppZecLqCg8hTPEVqU8fRUGz/s/3ofrp+E5EOwdjiuT/+Gi1JOoc5ARp62QkVYqNMz+dtEujf3Y+vxa+y9kM4dLRrzcKeQKn+uhkp2gXH+trACRWhXGjVJMsYKXkkwptMauhrUdXuJ6a3gsCKUy+W88sorvPLKK+ZlZrV1jqyhozArQkNxYXfT8jqVGwxZDZ/da7QO/zcZP7ehpGQXkpGnJayCOjwHLmbw27FUfjuWah5WX7qZV1WPIqB4aGyZtLckchmo7clctPdzOLgaZHL4vxXQKKLia+oxtxQkVNsDixs6JotQZ5BsZ57xDYMnvgK5Eo6s52nFz4B9640tl3OZYg8vC0VYpZgVoa50tToT7mplxUtdz/8Jm6cZ9x94A1re5ywR6yyVKkL6/fff891335GUlIRGY/2lOXDggFMEE9w6CovwGauhsSXhd0K/+fDLVJ4pWMlOeRAZ+V0r7DunsHSYzZWb+bcutKBMTEPj8uYIKxwWZ1yC9SNB0kPHwRBle+FCQ8Nhi/CDDz5g9OjRBAUFkZiYSK9evWjcuDHnzp2jf//6vyaxLmFlEZpScNlIuEDPZ6Drv5Fj4CPVB+jSzlXYd05B6UJBl4UirDIkSSp2lpQzNC7XUaLJg7VPQt4NCO4E//qgwTpHSuKwIvz444/57LPP+PDDD1Gr1bzyyits2bKFyZMnk5mZWRUyCiqJomiuSK+3HBrbSM0kk8FDi7jk1g5fWS73JLwAmtxy+862UTHtSka+CL2pIvI0egxFb61WV/Z7XOaqEkmC/71gnA92b1zkHHGvAknrJg4rwqSkJHr37g2Am5sb2dnZADz11FN8++23zpVOcEsoin7t9ZJEWjm5CAFQufJTu3e5Lvngn3cGNkwsXrxqA1ulI/M0epG0wUG05Vh3lmRbWOCVsgj//hiOfAcyBQxeBb7NbLdroDisCIODg0lPTweMWar//vtvAM6fPy+sgVqG5RK79FxTdmobQ2NTe58mPKt5ER1KOL4Bdr1XZltbQ2MQDhNHWL//Eq1f+5XfK1gyB8XzgwD5mvKdJaU4uw1+L6on1G8eRNztsKz1HYcV4f33389PP/0EGIOrp0yZwgMPPMCQIUN47LHHnC6goPKYnCVavaFsZ4kFfu5qEqQ2fNOoaAI97g04vcVmW1tDYxAOE0d4+fvDAIz7OqHCtpbvd57G9nsPlA5ov3kBvh8NkgG6DIde4yola33HYa/xZ599Zl5K99xzz9G4cWN2797NgAEDGD9+vNMFFFQepcKoCG/madAVTTCVOTSmOPHCBsWDjOqeCQkr4PsxMG5bqUX4ZVuEQhFWBZZD47xyLUILRajJhbXDIf8mhHaDhxcL50gZOGQR6nQ6/vOf/1ilxR86dCgffPABzz//PGq1E+uoCm4Zk0WYmmUcFnu5KnFRlu1VNOUkzMzXQv93IewOKMyEb4dBYbZV29wyrJIrGUIROoo9usnyh8f0o2YLsyKUJNj4HKQeBY8AGPINqFzLvK6h45AiVCqVvPvuu+h0ZZvmgtqDaY7wRk6JVSVlYMpJeDNPA0q1MdjaKwTSTsGPz4JFUg0xR+g85HZoQss5wvIwxxH+tQSO/WgMln/ia/BpcgsS1n8cniPs27cvO3bsqApZBE7GlHTBZEDYqmdsiWlonJmvNeYw9AoyWhIKNZzcBDsXIEkSo1bs5dBl26FSYmjsOHI7LMLsMn54SuKuVsDprbC1qNBY/3eheens7QJrHJ4j7N+/P9OmTePIkSN07969VC3gAQMGOE04wa2hLPENq8gi9HUznpckowXi666Gpj3gkfeMw6ztb5Pr147tp8oeXgtniePYU/2xLOdUSUL0yfDfpwEJuo2EHk/fonQNA4cV4cSJEwHb9YtlMhl6fdkTuYLqRV5SEVZgEaqVcjzUCnKL4gHNdXG7/tuYpWbvZ7j/PIGWstmclayHWj5uKjLztWQX6sjM15qLQQkqxj6LsOKhsQf5PHzsdSjIhKa94KEFwjliJw4PjQ0GQ5mbUIK1i5IWoa3qdSXxNRdxKpF4IeZtaH4nck0On6kW44X1XKCfu8pscYp5Qsewb46wfItQhoFFqmX45Z4Fz2AY8jUoy//hExTTMEpUNVAUJRRheaEzJiwzVVt3poLBq9B7NaGlPJklqqXIKHaeqJVymvgZi0WJ4XHZrNp9gcnfJhrnYItQ2KEIy3JOmXhOsZF+in3o5SrjvK6X7XK4AtvYrQjz8/PZtGmT+bWp4Llpe/nllykoKCinh7JZunQp4eHhuLq6EhkZyd69e8ttv379etq2bYurqysdO3bkl19+sTo/atQoZDKZ1davXz+rNunp6QwfPhxvb298fX0ZM2YMOTk5lZK/tlJqjrCCoTFYlPXMt5GKyzOAzAErKJBU9FUkMkX5vfmUi1JB0yJFKBwmZTP7p2P8dOiqdQEme4bGNrL9mOgrTyC26H9xsttsCOt5q2I2OOxWhKtWreLTTz81v/7oo4/YvXs3iYmJJCYm8s033/DJJ584LMC6deuIjY1l9uzZHDhwgM6dOxMTE8O1a9dstt+9ezfDhg1jzJgxJCYmMnDgQAYOHMjRo0et2vXr14/k5GTzVnId9PDhwzl27Bhbtmxh06ZN7Ny5k3Hj6lfUfUmL0N8Bi/Bmru0vniawM9O0YwGYrNxAP7nxR8tFKadJUflQEUtYMZZB0bcyNG4pu8J7qo+RyyS+0j3AzTalqzYKKsZuRbh69epSimLNmjVs27aNbdu2sWDBAr777juHBVi8eDFjx45l9OjRtG/fnmXLluHu7s7y5ctttn///ffp168fL7/8Mu3atePNN9+kW7dufPTRR1btXFxcCA4ONm+W9YpPnDjB5s2b+eKLL4iMjOSuu+7iww8/ZO3atVy9etXhZ6itKEsU57bHIvStoL6xQZLYYLiLL3TGlGuLVJ9wm+wSaqWcpn7GbCZijrBiLJNIl/zBsoWtobEXeXymWoy3LJ89hra8qXvKvjT9glLYrQjPnDlDx44dza9dXV2t6g336tWL48ePO3RzjUZDQkIC0dHRxQLJ5URHRxMfH2/zmvj4eKv2ADExMaXab9++ncDAQNq0acOECROsyozGx8fj6+tLjx49zMeio6ORy+Xs2bPH5n0LCwvJysqy2mo7Jb9g5a0zNlE8NLZtEZpmtubpniRB3hEPWSGfqRbjJ8s1W4SODI3X7UtixV/n7W5fX7C0Au3x62aVUIQyDCxWfUxLeTJXpUY8p3kBLUrHKtgJzNitCDMyMigsLDS/vn79OuHh4ebXBoPB6rw9pKWlodfrzcXcTQQFBVkt47MkJSWlwvb9+vXjq6++Ii4ujnfeeYcdO3bQv39/s1c7JSWFwMBAqz6USiWNGjUq877z5s3Dx8fHvIWFhTn0rDWBpSKUyYqVXHmYwl7KSqdlKJrk16Ngofc0rhBAuDyViTfm0dTX2L+tofGVjHwGL9vNr0eSzcd0egOv/vcIc/93nGtZlZtfritkF2j5fGdxwlvr/439K0tMiu5F5Q88oDhAoaRivCaWNHyszgscw25F2LRp01LzcJYcPnyYpk2bOkWoW2Xo0KEMGDCAjh07MnDgQDZt2sS+ffvYvn17pfucPn06mZmZ5u3SpUvOE7iKsPyy+bmr7RqC+ZUVPmMDvVtjpqumkS+p6ZC/j4jDi4uu1VrVNAF4++cT7Ltwkwmri0s5WObVq6gyW12n89zfeeuXE+bXlp7iimotaXQG8/vTyENNjHwfLyh/AGC6dgxHpBbmtmUmZhWUi92K8KGHHmLWrFk2PcP5+fnMnTuXhx9+2KGb+/v7o1AoSE1NtTqemppKcLBt939wcLBD7QFatGiBv78/Z86cMfdR0hmj0+lIT08vsx8XFxdzsaq6UrTK0mtc0aoSE2WGzxRhmXLS00VJsltrXtYasw65/P0BT7ganSclQ2gKtKVjTC1rb9ijpOsyJfMkWAa7V+QssfxR6eySzCKV0Sn5pa4/PxjusWor5ggrh92KcMaMGaSnp9OmTRsWLFjAxo0b2bhxI++++y5t2rTh5s2bzJgxw6Gbq9VqunfvTlxcnPmYwWAgLi6OqCjb6yOjoqKs2gNs2bKlzPYAly9f5saNG4SEhJj7yMjIICGhOA/cH3/8gcFgIDIy0qFnqM1YKhd75gfBIqDaVvgMIFH8jfZ0UeLuomSTIYodAcMBeINPaCe7WMph4m1jpUlFhcrrM5YWYUWK0DQsDlbnMzP7P3jKCtitb8/buidLtbWZmFVQIXa/a0FBQezevZsJEyYwbdo0czZqmUzGAw88wMcff1xq7s4eYmNjGTlyJD169KBXr14sWbKE3NxcRo8eDcCIESNo0qQJ8+bNA+CFF16gT58+LFq0iIcffpi1a9eyf/9+PvvsMwBycnKYO3cugwYNIjg4mLNnz/LKK6/QqlUrYmJiAGjXrh39+vVj7NixLFu2DK1Wy6RJkxg6dCihoaEOP0NtxZSPEOzzGENxBpqMMsJnLC0bT1clni5GC2Rn2AT6eKfgejaOz1SL2X0tCtoVfx68XUt/1CyHw/pyUkvVR0rO35ZHdoEOOQYWKz4iRH+Vy5I/k7ST0WNt/amV8npvWVcVDv18REREsHnzZtLT083DzFatWtGoUQXVwMthyJAhXL9+nVmzZpGSkkKXLl3YvHmzWakmJSVZead79+7NmjVrmDlzJjNmzKB169Zs2LCB22+/HQCFQsHhw4dZtWoVGRkZhIaG8uCDD/Lmm2/i4lKsDFavXs2kSZPo27cvcrmcQYMG8cEHH1T6OWojikoNjY3tsgt1aPUGVCUmsCzLMXi5KM0WiFqtgkFfkP7+XYQVXiEyYSrc9RsojOctLUKDQUIulzVoRWhraLxq9wVSswp46cE2Vv+77AIdU5Xf0VtKpFDmwvjCWNIpPTUjHCWVp1J2dKNGjejVq5fThJg0aRKTJtmur2rLwTF48GAGDx5ss72bmxu//fZbhfds1KgRa9ascUjOuoZlHGFjW2U8bWCZLCEzX1uqxomlvvJwUeLpUqQIFXJwb8T2ru8TEz+c8Kz9sHU2xLwFGJPCmsgu0OHjrrIaGusbWL0b66GxMfZy9k/HAON7Mb1/O/N5jzM/MVFpLI/xTeBUjl0Mt9mnu3CUVBqx1rgeU5k5QoVcZh7G2vYcW88RdmpqDNtoH2q0UDzCOhKrnWBsEP8RHFpX1G/xR800/2jpNTY0OIvQYl8m4/djxQ7AT3ecKw4zSjlK+73TAfjZazCH/R4os0/hKKk8QhHWYywVoT2ZZ0z4eZhCaErPE5acIxx9ZwSHZj1ITAejt72Jrxu/GXrxhWyQsdH/JsPVRKshtalfYREakcmKyiNYMHX9Ic5eTIK1T6LU57NT35FfgsYZLe8yEIqw8ghFWI+xCp+x01kC4FtOULWlvvIqGhabMlsDhBUts3sr/zH0rWJAVwBr/426sHhlj2nViqUi1OkbliK0fFq5TGaub/zUHc2JatGYAo2Gm1/9GzIukunahOe1z+Pp6opKWfZX1l0lPMaVRSjCeoylRWhPCi4TZeYkxLjW2ISHS+kvnrebEi8XJRJyLvZZAo1bQ9Zloo+8ghKdVb8ai/yVhgZmEVo+rlwmMxdkclMr+OjJrrzpsZ4e+kMUyFxZ3/odMvHEy1UpLMIqQijCeoxSLqOxhxp3tYJgb/srmPmVE1QtlRgal0Qmk5nzEiblKWHoGgrl7oRmHmCm8hugeBhYqG24XmPLeEy5XGa2jpVyGY3PbeRJvdE5MqVwPJuvNQaM77dLeRahUISVRijCeoxMJuO7Z6P4YWJvm9ZbWZgswpsVWIReZfRpmZcwza05zxUYnSejlL8zWLGdGzka9py7QZZF+vmGZhFajo3VSjm6ogqBTfL/gZ+eB+Bbl8H8aojkfFouAF6uKtQ2FKHJShQWYeURkwr1nJYBng5fY15mV0YGGhN+ZQy3LfMSZuRp2WrozmLt/xGr+p7/KJcz5I+mvB/XyuqahjZHuHT7GfN+Yw81Wp1EI7IYcHKOcV619YNsyBoJmVncyDX+IHm5KsmzUcTJx13F9exCc/EtgeMIi1BQCpOzxNYcoWWAdckYQxPFeQnzzcvDPtQP5Dd9D1xkOpap3yOADKtr6pPX+Gauhv8dumpzfbWJv84UO48aeajR6zUsVX2AV2EKNGoJj3+Op5v1dIaXi9KmRTg8shkT7m3J6DvDnfYMDQ2hCAWlKC98xjS3VZYSBCxql+SZ8+hJyInVTuAfQxOCZTf5RL0EFcXWjaEeLTt+8os9PP9tIu9sPmlXe183FY8kf0yU4jgahQcM+xbcfK2C0KHsoXFTP3de7deWsEbuTpG/ISIUoaAU5eUkNCms8tbHWs4RWpahzMWNcdpYsiR3esj/YY5ylfmcrh5pwhPJxqS9mw4nV9DSSOcbP3Nvxn8BiO/8NgS0AUonqvByVZZa8gjYVI4CxxDvoKAUppyEmTaGxiaLsLy1/aY5wmvZhaRlWyfrvSCFMFk7CYMkY7gyjicVxkxC9dFZUljO0NhEZ9kZ+l94F4D3tIOQtS1OZeftaq0IPV1tD43L8yQL7EO8g4JSmAs4lRM+IysnwXwjD7U5Qeip1OxS57cburBQ9wQAc5Qr6S47hd4Aa/cmWWWwLsmvR5K5b+F2jl7JtPtZapKsAh1HLpctawAZLFMvQSlp2GrowQf6x4jw9zCfLz00th0+IyzCW0e8g4JSmMJn8rX6UhP+JkVYnkUok8nMw+PjyaUVIcDH+gH8rO+FWqZnmXoJaVfPM+2HI0xYfcBqOZ4lE1Yf4HxaLpO/TXTwiWqOJVv/sXlchY6P1UsIkaWTrGrGi5pnUSmVhBZZ02BjaOyishlQLSzCW0e8g4JSeLsqzatSSq6BNVjkoSwPk8PENF9WGhkva5/lhCGMAFkm9x6MxQXjUFxXQXB1ed7Y2oa8jF+MWcqv6Cn/hyzJjddcZpCDO80buVutBrK0CJVyGa4quRgaVxHiHRSUQiaTWThMrOcJJXOb8vswWYTlZaHOw5Vx2lgyJA9Cco7xhnIlINm8ZkPiFfN+eettq4uUzAL2nk+vsJ3ShiIcqviDp5RbMUgyXtBOYleGL4DVsBis5wi9XJXIZLIyAqpFIPWtUvOfKEGtxLTMbtvJ61bHiy3C8q9v4mtfKMclKYhJ2skYkDNEuZ2nFFusVpyYeHHdQfO+LeVS3dwxL44nPo1n34XylWFJi7Cb7B/eUK4AYKFuMNsMXc2KPyKghCK0GBqbljPaHBqrxNf4VhHvoMAmT/Qwlit9Z/NJvv77ovl48Rxh+crIZBHawy5DR3a3mAzALOXXzP3ws3Lb2wohqSl2WwRG28JSaQdyk2XqJahlen7W9+Jj/aNWbSMaWytCy6Gxl4tRKdqyhstLxCCwD/EOCmwy7p4WPNunJQCvbzjK+v3G8qUmR0ZFirCJA4oQYH/Ik2zQ90Yl0/Mf7ULIKLtcam1ShBWtiDHlHVSj5VP1ewTKMjhpCONl7bOULO1e0dAYhEVYVdSKd3Dp0qWEh4fj6upKZGQke/fuLbf9+vXradu2La6urnTs2JFffvnFfE6r1fLqq6/SsWNHPDw8CA0NZcSIEVy9etWqj/DwcGQymdU2f/78Knm+uohMJuPVfm3My7Ze/e9h/nfoavEcYQXXO2IRAuglmKYdyzFDc/xlWbBuOGhLF4oHUBUVpSrQ6m0Oo6sTfQWB4Aq5DCSJN5Ur6Co/Q4bkwThtLHmUzgZUcmhsZREW7dsMn6lFPwx1lRp/B9etW0dsbCyzZ8/mwIEDdO7cmZiYmFJ1h03s3r2bYcOGMWbMGBITExk4cCADBw40F5/Py8vjwIEDvP766xw4cIAffviBU6dOMWDAgFJ9vfHGGyQnJ5u3559/vkqfta4hk8mY9Uh7hvYMwyDBlHUH2XI8tehc+dcGeLo45M3UGSQKcGGcJpYbkhckH4L/vWCd96sIZdEXv8d/ttJpzu9Wq1eqG30FC2J2n70B+75giHI7eknGZO0kkqTS1R491AoCSixbdFUpzM4RryLr0KbXWNQquWVqXBEuXryYsWPHMnr0aNq3b8+yZctwd3dn+fLlNtu///779OvXj5dffpl27drx5ptv0q1bNz766CMAfHx82LJlC0888QRt2rThjjvu4KOPPiIhIYGkpCSrvry8vAgODjZvHh4etm7ZoJHJZLz1WEce7RKKziDx2c5z5uMVXRfsY38ORF2RRrlCgNF5IlPA4XXw9yel2posIFPh85MptmMVq4OSFmGexjo7TGhmImyeBsA7uqHsNHS22U9EgIfN99RUP8Y8NBYWYZVQo++gRqMhISGB6Oho8zG5XE50dDTx8fE2r4mPj7dqDxATE1Nme4DMzExkMhm+vr5Wx+fPn0/jxo3p2rUrCxYsQKcrneJIYBzeLRrcmZgOxZaMPY5bXxtF3cvCMnYw3tCBNzTGgvHS7zPh3Hartpb1msGm0VhtWFqE6bkaJq4+YH4dwg0+Vi8Bg46f9FF8pn+kzH4i/G2nSzPNE1pVC7RAJiueKhBUnhpVhGlpaej1+lKF4YOCgkhJSbF5TUpKikPtCwoKePXVVxk2bBje3sW1YCdPnszatWvZtm0b48eP5+233+aVV14pU9bCwkKysrKstoaEUiHng2Fd6XNbAIB5CV15lFwZUR4lq9it1Mfwvf4eZJIezdqRNJUVT5WUV2u5urFcI33n/D/YfsoYbuSChk/ViwmQZZHr155XtOMob2a1pKPEhFfRe1jW0FitkFdonQsqpl4nZtVqtTzxxBNIksQnn1gPsWJjY837nTp1Qq1WM378eObNm2dVCN7EvHnzmDt3bpXLXJtxUSr49KnufL7zHL0iGlXYPtDL/qGxttRqEhmvaZ+mlewyXTTn+Ez1HoM0s8nHtZQFVJPpGixLDOSbV7xIvK36kk7y86RLnsx3n0FBBTZHhL/tuMuSQ+OSPwJinbFzqNF30d/fH4VCQWpqqtXx1NRUgoODbV4THBxsV3uTErx48SJbtmyxsgZtERkZiU6n48KFCzbPT58+nczMTPN26VLZ4R31GVeVguf7tiayReMK2z5zdwQAkXYoTb2NDNWFqHlWM4UshS/t5Rd5V/UZIJVSBiv/usC56zn2PYCTsRU+M1qxmUGKP9FJcp7TvkCKorRzpCRlDY0HdmlC22Av7m7tD5T2Ggd62V+dUFA2NaoI1Wo13bt3Jy4uznzMYDAQFxdHVFSUzWuioqKs2gNs2bLFqr1JCZ4+fZqtW7fSuHHFX9qDBw8il8sJDAy0ed7FxQVvb2+rTVA+7UK82fnyfSwf1bPCtmWtL06hMWPyJqOVFPxL8TfjFZtQyq0/tpuPpXD/oh01sgbZpMDTi9LpR8mP8ZpyNQBv64YTb+hAcobtMCBLSgZTmxjUvSmbX7yH5kXnZTIZO1++j62xffhmTCSfPtXDGY/R4Klxuzo2NpbPP/+cVatWceLECSZMmEBubi6jR48GYMSIEUyfPt3c/oUXXmDz5s0sWrSIkydPMmfOHPbv38+kSZMAoxL8v//7P/bv38/q1avR6/WkpKSQkpKCRmP8sMbHx7NkyRIOHTrEuXPnWL16NVOmTOHf//43fn5+1f8m1GOaNXa3q3BUSe+ri1LOrEfaA7BPastc3QgAXlWupU2O7TjTb/cm2TzuLP5JzS6lbPWSxI5/rtPjP1toKrvOUtX7KGUG/qu/i+X6fgBctUMRWtaGrohmjd1pFejJXa39aRXoeE0aQWlqfI5wyJAhXL9+nVmzZpGSkkKXLl3YvHmz2SGSlJSE3MIC6N27N2vWrGHmzJnMmDGD1q1bs2HDBm6//XYArly5wk8/GUshdunSxepe27Zt495778XFxYW1a9cyZ84cCgsLiYiIYMqUKVbzhoLqpaRF6O/pQohF+M03+mg6yC4wTLmN4ZfmwI37S/VRoK26LNdbj6fyzFf7ub2JN5uev9t83GCQGP/1ftRSIZ+qF9NIlsNhQwQztM9gco7kaupOtpyGSo0rQoBJkyaZLbqSbN++vdSxwYMHM3jwYJvtw8PDK/QiduvWjb///tthOQWVx9NFSU6hjo5NfDhiI7FqybrGXq5KXK080zJm60bRRn6JbvIzsHY47rxstULDtQqXmq0rWmJ49EqWOeYR4IfEK4DE+6rP6SC/SJrkzXhNLIXYV1Humbsi6N/R9ny4oPqo8aGxoGHw6wt3M/PhdrxeNNwtSXJmgdVrb1dVCUUIGlQ8q5lCquQL10+wSPUJlj7jXafTmLXxKBsPXuFSep5T5bf8cd140Hq55ljFzzyq2I1WUjBR8wLJVDwnbWLmI+3p3rxiZ5KgahGKUFAthDVy55m7W9DY07aldPBShtVro0VY+uN5DT+e1UyhUFLSX7GP5xQbzefiTl7jq/iLvLD2IHe/u82p8ltarCbHCMDd8sNMU34LwFzdCPZK7crtx8uO+VJB9SMUoaBasQzEHtStKe5q24HZXq5K3Mo4lyi1ZpbO6Ex7Sbme++UHbLZzBpfS89h8NBnL6B5T/HIzWSofqj5EIZNYq7uXb/TRtjuxoKyM1YKaRfw8CaoVS0U45q4IQnxc+WjbmVLtPF2VpcJkLFmnv4/bZed5SrmVJaqlDNS8yTkp1Ony2rIsz17PwZ0CPlMtxleWywFDqyLFXLGSUwhFWCsRFqGgWrG08nQGg5WFNL5PC/O+l2vF4SRv6Eaw19AGb1k+n6sW4YVz5wXL4tu9SSxQLaOt/BLXJF+e1UxBg33hLxXlcRTUDEIRCqoVy5URWr3BKoPzY12bmPe9XJW0KGP9rfl6lEzUvEiy1IiW8mQWqz5GRtUXip+o+ImHFXvRSAqe1bzINeyPPS1rKkBQswhFKKhWLBMEtAzwNA8V24V40za4eLWOl4vSrvm0NHwYr5lCoaTiAcUBXlD+4HyhLbhXnshU5XcAzNKN5oB0m13X/fuOZky4tyXNG9tXy0VQvQhFKKh2Ds16kD0z+uLrriamQxCdm/owrX9bqza9W/nb3d9hqSXTtWMAeFH5Aw/K9zlVXhMRsmQ+UC1FLpP4RteXtfrSQd22UClkzB1wO6/2a1urygwIihHOEkG14+OuwqdoTq1VoBcbJ91lPrfvtWjScgppGVB66ViHUG+OXbWd/uwHwz101J1ntPI3Fqs+YaAmhAW/nSTuxDW+n9DbnM+vsniSx2eqxXjL8thnuI25upF2Xxvq62a2fGtDBT5BacTPk6BWEeDlQruQ0gktmvi60SG0/EQXb+mGE69vj6esgM9Vi/h622FOpmSz9hbXIMswsFj1Ca3lV0iWGjFR8yJaB2wIy/otIQ5k7RZUH0IRCmo165+N4s5WjVk5umeF9UF0KHlOO5nLkj8R8lTeV32EHAO/H08t/8IKmKz4kQcVCRRKKsZrpnAdX4euz7dYaxz7YJtbkkVQNQhFKKjV9AxvxOpn7qB1kBfebhVbYekY1/rmS2ruUxziJeV37D2fztiv9tuVydpgkMwF1wEekO9niuq/ALyme5rDUkuHn2FQ96bmfR8HsnYLqg+hCAV1BnvKAwAck8J5VTsWgOeUP/GQ/G+2HE/lWnZhhdcOWrab7m9uMRZhun6KxSpjZvMVuhi+1/eplNxDeoRV6jpB9SEUoaDOMP6elnRq6sPEeyu2yn4y3MmnuocBWKj6lLayJC7fzOe7/Ze4abFWuCSJSRlkF+o4cOoC+jVD8ZLl87ehHW/phldK5vYh3ubyo4Lai/gPCeoMPu4qfpp0F6/0a1txY+Bd3VB26jviLivkM9Uinv7kd175/jDPfLW/3OvkGLjtr1gUN89xWfJnouYFdHY4R9rbcPK8N6RLqWPLR/WgkYeaFXZk7hZUD0IRCuokg7s3JaqCuil6FDyvfZ4kQwDN5NeNCRLQk3DxJn+evl6qvbbIGzNF+T2BKTsoKHKOpGNfWYavxvQy73u5Kol7qQ9tgr1Ktbu/bRAJM6O5r63tshCC6kcoQkGdZMHgznw77o4K22XiyTjtS+RJLtyjOMIryrUAPPVl6XT/d87/g/7yPTyv3ADAq9qxHJMi7JbJMlj69yn32IyFNCFKcNYuhCIU1HtOSs2Yqh0PwHjlzwyQ/1Wqza9HkvHLOc1C1TIAPtM9zEbDXaXalcWIqOZ4uShp4utGgJcLAZ6iulxdQihCQZ1m5sPlJ0I18YvhDpbqBgDwruozOsjOs+1kcdH4d36M5zPVYjxkhezSd+Ad3VCH5Jh0fyvkchnbX76X3dPuFw6SOkat+G8tXbqU8PBwXF1diYyMZO9e21XKTKxfv562bdvi6upKx44d+eWXX6zOS5LErFmzCAkJwc3NjejoaE6fPm3VJj09neHDh+Pt7Y2vry9jxowhJ6dmauMKKs8zd7fgwvyH+XxEDxYO7lxu20W6J9im74yrTMun6veYuqqoLKxBzxvaxTSXX+OSIYBJ2snosS9UR62QM7RnmLmYvUohF+uJ6yA1/h9bt24dsbGxzJ49mwMHDtC5c2diYmK4du2azfa7d+9m2LBhjBkzhsTERAYOHMjAgQM5evSouc27777LBx98wLJly9izZw8eHh7ExMRQUFBcF2P48OEcO3aMLVu2sGnTJnbu3Mm4ceOq/HkFVcMD7YP4v+5NOTLnwTLbGJDzgnYS5wzBNJWl8aHyAzSFhSybNZJ7FEfIk1wYp40lg9IODktub+JNwsxofp9yD/+81Z/5gzo5+3EE1YxMsifcvgqJjIykZ8+efPTRR4CxwHtYWBjPP/8806ZNK9V+yJAh5ObmsmnTJvOxO+64gy5durBs2TIkSSI0NJSXXnqJqVOnApCZmUlQUBArV65k6NChnDhxgvbt27Nv3z569DAWyN68eTMPPfQQly9fJjS04kzHWVlZ+Pj4kJmZKYq91zL0BonDlzM4cy2Hl78/XOp8K9llNqhn4Skr4LrkQ4DMWFXvOc1kfjaUdsD8Mvlunv/2AM/f3xqAPrcF4OdhX5U6Qc3g6PezRi1CjUZDQkIC0dHFtR7kcjnR0dHEx8fbvCY+Pt6qPUBMTIy5/fnz50lJSbFq4+PjQ2RkpLlNfHw8vr6+ZiUIEB0djVwuZ8+ePTbvW1hYSFZWltUmqJ0o5DK6NvNjcI8wjr8RQ4S/B40sFNcZqSmx2gkAZiX4sW6AlRK8u7U/h+c8yJm3+tM+1Ju4l+5lYNcmDOzaRCjBekiNpuFKS0tDr9ebi7mbCAoK4uTJkzavSUlJsdk+JSXFfN50rLw2gYHWMVxKpZJGjRqZ25Rk3rx5zJ07184nE9QW3NVKtk291/xaqzew63Qao1fCe9pBTFH9lz/0XVioe4IIfw9+mXw359NyaRfiJUJcGhAiH6GdTJ8+ndjYWPPrrKwswsLEGtK6hkoh5762gVyY/zBafX/2HpxA1O1dOOdSnAyhfQXpvgT1jxpVhP7+/igUClJTrdMkpaamEhwcbPOa4ODgctub/qamphISEmLVpkuXLuY2JZ0xOp2O9PT0Mu/r4uKCi4uIDatPqBRyenUXy9wENTxHqFar6d69O3FxceZjBoOBuLg4oqKibF4TFRVl1R5gy5Yt5vYREREEBwdbtcnKymLPnj3mNlFRUWRkZJCQkGBu88cff2AwGIiMjHTa8wkEgjqCVMOsXbtWcnFxkVauXCkdP35cGjdunOTr6yulpKRIkiRJTz31lDRt2jRz+7/++ktSKpXSwoULpRMnTkizZ8+WVCqVdOTIEXOb+fPnS76+vtLGjRulw4cPS48++qgUEREh5efnm9v069dP6tq1q7Rnzx5p165dUuvWraVhw4bZLXdmZqYESJmZmU54FwQCgTNx9PtZ44pQkiTpww8/lJo1ayap1WqpV69e0t9//20+16dPH2nkyJFW7b/77jvptttuk9RqtdShQwfp559/tjpvMBik119/XQoKCpJcXFykvn37SqdOnbJqc+PGDWnYsGGSp6en5O3tLY0ePVrKzs62W2ahCAWC2ouj388ajyOsq4g4QoGg9lKn4ggFAoGgNiAUoUAgaPAIRSgQCBo8IqC6kpimVsVSO4Gg9mH6XtrrAhGKsJJkZ2cDiNUlAkEtJjs7Gx8fnwrbCa9xJTEYDFy9ehUvr7LXpJqW4V26dKleeJbr2/NA/Xsm8TxGJEkiOzub0NBQ5PKKZwCFRVhJ5HI5TZs2rbgh4O3tXS8+lCbq2/NA/Xsm8TzYZQmaEM4SgUDQ4BGKUCAQNHiEIqxCXFxcmD17dr3JWlPfngfq3zOJ56kcwlkiEAgaPMIiFAgEDR6hCAUCQYNHKEKBQNDgEYpQIBA0eIQivAXS09MZPnw43t7e+Pr6MmbMGHJycsq95rPPPuPee+/F29sbmUxGRkaGU/p1FpW5d0FBAc899xyNGzfG09OTQYMGlaorI5PJSm1r1651uvxLly4lPDwcV1dXIiMj2bt3b7nt169fT9u2bXF1daVjx4788ssvVuclSWLWrFmEhITg5uZGdHQ0p0+fdrrcZeHs5xk1alSp/0O/fv2q8hFK4cgzHTt2jEGDBhEeHo5MJmPJkiW33KdNqiQ9bAOhX79+UufOnaW///5b+vPPP6VWrVpVmO7/vffek+bNmyfNmzdPAqSbN286pV9nUZl7P/vss1JYWJgUFxcn7d+/X7rjjjuk3r17W7UBpBUrVkjJycnmzbJ0gjNYu3atpFarpeXLl0vHjh2Txo4dK/n6+kqpqak22//111+SQqGQ3n33Xen48ePSzJkzbZZ98PHxkTZs2CAdOnRIGjBgQKmyD1VFVTzPyJEjpX79+ln9H9LT06v8WUw4+kx79+6Vpk6dKn377bdScHCw9N57791yn7YQirCSHD9+XAKkffv2mY/9+uuvkkwmk65cuVLh9du2bbOpCG+131uhMvfOyMiQVCqVtH79evOxEydOSIAUHx9vPgZIP/74Y5XJLkmS1KtXL+m5554zv9br9VJoaKg0b948m+2feOIJ6eGHH7Y6FhkZKY0fP16SJGPJh+DgYGnBggXm8xkZGZKLi4v07bffVsETWOPs55EkoyJ89NFHq0Ree3D0mSxp3ry5TUV4K32aEEPjShIfH4+vry89evQwH4uOjkYul7Nnz55a129V3TshIQGtVkt0dLT5WNu2bWnWrBnx8fFWbZ977jn8/f3p1asXy5cvtztFkj1oNBoSEhKs5JDL5URHR5eSw0R8fLxVe4CYmBhz+/Pnz5OSkmLVxsfHh8jIyDL7dBZV8Twmtm/fTmBgIG3atGHChAncuHHD+Q9gg8o8U3X1KZIuVJKUlBQCAwOtjimVSho1akRKSkqt67eq7p2SkoJarcbX19fqeFBQkNU1b7zxBvfffz/u7u78/vvvTJw4kZycHCZPnuwU2dPS0tDr9QQFBZWS4+TJk2XKbqu9SW7T3/LaVBVV8TwA/fr14/HHHyciIoKzZ88yY8YM+vfvT3x8PAqFwvkPYkFlnqm6+hSKsATTpk3jnXfeKbfNiRMnqkka51Abnun1118373ft2pXc3FwWLFjgNEUosI+hQ4ea9zt27EinTp1o2bIl27dvp2/fvjUoWc0iFGEJXnrpJUaNGlVumxYtWhAcHMy1a9esjut0OtLT0wkODq70/aui36p8puDgYDQaDRkZGVZWYWpqarnyRkZG8uabb1JYWOiUdaT+/v4oFIpS3ury5AgODi63velvamoqISEhVm26dOlyyzKXR1U8jy1atGiBv78/Z86cqXJFWJlnqq4+xRxhCQICAmjbtm25m1qtJioqioyMDBISEszX/vHHHxgMBiIjIyt9/6rotyqfqXv37qhUKuLi4szHTp06RVJSElFRUWXKdPDgQfz8/Jy2mF6tVtO9e3crOQwGA3FxcWXKERUVZdUeYMuWLeb2ERERBAcHW7XJyspiz5495T6bM6iK57HF5cuXuXHjhpWiryoq80zV1qfdbhVBKfr16yd17dpV2rNnj7Rr1y6pdevWVqEmly9fltq0aSPt2bPHfCw5OVlKTEyUPv/8cwmQdu7cKSUmJko3btywu9/a9kzPPvus1KxZM+mPP/6Q9u/fL0VFRUlRUVHm8z/99JP0+eefS0eOHJFOnz4tffzxx5K7u7s0a9Ysp8q+du1aycXFRVq5cqV0/Phxady4cZKvr6+UkpIiSZIkPfXUU9K0adPM7f/66y9JqVRKCxculE6cOCHNnj3bZviMr6+vtHHjRunw4cPSo48+Wq3hM858nuzsbGnq1KlSfHy8dP78eWnr1q1St27dpNatW0sFBQVV/jyVeabCwkIpMTFRSkxMlEJCQqSpU6dKiYmJ0unTp+3u0x6EIrwFbty4IQ0bNkzy9PSUvL29pdGjR0vZ2dnm8+fPn5cAadu2beZjs2fPloBS24oVK+zut7Y9U35+vjRx4kTJz89Pcnd3lx577DEpOTnZfP7XX3+VunTpInl6ekoeHh5S586dpWXLlkl6vd7p8n/44YdSs2bNJLVaLfXq1Uv6+++/zef69OkjjRw50qr9d999J912222SWq2WOnToIP38889W5w0Gg/T6669LQUFBkouLi9S3b1/p1KlTTpe7LJz5PHl5edKDDz4oBQQESCqVSmrevLk0duxYhxSGM3DkmUyft5Jbnz597O7THkQaLoFA0OARc4QCgaDBIxShQCBo8AhFKBAIGjxCEQoEggaPUIQCgaDBIxShQCBo8AhFKBAIGjxCEQoEggaPUISCeoOtNPQ1kYpeUPcQ2WcE9Yp+/fqxYsUKq2NlJXbQarWoVCqrYxqNBrVa7fB9K3udoHYgLEJBvcLFxYXg4GCrzc/PDzAWkPrkk08YMGAAHh4evPXWW8yZM4cuXbrwxRdfEBERgaurKwBJSUk8+uijeHp64u3tzRNPPGGV6qms6wR1E6EIBQ2KOXPm8Nhjj3HkyBGefvppAM6cOcN///tffvjhBw4ePIjBYODRRx8lPT2dHTt2sGXLFs6dO8eQIUOs+ip5naDuIobGgnrFpk2b8PT0tDo2Y8YMZsyYAcCTTz7J6NGjrc5rNBq++uorAgICAGMOvyNHjnD+/HnCwsIA+Oqrr+jQoQP79u2jZ8+eNq8T1F2EIhTUK+677z4++eQTq2ONGjUy71sWpjLRvHlzK2V24sQJwsLCzEoQoH379vj6+nLixAmzIix5naDuIhShoF7h4eFBq1atyj1vzzF77yWoH4g5QoGgBO3atePSpUtcunTJfOz48eNkZGTQvn37GpRMUFUIi1BQrygsLCxValOpVOLv7293H9HR0XTs2JHhw4ezZMkSdDodEydOpE+fPjaH1oK6j7AIBfWKzZs3ExISYrXdddddDvUhk8nYuHEjfn5+3HPPPURHR9OiRQvWrVtXRVILahqRql8gEDR4hEUoEAgaPEIRCgSCBo9QhAKBoMEjFKFAIGjwCEUoEAgaPEIRCgSCBo9QhAKBoMEjFKFAIGjwCEUoEAgaPEIRCgSCBo9QhAKBoMEjFKFAIGjw/D/7RbtG+mxk7AAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.018843, + "end_time": "2024-03-26T10:56:00.541572", + "exception": false, + "start_time": "2024-03-26T10:56:00.522729", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 3209.216375, + "end_time": "2024-03-26T10:56:03.283907", + "environment_variables": {}, + "exception": null, + "input_path": "eval/insurance/tvae/4/mlu-eval.ipynb", + "output_path": "eval/insurance/tvae/4/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "insurance", + "dataset_name": "insurance", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/insurance/tvae/4", + "path_prefix": "../../../../", + "random_seed": 4, + "single_model": "tvae" + }, + "start_time": "2024-03-26T10:02:34.067532", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/insurance/tvae/model.pt b/insurance/tvae/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..213164563e784b8502a405ad788549b78d643ef4 --- /dev/null +++ b/insurance/tvae/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a6fb8c78d9010f71d8bca74b7270e5c542ab122517161aacf0e58d42ef38665 +size 38610389 diff --git a/insurance/tvae/params.json b/insurance/tvae/params.json new file mode 100644 index 0000000000000000000000000000000000000000..8229c600276c9af0f22bb842bcc388fdd1e85115 --- /dev/null +++ b/insurance/tvae/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 16, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "fixnorm", "grad_clip": 0.7, "head_final_mul": "identity", "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "loss_balancer_beta": 0.79, "loss_balancer_r": 0.95, "dataset_size": 2048, "batch_size": 8, "epochs": 100, "n_warmup_steps": 100, "Optim": "diffgrad", "fixed_role_model": "tvae", "mse_mag": true, "mse_mag_target": 0.1, "mse_mag_multiply": true, "d_model": 256, "attn_activation": "leakyrelu", "tf_d_inner": 512, "tf_n_layers_enc": 4, "tf_n_head": 64, "tf_activation": "leakyhardsigmoid", "tf_activation_final": "leakyhardsigmoid", "tf_num_inds": 32, "ada_d_hid": 1024, "ada_n_layers": 7, "ada_activation": "relu", "ada_activation_final": "softsign", "head_d_hid": 128, "head_n_layers": 9, "head_n_head": 64, "head_activation": "prelu", "head_activation_final": "softsign", "models": ["tvae"], "max_seconds": 3600} \ No newline at end of file diff --git a/treatment/lct_gan/eval.csv b/treatment/lct_gan/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..933bed2b5866a67f1ab1f4a2901beb16b3b07b8f --- /dev/null +++ b/treatment/lct_gan/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +lct_gan,0.0,0.12000000000000194,0.004340556185786303,4.989878177642822,0.09192690253257751,1.7987895011901855,0.13193973898887634,3.62990285793785e-05,6.636855602264404,0.04716583713889122,836807.125,0.06588289886713028,0.21109354496002197,0.0002487990423105657,11.626733779907227 diff --git a/treatment/lct_gan/history.csv b/treatment/lct_gan/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..d2dd8807a6d6a2563f316d5d4cb30060c9c2613b --- /dev/null +++ b/treatment/lct_gan/history.csv @@ -0,0 +1,5 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.04985262780790322,3.6183568816933374,0.00821856617848477,0.08611974312061244,0.0,0.0,0.0,0.0,0.21120899681122424,900,450,569.3217799663544,1.265159511036343,0.6325797555181715,0.11481869970826059,0.02072129439680187,1.9086491901219171,0.0010903308281523874,0.0,0.0,0.0,0.0,0.0,0.02072129439680187,450,225,196.6758050918579,0.8741146892971463,0.43705734464857315,0.09171225249049408 +1,0.013151365245423803,0.6379314760027313,0.0008179183341412702,0.19476993432928189,0.0,0.0,0.0,0.0,0.013397588028950243,900,450,571.9026720523834,1.2708948267830742,0.6354474133915371,0.20376968434217915,0.009131540145187592,0.9951137590999442,0.00024677024046790486,0.0,0.0,0.0,0.0,0.0,0.009131540145187592,450,225,198.25886011123657,0.8811504893832737,0.44057524469163684,0.10169168976094321 +2,0.008621372499677054,0.8382067257209166,0.00026993465244648865,0.17589655292686074,0.0,0.0,0.0,0.0,0.008767833580902504,900,450,570.2824792861938,1.2672943984137641,0.6336471992068821,0.1927573654050098,0.008410379777208779,0.9130473620787616,0.0003690813285744622,0.0,0.0,0.0,0.0,0.0,0.008410379777208779,450,225,195.47286939620972,0.8687683084275988,0.4343841542137994,0.11685342669800876 +3,0.0067645793513913645,0.7113772220321888,0.00023273553968767465,0.16243632634791236,0.0,0.0,0.0,0.0,0.006880470157437446,900,450,570.8296418190002,1.268510315153334,0.634255157576667,0.20007731687348698,0.007127348931724993,1.0058974532183806,0.00017908157135019585,0.0,0.0,0.0,0.0,0.0,0.007127348931724993,450,225,198.4686758518219,0.8820830037858751,0.44104150189293756,0.1067870712840603 diff --git a/treatment/lct_gan/mlu-eval-load.ipynb b/treatment/lct_gan/mlu-eval-load.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..c838ccec17e68d3bc8739eb09da02e838c0869ef --- /dev/null +++ b/treatment/lct_gan/mlu-eval-load.ipynb @@ -0,0 +1,1982 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:45:58.199314Z", + "iopub.status.busy": "2024-03-26T12:45:58.198943Z", + "iopub.status.idle": "2024-03-26T12:45:58.235333Z", + "shell.execute_reply": "2024-03-26T12:45:58.234417Z" + }, + "papermill": { + "duration": 0.050549, + "end_time": "2024-03-26T12:45:58.237366", + "exception": false, + "start_time": "2024-03-26T12:45:58.186817", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:45:58.260770Z", + "iopub.status.busy": "2024-03-26T12:45:58.260400Z", + "iopub.status.idle": "2024-03-26T12:45:58.267822Z", + "shell.execute_reply": "2024-03-26T12:45:58.266705Z" + }, + "papermill": { + "duration": 0.021256, + "end_time": "2024-03-26T12:45:58.269760", + "exception": false, + "start_time": "2024-03-26T12:45:58.248504", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:45:58.292414Z", + "iopub.status.busy": "2024-03-26T12:45:58.291641Z", + "iopub.status.idle": "2024-03-26T12:45:58.295974Z", + "shell.execute_reply": "2024-03-26T12:45:58.295109Z" + }, + "papermill": { + "duration": 0.017503, + "end_time": "2024-03-26T12:45:58.297981", + "exception": false, + "start_time": "2024-03-26T12:45:58.280478", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:45:58.319450Z", + "iopub.status.busy": "2024-03-26T12:45:58.319173Z", + "iopub.status.idle": "2024-03-26T12:45:58.323349Z", + "shell.execute_reply": "2024-03-26T12:45:58.322388Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017248, + "end_time": "2024-03-26T12:45:58.325392", + "exception": false, + "start_time": "2024-03-26T12:45:58.308144", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:45:58.346826Z", + "iopub.status.busy": "2024-03-26T12:45:58.346261Z", + "iopub.status.idle": "2024-03-26T12:45:58.351997Z", + "shell.execute_reply": "2024-03-26T12:45:58.351115Z" + }, + "papermill": { + "duration": 0.01848, + "end_time": "2024-03-26T12:45:58.353872", + "exception": false, + "start_time": "2024-03-26T12:45:58.335392", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "124915d4", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:45:58.376491Z", + "iopub.status.busy": "2024-03-26T12:45:58.376197Z", + "iopub.status.idle": "2024-03-26T12:45:58.381908Z", + "shell.execute_reply": "2024-03-26T12:45:58.380814Z" + }, + "papermill": { + "duration": 0.019825, + "end_time": "2024-03-26T12:45:58.384349", + "exception": false, + "start_time": "2024-03-26T12:45:58.364524", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"lct_gan\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 2\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/lct_gan/2\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011565, + "end_time": "2024-03-26T12:45:58.407875", + "exception": false, + "start_time": "2024-03-26T12:45:58.396310", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:45:58.433710Z", + "iopub.status.busy": "2024-03-26T12:45:58.432766Z", + "iopub.status.idle": "2024-03-26T12:45:58.444195Z", + "shell.execute_reply": "2024-03-26T12:45:58.443151Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.026928, + "end_time": "2024-03-26T12:45:58.446588", + "exception": false, + "start_time": "2024-03-26T12:45:58.419660", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/lct_gan/2\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:45:58.471887Z", + "iopub.status.busy": "2024-03-26T12:45:58.471518Z", + "iopub.status.idle": "2024-03-26T12:46:00.530583Z", + "shell.execute_reply": "2024-03-26T12:46:00.529561Z" + }, + "papermill": { + "duration": 2.073621, + "end_time": "2024-03-26T12:46:00.532823", + "exception": false, + "start_time": "2024-03-26T12:45:58.459202", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:46:00.559187Z", + "iopub.status.busy": "2024-03-26T12:46:00.558727Z", + "iopub.status.idle": "2024-03-26T12:46:00.575319Z", + "shell.execute_reply": "2024-03-26T12:46:00.574453Z" + }, + "papermill": { + "duration": 0.031393, + "end_time": "2024-03-26T12:46:00.577304", + "exception": false, + "start_time": "2024-03-26T12:46:00.545911", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:46:00.599839Z", + "iopub.status.busy": "2024-03-26T12:46:00.599160Z", + "iopub.status.idle": "2024-03-26T12:46:00.606801Z", + "shell.execute_reply": "2024-03-26T12:46:00.606078Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.020732, + "end_time": "2024-03-26T12:46:00.608747", + "exception": false, + "start_time": "2024-03-26T12:46:00.588015", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:46:00.630528Z", + "iopub.status.busy": "2024-03-26T12:46:00.630206Z", + "iopub.status.idle": "2024-03-26T12:46:00.727762Z", + "shell.execute_reply": "2024-03-26T12:46:00.726968Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.110868, + "end_time": "2024-03-26T12:46:00.730044", + "exception": false, + "start_time": "2024-03-26T12:46:00.619176", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:46:00.754718Z", + "iopub.status.busy": "2024-03-26T12:46:00.754361Z", + "iopub.status.idle": "2024-03-26T12:46:05.649081Z", + "shell.execute_reply": "2024-03-26T12:46:05.648117Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.910009, + "end_time": "2024-03-26T12:46:05.651560", + "exception": false, + "start_time": "2024-03-26T12:46:00.741551", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-26 12:46:03.119316: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-26 12:46:03.119365: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-26 12:46:03.121104: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:46:05.675200Z", + "iopub.status.busy": "2024-03-26T12:46:05.674582Z", + "iopub.status.idle": "2024-03-26T12:46:05.681339Z", + "shell.execute_reply": "2024-03-26T12:46:05.680586Z" + }, + "papermill": { + "duration": 0.020663, + "end_time": "2024-03-26T12:46:05.683356", + "exception": false, + "start_time": "2024-03-26T12:46:05.662693", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:46:05.707178Z", + "iopub.status.busy": "2024-03-26T12:46:05.706873Z", + "iopub.status.idle": "2024-03-26T12:46:28.590555Z", + "shell.execute_reply": "2024-03-26T12:46:28.589350Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 22.898833, + "end_time": "2024-03-26T12:46:28.593290", + "exception": false, + "start_time": "2024-03-26T12:46:05.694457", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-26T12:46:28.620728Z", + "iopub.status.busy": "2024-03-26T12:46:28.619561Z", + "iopub.status.idle": "2024-03-26T12:46:28.627315Z", + "shell.execute_reply": "2024-03-26T12:46:28.626281Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.023258, + "end_time": "2024-03-26T12:46:28.629341", + "exception": false, + "start_time": "2024-03-26T12:46:28.606083", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 95,\n", + " 'realtabformer': (69, 281, Embedding(281, 768), True),\n", + " 'lct_gan': 75,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:46:28.653205Z", + "iopub.status.busy": "2024-03-26T12:46:28.652887Z", + "iopub.status.idle": "2024-03-26T12:46:28.717510Z", + "shell.execute_reply": "2024-03-26T12:46:28.716520Z" + }, + "papermill": { + "duration": 0.079027, + "end_time": "2024-03-26T12:46:28.719466", + "exception": false, + "start_time": "2024-03-26T12:46:28.640439", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/aug_test/treatment 0\n", + "Caching in ../../../../treatment/_cache_bs_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/bs_test/treatment 0\n", + "Caching in ../../../../treatment/_cache_synth_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/synthetics/treatment 600\n", + "600\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:46:28.746433Z", + "iopub.status.busy": "2024-03-26T12:46:28.746104Z", + "iopub.status.idle": "2024-03-26T12:46:29.079518Z", + "shell.execute_reply": "2024-03-26T12:46:29.078546Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.349928, + "end_time": "2024-03-26T12:46:29.082164", + "exception": false, + "start_time": "2024-03-26T12:46:28.732236", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.75,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'loss_balancer_beta': 0.75,\n", + " 'loss_balancer_r': 0.96,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 2,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.06,\n", + " 'n_warmup_steps': 80,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'lct_gan',\n", + " 'g_loss_mul': 0.2,\n", + " 'd_model': 512,\n", + " 'attn_activation': torch.nn.modules.activation.SELU,\n", + " 'tf_d_inner': 256,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 2048,\n", + " 'ada_n_layers': 6,\n", + " 'ada_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 7,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['lct_gan'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.5, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "# params = {\n", + "# **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "# }\n", + "params = getattr(PARAMS, dataset_name).BEST_DICT[gp][gp_multiply][single_model]\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).DEFAULTS,\n", + " **params,\n", + "}\n", + "if isinstance(params, (list, tuple)):\n", + " params = params[param_index]\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-26T12:46:29.107844Z", + "iopub.status.busy": "2024-03-26T12:46:29.107551Z", + "iopub.status.idle": "2024-03-26T12:46:29.754839Z", + "shell.execute_reply": "2024-03-26T12:46:29.753780Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.662308, + "end_time": "2024-03-26T12:46:29.756867", + "exception": false, + "start_time": "2024-03-26T12:46:29.094559", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n", + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['lct_gan'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:46:29.785757Z", + "iopub.status.busy": "2024-03-26T12:46:29.784922Z", + "iopub.status.idle": "2024-03-26T12:46:29.789475Z", + "shell.execute_reply": "2024-03-26T12:46:29.788615Z" + }, + "papermill": { + "duration": 0.02141, + "end_time": "2024-03-26T12:46:29.791465", + "exception": false, + "start_time": "2024-03-26T12:46:29.770055", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:46:29.817032Z", + "iopub.status.busy": "2024-03-26T12:46:29.816701Z", + "iopub.status.idle": "2024-03-26T12:46:29.824659Z", + "shell.execute_reply": "2024-03-26T12:46:29.823655Z" + }, + "papermill": { + "duration": 0.022886, + "end_time": "2024-03-26T12:46:29.826684", + "exception": false, + "start_time": "2024-03-26T12:46:29.803798", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "29688705" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:46:29.853279Z", + "iopub.status.busy": "2024-03-26T12:46:29.853040Z", + "iopub.status.idle": "2024-03-26T12:46:29.951480Z", + "shell.execute_reply": "2024-03-26T12:46:29.950581Z" + }, + "papermill": { + "duration": 0.114124, + "end_time": "2024-03-26T12:46:29.953658", + "exception": false, + "start_time": "2024-03-26T12:46:29.839534", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 75] --\n", + "├─Adapter: 1-1 [2, 2648, 75] --\n", + "│ └─Sequential: 2-1 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 2048] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 2048] 155,648\n", + "│ │ │ └─LeakyHardtanh: 4-2 [2, 2648, 2048] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 2048] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 2048] 4,196,352\n", + "│ │ │ └─LeakyHardtanh: 4-4 [2, 2648, 2048] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 2048] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 2048] 4,196,352\n", + "│ │ │ └─LeakyHardtanh: 4-6 [2, 2648, 2048] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 2048] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 2048] 4,196,352\n", + "│ │ │ └─LeakyHardtanh: 4-8 [2, 2648, 2048] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 2048] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 2048] 4,196,352\n", + "│ │ │ └─LeakyHardtanh: 4-10 [2, 2648, 2048] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 512] 1,049,088\n", + "│ │ │ └─LeakyHardsigmoid: 4-12 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 75] (recursive)\n", + "│ └─Sequential: 2-2 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-7 [2, 661, 2048] (recursive)\n", + "│ │ │ └─Linear: 4-13 [2, 661, 2048] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-14 [2, 661, 2048] --\n", + "│ │ └─FeedForward: 3-8 [2, 661, 2048] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 2048] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-16 [2, 661, 2048] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 2048] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 2048] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-18 [2, 661, 2048] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 2048] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 2048] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-20 [2, 661, 2048] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 2048] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 2048] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-22 [2, 661, 2048] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-24 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 8192] --\n", + "│ └─Encoder: 2-3 [2, 16, 512] --\n", + "│ │ └─ModuleList: 3-14 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-25 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-6 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 2648, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 256] 131,328\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 256] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 131,584\n", + "│ │ │ └─EncoderLayer: 4-26 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-18 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 2648, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 256] 131,328\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 256] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 131,584\n", + "│ │ │ └─EncoderLayer: 4-27 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-30 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 2648, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 256] 131,328\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 256] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 131,584\n", + "│ │ │ └─EncoderLayer: 4-28 [2, 16, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-42 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 2648, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 256] 131,328\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 256] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 131,584\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 512] 8,192\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 16, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 16, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-54 [2, 16, 512] --\n", + "│ └─Encoder: 2-4 [2, 16, 512] (recursive)\n", + "│ │ └─ModuleList: 3-14 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-60 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 661, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 661, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-31 [2, 661, 256] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-72 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 661, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 661, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 661, 256] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-40 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-84 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 32, 661, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 32, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-42 [2, 661, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-43 [2, 661, 256] --\n", + "│ │ │ │ │ └─Linear: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-45 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-46 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 32, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-96 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 32, 661, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 32, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-48 [2, 661, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-49 [2, 661, 256] --\n", + "│ │ │ │ │ └─Linear: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-18 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-51 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-52 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 32, 16, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 32, 16, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-108 [2, 16, 512] --\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-15 [2, 128] --\n", + "│ │ │ └─Linear: 4-33 [2, 128] 1,048,704\n", + "│ │ │ └─LeakyHardsigmoid: 4-34 [2, 128] --\n", + "│ │ └─FeedForward: 3-16 [2, 128] --\n", + "│ │ │ └─Linear: 4-35 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-36 [2, 128] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 1] --\n", + "│ │ │ └─Linear: 4-45 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 29,688,705\n", + "Trainable params: 29,688,705\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 116.20\n", + "========================================================================================================================\n", + "Input size (MB): 1.99\n", + "Forward/backward pass size (MB): 1231.11\n", + "Params size (MB): 118.75\n", + "Estimated Total Size (MB): 1351.85\n", + "========================================================================================================================" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = test_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "90576617", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:46:29.985252Z", + "iopub.status.busy": "2024-03-26T12:46:29.984910Z", + "iopub.status.idle": "2024-03-26T12:46:30.125419Z", + "shell.execute_reply": "2024-03-26T12:46:30.124416Z" + }, + "papermill": { + "duration": 0.159533, + "end_time": "2024-03-26T12:46:30.127703", + "exception": false, + "start_time": "2024-03-26T12:46:29.968170", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torch\n", + "model.load_state_dict(torch.load(\"model.pt\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:46:30.156028Z", + "iopub.status.busy": "2024-03-26T12:46:30.155659Z", + "iopub.status.idle": "2024-03-26T12:51:27.019967Z", + "shell.execute_reply": "2024-03-26T12:51:27.018975Z" + }, + "papermill": { + "duration": 296.881691, + "end_time": "2024-03-26T12:51:27.022857", + "exception": false, + "start_time": "2024-03-26T12:46:30.141166", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:51:27.056139Z", + "iopub.status.busy": "2024-03-26T12:51:27.055710Z", + "iopub.status.idle": "2024-03-26T12:51:27.082062Z", + "shell.execute_reply": "2024-03-26T12:51:27.080982Z" + }, + "papermill": { + "duration": 0.046009, + "end_time": "2024-03-26T12:51:27.084294", + "exception": false, + "start_time": "2024-03-26T12:51:27.038285", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
lct_gan0.00.120.0043414.9898780.0919271.798790.131940.0000366.6368560.047166836807.1250.0658830.2110940.00024911.626734
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "lct_gan 0.0 0.12 0.004341 4.989878 0.091927 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "lct_gan 1.79879 0.13194 0.000036 6.636856 0.047166 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "lct_gan 836807.125 0.065883 0.211094 0.000249 11.626734 " + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:51:27.115770Z", + "iopub.status.busy": "2024-03-26T12:51:27.115217Z", + "iopub.status.idle": "2024-03-26T12:51:27.511473Z", + "shell.execute_reply": "2024-03-26T12:51:27.510422Z" + }, + "papermill": { + "duration": 0.414184, + "end_time": "2024-03-26T12:51:27.513543", + "exception": false, + "start_time": "2024-03-26T12:51:27.099359", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:51:27.543833Z", + "iopub.status.busy": "2024-03-26T12:51:27.543516Z", + "iopub.status.idle": "2024-03-26T12:56:34.602968Z", + "shell.execute_reply": "2024-03-26T12:56:34.602114Z" + }, + "papermill": { + "duration": 307.077605, + "end_time": "2024-03-26T12:56:34.605828", + "exception": false, + "start_time": "2024-03-26T12:51:27.528223", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/lct_gan/all inf False\n", + "Caching in ../../../../treatment/_cache_bs_test/lct_gan/all inf False\n", + "Caching in ../../../../treatment/_cache_synth_test/lct_gan/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:56:34.636779Z", + "iopub.status.busy": "2024-03-26T12:56:34.636017Z", + "iopub.status.idle": "2024-03-26T12:56:34.657091Z", + "shell.execute_reply": "2024-03-26T12:56:34.656335Z" + }, + "papermill": { + "duration": 0.038445, + "end_time": "2024-03-26T12:56:34.659109", + "exception": false, + "start_time": "2024-03-26T12:56:34.620664", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:56:34.686668Z", + "iopub.status.busy": "2024-03-26T12:56:34.686396Z", + "iopub.status.idle": "2024-03-26T12:56:34.691769Z", + "shell.execute_reply": "2024-03-26T12:56:34.690980Z" + }, + "papermill": { + "duration": 0.021649, + "end_time": "2024-03-26T12:56:34.693762", + "exception": false, + "start_time": "2024-03-26T12:56:34.672113", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'lct_gan': 0.4631171667436077}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:56:34.721244Z", + "iopub.status.busy": "2024-03-26T12:56:34.720741Z", + "iopub.status.idle": "2024-03-26T12:56:35.156277Z", + "shell.execute_reply": "2024-03-26T12:56:35.155385Z" + }, + "papermill": { + "duration": 0.451329, + "end_time": "2024-03-26T12:56:35.158354", + "exception": false, + "start_time": "2024-03-26T12:56:34.707025", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/o0lEQVR4nO3deXhU9b348feZfSYzk32FQNgEBEFEUNxA5bpxVWqv116tghS1t1q11Hsrt0+12vZiferW1svjr1dBfy619rHa1qrlZ0VcUdlkkz0JBEL2zJLZ5/z+OJnREALJZDJnknxezzNPyJkzZz4zzHzy3b+KqqoqQgihI4PeAQghhCQiIYTuJBEJIXQniUgIoTtJREII3UkiEkLoThKREEJ3koiEELqTRCSE0J0kInFCq1evRlEUqqur9Q5FDGGSiETavfjiizz++ON6hyEGEUlEIu0kEYm+kkQkhNCdJCLRZ2+++SZz587F5XLhdruZNWsWL774IgDz5s3jjTfeoKamBkVRUBSFqqqqXl87Ho/z05/+lIqKChwOBxdeeCE7duygqqqKxYsXJ89raWnhnnvu4bTTTsPpdOJ2u7n88svZsmVLl+utXbsWRVH4wx/+wC9+8QtGjhyJzWbj4osvZu/evel4O0QamPQOQAwuq1evZsmSJUyZMoXly5eTl5fHpk2beOutt7j++uv58Y9/THt7O4cOHeKxxx4DwOl09vr6y5cv5+GHH+bKK6/k0ksvZcuWLVx66aUEg8Eu5+3fv5/XXnuNa6+9ljFjxnD06FGeeuop5s6dy44dO6ioqOhy/kMPPYTBYOCee+6hvb2dhx9+mBtuuIH169f3/00R/acKcQKrVq1SAfXAgQNqW1ub6nK51LPOOksNBAJdzovH48l/L1iwQB09enSfn6u+vl41mUzqwoULuxz/6U9/qgLqokWLkseCwaAai8W6nHfgwAHVarWqDz74YPLYu+++qwLq5MmT1VAolDz+xBNPqIC6devWPscp0k+qZqLX1qxZg9fr5d5778Vms3W5T1GUfl//nXfeIRqN8r3vfa/L8e9///vdzrVarRgM2sc3FovR3NyM0+lk4sSJbNy4sdv5N998MxaLJfn7+eefD2glK6E/SUSi1/bt2wfA1KlTB+T6NTU1AIwfP77L8YKCAvLz87sci8fjPPbYY0yYMAGr1UpRURHFxcV88cUXtLe3d7v2qFGjuvyeuF5ra2s6X4JIkSQiMSj993//N8uWLeOCCy7g+eef5+2332bNmjVMmTKFeDze7Xyj0Xjc66iyUnJWkMZq0Wvjxo0DYNu2bd1KLV+XajVt9OjRAOzdu5cxY8Ykjzc3N3crufzxj3/kwgsv5Omnn+5yvK2tjaKiopSeX+hHSkSi1y655BJcLhcrVqzo1ov19ZJFTk7OcatHJ3PxxRdjMplYuXJll+O//e1vu51rNBq7lWZeeeUV6urq+vy8Qn9SIhK95na7eeyxx1i6dCmzZs3i+uuvJz8/ny1bttDR0cGzzz4LwMyZM3n55ZdZtmwZs2bNwul0cuWVV570+qWlpdx111088sgjXHXVVVx22WVs2bKFN998k6Kioi4lrX/+53/mwQcf5Oabb+acc85h69atvPDCC4wdO3bAXr8YQDr32oks9/Xu+4Q///nP6jnnnKPa7XbV7Xars2fPVl966aXk/T6fT73++uvVvLw8FehTV340GlV/8pOfqGVlZardblcvuugidefOnWphYaH63e9+N3leMBhUf/jDH6rl5eWq3W5Xzz33XPXjjz9W586dq86dOzd5XqL7/pVXXunyPAcOHFABddWqVX19S8QAUFRVWutEdmtrayM/P5+f//zn/PjHP9Y7HDEApI1IZJVAINDtWGIC7bx58zIbjMgYaSMSGdHY2EgsFuvxfovFQkFBAS+//DKrV6/miiuuwOl08sEHH/DSSy9xySWXcO6552YwYpFJkohERsyaNSs5YPF45s6dy9q1a5k2bRomk4mHH34Yj8eTbMD++c9/nsFoRaZJG5HIiA8//PC41a6E/Px8Zs6cmcGIRDaRRCSE0J00VgshdDeo24ji8TiHDx/G5XKlZfa3ECK9VFXF6/VSUVGRXC3heAZ1Ijp8+DCVlZV6hyGEOImDBw8ycuTIHu8f1InI5XIB2ot0u906RyOEOJbH46GysjL5Xe3JoE5EieqY2+2WRCREFjtZ04k0VgshdCeJSAihO0lEQgjdDeo2ot5QVZVoNHrCeU7i+IxGIyaTSYZGiAE3pBNROBzmyJEjdHR06B3KoOVwOCgvL++yA4YQ6ZY1ieihhx5i+fLl3HXXXWnZNz0ej3PgwAGMRiMVFRVYLBb5y94HqqoSDodpbGzkwIEDTJgw4YQD0oToj6xIRJ999hlPPfUU06ZNS9s1w+Ew8XicyspKHA5H2q47nNjtdsxmMzU1NYTD4W57mQ1Vext8bKxtZUZlHhNKTzz+RaSH7n/ifD4fN9xwA7/73e+67V2VDvJXvH+G4/s3Mt9Oqz/MG1uPcLit5xUDRPro/im7/fbbWbBgAfPnzz/puaFQCI/H0+UmRLpZTQbGFjsxxEJs3bYFoiG9QxrydK2a/f73v2fjxo189tlnvTp/xYoVPPDAAwMclRiuVFVFURQURWF2mQHrp69gjnUQiH+KffZisMno/YGiW4no4MGD3HXXXbzwwgu9bntYvnw57e3tydvBgwcHOMqhr6qqKi2dA0PBK58f4tWNh2j2hcite49CcxiAtpYm2P+uztENbbqViDZs2EBDQwNnnHFG8lgsFmPdunX89re/JRQKddsm2Gq1YrVaMx2qGAb8oSh1ne1B1kgrNO0h32llQ84F5Ha8T3nDlzB2Hthy9Q10iNItEV188cVs3bq1y7Gbb76ZSZMm8aMf/ajHvcpFd+FwWMb59FNtizbWrMRtxdm2BwBXxUSa28ZzJPAlk+NBlMZdUDlbzzCHLN2qZi6Xi6lTp3a55eTkUFhYyNSpUwf0ucPReI+3aCze63MjvTg3FfPmzeOOO+7gjjvuIDc3l6KiIn7yk58kt1iuqqriZz/7GTfddBNut5tbb70VgA8++IDzzz8fu91OZWUld955J36/P3ndhoYGrrzySux2O2PGjOGFF15IKb6h6Ei7VhqqzHdA45cAOEdOoSLPhrtyKjFVhabdeoY4pGXFOKJMe/LdvT3eN6Yoh4UzRiR//z/r9hGJHX9Z75H5dq4986uF2Z758ACBcNepJD/4p1NSivHZZ5/lO9/5Dp9++imff/45t956K6NGjeKWW24B4Fe/+hX33Xcf999/PwD79u3jsssu4+c//znPPPMMjY2NyWS2atUqABYvXszhw4d59913MZvN3HnnnTQ0NKQU31BT3671jJVbw+BvAsWAUnQK15XbocMJ6zeA5zDEImA06xzt0JNViWjt2rV6h5A1Kisreeyxx1AUhYkTJ7J161Yee+yxZCK66KKL+OEPf5g8f+nSpdxwww3cfffdAEyYMIFf//rXzJ07l5UrV1JbW8ubb77Jp59+yqxZswB4+umnmTx5csZfW7aJxuI0+bREVKY2agddpWC2a/+254PVCSEfeI9A3iidIh26sioRZcrtF47v8T7DMbNAbr1gXI/nHjtjZMm5Y/oTVhdnn312lykpc+bM4ZFHHklO3j3zzDO7nL9lyxa++OKLLtUtVVWTU112796NyWTqsmXPpEmTyMvLS1vMg1WTL0wsrmK3GHEGD2sHc79a1jQaVwnYynCF9kL7IUlEA2BYJiKLqfdNYwN1bn/l5OR0+d3n83Hbbbdx5513djt31KhR7N4t7RsnUlXkwGYyonjqtAO5WrLxhaI8/f4ByjxGrs1TMbTX6Rjl0DUsE9FgsH79+i6/f/LJJ0yYMKHH3sQzzjiDHTt2MH788Ut7kyZNIhqNsmHDhmTVbNeuXbS1taU17sGoLNfGN2aMhGgYPmjWDrrLAcixGDGbFLyWIgKRGDl+aVMbCLpP8RDHV1tby7Jly9i1axcvvfQSv/nNb7jrrrt6PP9HP/oRH330EXfccQebN29mz549vP7669xxxx0ATJw4kcsuu4zbbruN9evXs2HDBpYuXYrdbs/US8p+/kZQVbDkgFWb7KooCkVOKx3mfDpCMQh6ICLzz9JNElGWuummmwgEAsyePZvbb7+du+66K9lNfzzTpk3jvffeY/fu3Zx//vnMmDGD++67j4qKiuQ5q1atoqKigrlz53LNNddw6623UlJSkomXk9WCkc6eTt9R7aeztMv9hTkWYgYrXqWzOuyTUlG6SdUsS5nNZh5//HFWrlzZ7b7q6urjPmbWrFn8/e9/7/GaZWVl/PWvf+1y7MYbb+xXnINdRzjKU+/tx2k1saTkKEYAZ9fknOfQBou2GfKAJq3klD8606EOaVIiEsNas0+bT2Y0KBgT7T/HlIjyHdq4oWY6p3f4GzMW33AhiUgMa60dWiIqzDFDj4lIKxE1xXJQUSHQmtEYhwOpmmUhGdiZOS1+LREVmcMQi4JiAHtel3PcdjOTy92UqqNRG7egdLToEOnQJolIDGvtgQgAhQavdsCeD4auQySMBoXLppZB2A1NCoS8MtUjzaRqJoa1RCLKVTsTkaOg55MtDjB3rp0l1bO0kkQkhi1VVWnv0BKRK96uHewhEUVjcVr8YfzGzlUapXqWVlI1E8NWNK5yaoWb9kAER6xz/XNH4XHP/aKunfd2NXJB1MRMG1IiSjNJRGLYMhsNXDy5s4fs484STg+JyG3TviqtOLUDASkRpZNUzYSIhrWpGwD241fN3DatYbot3rlHXlB2kEknSURi2OoIR7VVNBPVLLNNa5A+DrddS0Ttcbu2WmOwPVNhDgtSNRPD1rrdjew84uXS0nZOBa3rvgdWkwGLyUDI5CIcjWMPebQJsrKNeVpIiUgMW4mue6faua63La/HcxVFwW0zETY6CEbjEI9B2JeBKIeH4ZWIVFVrD8j0TT3+mtfH89xzz1FYWEgo1HV30YULFw77CarplkhELrUzoZxkqyC33YyqGAkkZuFLO1HaDK+qWSwC7z+S+ec9/4dg6t12P9deey133nknf/7zn7n22msBbfeNN95444Qz60XfRGJx/CFt+Y+cRInomKkdxzql1EWp24azPh9CR7V2otwRJ3yM6J3hVSIaBOx2O9dff31y5w2A559/nlGjRjFv3jz9Ahti/KEoAGajgjnc2fB8khLR5HI3Z48txJ1XpB0ISYkoXYZXicho1konejxvH9xyyy3MmjWLuro6RowYwerVq1m8eHGXxfRF/3iDWiJyWowooUQiyuvdg22do6ul5yxthlciUpReV5H0NGPGDKZPn85zzz3HJZdcwvbt23njjTf0DmtI8Ye1ROQ2RSAQ1T4bJykRxeIq7YEI0YiNEpA2ojQaXoloEFm6dCmPP/44dXV1zJ8/n8rKypM/SPSa22bmtBG5lKiNEEBbo9pw4m3OfaEoz35UTUGog5tyVJRgW0ZiHQ6kjShLXX/99Rw6dIjf/e53LFmyRO9whpyKPDvzTy1lWlFndbcX1TKnVfu7HVAcRGOqdN+nkSSiLJWbm8s3v/lNnE4nCxcu1DucoSvQpv08SbUMtHWJHBYjYVMO4VgcIkGtJ1b0mySiLFZXV8cNN9yA1WrVO5QhxxOMEIrGIFG9OknXfYLTZiKmWAjFO786Ie+AxDfcSBtRFmptbWXt2rWsXbuW//mf/9E7nCHplc8P4QlEWJzTSD70usfMaTXRoCgEFQcQ1KpnJ1pMTfSKJKIsNGPGDFpbW/nlL3/JxIkT9Q5nyFFVNTmOyBbr3ajqhEQ7UYdiB4IQknaidJBElIV62rdMpEcgEiMWV0FVscYS88zcvXpsTmci8mEHWqXBOk0kEYlhx9c5mNFtDGNQ49oYIouzV48dVaAtE1LeXgRth2V0dZoM+USk9mHCqehuKL5/vs5qWb4pCHG0ve5PMoYooSLPTkWeHQ4WQxtSNUuTIdtrZjZr0yo6Ojp0jmRwS7x/ifdzKEgkolxDQDtg7V21rItECUqqZmkxZEtERqORvLw8Ghq03TsdDofM1eoDVVXp6OigoaGBvLw8jMbelRgGg0QicitB7YDV1evHqqpKiz9MMGCiXFUxSIkoLYZsIgIoKysDSCYj0Xd5eXnJ93GoKHXbOG1ELmX+CEToc4nopU9rMYQ8LDHGsYW8slJjGgzpRKQoCuXl5ZSUlBCJyAjYvjKbzUOqJJQwrtjJuGIn7IiCj173mIH2mcqxmmiPOghF49jMUYgGwWwfuICHgSGdiBKMRuOQ/EKJfkrMnu9D1Qy0Lvy2DhNhgwVQtQZrSUT9MmQbq4XoSYs/rE3vSEzP6Gsismh/v7XR1UgXfhoMixKREAmRWJxnP6oGVeUOUztmhT4nIodVK113YAf8EPanPc7hRkpEYlhJTO2wqwFMSmcjsyW1ElGHYtMOSCLqN0lEYlj5+mBGhc4R1Ya+fQ0cFq1E5Fc7V0WISCLqL10T0cqVK5k2bRputxu3282cOXN488039QxJDHGJnTvchr6PIUooy7VxzrhCqspLtANSIuo3XduIRo4cyUMPPcSECRNQVZVnn32Wq6++mk2bNjFlyhQ9QxNDlC+kDePIUwKg0qeu+4Qip5UipxXqi6AZSURpoGsiuvLKK7v8/otf/IKVK1fyySefSCISA8LXWSJyJhJRCiWiJEvnRouSiPota3rNYrEYr7zyCn6/nzlz5hz3nFAo1GUHVI9Huk1F3yRm3jvoxzwzoNkXIuA3Uq6qGCUR9ZvuiWjr1q3MmTOHYDCI0+nkT3/6E6eeeupxz12xYgUPPPBAhiMUQ0lVkQOryUB+S+fM+xQT0R83HCIc8LNEiZGjBCAe6/UMftGd7r1mEydOZPPmzaxfv55///d/Z9GiRezYseO45y5fvpz29vbk7eDBgxmOVgx2UypymX9qKQXG1BurQRtdHTVYCcc6D0ipqF90LxFZLBbGjx8PwMyZM/nss8944okneOqpp7qda7VaZSF50X/x+FfrCKWciIw0Kgohgw2IaokohYZvodG9RHSseDzepR1IiHSJxuI0+0KEAh5Q46AYer0y47EcyWkenXPMpETUL7qWiJYvX87ll1/OqFGj8Hq9vPjii6xdu5a3335bz7DEENXSEeaFT2opjjfxbQtg7ftgxoTk6GqsgFcGNfaTromooaGBm266iSNHjpCbm8u0adN4++23+ad/+ic9wxJDVGIwY24/24fg2PlmSImon3RNRE8//bSeTy+GmeSi+SmszHisRInIH7doByQR9UvWtREJMVAS88xcSuc65il23QOUuKycM66QsRWJaR6yZGx/6N5rJkSmJBJRTj8HMwLk51g4a2whHC2BFiAsmzT0h5SIxLCRWALEqSZKRP2Y3pEg0zzSQhKRGDa8iW2m451Jo5+JqNkX4pBfIRqPS9Wsn6RqJoaNU8vdtLmD5NQFAaXfAxBf23wYv8/LzWoMlyEEsSgY5SuVCnnXxLAxc3S+tmD+UUUbzGjO6df1cixGPAYr4WjnVkIRPxhz0xDp8CNVMzG8JBfMT30wY4LDagJFkdHVaSCJSAwLwUiMZl+IcEerdqAfPWYJDrM2qDGIrF3dX1I1E8NCTXMHf9t6hGkc4GITaekxS4yuDig2wCuJqB+kRCSGheRgxuQYov4nouR8s8Qi+pKIUiaJSAwLiUTkpHMMka3/jco5nSUiP4ndPGRQY6okEYlhITGY0ZHGElGx08a544sYL9M8+k3aiMSwkCgROdI4qjrXYWb2mAI4WizTPPpJSkRiWPAFo6DGscXTOL0jQaZ59JskIjHkqaqKPxTFEuvAYlC0Re5TXJnxWE2+EAd9ndM8ZHG0lEnVTAx5cRVmjykg0urH0t65PKyipOXaf+6c5rFIjWm7x8puHimREpEY8owGhbPGFnLeSAsGRUlrtSzHaiRqsBKJdx6Q6llKJBGJ4SMxvSONu204LNo0j5DSObpauvBTIolIDHm+UFSb3hFo0w6kuUQEidHVSIkoRdJGJIa8bXXtfLyvmfmxOk6zAtb0zZBPbCsUwAb4JRGlSEpEYsjrvt99+kpEDkvnbh4yzaNfUkpE+/fvT3ccQgwYf7gzEaVpZcavS5SI/IlEJF34KUkpEY0fP54LL7yQ559/nmAwmO6YhEgrbzCKosaw0flZTWNjdbHTqk3zGFmqHZASUUpSSkQbN25k2rRpLFu2jLKyMm677TY+/fTTdMcmRFokBzMaFTCYwOxI27UT0zwqS4u1AzLNIyUpJaLTTz+dJ554gsOHD/PMM89w5MgRzjvvPKZOncqjjz5KY2NjuuMUIiXRWJyOcAxL1IfFaNCqZWkazNiFpTO5ycTXlPSrsdpkMnHNNdfwyiuv8Mtf/pK9e/dyzz33UFlZmdxKWgg9+cPaNtMOtQOTMb2DGRO6TvOQElEq+pWIPv/8c773ve9RXl7Oo48+yj333MO+fftYs2YNhw8f5uqrr05XnEKkxGxUOGdcIVOLFBQGJhH9dcthXt/RTkc4BpEAxOMnf5DoIqVxRI8++iirVq1i165dXHHFFTz33HNcccUVGDoXIx8zZgyrV6+mqqoqnbEK0WcOi0nbkTVqgDrS2lCdfA6riVa/jXBYBVXVSkXW9EyqHS5SSkQrV65kyZIlLF68mPLy8uOeU1JSwtNPP92v4IRIm5BH+zkAJSKHxQiKgZBiBSQRpSKlRLRmzRpGjRqVLAElqKrKwYMHGTVqFBaLhUWLFqUlSCFS1eoPE1NV8jratQ97GkdVJyTWrtZ28wh0NliXpP15hrKU2ojGjRtHU1NTt+MtLS2MGTOm30EJkS6f7G/m/35cQ31jg3ZgoEpEfH2+mTRY91VKiUhV1eMe9/l82Gy2fgUkRDp5Q1EUNYpNDWkHBiAR5VhlN4/+6lPVbNmyZQAoisJ9992Hw/HVwLBYLMb69es5/fTT0xqgEP3hC0axRP1YTAZtX3qzPe3PYe8sEfnVxFIgkoj6qk+JaNOmTYBWItq6dSsWiyV5n8ViYfr06dxzzz3pjVCIFCWWiHXE/FhsBm131wEYzFjktHLehCLK2kqgtVpKRCnoUyJ69913Abj55pt54okncLvT3xUqRLoEI3GicRVLLDGqemA+r7l2M7OqCuBIMbQibUQpSKnXbNWqVemOQ4i084YiALiVQNqXiD2u5G4eMs2jr3qdiK655hpWr16N2+3mmmuuOeG5r776ar8DE6K/EusQuQ3pn3V/rCZfiIAXyuNxTDLNo896nYhyc3NROuvXubnpH4shRLrlOSycM66QstoYqAxoieiNL47ga2/n22qMXGOHNsJ6ICbXDlG9TkRfr45J1UwMBgU5Fm16R3MMfAxYGxFoY4lajXYioTiocW3OmSV9y40MdSmNIwoEAnR0fFX8rKmp4fHHH+fvf/972gITIm2S0zsGMhGZUBUjIaWzJ1l6zvokpUR09dVX89xzzwHQ1tbG7NmzeeSRR7j66qtZuXJlWgMUIlVHPUGa2r3Ew+lfq/pYjs7dPLRpHshYoj5KeYXG888/H4A//vGPlJWVUVNTw3PPPcevf/3rXl9nxYoVzJo1C5fLRUlJCQsXLmTXrl2phCREN29vr+ePH2zHG4yCyQrmgRv1n5hvJtM8UpNSIuro6MDl0v66/P3vf+eaa67BYDBw9tlnU1NT0+vrvPfee9x+++188sknrFmzhkgkwiWXXILfL39NRP95g1GsMZ82qto2sB0ssptH/6Q0jmj8+PG89tprfOMb3+Dtt9/mBz/4AQANDQ19GuT41ltvdfl99erVlJSUsGHDBi644IJUQhMCgFA0RjgaJz/q1QYzZigR+ZHdPFKRUonovvvu45577qGqqoqzzjqLOXPmAFrpaMaMGSkH097eDkBBQcFx7w+FQng8ni43IY7HH9KWiHXix2hQwJY3oM9X5NKmeYwfIbt5pCKlEtG//Mu/cN5553HkyBGmT5+ePH7xxRfzjW98I6VA4vE4d999N+eeey5Tp0497jkrVqzggQceSOn6YnhJDGbMVTrbaga4ROS2dU7zsBRDG9JG1EcpbzldVlZGWVlZl2OzZ89OOZDbb7+dbdu28cEHH/R4zvLly5MrAAB4PB4qKytTfk4xdCWmd7joLJkM4KjqLmSaR0pSSkR+v5+HHnqId955h4aGBuLHLBbe151g77jjDv7617+ybt06Ro4c2eN5VqsVq9WaSshimEmUiHLUDmDg24igc5qHT6Z5pCKlRLR06VLee+89brzxRsrLy5NTP/pKVVW+//3v86c//Ym1a9fK6o4ibUYVOjCoUYq/jAKWjCSiv209gq/Nww1qjFyTX6Z59EFKiejNN9/kjTfe4Nxzz+3Xk99+++28+OKLvP7667hcLurr6wFtLpvdnv4FrMTwUZ5rp9zUAbUWMFnANPArhzosJloNdiLhOMRjEA0N6NiloSSlXrP8/Pwee7b6YuXKlbS3tzNv3jzKy8uTt5dffrnf1xaCYJv205abkZJJjsVI3GAilPj7Lj1nvZZSIvrZz37Gfffd12W+WSpUVT3ubfHixf26rhDVTX7aWpu09dUHYOeO40ksGSvTPPouparZI488wr59+ygtLaWqqgqz2dzl/o0bN6YlOCFSEYzE+NOmOka17WZhARgz0D4EX1tEHxsQkS78PkgpES1cuDDNYQiRPt7OHjMngc7BjJlJRF9tK2QFvFI164OUEtH999+f7jiESBtvUBtDlKskxhBlqETUOfHVH5eqWV+l1EYE2vIf//u//8vy5ctpaWkBtCpZXV1d2oITIhWezhKRi8yMqk4odFq0aR4jO3d5lRJRr6VUIvriiy+YP38+ubm5VFdXc8stt1BQUMCrr75KbW1tcq0iIfTgDUZQ1CgOAkBOxkZVuxLTPMzF0I4koj5IqUS0bNkyFi9ezJ49e7rs7HrFFVewbt26tAUnRCq8wSjWqA+ryQBGM5gzvGRrcpqHJKLeSikRffbZZ9x2223djo8YMSI5KFEIvXgCEWzRdi0R2fMyOrq50Rui1qcQicVBpnn0WkpVM6vVetwlOHbv3k1xcXG/gxKiP86syieEgZw2E9jzM/rcb22vx9vSzvVEyZOJr72WUonoqquu4sEHHyQS0XonFEWhtraWH/3oR3zzm99Ma4BC9NX4EhdT8uPYzMaMJyKH2UjUaCcSVSEWhWg4o88/WKWUiB555BF8Ph/FxcUEAgHmzp3L+PHjcblc/OIXv0h3jEL0XaBV+znAC6IdK8dqJKaYCSe+WlIq6pWUqma5ubmsWbOGDz/8kC1btuDz+TjjjDOYP39+uuMTok+8wQhNvjClniYckPkSkcUEitI5zSPW2U7U/3mZQ12fE1E8Hmf16tW8+uqrVFdXoygKY8aMoaysDFVVU14SRIh0qG3p4O/bjnBpcx2nljl1SESJ0dU2wC89Z73Up6qZqqpcddVVLF26lLq6Ok477TSmTJlCTU0NixcvTnmZWCHSxROIYo35sRoBg3FAN1U8HkdidLXs5tEnfSoRrV69mnXr1vHOO+9w4YUXdrnvH//4BwsXLuS5557jpptuSmuQQvSWNxjBGvVoXfe2XDCkPHkgJTmdGy12JGfgSxd+b/Tpf+mll17iv/7rv7olIYCLLrqIe++9lxdeeCFtwQnRV22BCLaoR5ceM4CCHAvny24efdanRPTFF19w2WWX9Xj/5ZdfzpYtW/odlBCp8nQmIqvZoEsictnMnFlVQGVp53g6SUS90qdE1NLSQmlpaY/3l5aW0tra2u+ghEhFJBbHG4xqJSKTMeNd911YOqeVSCLqlT4lolgshsnUc7OS0WgkGo32OyghUuEJaANsc+JeTEZFlxIRQLMvRK1XISzTPHqtT43ViWVce9rSJxQKpSUoIVKRYzVx2ZRS8kJRFIzaPDMdrNlxlNYmD9fFIxRYZUBjb/QpES1atOik50iPmdCLzWxkcoECOUZQ9GkjAi0hNhhshMJxbYpHLArGlPcyHRb69O6sWrVqoOIQIj06mrWf9jxtHJEOnDYTMYOVcLxzcG/Yp1vpbLDI7CALIQbQvkYfdUfqiMbj4CjULQ6XVZvmEaCzCUPaiU5KEpEYMt7f3cin2/bgD8XAod/8rsRuHj61c1Cj9JydlCQiMSTE4yqeYBR7tE0bQ6Rjich5bCIKdV+7S3QliUgMCd5QlFhcJSfahtWobyJy2bRE5FXtqKgQkp6zk5FEJIYETyCCMR7GpQS0FSB0LhGdP6GI08aNBBUIeXWLZbCQPkUxJLR2hLFF2rU5ZpYcMNt1i8VkNHBmVQFYS6FdkcXRekFKRGJIaPGHsUfbsJuNupaGurC6tJ9SIjopSURiSGjtCGOPtGG3ZEciausIU+0zEIjEpLG6F6RqJoaEc8cXEfapuAPmrEhEn1W3svOgl2v8IUaajdoIa5NF77CyliQiMSSUuGxg9kHUCDlFeoeD02oiZrAQVDu/YiEvmPRPkNlKqmZiaIiGIdim/dtZomso8FUXvp/ORvOwtBOdiCQiMegd9QTZsXcfvmBE6zFLbPmso+SgxkQikgbrE5JEJAa9/Y1+Nu3cw1FPMCtKQ/DVNA9vPDG6WhLRiUgiEoNea0cYR6RZ6zHLyY4tzxNVMx92YnEZXX0ykojEoNfsD+MIt2hjiLKkRGQzG7GaDYSNOYSi0oV/MpKIxKAWi6u0eEM4Ii3a5oY52ZGIAC6YUMycyVVYTAYZXX0S0n0vBrUWfxhT1IeNMBazKyvGECVMHZELrjKoN0gb0UlIiUgMak2+r0pDiqMw+5ZkTUzzCPshHtM3liwmiUgMak2+EDnhRm2r5yxpH0roCEfZ367SGoyDqko70QlIIhKD2uwxBcwfqVLqtoKrQu9wuqhrDfD6liPs93Z+zYLt+gaUxXRNROvWrePKK6+koqICRVF47bXX9AxHDEJWk5GCeLNWInKV6R1OF267GQCP2rnZoiSiHumaiPx+P9OnT+fJJ5/UMwwxmIW82hgdRQFnz7sQ68Ft60xE5BBXVQhK1awnurbsXX755Vx++eV6hjB4qSr4G7X9uxyF2hdxmDnqCVK9azsT/CEKSkZk3ex2m9mA2agQMroIReuxS4moR1nWxXBioVCoy26yHs8w/QsTbIdtr4K3Xvs9rxJOvfqrHpphoqa5g9rq/RTEwxS4s6t9CEBRFNx2MyGTk1A0JonoBAZVY/WKFSvIzc1N3iorK/UOKfMiQQKf/V9CrXVgMGmbCLYdJLjx99oM9GGk3hPEGW7Q5nVlWftQgttmJmRyEY7GpY3oBAZVIlq+fDnt7e3J28GDB/UOKePCX77Nlwdq2dSo0j59CcxaytGgkS++3I1n5zt6h5dRR9sCuMIN2kz3LOsxS3DbTYSMTkKRuNaeFY/rHVJWGlSJyGq14na7u9yGFW89h3Z+SjAS50DJP2G056La89nsPI9YXOXAlveIexv1jjIjvMEIMV8jpngYh92edQ3VCZPL3Vx8+liK3A5Q47IuUQ8GVSIa7rxf/oMGb4gmx3jOnzkNp9WEoiicN3s2PudYOkJRarcMj1LRUU8Id6geu8WIKW8kGLLzo1yea2dSeR52V552QKpnx6Xr/57P52Pz5s1s3rwZgAMHDrB582Zqa2v1DCs7BVo5sn87KmAedz4j8x3Ju3KsJsqmXwRA875NxDpadQoyc456grjC9Vq1LHek3uGcnC1X+yld+MelayL6/PPPmTFjBjNmzABg2bJlzJgxg/vuu0/PsLKSv/pzWnwh2mwjOWPyuG73T55wCkFnJeFojIPbP9EhwszyBiO4QoMjEVU3+dnvNRKNS4N1T3Ttvp83bx6qquoZwuAQj9G05zNUgBFnaAvFH8NkNFB0yln4Nh6ked/nVM28NGurK+lw2XgHoSMmDEYjZGHX/det2XGU3IY4Jc44TklExzV0P6lDSVstkaCfiMHOmInTejxt/OQZxIw2Iv522ut2ZTBAHbQdxGoyYnaXgcmqdzQnlOswEzK5CUZiEBj61eZUDKoBjcNW817GFzspnzgFa0nPPYV2m5WqKbMpavsCm2cXMDlzMWZa6wHtZ36VrmH0Rr7DQrXJTSAcg0CL3uFkJSkRZTtVhea9AORUTMJkPPF/2ciJZ2IzGbXHxCKZiDDj/vbFYbZt24IvFB0UiSjPYSZgyiUUiWnz4obZwNPekESU5VR/I1F/izaKujdfOle51kMTi6A27xvw+DItHlc5cqQOr6cNjEbIzf7R9Xl2MzGjDV+8cy6cVM+6kUSU5doO7uDz6lY+97hRjeaTP0BRaMsZy5f1HjZv/HjgA8ywBm8Ih78Wk0HBUTwm+1ZkPI48h5aA2lSndkCqZ91IIspyLbU7UAGfawxKL2fYRwsn0toRIVi/m1AoOLABZlh1s5+8wCFy7WYMBWP0DqdXchPrEhlcWhe+lIi6kUSUzUI+Ohq1wZ0lo6f0+mGF5aMx2rXq2aF92wcqOl3UHm3FHTpMnsMMBd3HU2Uji8nA5aeVce5pEzAoCnRIiehYkoiymP/Il/hCUfzWYkZX9H4ulWIw4KzUEldz9baBCi/jOsJRgo17MagxcgtKIKdI75B6bVKZm8KiMi0RSYmoG0lEWay5RivNKEUTklsY91b5uOkARBp2EwwNjV6amuYO8juqybEYsZZNGnyLwTkKtJ/SRtSNJKJsFYvgr9e67fNH9b5alpBfPhaL3YUhFqJ23850R6cLq0GliiPk51ig6BS9w+kTXyjKF80KR9oDEO6AyNBqu+svSURZKty0H48/QMjkZFTl6L5fQFFwjtQGNDZVf5Hm6PQx1tTEaaU2KkuLwD1C73D6xBeM8s6edqp9Ru1AR5O+AWUZSURZSmneQ1VhDq4Rkyl0pjaFoWzsdNw2E1Vq3dBYkOtoZ8N78eRBN4+uIKezC1/JJRKLa+uNi6TB9b85XKgq5tb9lLptzDxjVq+77Y+VWzGeKaNLqXCo4DmU5iAzq/poK+H6zipmad+rqnqzmAy47WY6zAV0hGPglxLR10kiykaew9oWxSYL5KVQLUswGKFwgvbvxt3piU0H4Wicj9d/zMYDjXQYXVk/274nRU4LAXM+gUhMSkTHkESUhdoP7aDeE6TDNVpLJv1RPJFILM7hvVvwBwfn3LM9DV4KvF9iNRuwV04bfL1lnQpzrJ0loqgkomNIIspCTdXbOdDkZ4MvDeNk8qvY2Rik5vARDlTv7//1dLBn/wHcwcOUuO0o5dP1DidlxS4rAXM+/nBc6zkL+fQOKWtIIso2gVZ8LUdQUSipSsMyHkYzOeUTAWg6sKX/18uwJl8ItW4TClA4espXS64OQiUuK3GDida4Q1sQUEpFSZKIsoyvbhcd4Rg+WzmjS9Mzcrh0nLaYmtq4C29gcA1u3F7TQHHHbvJzLNhGz9I7nH7Jc5j5l5kjOXvaJK0Dwtegd0hZQxJRlmmq0aZkmEpOwWbuZ/tQJ2f5RFx2K7ZIOwdqa9JyzUwIhGM07/oIYzxCcemIQbH20IkoikJlgQNLXucYKO8RfQPKIpKIskkkiL9Ba8cpHH1q+q5rsuIs13rPGvcPnsGN9a0eSr1bybEayZ88d9A2UnfjKtd+SiJKkkSURUINe/B2hAmY8xg9Mr07U5SOm44C0LibZl8ordceKGOCXzJ7hJ0xlaNQStKYmHXkCUb4oN7E/iYfBNq0RmshiSibeA5qI4djBROSi2mli71sInlOKzmRZhqO1qf12gMi7IfajzAbDbgmXjDoRlL3RFXhs7oAtQEbsbgKvkHwf5EB2b+83XARi1IcPkR+VT7tk89K//UtDiqrJjHWW4vFXA/0Y6DkAOsIR/F/sYbiaBhcZVA6Ve+Q0sZtM+GymfCai/GFGsj1HIGCsXqHpbuh8WdmKGg9ALEIJnsuheUDkyRyRp6KxWiAxuyejb9x6zb2bvmQmmY/jJ8/dNqG0BqsR+bb8VmL8QQj4KnTO6SsIIkoS8QbO/chKzpl4L54RRNBMYDnCC1HD2Xl5paHmz1Etv8VUMkZfTrkZf/i+H01Is+B11qOJxCB9oNDY0JyP0kiygbxOHt3bGJrXTt1pgH84lmdqIXj2X3Uy3vvreFgS2DgnisF0VicnR++ji3SRn5+AUWnL9A7pAExIt+O31xAe8RIPBKSdiIkEWWFaEs1bR4PbRET8QHeHkepOB2z0UCxfw+f7z86oM/VVxs/ex9H4xbMRoWqc/4FzHa9QxoQ+Q4zOTYzbZYyvMEotNXqHZLuJBFlgca9G4jFVTryxjOyIGdgn6xgLGWlpZjiYQKHvuBgS3Z0H1fv20Vkx5sAVEyfj61scK3A2BeKojC6MIewa4S2NpEkIklEuotFaKvVRlPnjZmR8tpDvaYo2MecRYnbSoVnC+/vbtC9rSjYepgjHzyHQY2SVzmZitMv0TWeTJh7SjFXzzuHIqdVS0SxqN4h6UoSkc5C9V/i8fkJmVyMGZuhUkD56YwsKSQn7iVav4Otde2Zed7jaT+EbfvLTCqy4CiuYuKFNw6pXrKe2MxGFGcpWF3a1uCt1XqHpCtJRDqr/3I9cRXChZMpdtky86QmC5bRZ1GZb2ekZyMf7D5Ke0fm1yqK12+HzS9BJEh+2WimX74Ugzm9AzmzmqIQLxhPKBqD5j16R6MrSUQ6Un2NtB3eAyiUTzpr4KtlXzdiJmVFhZQafVR27NB2IM2UaJiaj/7IlrefJRgKQeF4mH49mDOUiLPE4bYAf6i2s+uoF5r2aMOuhykZWa2nwxupLHCwnwpOqUrv3LKTMttQxs1jfOAvTDDtwWLwA6kt0t9rqor34DZqP/0LnvYWQGGHZSpnTL1qyEzh6It8h4VGUymeiJEOXzuOtppBv8JAqiQR6SUSRKnfSoHDQsH0+WBKz5IffVI2DWv9Vmg7CDtehxk3sq8lSJHTmtyvPS3iMVprtnJ027u0N9ZpVVGTm7wZVzHttOnDok3oeOwWI6OL3TQ1j6PRd4DRR7ZIIhIZdugzrZEyp0i/D5+iwOSr4PNnwFtPw/rf84b/TIwmEzMq85hemdfnHWYBrRcuFkHx1EHTbnZuWU+bxwNATDETHXEmU86+lOI8Z7pf0aBzarmbfxycREPjl4w4ugvThMCQHT91IpKI9BAJsmfju9iIUHLK2Vj1LBHY3DDlG/DFH3D7qjnb18Jn9vNYfyDOp9UtlLptlLq1EtLEMjfOzsTU4AlS1xYgGIkTjESJd7Sgeo8S8xzF4D3M+SUhbEatzcNljNBkysEwcgZV085nRFFBZtvDstjYohzezyvH01JAY7uP8iNbYNTZeoeVcZKIdNC8/R80tXkIWPKxuSZQrHdA+aPhtG9i2/4nZpl9jAu8ybbQaPaqI2iKFlLfpvVkVTpVnJE4BNto2Luf2upa7NF2HJFmjPGuvW6hsBtbXgEUjKF4ykSK8kZjs6SxujdEGAwKZ4wuYGvjVPLb36ekdj3GETPBOLzeK0lEGRb3HOXQtvcBsJ4yn2J3lvQUFYyFmTejfPkGhRxiLgc5O1qNJxglEI4RjsbI25QDRq1RudwfwkIYk1XBZDdhNFkhpwRLXjn2/AocpeMgpxAUZaCbwAe9UyvcfFYwCY9nA+3tbRTUbRh2pSJJRJkUi1L94ct0hCJ4XGM5f8ZMvSPqylEAM76tLUlydDvWtlqKTZ6u55htYMulsKSAQkcB2AvAWQKOwv7vwTZMmY0G5k+pwFnxzxQcWgPVH0DJ5EG9Y0lfSSLKFFWlcdNfaThcS9Rgo+qshTgsWfj2K4pWOkos1hWLaDdUMNkk2QyQqqIcKJwJ3p3Qfgh2/hWm/9uwGdYwPF6l3lSVlq1/58DWD1FRUE+9iomjB8m2yUYzWBxgyZEkNNAUBSYtwBdT2LdnB7Fdbw2bQY5ZkYiefPJJqqqqsNlsnHXWWXz66ad6h5Q+0TDs/AvWI9prClRdxDlnnqlzUCJbRax5vBmZSYM3zO7N7xPa+pr2GRridE9EL7/8MsuWLeP+++9n48aNTJ8+nUsvvZSGhkG++VwsSsvez1A//T9wdDs5Vgujz/4G5869BJNR97ddZCmz0cCZs86humgebR1Rtm78iEP/70nCR7ZBPKZ3eANGUXVeA+Kss85i1qxZ/Pa3vwUgHo9TWVnJ97//fe69994TPtbj8ZCbm0t7eztutzsT4XanqsQiIYIdPgK+ZnzN9QSaagjU7yEUDDC53EVefjFMumLYjpoVfdfoDbHus88pqn0bS6wDo0HB6XQzYtxUcsuqwFFIzOLGYHWiGLOwrbFTb7+jur6CcDjMhg0bWL58efKYwWBg/vz5fPzxx93OD4VChEJf7cnl8Xi6ndOTxq1r2L9jE6BqN1XtrH5rv1cV5lDmtoKq0h6MsPNwe+d9fFVPV7WJoZX5dkbkad3uHn8H2w+1Hvc5IyYn9UXnkDdz3rAbFyL6p9hlZeG8c9heO57DW9/D0bSVmKedMU3bof1LAOpaOjjcFkAxGsFoQTWYtDXJUVAUhWkj87FbTKAoHGwNcLit56WBp47IJceitQHWtQU51NrzgnmTK0twn3NzWl+vromoqamJWCxGaWlpl+OlpaV8+eWX3c5fsWIFDzzwQErPpQbaiZ9gr3HVHkoOrVeCEdTgCZJcpPMGmNCSk6oYiVtdGHOKsOSVkztyIiNHjZNBfCJlRoPCtKpSTht9LUfbFtB0aDc2R7u2Q2ywjajaof0ZjcUg1jXJqIASUCGqJRfV10Hcd4I1yv1hiHSmA3+AuK/nRKR2pL/TInvLdMexfPlyli1blvzd4/FQWdm7NZ5dE85jVNEUbWpB5/QC7Z9ae43LboHO7nRrLM64QKzzNO2vi9o5I0FRDFq3u8UEKNgUI6crNqxWKwaDTFsQ6acoCmX5OZTlz+hyfFQ0SmkoSDgUJBoJosSixFWVeDyOqsYxOi1o2/uq5AXCmEI9tzGZnJbkYFVXKMqoQM8rRlrd6Z8Lp2siKioqwmg0cvRo10Xcjx49SllZWbfzrVYrVmtq43Tt+WWMyO9+zeOxAiW9vK4BGH5TFEU2MJlMmExOcnJOPnnYnQ+9bUV10/tz00XX7huLxcLMmTN55513ksfi8TjvvPMOc+bM0TEyIUQm6V41W7ZsGYsWLeLMM89k9uzZPP744/j9fm6+Ob2NYUKI7KV7IrruuutobGzkvvvuo76+ntNPP5233nqrWwO2EGLo0n0cUX9kxTgiIUSPevsdlSG+QgjdSSISQuhOEpEQQne6N1b3R6J5qy9TPYQQmZP4bp6sKXpQJyKv1wvQ69HVQgh9eL1ecnN7XnFyUPeaxeNxDh8+jMvlGpa7QiSmuBw8eFB6DXtB3q/eS9d7paoqXq+XiooKDCdYbXJQl4gMBgMjR2Z4h9Qs5Ha75YvVB/J+9V463qsTlYQSpLFaCKE7SURCCN1JIhrErFYr999/f8orEgw38n71Xqbfq0HdWC2EGBqkRCSE0J0kIiGE7iQRCSF0J4lICKE7SURZrq+74L7yyitMmjQJm83Gaaedxt/+9rcMRZod+vJ+rV69GkVRutxsNlsGo9XPunXruPLKK6moqEBRFF577bWTPmbt2rWcccYZWK1Wxo8fz+rVq9MWjySiLNbXXXA/+ugj/u3f/o3vfOc7bNq0iYULF7Jw4UK2bduW4cj1kcquwW63myNHjiRvNTU1GYxYP36/n+nTp/Pkk0/26vwDBw6wYMECLrzwQjZv3szdd9/N0qVLefvtt9MTkCqy1uzZs9Xbb789+XssFlMrKirUFStWHPf8f/3Xf1UXLFjQ5dhZZ52l3nbbbQMaZ7bo6/u1atUqNTc3N0PRZS9A/dOf/nTCc/7zP/9TnTJlSpdj1113nXrppZemJQYpEWWpxC648+fPTx470S64AB9//HGX8wEuvfTSHs8fSlJ5vwB8Ph+jR4+msrKSq6++mu3bt2ci3EFnoD9bkoiy1Il2wa2vrz/uY+rr6/t0/lCSyvs1ceJEnnnmGV5//XWef/554vE455xzDocOHcpEyINKT58tj8dDIHCCHWR7aVDPvheiP+bMmdNl/7xzzjmHyZMn89RTT/Gzn/1Mx8iGHykRZam+7oILUFZW1qfzh5JU3q9jmc1mZsyYwd69ewcixEGtp8+W2+3Gbu//XseSiLJUKrvgzpkzp8v5AGvWrBkWu+amY9fgWCzG1q1bKS8vH6gwB60B/2ylpclbDIjf//73qtVqVVevXq3u2LFDvfXWW9W8vDy1vr5eVVVVvfHGG9V77703ef6HH36omkwm9Ve/+pW6c+dO9f7771fNZrO6detWvV5CRvX1/XrggQfUt99+W923b5+6YcMG9Vvf+pZqs9nU7du36/USMsbr9aqbNm1SN23apALqo48+qm7atEmtqalRVVVV7733XvXGG29Mnr9//37V4XCo//Ef/6Hu3LlTffLJJ1Wj0ai+9dZbaYlHElGW+81vfqOOGjVKtVgs6uzZs9VPPvkked/cuXPVRYsWdTn/D3/4g3rKKaeoFotFnTJlivrGG29kOGJ99eX9uvvuu5PnlpaWqldccYW6ceNGHaLOvHfffVcFut0S78+iRYvUuXPndnvM6aefrlosFnXs2LHqqlWr0haPLAMihNCdtBEJIXQniUgIoTtJREII3UkiEkLoThKREEJ3koiEELqTRCSE0J0kIiGE7iQRiUFl9erV5OXlJX//6U9/yumnn578ffHixSxcuDDjcYn+kUQkjmvx4sUoisJ3v/vdbvfdfvvtKIrC4sWLu5yf7gRQVVXF448/3uXYddddx+7du3t8zBNPPNFlLeV58+Zx9913pzUukX6SiESPKisr+f3vf99l4atgMMiLL77IqFGjdInJbrdTUlLS4/25ubldSkxicJBEJHp0xhlnUFlZyauvvpo89uqrrzJq1ChmzJjRr2sfr6SycOHCZClr3rx51NTU8IMf/CC5wwZ0r5od6+sls8WLF/Pee+/xxBNPJK9x4MABxo8fz69+9asuj9u8eTOKoshaRDqRRCROaMmSJaxatSr5+zPPPMPNN9884M/76quvMnLkSB588MHkDht99cQTTzBnzhxuueWW5DVGjRrV7TUBrFq1igsuuIDx48en6yWIPpBEJE7o29/+Nh988AE1NTXU1NTw4Ycf8u1vf3vAn7egoACj0YjL5aKsrCylVSZzc3OxWCw4HI7kNYxGI4sXL2bXrl3JPc8ikQgvvvgiS5YsSffLEL0ka1aLEyouLmbBggWsXr0aVVVZsGABRUVFeofVLxUVFSxYsIBnnnmG2bNn85e//IVQKMS1116rd2jDlpSIxEktWbKE1atX8+yzz6at1GAwGDh2KaxIJJKWa/fG0qVLkw3xq1at4rrrrsPhcGTs+UVXkojESV122WWEw2EikQiXXnppWq5ZXFzcpd0nFot125HWYrEQi8X69Tw9XeOKK64gJyeHlStX8tZbb0m1TGdSNRMnZTQa2blzZ/LfPWlvb2fz5s1djhUWFlJZWdnt3Isuuohly5bxxhtvMG7cOB599FHa2tq6nFNVVcW6dev41re+hdVqTalKWFVVxfr166mursbpdFJQUIDBYEi2FS1fvpwJEyYMiw0GspmUiESvuN1u3G73Cc9Zu3YtM2bM6HJ74IEHjnvukiVLWLRoETfddBNz585l7NixXHjhhV3OefDBB6murmbcuHEUFxenFPc999yD0Wjk1FNPpbi4mNra2uR93/nOdwiHwxnpBRQnJmtWi2Hr/fff5+KLL+bgwYPddjEVmSWJSAw7oVCIxsZGFi1aRFlZGS+88ILeIQ17UjUTw85LL73E6NGjaWtr4+GHH9Y7HIGUiIQQWUBKREII3UkiEkLoThKREEJ3koiEELqTRCSE0J0kIiGE7iQRCSF0J4lICKG7/w+w5+kh3A5tgwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:56:35.191132Z", + "iopub.status.busy": "2024-03-26T12:56:35.190766Z", + "iopub.status.idle": "2024-03-26T12:56:35.542909Z", + "shell.execute_reply": "2024-03-26T12:56:35.541778Z" + }, + "papermill": { + "duration": 0.371185, + "end_time": "2024-03-26T12:56:35.545300", + "exception": false, + "start_time": "2024-03-26T12:56:35.174115", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA9/ElEQVR4nO3deXhU9b348feZfclM9pUkBAiyKYsiCGrdaKlyrdQudlMQte2t7VPLz1a5fS5WvV7tvWrF1sfbX1tBb7VWW2sXWyk/K2hREdlkR4EkLAnZZ8vsc35/nMxogEAymeRMks/reeZJcubMmc9MMp9896+iqqqKEELoyKB3AEIIIYlICKE7SURCCN1JIhJC6E4SkRBCd5KIhBC6k0QkhNCdJCIhhO5MegcwEIlEguPHj+NyuVAURe9whBAnUVUVn89HRUUFBkPv5Z5hnYiOHz9OVVWV3mEIIc7iyJEjVFZW9nr/sE5ELpcL0F6k2+3WORohxMm8Xi9VVVWpz2pvhnUiSlbH3G63JCIhstjZmk6ksVoIoTtJREII3UkiEkLobli3EfWFqqrEYjHi8bjeoYxqZrMZo9GodxgiS2VNInrooYdYsWIF3/3ud3nssccycs1IJEJjYyNdXV0ZuZ5In6IoVFZWkpOTo3coIgtlRSLavHkzP//5z5k+fXrGrplIJDh8+DBGo5GKigosFosMetSJqqq0tLRw9OhRJk6cKCUjcQrdE5Hf7+erX/0qv/jFL/iP//iPjF03EomQSCSoqqrC4XBk7LoiPcXFxdTV1RGNRrM+EX3Y7GdrQwezqvKYWHrm8S8iM3RvrL799ttZtGgRCxYsGJTrn2lYuRg6w6k0WplvpyMQ4ZWdjRzvDOodzqiga4no+eefZ+vWrWzevLlP54fDYcLhcOpnr9c7WKGJUcxqMjC+OIe9R5rZuWsHFRfNApNV77BGNN2KC0eOHOG73/0uzz77LDabrU+PefDBB8nNzU3dZJ6ZyKTkhjaKojCnzMDMxhdx7/8dwbd+DiH5pzeYdEtEW7Zsobm5mfPPPx+TyYTJZGLDhg08/vjjmEym03a3r1ixAo/Hk7odOXJEh8gH3+WXX84dd9yhdxijzovvHeWlrUdp84fJPbaBQnMEgM72Vjj0us7RjWy6Vc2uuuoqdu7c2ePYzTffzOTJk7nrrrtO26BptVqxWqWInLR+/XquuOIKOjo6yMvL0zucYS0QjnGsuz3IGu2A1g/Iz7GyxfkJcrvepLx5H4y/HGy5+gY6QumWiFwuF+eee26PY06nk8LCwlOOCzHYGtq1sWYlbis5nR8A4KqYRFtnLY3BfUxJhFBa9kPVHD3DHLFGZZdSJJbo9RaLJ/p8brQP5w5UOBzmrrvuoqqqCqvVSm1tLb/61a+oq6vjiiuuACA/Px9FUVi6dOlZr+fz+fjqV7+K0+mkvLycn/zkJ6dUBf/3f/+X2bNn43K5KCsr4ytf+QrNzc2p+9evX4+iKLz22mvMnj0bh8PB/Pnz2b9//4Bfr14aPVppqCrfAS37AMipnEZFng131bnEVRVaD+gZ4oim+ziij1u/fv2QPM8Tr3/Y633jipwsnjUm9fP/feMg0fjpd+WuzLfzhdkfNZg/tfEwwUjPtq3vffKcAcV600038fbbb/P4448zY8YMDh8+TGtrK1VVVfz+97/nc5/7HPv378ftdmO32896veXLl7Nx40b+9Kc/UVpaysqVK9m6dSszZ85MnRONRrn//vuZNGkSzc3NLF++nKVLl/LXv/61x7V++MMf8sgjj1BcXMw3v/lNli1bxsaNGwf0evXS5NF6Y8utEQi0gmJAKTqHG8rt0JUDm7aA9zjEo2A06xztyJNViUj0dODAAV544QXWrVuXGmc1fvz41P0FBQUAlJSU9KmNyOfz8fTTT/Pcc89x1VVXAbB69WoqKip6nLds2bLU9+PHj+fxxx/nwgsvxO/395ii8cADD3DZZZcBcPfdd7No0SJCoVCfe0GzRSyeoNWvJaIytUU76CoFc3dit+eDNQfCfvA1Ql61TpGOXKMyEd1+RW2v9xlOGnf39U9M6PXck8foLbt43EDCOsX27dsxGo2pD/tAHTp0iGg0ypw5H7Vz5ObmMmnSpB7nbdmyhR/96Efs2LGDjo4OEgmtitnQ0MDUqVNT5318Sk55eTkAzc3NVFcPrw9qqz9CPKFitxjJCR3XDuZ+tKxpLKEStJXhCn8InqOSiAbBqExEFlPfm8YG69y+6EtVK9MCgQALFy5k4cKFPPvssxQXF9PQ0MDChQuJRCI9zjWbP6qiJEdOJ5PWcFNT5MBmMqJ4j2kHcrVk4w/H+NWbhynzGvlCnorBc0zHKEeuUdlYPVycd955JBIJNmzYcNr7LRYLQJ+XOBk/fjxms7nHSHaPx8OBAx81wu7bt4+2tjYeeughLr30UiZPntyjoXokKsu18dlZlVw9pRC62rSDbq2E57QYMZsUfJYigtE4BEb2e6EXSURZrKamhiVLlrBs2TJefvllDh8+zPr163nhhRcAGDt2LIqi8Je//IWWlhb8fv8Zr+dyuViyZAnf//73ef3119m9eze33HILBoMhVaKprq7GYrHw05/+lEOHDvGnP/2J+++/f9Bfa1YItICqgsUJVm2yq6IoFOVY6TLn0xWOayOsozL/LNMkEWW5J598ks9//vN861vfYvLkydx2220EAgEAxowZw7333svdd99NaWkp3/72t896vUcffZR58+bxL//yLyxYsICLL76YKVOmpBqYi4uLWbNmDS+++CJTp07loYce4uGHHx7U16i3ULS7ROk/oX3NKe1xf6HTQtxgxac4u8+TUlGmKWpygs0w5PV6yc3NxePxnLKLRygU4vDhw4wbN27Y9eIMpUAgwJgxY3jkkUe45ZZbBu15svX30RWJ8fMNh8ixmlhWsh9j43aovggmXJE6Z0t9B28caOGyyAbOd7TCxE9C5Wz9gh5GzvQZ/bhR2Vg9mm3bto19+/YxZ84cPB4P9913HwDXXXedzpHpo82vNcAbDQrGZPvPSSWifIfWKN9GLtCqVeFERkkiGkFO7l4/2Z49ewB4+OGH2b9/PxaLhQsuuIA333yToqKioQozq3R0aYmo0GkGX2+JSOsUaI07UVFRgh1DGuNoIIloBKmoqGD79u1nvL+6upotW7YMXVBZrj2gJaIicwTiMVAMYM/rcY7bbmZKuZtSdSxqyw6UrnYdIh3ZJBGNICaTidra3gdrilN5glEACg0+7YA9Hww9V34wGhQ+fW4ZRNzQqkDYJ1M9Mkx6zcSolkxEuWp3InIU9H6yxQHm7oZ2qZ5llCQiMWqpqoqnS0tEroRHO9hLIorFE7QHIgSM3T0/Uj3LKKmaiVErllCZWuHGE4ziiHcvBesoPO257x/zsGF/C5+ImbjAhpSIMkwSkRi1zEYDV03p7iF7u7uE00sictu0j0oH3asPBKVElElSNRMiFvlocXz76atmbpvWMN2Z6N4jTxbTzyhJRGLU6orEtFU0k9Uss01rkD4Nt11LRJ6EXVutMeQZqjBHBUlEWSjbdvHItngy5Y0DLTzx+ofsOdygHbDn93qu1WTAYjIQNrm05BX2ahNkRUZIIhqhTl47SJwq2XWfo2qTiLHl9Xquoii4bSYiRgehWAIScYicebUD0XejKxGpqtYeMNS3fvznXLp0KRs2bGDVqlUoioKiKBw8eJBbbrmFcePGYbfbmTRpEqtWrTrlcYsXL+aBBx6goqIiteriW2+9xcyZM7HZbMyePZuXX34ZRVF6jMDetWsXV199NTk5OZSWlnLjjTfS2traazx1dXUD/lVkg2QicqndCeUsWwW57WZUxUgwOQtf2okyZnT1msWj8OYjQ/+8l/4fMFn6dOqqVas4cOAA5557bmpCan5+PpWVlbz44osUFhby1ltv8fWvf53y8nK++MUvph772muv4Xa7WbduHaDNfL722mu55ppreO6556ivrz+litXZ2cmVV17Jrbfeyk9+8hOCwSB33XUXX/ziF/nHP/5x2niKi4sz8KboKxpPEAhry384kyWik6Z2nOycUhelbhs5TfkQPqG1E+WOOeNjRN+MrkQ0DOTm5mKxWHA4HJSVlaWO33vvvanvx40bx9tvv80LL7zQIxE5nU5++ctfplZu/J//+R8UReEXv/gFNpuNqVOncuzYMW677bbUY372s58xa9Ys/vM//zN17KmnnqKqqooDBw5wzjnnnDae4S4QjgFgNiqYI90Nz2cpEU0p7x7MGCqCEye0diKREaMrERnNWulEj+cdoCeeeIKnnnqKhoYGgsEgkUikxxZAoC0tm0xCAPv372f69Ok91v/5+ML5ADt27OD111/vsTtH0sGDBznnnIFth5StfCEtEeVYjCjhZCLK69uDbcmEJD1nmTK6EpGi9LmKlE2ef/557rzzTh555BHmzZuHy+Xiv//7v9m0aVOP85xOZ7+v7ff7ufbaa/nxj398yn3JnTlGokBES0RuUxSCMe1v4ywlonhCxROMEovaKAFpI8qg0ZWIhgmLxdJjQfyNGzcyf/58vvWtb6WOHTx48KzXmTRpEr/+9a8Jh8NYrVaAHgvnA5x//vn8/ve/p6amBpPp9H8OJ8czErhtZs4bk0uJ2gJBtDWqT5p1fzJ/OMbTb9VREO7iJqeKEuocklhHg9HVazZM1NTUsGnTJurq6mhtbWXixIm89957rF27lgMHDvDv//7vpySU0/nKV75CIpHg61//Onv37mXt2rWp9aeTi+XffvvttLe38+Uvf5nNmzdz8OBB1q5dy80335xKPifHM1y3DPq4ijw7C6aWMr2oe3O6PlTLcqxaog4qDmJxVbrvM0gSURa68847MRqNTJ06leLiYhYuXMj111/PDTfcwNy5c2lra+tROuqN2+3mz3/+M9u3b2fmzJn88Ic/ZOXKlQCpdqOKigo2btxIPB7nU5/6FOeddx533HEHeXl5GAyG08bT0NAweC9+qAU7ta9nqZaBti6Rw2IkYnISiScgGtJ6YsWAyeL5o8yzzz7LzTffjMfjGdINHLPt9+ENRbGaDFg/fBUa34dxl0LNJWd93LOb6mn2hFgS/x0FNgPM/caZ1zAa5WTxfAHAM888w/jx4xkzZgw7duxIjRHSYxfZbPLie0fxBqMsdbaQD33uMcuxmmhWFEKKAwhp1TNJRAMmiWiEa2pqYuXKlTQ1NVFeXs4XvvAFHnjgAb3D0pWqqqlxRLZ430ZVJyXbiboUOxCCsLQTZYIkohHuBz/4AT/4wQ/0DiOrBKNx4gkVVBVrPDnPrPdqw8c5uxORHzvQIQ3WGSKJSIw6/u7BjG5jBIOa0MYQWU4d0Hk61QXaMiHlniLoPC6jqzNkxCeiYdwWP6Jk0+/B310tyzeFIIG21/1ZxhAlVeTZqcizw5Fi6ESqZhkyYrvvzWZtWkVXV5fOkQj4aFkSo7FvH/jBlExEuYagdsDat2pZD8kSlFTNMmLEloiMRiN5eXk0N2u7dzocjtQgPjG0EokELS0tOByOXkdvD6VkInIrIe2A1dXnx6qqSnsgQihoolxVMUiJKCP0/6sYRMnZ4slkJPRjMBiorq7Oin8GpW4b543JpSwQhSj9LhH95t0GDGEvy4wJbGGftt5UFryu4WxEJyJFUSgvL6ekpIRoVEbA6slisaRGauttQnEOE4pzYE8M/PS5xwy0vymn1YQn5iAcS2AzxyAWAvPoHpc1UCM6ESUZjcasaJsQWSY5e74fVTPQuvA7u0xEDBZA1RqsJRENSHb8ixJiCLUHIoRjcW0Pe+h/IrJo/7+10dVIF34GjIoSkRBJ0XiCp9+qA1Xl2yYPZoV+JyKHVStdd2EHAhAJZDzO0UZKRGJUSU7tsKtBTEp3I7MlvRJRl9I9eVcS0YBJIhKjyscHMyp0j6juZyO6w6KViAKqttgcUUlEA6VrInryySeZPn06brcbt9vNvHnz+Nvf/qZnSGKES+7c4Tb0fwxRUlmujfkTCqkpL9EOSIlowHRtI6qsrOShhx5i4sSJqKrK008/zXXXXce2bduYNm2anqGJEcof1oZx5ClBUOlX131SUY6VohwrNBVBG5KIMkDXRHTttdf2+PmBBx7gySef5J133pFEJAaFv7tElJNMRGmUiFIs3ZsVSCIasKzpNYvH47z44osEAgHmzZt32nPC4TDhcDj1s9cr3aaif5Iz7x0MYJ4Z0OYPEwwYKVdVjJKIBkz3RLRz507mzZtHKBQiJyeHP/zhD0ydOvW05z744IM9NhoUor9qihxYTQby27tn3qeZiH635SiRYIBlShynEoREvM8z+MWpdO81mzRpEtu3b2fTpk3867/+K0uWLGHPnj2nPXfFihV4PJ7U7ciRI0McrRjuplXksmBqKQXG9BurQRtdHTNYiSR3WZJS0YDoXiKyWCzU1tYCcMEFF7B582ZWrVrFz3/+81POtVqtqf25hEhbIvHROkJpJyIjLYpC2GADYloiSqPhW2h0LxGdLJFI9GgHEiJTYvEEbf4w4aAX1AQohj6vzHgyR2qaR/ccMykRDYiuJaIVK1Zw9dVXU11djc/n47nnnmP9+vWsXbtWz7DECNXeFeHZdxooTrTyNQtg7f9gxqTU6GqsgE8GNQ6QromoubmZm266icbGRnJzc5k+fTpr167lk5/8pJ5hiREqOZgxd4DtQ3DyfDOkRDRAuiaiX/3qV3o+vRhlUovmp7Ey48mSJaJAwqIdkEQ0IFnXRiTEYEnOM3Mp3euYp9l1D1DisjJ/QiHjK5LTPGTJ2IHQvddMiKGSTETOAQ5mBMh3Wpg7vhBOlEA7EJFNGgZCSkRi1EguAZKjJktEA5jekSTTPDJCEpEYNXzJbaYT3UljgImozR/maEAhlkhI1WyApGomRo2p5W463SGcx0KAMuABiC9vP07A7+NmNY7LEIZ4DIzykUqHvGti1LhgbL62YP4JRRvMaHYO6HpOixGvwUok1r2VUDQAxtwMRDr6SNVMjC6pBfPTH8yY5LCaQFFkdHUGSCISo0IoGqfNHybS1aEdGECPWZLDrA1qDCFrVw+UVM3EqFDf1sVfdzYyncNcZSIjPWbJ0dVBxQb4JBENgJSIxKiQGsyYGkM08ESUmm+WXERfElHaJBGJUSGZiHLoHkNkG3ijsrO7RBQguZuHDGpMlyQiMSokBzM6MlgiKs6xcXFtEbUyzWPApI1IjArJEpEjg6Oqcx1m5owrgBPFMs1jgKREJEYFfygGagJbIoPTO5JkmseASSISI56qqgTCMSzxLiwGRVvkPs2VGU/W6g9zxN89zUMWR0ubVM3EiJdQYc64AqIdASye7uVhFSUj1/5T9zSPJWpc2z1WdvNIi5SIxIhnNCjMHV/IJZUWDIqS0WqZ02okZrASTXQfkOpZWiQRidEjOb0jg7ttOCzaNI+w0j26Wrrw0yKJSIx4/nBMm94R7NQOZLhEBMnR1UiJKE3SRiRGvF3HPLx9sI0F8WOcZwWsmZshn9xWKIgNCEgiSpOUiMSId+p+95krETks3bt5yDSPAUkrER06dCjTcQgxaAKR7kSUoZUZPy5ZIgokE5F04aclrURUW1vLFVdcwa9//WtCoVCmYxIio3yhGIoax0b332oGG6uLc6zaNI/KUu2AlIjSklYi2rp1K9OnT2f58uWUlZXxjW98g3fffTfTsQmREanBjEYFDCYwOzJ27eQ0j6rSYu2ATPNIS1qJaObMmaxatYrjx4/z1FNP0djYyCWXXMK5557Lo48+SktLS6bjFCItsXiCrkgcS8yPxWjQqmUZGszYg6U7ucnE17QMqLHaZDJx/fXX8+KLL/LjH/+YDz/8kDvvvJOqqqrUVtJC6CkQ0baZdqhdmIyZHcyY1HOah5SI0jGgRPTee+/xrW99i/Lych599FHuvPNODh48yLp16zh+/DjXXXddpuIUIi1mo8L8CYWcW6SgMDiJ6C87jvPHPR66InGIBiGROPuDRA9pjSN69NFHWb16Nfv37+eaa67hmWee4ZprrsHQvRj5uHHjWLNmDTU1NZmMVYh+c1hM2o6sMQMcI6MN1annsJroCNiIRFRQVa1UZM3MpNrRIq1E9OSTT7Js2TKWLl1KeXn5ac8pKSnhV7/61YCCEyJjwl7t6yCUiBwWIygGwooVkESUjrQS0bp166iurk6VgJJUVeXIkSNUV1djsVhYsmRJRoIUIl0dgQhxVSWvy6P9sWdwVHVScu1qbTePYHeDdUnGn2ckS6uNaMKECbS2tp5yvL29nXHjxg04KCEy5Z1Dbfzv2/U0tTRrBwarRMTH55tJg3V/pZWIVFU97XG/34/NZhtQQEJkki8cQ1Fj2NSwdmAQEpHTKrt5DFS/qmbLly8HQFEUVq5cicPx0cCweDzOpk2bmDlzZkYDFGIg/KEYllgAi8mg7Utvtmf8OezdJaKAmlwKRBJRf/UrEW3btg3QSkQ7d+7EYrGk7rNYLMyYMYM777wzsxEKkabkErGOeACLzaDt7joIgxmLcqxcMrGIss4S6KiTElEa+pWIXn/9dQBuvvlmVq1ahdud+a5QITIlFE0QS6hY4slR1YPz95prN3NhTQE0FkMH0kaUhrR6zVavXp3pOITIOF84CoBbCWZ8idjTSu3mIdM8+qvPiej6669nzZo1uN1urr/++jOe+9JLLw04MCEGKrkOkduQ+Vn3J2v1hwn6oDyRwCTTPPqtz4koNzcXpbt+nZub+bEYQmRansPC/AmFlDXEQWVQS0SvvN+I3+Pha2qcXGOXNsJ6MCbXjlB9TkQfr45J1UwMBwVOiza9oy0OfgatjQi0sUQdRjvRcALUhDbnzJK55UZGurTGEQWDQbq6Pip+1tfX89hjj/H3v/89Y4EJkTGp6R2DmYhMqIqRsNLdkyw9Z/2SViK67rrreOaZZwDo7Oxkzpw5PPLII1x33XU8+eSTGQ1QiHSd8IZo9fhIRDK/VvXJHN27eWjTPJCxRP2U9gqNl156KQC/+93vKCsro76+nmeeeYbHH3+8z9d58MEHufDCC3G5XJSUlLB48WL279+fTkhCnGLt7iZ+98/d+EIxMFnBPHij/pPzzWSaR3rSSkRdXV24XNp/l7///e9cf/31GAwGLrroIurr6/t8nQ0bNnD77bfzzjvvsG7dOqLRKJ/61KcIBOS/iRg4XyiGNe7XRlXbBreDRXbzGJi0xhHV1tby8ssv89nPfpa1a9fyve99D4Dm5uZ+DXJ89dVXe/y8Zs0aSkpK2LJlC5/4xCfSCU0IAMKxOJFYgvyYTxvMOESJKIDs5pGOtEpEK1eu5M4776Smpoa5c+cyb948QCsdzZo1K+1gPB4PAAUFBae9PxwO4/V6e9yEOJ1AWFsiNocARoMCtrxBfb4ilzbNo3aM7OaRjrRKRJ///Oe55JJLaGxsZMaMGanjV111FZ/97GfTCiSRSHDHHXdw8cUXc+655572nAcffJB77703reuL0SU5mDFX6W6rGeQSkdvWPc3DUgydSBtRP6W95XRZWRllZWU9js2ZMyftQG6//XZ27drFP//5z17PWbFiRWoFAACv10tVVVXazylGruT0DhfdJZNBHFXdg0zzSEtaiSgQCPDQQw/x2muv0dzcTOKkxcL7uxPst7/9bf7yl7/wxhtvUFlZ2et5VqsVq9WaTshilEmWiJxqFzD4bUTQPc3DL9M80pFWIrr11lvZsGEDN954I+Xl5ampH/2lqirf+c53+MMf/sD69etldUeRMdWFDgxqjOJ9McAyJInorzsb8Xd6+aoaJ9cUkGke/ZBWIvrb3/7GK6+8wsUXXzygJ7/99tt57rnn+OMf/4jL5aKpqQnQ5rLZ7ZlfwEqMHuW5dspNXdBgAZMFTIO/cqjDYqLDYCcaSUAiDrHwoI5dGknS6jXLz8/vtWerP5588kk8Hg+XX3455eXlqdtvf/vbAV9bCEKd2ldb7pCUTJwWIwmDiXDy/7v0nPVZWono/vvvZ+XKlT3mm6VDVdXT3pYuXTqg6wpR1xqgs6NVW199EHbuOJ3kkrEyzaP/0qqaPfLIIxw8eJDS0lJqamowm8097t+6dWtGghMiHaFonD9sO0Z15wEWF4BxCNqH4GOL6GMDotKF3w9pJaLFixdnOAwhMsfX3WOWQ7B7MOPQJKKPthWyAj6pmvVDWononnvuyXQcQmSML6SNIcpVkmOIhqhE1D3xNZCQqll/pdVGBNryH7/85S9ZsWIF7e3tgFYlO3bsWMaCEyId3u4SkYuhGVWdVJhj0aZ5VHbv8ioloj5Lq0T0/vvvs2DBAnJzc6mrq+O2226joKCAl156iYaGhtRaRULowReKoqgxHAQB55CNqnYlp3mYi8GDJKJ+SKtEtHz5cpYuXcoHH3zQY2fXa665hjfeeCNjwQmRDl8ohjXmx2oygNEM5iFesjU1zUMSUV+llYg2b97MN77xjVOOjxkzJjUoUQi9eINRbDGPlojseUM6urnFF6bBrxCNJ0CmefRZWlUzq9V62iU4Dhw4QHFx8YCDEmIgZtfkE8aAs9ME9vwhfe5Xdzfha/fwFWLkycTXPkurRPSZz3yG++67j2hU651QFIWGhgbuuusuPve5z2U0QCH6q7bExbT8BDazccgTkcNsJGa0E42pEI9BLDKkzz9cpZWIHnnkEfx+P8XFxQSDQS677DJqa2txuVw88MADmY5RiP4LdmhfB3lBtJM5rUbiiplI8qMlpaI+Satqlpuby7p169i4cSM7duzA7/dz/vnns2DBgkzHJ0S/+EJRWv0RSr2tOGDoS0QWEyhK9zSPeHc70cDnZY50/U5EiUSCNWvW8NJLL1FXV4eiKIwbN46ysjJUVU17SRAhMqGhvYu/72pkYdsxppbl6JCIkqOrbUBAes76qF9VM1VV+cxnPsOtt97KsWPHOO+885g2bRr19fUsXbo07WVihcgUbzCGNR7AagQMxkHdVPF0HMnR1bKbR7/0q0S0Zs0a3njjDV577TWuuOKKHvf94x//YPHixTzzzDPcdNNNGQ1SiL7yhaJYY16t696WC4a0Jw+kxdm90WJXaga+dOH3Rb9+S7/5zW/4t3/7t1OSEMCVV17J3XffzbPPPpux4ITor85gFFvMq0uPGUCB08KlsptHv/UrEb3//vt8+tOf7vX+q6++mh07dgw4KCHS5e1ORFazQZdE5LKZmV1TQFVp93g6SUR90q9E1N7eTmlpaa/3l5aW0tHRMeCghEhHNJ7AF4ppJSKTcci77nuwdE8rkUTUJ/1KRPF4HJOp92Ylo9FILBYbcFBCpMMb1AbYOhM+TEZFlxIRQJs/TINPISLTPPqsX43VyWVce9vSJxwOZyQoIdLhtJr49LRS8sIxFIzaPDMdrNtzgo5WLzckohRYZUBjX/QrES1ZsuSs50iPmdCLzWxkSoECTiMo+rQRgZYQmw02wpGENsUjHgNj2nuZjgr9endWr149WHEIkRldbdpXe542jkgHOTYTcYOVSKJ7cG/Er1vpbLgY2kEWQgyigy1+jjUeI5ZIgKNQtzhcVm2aR5DuJgxpJzorSURixHjzQAvv7vqAQDgODv3mdyV38/Cr3YMapefsrCQRiREhkVDxhmLYY53aGCIdS0Q5Jyei8Klrd4meJBGJEcEXjhFPqDhjnViN+iYil01LRD7VjooKYek5OxtJRGJE8AajGBMRXEpQWwFC5xLRpROLOG9CJahA2KdbLMOF9CmKEaGjK4It6tHmmFmcYLbrFovJaGB2TQFYS8GjyOJofSAlIjEitAci2GOd2M1GXUtDPVhd2lcpEZ2VJCIxInR0RbBHO7FbsiMRdXZFqPMbCEbj0ljdB1I1EyPCxbVFRPwq7qA5KxLR5roO9h7xcX0gTKXZqI2wNln0DitrSSISI0KJywZmP8SM4CzSOxxyrCbiBgshtfsjFvaBSf8Ema2kaiZGhlgEQp3a9zkluoYCH3XhB+huNI9IO9GZSCISw94Jb4g9Hx7EH4pqPWbJLZ91lBrUmExE0mB9RpKIxLB3qCXAtr0fcMIbyorSEHw0zcOXSI6ulkR0JpKIxLDX0RXBEW3Tesyc2bHlebJq5sdOPCGjq89GEpEY9toCERyRdm0MUZaUiGxmI1azgYjRSTgmXfhnI4lIDGvxhEq7L4wj2q5tbujMjkQE8ImJxcybUoPFZJDR1Wch3fdiWGsPRDDF/NiIYDG7smIMUdK5Y3LBVQZNBmkjOgspEYlhrdX/UWlIcRRm35KsyWkekQAk4vrGksUkEYlhrdUfxhlp0bZ6zpL2oaSuSIxDHpWOUAJUVdqJzkASkRjW5owrYEGlSqnbCq4KvcPp4VhHkD/uaOSQr/tjFvLoG1AW0zURvfHGG1x77bVUVFSgKAovv/yynuGIYchqMlKQaNNKRK4yvcPpwW03A+BVuzdblETUK10TUSAQYMaMGTzxxBN6hiGGs7BPG6OjKJDT+y7EenDbuhMRThKqCiGpmvVG15a9q6++mquvvlrPEIYvVYVAi7Z/l6NQ+yCOMie8Ier272ZiIExByZism91uMxswGxXCRhfhWBN2KRH1Ksu6GM4sHA732E3W6x2l/2FCHtj1EviatJ/zqmDqdR/10IwS9W1dNNQdoiARocCdXe1DAIqi4LabCZtyCMfikojOYFg1Vj/44IPk5uamblVVVXqHNPSiIYKb/5dwxzEwmLRNBDuPENr6vDYDfRRp8obIiTRr87qyrH0oyW0zEza5iMQS0kZ0BsMqEa1YsQKPx5O6HTlyRO+Qhlxk31r2HW5gW4uKZ8YyuPBWToSMvL/vAN69r+kd3pA60RnEFWnWZrpnWY9ZkttuImzMIRxNaO1ZiYTeIWWlYZWIrFYrbre7x21U8TVxdO+7hKIJDpd8EqM9F9Wez/acS4gnVA7v2EDC16J3lEPCF4oS97dgSkRw2O1Z11CdNKXczVUzx1PkdoCakHWJejGsEtFo59v3D5p9YVodtVx6wXRyrCYUReGSOXPw54ynKxyjYcfoKBWd8IZxh5uwW4yY8irBkJ1/yuW5diaX52F35WkHpHp2Wrr+9vx+P9u3b2f79u0AHD58mO3bt9PQ0KBnWNkp2EHjod2ogHnCpVTmO1J3Oa0mymZcCUDbwW3Euzp0CnLonPCGcEWatGpZbqXe4ZydLVf7Kl34p6VrInrvvfeYNWsWs2bNAmD58uXMmjWLlStX6hlWVgrUvUe7P0ynrZLzp0w45f4pE88hlFNFJBbnyO53dIhwaPlCUVzh4ZGI6loDHPIZiSWkwbo3unbfX3755aiqqmcIw0MiTusHm1EBxpyvLRR/EpPRQNE5c/FvPULbwfeouWBh1lZXMuHTtQ7CjSYMRiNkYdf9x63bc4Lc5gQlOQlyJBGd1sj9Sx1JOhuIhgJEDXbGTZre62m1U2YRN9qIBjx4ju0fwgB10HkEq8mI2V0GJqve0ZxRrsNM2OQmFI1DcORXm9MxrAY0jlptH1JbnEP5pGlYS3rvKbTbrNRMm0NR5/vYvPuBKUMX41DrOKx9za/RNYy+yHdYqDO5CUbiEGzXO5ysJCWibKeq0PYhAM6KyZiMZ/6VVU6ajc1k1B4Tjw5FhEPur+8fZ9euHfjDsWGRiPIcZoKmXMLRuDYvbpQNPO0LSURZTg20EAu0a6Oo+/Khc5VrPTTxKGrbwUGPb6glEiqNjcfweTvBaITc7B9dn2c3Ezfa8Ce658JJ9ewUkoiyXOeRPbxX18F7Xjeq0Xz2BygKnc7x7Gvysn3r24Mf4BBr9oVxBBowGRQcxeOyb0XG08hzaAmoU83RDkj17BSSiLJce8MeVMDvGofSxxn2scJJdHRFCTUdIBwODW6AQ6yuLUBe8Ci5djOGgnF6h9Mnucl1iQwurQtfSkSnkESUzcJ+ulq0wZ0lY6f1+WGF5WMx2rXq2dGDuwcrOl00nOjAHT5OnsMMBaeOp8pGFpOBq88r4+LzJmJQFOiSEtHJJBFlsUDjPvzhGAFrMWMr+j6XSjEYyKnSEldb3a7BCm/IdUVihFo+xKDGyS0oAWeR3iH12eQyN4VFZVoikhLRKSQRZbG2eq00oxRNTG1h3FflE2YAEG0+QCg8Mnpp6tu6yO+qw2kxYi2bPPwWg3MUaF+ljegUkoiyVTxKoEnrts+v7nu1LCm/fDwWuwtDPEzDwb2Zjk4XVoNKDY3kOy1QdI7e4fSLPxzj/TaFRk8QIl0QHVltdwMliShLRVoP4Q0ECZtyqK4a2/8LKAo5ldqAxta69zMcnT7Gm1o5r9RGVWkRuMfoHU6/+EMxXvvAQ53fqB3oatU3oCwjiShLKW0fUFPoxDVmCoU56U1hKBs/A7fNRI16bGQsyHWiu+G9eMqwm0dX4OzuwldyicYT2nrjImV4/TZHC1XF3HGIUreNC86/sM/d9ifLrahl2thSKhwqeI9mOMihVXeig0hTdxWztP9VVb1ZTAbcdjNd5gK6InEISIno4yQRZSPvcW2LYpMF8tKoliUZjFA4Ufu+5UBmYtNBJJbg7U1vs/VwC11GV9bPtu9NUY6FoDmfYDQuJaKTSCLKQp6je2jyhuhyjdWSyUAUTyIaT3D8wx0EQsNz7tkHzT4KfPuwmg3Yq6YPv96yboVOa3eJKCaJ6CSSiLJQa91uDrcG2OLPwDiZ/Br2toSoP97I4bpDA7+eDj44dBh36DglbjtK+Qy9w0lbsctK0JxPIJLQes7Cfr1DyhqSiLJNsAN/eyMqCiU1GVjGw2jGWT4JgNbDOwZ+vSHW6g+jHtuGAhSOnfbRkqvDUInLSsJgoiPh0BYElFJRiiSiLOM/tp+uSBy/rZyxpZkZOVw6QVtMTW3Zjy84vAY37q5vprjrAPlOC7axF+odzoDkOcx8/oJKLpo+WeuA8DfrHVLWkESUZVrrtSkZppJzsJkH2D7ULad8Ei67FVvUw+GG+oxccygEI3Ha9r+FMRGluHTMsFh76EwURaGqwIElr3sMlK9R34CyiCSibBINEWjW2nEKx07N3HVNVnLKtd6zlkPDZ3BjU4eXUt9OnFYj+VMuG7aN1KdwlWtfJRGlSCLKIuHmD/B1RQia8xhbmdmdKUonzEABaDlAmz+c0WsPlnGhfcwZY2dcVTVKSQYTs468oSj/bDJxqNUPwU6t0VpIIsom3iPayOF4wcTUYlqZYi+bRF6OFWe0jeYTTRm99qCIBKDhLcxGA65Jnxh2I6l7o6qw+ViQhqCNeEIF/zD4XQyB7F/ebrSIxyiOHCW/Jh/PlLmZv77FQVXNZMb7GrCYm4ABDJQcZF2RGIH311Eci4CrDErP1TukjHHbTLhsJnzmYvzhZnK9jVAwXu+wdDcy/s2MBB2HIR7FZM+lsHxwkoSzcioWowFasns2/tadu/hwx0bq2wJQu2DktA2hNVhX5tvxW4vxhqLgPaZ3SFlBElGWSLR070NWdM7gffCKJoFiAG8j7SeOZuXmlsfbvER3/wVQcY6dCXnZvzh+f43Jc+CzluMNRsFzZGRMSB4gSUTZIJHgwz3b2HnMwzHTIH7wrDmohbUcOOFjw4Z1HGkPDt5zpSEWT7B34x+xRTvJzy+gaOYivUMaFGPy7QTMBXiiRhLRsLQTIYkoK8Ta6+j0eumMmkgM8vY4SsVMzEYDxYEPeO/QiUF9rv7auvlNHC07MBsVauZ/Hsx2vUMaFPkOM06bmU5LGb5QDDob9A5Jd5KIskDLh1uIJ1S68mqpLHAO7pMVjKestBRTIkLw6Pscac+O7uO6g/uJ7vkbABUzFmArG14rMPaHoiiMLXQScY3R1iaSRCSJSHfxKJ0N2mjqvHGz0l57qM8UBfu4uZS4rVR4d/DmgWbd24pCHcdp/OczGNQYeVVTqJj5KV3jGQqXnVPMdZfPpyjHqiWieEzvkHQliUhn4aZ9eP0BwiYX48YPUSmgfCaVJYU4Ez5iTXvYecwzNM97Op6j2Hb/lslFFhzFNUy64sYR1UvWG5vZiJJTClaXtjV4R53eIelKEpHOmvZtIqFCpHAKxS7b0DypyYJl7Fyq8u1UerfyzwMn8HQN/VpFiabdsP03EA2RXzaWGVffisGc2YGcWU1RSBTUEo7Foe0DvaPRlSQiHan+FjqPfwAolE+eO/jVso8bcwFlRYWUGv1Ude3RdiAdKrEI9W/9jh1rnyYUDkNhLcz4CpiHKBFnieOdQV6os7P/hA9aP9CGXY9SMrJaT8e3UlXg4BAVnFOT2bllZ2W2oUy4nNrgn5lo+gCLIQCkt0h/n6kqviO7aHj3z3g97YDCHsu5nH/uZ0bMFI7+yHdYaDGV4o0a6fJ7cHTWD/sVBtIliUgv0RBK004KHBYKZiwAU2aW/OiXsulYm3ZC5xHY80eYdSMH20MU5VhT+7VnRCJOR/1OTux6HU/LMa0qanKTN+szTD9vxqhoEzodu8XI2GI3rW0TaPEfZmzjDklEYogd3aw1UjqL9PvjUxSY8hl47ynwNdG86XleCczGaDIxqyqPGVV5/d5hFtB64eJRFO8xaD3A3h2b6PR6AYgrZmJjZjPtooUU5+Vk+hUNO1PL3fzjyGSaW/Yx5sR+TBODI3b81JlIItJDNMQHW1/HRpSScy7CqmeJwOaGaZ+F91/A7a/jIn87m+2XsOlwgnfr2il12yh1ayWkSWVucroTU7M3xLHOIKFoglA0RqKrHdV3grj3BAbfcS4tCWMzam0eLmOUVpMTQ+UsaqZfypiigqFtD8ti44ucvJlXjre9gBaPn/LGHVB9kd5hDTlJRDpo2/0PWju9BC352FwTKdY7oPyxcN7nsO3+Axea/UwI/o1d4bF8qI6hNVZIU6fWk1WVo5ITTUCok+YPD9FQ14A95sERbcOY6NnrFo64seUVQME4iqdNoihvLDZLBqt7I4TBoHD+2AJ2tpxLvudNSho2YRxzARhH13sliWiIJbwnOLrrTQCs5yyg2J0lPUUF4+GCm1H2vUIhR7mMI1wUq8MbihGMxInE4uRtc4JRa1QuD4SxEMFkVTDZTRhNVnCWYMkrx55fgaN0AjgLQVEGuwl82Jta4WZzwWS83i14PJ0UHNsy6kpFkoiGUjxG3cbf0hWO4nWN59JZF+gdUU+OApj1NW1JkhO7sXY2UGzy9jzHbANbLoUlBRQ6CsBeADkl4Cgc+B5so5TZaGDBtApyKv6FgqProO6fUDJlWO9Y0l+SiIaKqtKy7S80H28gZrBRM3cxDksWvv2KopWOkot1xaPaDRVMNkk2g6SmyAmFF4BvL3iOwt6/wIwvj5phDaPjVepNVWnf+XcO79yIioI69TNMGjtMtk02msHiAItTktBgUxSYvAh/XOHgB3uI73911AxyzIpE9MQTT1BTU4PNZmPu3Lm8++67eoeUObEI7P0z1kbtNQVrrmT+7Nk6ByWyVdSax9+iF9Dsi3Bg+5uEd76s/Q2NcLonot/+9rcsX76ce+65h61btzJjxgwWLlxIc/Mw33wuHqP9w82o7/5fOLEbp9XC2Is+y8WXfQqTUfe3XWQps9HA7AvnU1d0OZ1dMXZufYuj/+8JIo27IBHXO7xBo6g6rwExd+5cLrzwQn72s58BkEgkqKqq4jvf+Q533333GR/r9XrJzc3F4/HgdruHItxTqSrxaJhQl5+gvw1/WxPB1nqCTR8QDgWZUu4iL78YJl8zakfNiv5r8YV5Y/N7FDWsxRLvwmhQyMlxM2bCueSW1YCjkLjFjcGag2LMwrbGbn39jOr6CiKRCFu2bGHFihWpYwaDgQULFvD222+fcn44HCYc/mhPLq/Xe8o5vWnZuY5De7YBqnZT1e7qt/ZzTaGTMrcVVBVPKMre457u+/ionq5qE0Or8u2MydO63b2BLnYf7Tjtc0ZNOTQVzSfvgstH3bgQMTDFLiuLL5/P7oZaju/cgKN1J3Gvh3Gtu8GzD4Bj7V0c7wyiGI1gtKAaTNqa5CgoisL0ynzsFhMoCkc6ghzv7H1p4HPH5OK0aG2AxzpDHO3ofcG8KVUluOffnNHXq2siam1tJR6PU1pa2uN4aWkp+/btO+X8Bx98kHvvvTet51KDHhJn2GtctYdTQ+uVUBQ1dIYkF+2+ASa05KQqRhJWF0ZnEZa8cnIrJ1FZPUEG8Ym0GQ0K02tKOW/sFzjRuYjWowewOTzaDrGhTmJql/ZvNB6HeM8kowJKUIWYllxUfxcJ/xnWKA9EINqdDgJBEv7eE5HalflOi+wt053GihUrWL58eepnr9dLVVXf1nh2TbyE6qJp2tSC7ukF2rdae43LboHu7nRrPMGEYLz7NO2/i9o9I0FRDFq3u8UEKNgUIzMVG1arFYNBpi2IzFMUhbJ8J2X5s3ocr47FKA2HiIRDxKIhlHiMhKqSSCRQ1QTGHAva9r4qecEIpnDvbUymHEtqsKorHKM62PuKkVZ35ufC6ZqIioqKMBqNnDjRcxH3EydOUFZWdsr5VqsVqzW9cbr2/DLG5J96zdOxAiV9vK4BGH1TFEU2MJlMmEw5OJ1nnzzszoe+tqK66fu5maJr943FYuGCCy7gtddeSx1LJBK89tprzJs3T8fIhBBDSfeq2fLly1myZAmzZ89mzpw5PPbYYwQCAW6+ObONYUKI7KV7IrrhhhtoaWlh5cqVNDU1MXPmTF599dVTGrCFECOX7uOIBiIrxhEJIXrV18+oDPEVQuhOEpEQQneSiIQQutO9sXogks1b/ZnqIYQYOsnP5tmaood1IvL5fAB9Hl0thNCHz+cjN7f3FSeHda9ZIpHg+PHjuFyuUbkrRHKKy5EjR6TXsA/k/eq7TL1Xqqri8/moqKjAcIbVJod1ichgMFBZOcQ7pGYht9stH6x+kPer7zLxXp2pJJQkjdVCCN1JIhJC6E4S0TBmtVq555570l6RYLSR96vvhvq9GtaN1UKIkUFKREII3UkiEkLoThKREEJ3koiEELqTRJTl+rsL7osvvsjkyZOx2Wycd955/PWvfx2iSLNDf96vNWvWoChKj5vNZhvCaPXzxhtvcO2111JRUYGiKLz88stnfcz69es5//zzsVqt1NbWsmbNmozFI4koi/V3F9y33nqLL3/5y9xyyy1s27aNxYsXs3jxYnbt2jXEkesjnV2D3W43jY2NqVt9ff0QRqyfQCDAjBkzeOKJJ/p0/uHDh1m0aBFXXHEF27dv54477uDWW29l7dq1mQlIFVlrzpw56u233576OR6PqxUVFeqDDz542vO/+MUvqosWLepxbO7cueo3vvGNQY0zW/T3/Vq9erWam5s7RNFlL0D9wx/+cMZzfvCDH6jTpk3rceyGG25QFy5cmJEYpESUpZK74C5YsCB17Ey74AK8/fbbPc4HWLhwYa/njyTpvF8Afr+fsWPHUlVVxXXXXcfu3buHItxhZ7D/tiQRZakz7YLb1NR02sc0NTX16/yRJJ33a9KkSTz11FP88Y9/5Ne//jWJRIL58+dz9OjRoQh5WOntb8vr9RIMnmEH2T4a1rPvhRiIefPm9dg/b/78+UyZMoWf//zn3H///TpGNvpIiShL9XcXXICysrJ+nT+SpPN+ncxsNjNr1iw+/PDDwQhxWOvtb8vtdmO3D3yvY0lEWSqdXXDnzZvX43yAdevWjYpdczOxa3A8Hmfnzp2Ul5cPVpjD1qD/bWWkyVsMiueff161Wq3qmjVr1D179qhf//rX1by8PLWpqUlVVVW98cYb1bvvvjt1/saNG1WTyaQ+/PDD6t69e9V77rlHNZvN6s6dO/V6CUOqv+/Xvffeq65du1Y9ePCgumXLFvVLX/qSarPZ1N27d+v1EoaMz+dTt23bpm7btk0F1EcffVTdtm2bWl9fr6qqqt59993qjTfemDr/0KFDqsPhUL///e+re/fuVZ944gnVaDSqr776akbikUSU5X7605+q1dXVqsViUefMmaO+8847qfsuu+wydcmSJT3Of+GFF9RzzjlHtVgs6rRp09RXXnlliCPWV3/erzvuuCN1bmlpqXrNNdeoW7du1SHqoff666+rwCm35PuzZMkS9bLLLjvlMTNnzlQtFos6fvx4dfXq1RmLR5YBEULoTtqIhBC6k0QkhNCdJCIhhO4kEQkhdCeJSAihO0lEQgjdSSISQuhOEpEYVtasWUNeXl7q5x/96EfMnDkz9fPSpUtZvHjxkMclBkYSkTitpUuXoigK3/zmN0+57/bbb0dRFJYuXdrj/EwngJqaGh577LEex2644QYOHDjQ62NWrVrVYwnTyy+/nDvuuCOjcYnMk0QkelVVVcXzzz/fY72ZUCjEc889R3V1tS4x2e12SkpKer0/Nze3R4lJDA+SiESvzj//fKqqqnjppZdSx1566SWqq6uZNWvWgK59upLK4sWLU6Wsyy+/nPr6er73ve+lFraHU6tmJ/t4yWzp0qVs2LCBVatWpa5x+PBhamtrefjhh3s8bvv27SiKIkuA6EQSkTijZcuWsXr16tTPTz31FDfffPOgP+9LL71EZWUl9913X2ph+/5atWoV8+bN47bbbktdo7q6+pTXBLB69Wo+8YlPUFtbm6mXIPpBEpE4o6997Wv885//pL6+nvr6ejZu3MjXvva1QX/egoICjEYjLpeLsrKytBZ3y83NxWKx4HA4UtcwGo0sXbqU/fv3p7YaikajPPfccyxbtizTL0P0kSwVK86ouLiYRYsWsWbNGlRVZdGiRRQVFekd1oBUVFSwaNEinnrqKebMmcOf//xnwuEwX/jCF/QObdSSEpE4q2XLlrFmzRqefvrpjJUaDAYDJ69AE41GM3Ltvrj11ltTDfGrV6/mhhtuwOFwDNnzi54kEYmz+vSnP00kEiEajbJw4cKMXLO4uLhHu088Hj9lI0iLxUI8Hh/Q8/R2jWuuuQan08mTTz7Jq6++KtUynUnVTJyV0Whk7969qe974/F42L59e49jhYWFVFVVnXLulVdeyfLly3nllVeYMGECjz76KJ2dnT3Oqamp4Y033uBLX/oSVqs1rSphTU0NmzZtoq6ujpycHAoKCjAYDKm2ohUrVjBx4sRRsa53NpMSkegTt9uN2+0+4znr169n1qxZPW733nvvac9dtmwZS5Ys4aabbuKyyy5j/PjxXHHFFT3Oue+++6irq2PChAkUFxenFfedd96J0Whk6tSpFBcX09DQkLrvlltuIRKJDEkvoDgzWSpWjFpvvvkmV111FUeOHDll80AxtCQRiVEnHA7T0tLCkiVLKCsr49lnn9U7pFFPqmZi1PnNb37D2LFj6ezs5L/+67/0DkcgJSIhRBaQEpEQQneSiIQQupNEJITQnSQiIYTuJBEJIXQniUgIoTtJREII3UkiEkLoThKREEJ3/x/EZXOzJQwMWAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:56:35.578696Z", + "iopub.status.busy": "2024-03-26T12:56:35.578311Z", + "iopub.status.idle": "2024-03-26T12:56:35.830290Z", + "shell.execute_reply": "2024-03-26T12:56:35.829287Z" + }, + "papermill": { + "duration": 0.271201, + "end_time": "2024-03-26T12:56:35.832459", + "exception": false, + "start_time": "2024-03-26T12:56:35.561258", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA6FklEQVR4nO3df1xUVf4/8NfMAMNvRIFBEYUkRVKhYEGwUosfiboSYqy2SuTSptGaE22y+134aFu0iai1JmtJbpbhR2VtN01F/BEqpmGUmmCiSCo/VUAHnBmG8/3Dz9x1YoDBgDv3zvv5ePCAe++5M+/LnXnPuWfOPUfCGGMghBCBkfIdACGE3A9KXoQQQaLkRQgRJEpehBBBouRFCBEkSl6EEEGi5EUIESRKXoQQQbLiO4CB1tHRgWvXrsHJyQkSiYTvcAghP8MYw61btzBs2DBIpV3XrywueV27dg3e3t58h0EI6cFPP/2E4cOHd7nd4pKXk5MTgLv/GGdnZ56j6T9arRb79u1DdHQ0rK2t+Q6H9AFLOactLS3w9vbm3qtdsbjkpb9UdHZ2Fn3ysre3h7Ozs6hf6JbE0s5pT8061GBPCBEkSl6EEEEyi+S1bt06+Pj4wNbWFmFhYThx4kSXZadMmQKJRNLpZ/r06QMYMSGEb7wnr61bt0KpVCIzMxOnTp1CYGAgYmJiUF9fb7R8QUEBampquJ8zZ85AJpNhzpw5Axw5IYRPvCevnJwcpKSkIDk5GQEBAcjNzYW9vT3y8vKMlh88eDA8PT25n8LCQtjb21PyIsTC8Ppto0ajQWlpKdLT07l1UqkUkZGRKCkpMekxNm7ciN/85jdwcHAwul2tVkOtVnPLLS0tAO5+c6PVan9B9OZNf2xiPkZLYynn1NTj4zV5NTY2QqfTQaFQGKxXKBQoLy/vcf8TJ07gzJkz2LhxY5dlsrKysHz58k7r9+3bB3t7+94HbabUajWuXLnCLWs7gBt3gPIfK2H9s/r18OHDIZfLBzhC0lcKCwv5DqFftba2mlRO0P28Nm7ciPHjxyM0NLTLMunp6VAqldyyvgNcdHS0qPp5ffvtt0hMTDSp7Ndff42HH364nyMifU2r1aKwsBBRUVGi7uelvzrqCa/Jy83NDTKZDHV1dQbr6+rq4Onp2e2+KpUK+fn5WLFiRbfl5HK50VqGtbW1qF4A48aNQ2lpKbdcUdME5bbTyJkzHmOGDjIo6+/vL6pjF6PW1tZOVx+329Q4droSrm4ecLT772va399fVFcRpr42eU1eNjY2CA4ORlFREeLi4gDcvXG6qKgIqamp3e67bds2qNVq/Pa3vx2ASM2fvb09HnnkEW5Zevk65MVtGDsuEEEjh/AYGbkf5eXlCA4ONrrtnZ8tl5aWGpx7S8H7ZaNSqURSUhJCQkIQGhqKNWvWQKVSITk5GQCwYMECeHl5ISsry2C/jRs3Ii4uDkOG0BuTiI+/v79BTRroujbt7+8/wNGZB96TV2JiIhoaGpCRkYHa2loEBQVhz549XCN+dXV1p2ExKioqcOTIEezbt4+PkAnpdz+vSQNUm/453pMXAKSmpnZ5mXjo0KFO68aMGQOaK5cQy8Z7J1VCCLkflLwIIYJEyYsQIkiUvAghgkTJixAiSJS8CCGCRMmLECJIlLwIIYJEyYsQIkiUvAghgkTJixAiSJS8CCGCRMmLECJIlLwIIYJEyYsQIkiUvAghgkTJixAiSJS8CCGCRMmLECJIvCevdevWwcfHB7a2tggLC8OJEye6Ld/U1ISXXnoJQ4cOhVwux+jRo7F79+4BipYQYi54nYBj69atUCqVyM3NRVhYGNasWYOYmBhUVFTAw8OjU3mNRoOoqCh4eHhg+/bt8PLywuXLlzFo0KCBD54Qwitek1dOTg5SUlK4ORpzc3Oxa9cu5OXlYdmyZZ3K5+Xl4caNGzh27Bg3q66Pj89AhkxIv7nUqIJK3d7l9soGFffbysr4W9dBbgVfN4d+ic/c8Ja8NBoNSktLkZ6ezq2TSqWIjIxESUmJ0X3+/e9/Izw8HC+99BI+//xzuLu7Y968eXj99dchk8mM7qNWq6FWq7nllpYWAIBWq4VWq+3DIzIv7e3t3G8xH6dYVF1XIWrNUZPKvrr9dLfbC1+ZBJ8hwk1gpr5eeUtejY2N0Ol03OSyegqFAuXl5Ub3uXjxIg4cOIBnn30Wu3fvxoULF7B48WJotVpkZmYa3ScrKwvLly/vtH7fvn2wt7f/5Qdipn66DQBWOH78OK6e4Tsa0hP9+Zrvp4PCzvicpNoO4IYaGCwHrI20Vte1SbD5ggx7iw7D27Ffw+1Xra2tJpUzi0lnTdXR0QEPDw9s2LABMpkMwcHBuHr1KlauXNll8kpPT4dSqeSWW1pa4O3tjejoaDg7Ow9U6APuu+obwOlvMHHiRASOGMx3OKQHZ6+1IPv0ccyOmoSHhhl/XWq1WhQWFiIqKoprNvn5Y2y+cByPPvpol48hBPqro57wlrzc3Nwgk8lQV1dnsL6urg6enp5G9xk6dCisra0NLhHHjh2L2tpaaDQa2NjYdNpHLpdDLpd3Wm9tbW30BSAk3bWRXL6p5n7b2nb9SWZJbSTmTN+GZWVl1ePrsqvXbm8ew5yZGjtvycvGxgbBwcEoKipCXFwcgLs1q6KiIqSmphrdZ9KkSdiyZQs6Ojogld6tN58/fx5Dhw41mrjE7FKjClOzD/VYrqf2EQA4mDaFEhgRHF4vG5VKJZKSkhASEoLQ0FCsWbMGKpWK+/ZxwYIF8PLyQlZWFgBg0aJF+Pvf/44lS5bg5Zdfxo8//oi33noLf/jDH/g8DF7oa1xrEoPg59G5gUPVpsYXh0owY0o4HOw61zwB4EL9bbyytazbb7gIMVe8Jq/ExEQ0NDQgIyMDtbW1CAoKwp49e7hG/Orqaq6GBQDe3t7Yu3cvli5digkTJsDLywtLlizB66+/ztch8M7PwxHjvFw6rddqtah1Bx4Z6SroSwhCusJ7g31qamqXl4mHDh3qtC48PBzHjx/v56gIIeaO99uDCCHkflDyIoQIEu+XjYQQQK27A6ntVVxqqYDU1ngP0/b2dlxrv4ZzN84ZvT3oUsttSG2vQq27A6BzO6jYUPIixAxcU12Gg+97+FP3g6oAAN7f836X2xx8gWuqIARD0WUZsaDkRYgZGOYwEqpLL2NtYhBGGen6AtyteR09chSTHp1ktOZVWX8bS7aWYdjUkf0drlmg5EWIGZDLbNFxxwu+zmMQMMT4JZ9Wq8Ulq0sYO3is0e4vHXea0XGnAXKZbX+HaxYoeQlUT20kPbWPAJbXRkLEhZKXQJnaRtJd+whgWW0kRFwoeQlUT20kPbWPAJbXRkLEhZKXQPXURtJT+whgeW0kRFyokyohRJAoeRFCBImSFyFEkCh5EUIEiZIXIUSQKHkRQgSJkhchRJAoeRFCBImSFyFEkMwiea1btw4+Pj6wtbVFWFgYTpzo+oa9TZs2QSKRGPzY2lIPcUIsDe/Ja+vWrVAqlcjMzMSpU6cQGBiImJgY1NfXd7mPs7MzampquJ/Lly8PYMSEEHPAe/LKyclBSkoKkpOTERAQgNzcXNjb2yMvL6/LfSQSCTw9Pbkf/VRphBDLweuN2RqNBqWlpUhPT+fWSaVSREZGoqSkpMv9bt++jZEjR6KjowOPPPII3nrrLTz00ENGy6rVaqjVam65paUFwN0bl7VabR8dycBrb2/nfhs7Dv267o6xp8cgA8eUc9HTORXL+TQ1dl6TV2NjI3Q6Xaeak0KhQHl5udF9xowZg7y8PEyYMAHNzc3Izs5GREQEzp49i+HDh3cqn5WVheXLl3dav2/fPtjb2/fNgfDgp9sAYIUjR47gsvFRgwEAhYWFv/gxSP/rzbno6pyK5Xy2traaVE5wQ+KEh4cjPDycW46IiMDYsWPxj3/8A2+88Uan8unp6VAqldxyS0sLvL29ER0dDWdn5wGJuT+cvdaC7NPH8eijj+KhYZ2PQ6vVorCwEFFRUV0OidPTY5CBY8q56OmciuV86q+OesJr8nJzc4NMJkNdXZ3B+rq6Onh6epr0GNbW1nj44Ydx4cIFo9vlcjnkcrnR/bp6UwuBfoBBKyurbo+ju+M09TFI/+vNuejqnIrlfJoaO68N9jY2NggODkZRURG3rqOjA0VFRQa1q+7odDqcPn0aQ4cO7a8wCSFmiPfLRqVSiaSkJISEhCA0NBRr1qyBSqVCcnIyAGDBggXw8vJCVlYWAGDFihWYOHEi/Pz80NTUhJUrV+Ly5cv43e9+x+dhEEIGGO/JKzExEQ0NDcjIyEBtbS2CgoKwZ88erhG/uroaUul/K4g3b95ESkoKamtr4erqiuDgYBw7dgwBAQF8HQIhhAe8Jy8ASE1NRWpqqtFthw4dMlhevXo1Vq9ePQBREULMGe+dVAkh5H5Q8iKECBIlL0KIIJlFmxchlq5NqwMAnLna3GUZVZsa3zQAnpdvwsGuc9/FC/W3+y0+c0TJS6B6erH39EIHLO/Fbs4q/+9cLCs43UNJK2y+cLLbEg5yy3hbW8ZRipBpL/aeX+iA5bzYzVn0Q3fvKBnl4Qg7a5nRMhU1zXh1+2msShiPMUM7z5IO3D2Xvm4O/RanOaFXrUD19GI35YUOWNaL3ZwNdrDBb0JHdFtGP2rEKHcHjPPq+pxaCkpeAtXTi51e6ETsev1t48WLF/sjDkII6ZVeJy8/Pz9MnToVn3zyCe7cudMfMRFCSI96nbxOnTqFCRMmQKlUwtPTE7///e+7nTCDEEL6Q6+TV1BQENauXYtr164hLy8PNTU1ePTRRzFu3Djk5OSgoaGhP+IkhBAD993D3srKCvHx8di2bRv+9re/4cKFC0hLS4O3tzcWLFiAmpqavoyTEEIM3Hfy+uabb7B48WIMHToUOTk5SEtLQ2VlJQoLC3Ht2jXMmjWrL+MkhBADve4qkZOTg48++ggVFRWIjY3Fxx9/jNjYWG7MLV9fX2zatAk+Pj59HSshhHB6nbzWr1+P559/Hs8991yXQy97eHhg48aNvzg4QgjpSq+TV2FhIUaMGGEwuikAMMbw008/YcSIEbCxsUFSUlKfBUkIIT/X6zavUaNGobGxsdP6GzduwNfXt0+CIoSQnvQ6eTHGjK6/ffs2bG1tf3FAhBBiCpMvG/UTt0okEmRkZBjMNq3T6fD1118jKCjovoJYt24dVq5cidraWgQGBuK9995DaGhoj/vl5+dj7ty5mDVrFnbu3Hlfz00IESaTk9e3334L4G7N6/Tp07CxseG22djYIDAwEGlpab0OYOvWrVAqlcjNzUVYWBjWrFmDmJgYVFRUwMPDo8v9qqqqkJaWhscee6zXz0kIET6Tk9fBgwcBAMnJyVi7di2cnftmOvGcnBykpKRw8zTm5uZi165dyMvLw7Jly4zuo9Pp8Oyzz2L58uUoLi5GU1NTn8RCCBGOXn/b+NFHH/XZk2s0GpSWliI9PZ1bJ5VKERkZiZKSki73W7FiBTw8PLBw4UIUFxd3+xxqtRpqtZpbbmlpAQBotVpotdpfeATmSz8kTnt7u6iP05JYyjk19dhMSl7x8fHYtGkTnJ2dER8f323ZgoICk54YABobG6HT6bgJZvUUCgXKy8uN7nPkyBFs3LgRZWVlJj1HVlYWli9f3mn9vn37DNrtxOan2wBghePHj+PqGb6jIX3BUs5pa2urSeVMSl4uLi6QSCTc33y5desW5s+fjw8++ABubm4m7ZOens592QDcrXl5e3sjOjq6zy59zdF31TeA099g4sSJCBwxmO9wSB+wlHOqvzrqiUnJ695Lxb68bHRzc4NMJkNdXZ3B+rq6Onh6enYqX1lZiaqqKsycOZNb19HRAeDujeIVFRUYNWqUwT5yuRxyeecJKKytrWFtbd0Xh2GWrKysuN9iPk5LYinn1NRj43XeRhsbGwQHB6OoqIhb19HRgaKiIoSHh3cq7+/vj9OnT6OsrIz7+fWvf42pU6eirKwM3t7eAxk+IYRHJtW8Hn74Ye6ysSenTp3qVQBKpRJJSUkICQlBaGgo1qxZA5VKxX37uGDBAnh5eSErKwu2trYYN26cwf6DBg0CgE7rCSHiZlLyiouL67cAEhMT0dDQgIyMDNTW1iIoKAh79uzhGvGrq6s73UdJCCEmJa/MzMx+DSI1NRWpqalGtx06dKjbfTdt2tT3ARFCzB5VaQghgmRSzWvw4ME4f/483Nzc4Orq2m37140bN/osOEII6YpJyWv16tVwcnLi/ja18Z4QQvqLScnr3oEFn3vuuf6KhRBCTNbrNi+ZTIb6+vpO669fvw6ZTNYnQRFCSE/6bDBCtVptMEwOIYT0J5NHlXj33XcB3B2M8MMPP4SjoyO3TafT4auvvoK/v3/fR0gIIUaYnLxWr14N4G7NKzc31+AS0cbGBj4+PsjNze37CAkhxAiTk9elS5cAAFOnTkVBQQFcXV37LShCCOlJrwcj1I+oSgghfOp18nr++ee73Z6Xl3ffwRBCiKl6nbxu3rxpsKzVanHmzBk0NTXhiSee6LPACCGkO71OXv/61786revo6MCiRYs6DQRICCH9pU9uzJZKpVAqldw3koQQ0t/6bFSJyspKbnYTQgjpb72+bLx3Mgvgbr+vmpoa7Nq1y+AeSEII6U+9Tl76mbP1pFIp3N3dsWrVqh6/iSSEkL5C/bwIIYJEI6kSQgTJLJLXunXr4OPjA1tbW4SFheHEiRNdli0oKEBISAgGDRoEBwcHBAUFYfPmzQMYLSHEHPCevLZu3QqlUonMzEycOnUKgYGBiImJMTpmGHB3SOo///nPKCkpwffff4/k5GQkJydj7969Axw5IYRPvCevnJwcpKSkIDk5GQEBAcjNzYW9vX2XtxlNmTIFTz/9NMaOHYtRo0ZhyZIlmDBhAo4cOTLAkRNC+NTrBvuuXLlyBStWrMCGDRtM3kej0aC0tBTp6encOqlUisjISJSUlPS4P2MMBw4cQEVFBf72t78ZLaNWq6FWq7nllpYWAHdva9JqtSbHKjT6Pnft7e2iPk5LYinn1NRj67Pkdf36dWzcuLFXyauxsRE6nY6bYFZPoVCgvLy8y/2am5vh5eUFtVoNmUyG999/H1FRUUbLZmVlYfny5Z3W79u3D/b29ibHKjQ/3QYAKxw/fhxXz/AdDekLlnJOW1tbTSrXZ8lrIDk5OaGsrAy3b99GUVERlEolHnjgAUyZMqVT2fT0dIOOtS0tLfD29kZ0dDScnZ0HMOqB9V31DeD0N5g4cSICRwzmOxzSByzlnOqvjnrCa/Jyc3ODTCZDXV2dwfq6ujp4enp2uZ9UKoWfnx8AICgoCOfOnUNWVpbR5CWXyyGXyzutt7a2hrW19S87ADNmZWXF/RbzcVoSSzmnph4brw32NjY2CA4ORlFREbeuo6MDRUVFCA8PN/lxOjo6DNq1CCHiZ3LNKz4+vtvtTU1N9xWAUqlEUlISQkJCEBoaijVr1kClUiE5ORkAsGDBAnh5eSErKwvA3TaskJAQjBo1Cmq1Grt378bmzZuxfv36+3p+QogwmZy8XFxcety+YMGCXgeQmJiIhoYGZGRkoLa2FkFBQdizZw/XiF9dXQ2p9L8VRJVKhcWLF+PKlSuws7ODv78/PvnkEyQmJvb6uQkhwiVhXU3EKFItLS1wcXFBc3OzaBvsdTodPtr2BdK3HEXWvElInjODJgQWgbLL1xG3/jh2LpqIoJFD+A6n35j6HuW9kyrpWwUFBRg1ahRS5sah8T8rkTI3DqNGjUJBQQHfoRHSp0y+bDR1uBuagIM/BQUFmD17Nuzs7AzW19fXY/bs2dixY0ePbZeECIXJyWvTpk0YOXIkHn74YVjYlaYg3Lp1C7/73e8AAMHBwfAb9wgKvq9H/AQPXDhzCkeOHEFKSgq8vb3x0EMPibqDrhi0trZ26qhdUdMEde0FnDtjh47rg7j1/v7+lnk+mYkWL17MXF1dWVBQEFu7di27fv26qbualebmZgaANTc38x1Kn1q/fj0DYNJPaWkp3+GSHpSWllrs+TT1PWpyzWvdunXIyclBQUEB8vLykJ6ejunTp2PhwoWIjo6GRCLpm2xK7ktVVRX39+OPP47fzPstzl9pxOjhbsjf8gm++uorAMDChQvh7+/PU5TEVP7+/igtLTVYd7tNjV0HSzB9ajgc7eQGZS1Rr3rYy+VyzJ07F3PnzsXly5exadMmLF68GO3t7Th79iwcHR37K07SA/23ieHh4Th48CB0Oh12796N2NhY/D5lISZNmoTjx49DoVBY5iWGwNjb2+ORRx4xWKfVanGzsR7hoSGi7mFvqvv+tlEqlUIikYAxBp1O15cxkfswePDde91UKhV0Oh0OHz6Mr776CocPH4ZOp4NKpTIoR4jQ9armpVarucvGI0eOYMaMGfj73/+Op556yqAjKRl4+ntBv//+e7i4uKCtrQ3A3fHS7OzsuOXu7hklREhMTl6LFy9Gfn4+vL298fzzz+Ozzz6Dm5tbf8ZGesHLy4v7++f3eWo0GqPlCBEyk3vYS6VSjBgxAg8//HC3jfPm3hlSrD3sNRoNHBwc4ODggEGDBuHy5cvcNh8fH9y8eRMqlQoqlQo2NjY8RkruR1tbG5RKJY4fP46JEydyNWoxMvU9anLNa8GCBfSNohk7duwY2tvb0dzcjEcffRQzZ87E+fPnMXr0aFy6dAm7du3iyhkbOoiYr7i4OHz++efccllZGXJzczFr1izs3LmTv8D4NhD9NsyJWPt5bdmyhQFgS5YsYVZWVgb9gKysrNiSJUsYALZlyxa+QyW9MGvWrG77eM2aNYvvEPtcn/fzIuZt6NChAIB3330X06dPR1RUFH788Uc8+OCDKCwsxLvvvmtQjpi/trY2rsZ175cu9y5//vnnaGtrE+0lZHfoK0KRiIiIgJWVFTw8PLB9+3YEBATAxsYGAQEB2L59Ozw8PGBlZYWIiAi+QyUmunf48ieffBLFxcX47LPPUFxcjCeffNJoOUtCNS+R0Ld51dXVwdXVtcuuEtTmJRwnT54EcPde1c8//xw6nQ7Xr19HWFgYPv/8c/zqV7/CqVOnuHKWhmpeIlFTU9Pltnu/aOmuHDEv7P86Ajz44IOd+lHeO48Ds9CBEih5iYSHhwcA4NFHH0VzczMKCwuhVCpRWFiIpqYmTJo0yaAcMX+RkZEAgO3bt6Otrc3grom2tjauW5K+nKWhy0YLQd1chCc6OhrvvPMO2tvbDe5HzcnJ6VTOElHyEon6+noAwNGjR43eHnTnzh2DcsT8TZkyheus2RUXFxeLbcM0i8vGdevWwcfHB7a2tggLC8OJEye6LPvBBx/gscceg6urK1xdXREZGdlteUuh7wJhrP1DfwP9veWIMOjvhvh5Vwj9srE5SS0F78lr69atUCqVyMzMxKlTpxAYGIiYmJguawiHDh3C3LlzcfDgQZSUlHCzX1+9enWAIzcv+q4SLi4ucHd3N9jm5uYGFxcX6iohMMXFxWhoaEBWVlantkqFQoG33noL9fX1KC4u5ilCfvGevHJycpCSkoLk5GQEBAQgNzcX9vb2XY6F/+mnn2Lx4sUICgqCv78/PvzwQ26iWkt27+1B1dXVBtuqq6vR3NyM9vZ2HDt2jKcISW/pvxn29vbutI0xhhEjRhiUszS8tnlpNBqUlpYiPT2dWyeVShEZGYmSkhKTHqO1tRVarbbLcarUarXBKAstLS0A7g7sptVqf0H05uXeG7HvvUz8+fLly5dFddxipq9Bz58/H7Gxsdi0aRNqa2vh6emJ7OxszJ8/nysnpnNq6rHwmrwaGxuh0+m4CWb1FApFp8kHuvL6669j2LBhXX5dnJWVheXLl3dav2/fPlGNKHrw4EEAgLW1NVxcXNDY2MhtGzJkCJqbm6HVanHw4EEMGjSIpyhJb2g0GkilUjg7OyM5ORnNzc2ws7NDc3MzkpOTcfToUdy6dQs3b97E7t27+Q63z7S2tppUTtDfNr799tvIz8/HoUOHYGtra7RMenq6we0TLS0tXDuZmIbE2bp1K4C7n1qhoaF47bXXuE/plStXci9ujUaD2NhYPkMlJjp8+DA6OjrQ1NSEvLw8REZG4uLFi3jggQewf/9+NDU1AQBcXV0xefJkfoPtQ/qro57wmrzc3Nwgk8lQV1dnsL6urq7HET+zs7Px9ttvY//+/ZgwYUKX5eRyudFvZKytrUU1Dvi9n1ZSqZQb014mkxn0zm5tbRXVcYtZQ0MDAGDJkiVYt26dQe3KysoKS5Yswdq1a9HQ0CCqc2rqsfCavGxsbBAcHIyioiLExcUBANf4npqa2uV+77zzDt58803s3bsXISEhAxSteRs2bBiAu8M8nz59Go8//ji3zcfHB56enqitreXKEfNHI4X0oL/H5ulJfn4+k8vlbNOmTeyHH35gL7zwAhs0aBCrra1ljDE2f/58tmzZMq7822+/zWxsbNj27dtZTU0N93Pr1i2Tnk+s43l9/PHH3BhP06dPZ2vXrmWpqals7dq1bPr06dy2jz/+mO9QiYnUajWzsrJiCoWCabVaptFo2M6dO5lGo2FarZYpFApmZWXF1Go136H2KcGM55WYmIiGhgZkZGSgtrYWQUFB2LNnD9eIX11dbXDZs379emg0GiQkJBg8TmZmJv7nf/5nIEM3K/d+nf7ll19yI6cC/50W7efliHm7d6SQuLg4+Pr64vz589i/fz8uXbrENbdY7EghA5RMzYZYa17t7e3Mx8eHjRo1ikmlUoPRNmUyGRs1ahTz9fVl7e3tfIdKTKQfHffemvO9P7GxsaIcHVcwNS/SN2QyGVatWoWEhATExsZyn9L6Mex3796N7du3G9TCiHnTt2Xt2rULCoUC8+bNg0qlgoODA7Zs2cI14Ftqm5fJsweJhVhnD9IrKCjAq6++iqqqKm6dr68vsrOzER8fz19gpNfa2tpgb28PGxsb3Lp1CxKJhJsFnTEGJycnaDQatLa2imoYaFPfo7zfHkT6Vnx8PC5cuGAwntePP/5IiUuA/vGPfwC423cvISEBx48fR1tbG44fP46EhASuJ7q+nKWhy0YRkslkmDx5MlQqFSZPnkyXigJVWVkJ4O5IKn/9618Nur/4+vpiw4YNSElJ4cpZGqp5EWKmRo0aBeDuTdjGatMdHR0G5SwNtXmJlFar5dpHxNT72pLoZ0EfMmQIrly5AsYYd04lEgmGDx+O69evi24WdGrzsmA6nc5gvHOdTsd3SOQ+2NjYYOnSpairq8Pw4cPx4Ycf4saNG/jwww8xfPhw1NXVYenSpaJKXL3S3302zI1Y+3np7dixg/n4+Bj0B/Lx8WE7duzgOzRyn1577TWjs6C/9tprfIfWL0x9j1LNS0QKCgqQkJCA8ePHG0xQOn78eCQkJHCzzRBheeedd6BSqZCdnY3Y2FhkZ2dDpVLhnXfe4Ts0XlGbl0jodDr4+flh/Pjx2LFjBw4fPowvv/wS06ZNw+TJkzF79mycOXMGP/74I337KFCW0o5JbV4Wpri4GFVVVYiIiMDo0aMRFRWFnJwcREVFYfTo0QgPD8elS5csdrxzIj7Uz0sk9OOY/+lPf0JsbCxmzJhhcHvQn//8Z4NyRFg0Gg3ee+89HDhwABcuXMDLL79suQ31egPSAmdGxNpgv3//fgaAeXl5MZlM1unGbC8vLwaA7d+/n+9QSS9Rg71xVPMSmatXrxoMIQTc7eRo6VPDCdUf//hHrFy5EgqFAsuXL4dcLodarUZmZiZWrlwJABbbcE9tXiJx7do17u8hQ4Zg6dKleOGFF7B06VIMGTLEaDli3jQaDVavXg2FQoErV67g+eefh6urK55//nlcuXIFCoUCq1evhkaj4TtUXlDyEgn9fIyDBg3CzZs3sXr1amzYsAGrV6/GzZs34eLiYlCOmL/3338f7e3t+Otf/worK8OLJCsrK6xYsQLt7e14//33eYqQX3TZKBL6hvimpiZMnz4dMTExXIP93r17uZFVqcFeOPQ3XM+YMcPodv16ujGbCJqjoyP3t0QiQVBQECZNmoSgoCBIJBKj5Yh5099w/cUXXxjdrl9vqTdmU/ISiaCgIACAnZ0dN3vQ3Llz8fjjj+PMmTPcYHX6csT8LV68GFZWVvh//+//ob293WBbe3s7MjIyYGVlhcWLF/MUIb94T17r1q2Dj48PbG1tERYWhhMnTnRZ9uzZs5g9ezZ8fHwgkUiwZs2agQvUzOmHAm5ra0NbWxteeeUVvPDCC3jllVfQ2tqKtrY2g3LE/NGN2T0YmJ4bxuXn5zMbGxuWl5fHzp49y1JSUtigQYNYXV2d0fInTpxgaWlp7LPPPmOenp5s9erVvX5OsfbzOnjwINcHSCKRGPQJunf54MGDfIdKeon6eRnHa/IKDQ1lL730Eres0+nYsGHDWFZWVo/7jhw5kpLXPbqbPUgqldLsQQLX2trKXnzxRRYUFMRefPFF1trayndI/cbsR5XQaDQoLS1FZGQkt04qlSIyMhIlJSV8hSVYMpkMc+bMQWVlJdzd3Q36ebm7u6OyshIJCQl0U7YAFRQUICAgALm5uSgrK0Nubi4CAgIsfpQQ3rpKNDY2QqfTcZPL6ikUCpSXl/fZ86jVaqjVam65paUFwN079PUTGIiBTqfDtm3bEBwcjMbGRqxevZrb5uPjg+DgYGzfvh0rVqygBCYg//rXv/Cb3/wGsbGx+Oijj1BbWwtPT09kZ2cjISEB+fn5ePrpp/kOs0+Z+r4UfT+vrKwsLF++vNP6ffv2wd7enoeI+sfp06dRVVWFRYsWwc/PDz/88ANu3rwJV1dXBAQE4Mcff8SyZcuQnZ2N8ePH8x0uMYFOp8PLL7+MkJAQLFy4EM3NzbCzs0NzczMWLlyI+vp6/OEPf4CVlZWoPpBaW1tNKsdb8nJzc4NMJuOmLNerq6uDp6dnnz1Peno6lEolt9zS0gJvb29ER0eLajwvfY0yJSUFjo6OeOqpp1BYWIioqChYW1vj1q1bWLZsGUaOHInY2FieoyWmOHz4MOrr67Fjxw6EhYVBq9UanFM3Nzc8/vjjcHZ2xuTJk/kOt8/oX8s94S152djYIDg4GEVFRYiLiwMAdHR0oKioCKmpqX32PHK5HHK5vNN6a2trUQ3o5u3tDQCoqKjAxIkTufX646yoqODKiem4xayhoQHA3b55954z/TnV99lraGgQ1Tk19Vh47eelVCrxwQcf4J///CfOnTuHRYsWQaVSITk5GQCwYMECpKenc+U1Gg3KyspQVlYGjUaDq1evoqysDBcuXODrEMzGY489Bh8fH7z11lvQarUGE3BotVpkZWXB19cXjz32GN+hEhPp++SdOXPG6Hb9eovtuzdA33526b333mMjRoxgNjY2LDQ0lB0/fpzbNnnyZJaUlMQtX7p0yaALgP5n8uTJJj+fWLtKMHZ38g2JRMLs7OwM/j92dnZMIpHQJBwCo+/+MnPmTKbT6ZhGo2E7d+5kGo2G6XQ6NnPmTFF2fxHMeF6pqaldXiYeOnTIYNnHxwfMsobc7zVj/x+JREL/NwGSyWRYtWoVEhISEBcXh9deew1tbW04fvw4Vq5ciS+++ALbt28XVWN9rwxEJjUnYq153fsp3dbWxrKzs1lsbCzLzs5mbW1tov2UtgTGprPz9fUVbU1aED3s+SDW5KW/PSgrK8vovI1vvfUW3R4kYO3t7aywsJAplUpWWFgo6g8hwVw2kr5x7wQcM2bMwObNm3HlyhUMHz4c77zzDk3AIXAymQyTJ0+GSqXC5MmTLfdS8R68jypB+oaHhwcAYNKkSdixYwfu3LmDkydP4s6dO9ixYwcmTZpkUI4QoaOal8g0NjZi9OjRqKqqAgDk5ORwQw4RIiaUvESivr4eAFBeXt5p9qDq6mp0dHQYlCNE6OiyUSTuvRz8+eB0995hQJeNRCwoeYmETqcDAAwePBhNTU0oLCyEUqlEYWEhd4P2veUIETpKXiJRXFwMALh58ybmzJkDuVyOX/3qV5DL5ZgzZw6ampoMyhEidJS8RCYzM9PoBBx/+ctf+A6NkD5FyUskpkyZAgDYv38/zp8/b3DZWFFRgQMHDhiUI0ToKHmJxJQpU+Du7o4jR44gPj7e4LIxPj4eR44cgYeHByUvIhrUVUIkZDIZcnNzMXv2bBQVFRlMVKofMXb9+vXUM1ugdDodN8yRg4MDpk6davHnkmpeIhIfH48dO3Z06g7h4eGBHTt2ID4+nqfIyC9RUFAAPz8/REVFIScnB1FRUfDz87P4CTgoeYlMfHw8zp8/j+zsbMTGxiI7OxsVFRWUuASqoKAACQkJGD9+PIqLi/HZZ5+huLgY48ePR0JCgmUnsAG6UdxsiHVUCT1jw6f4+PiIdvgUMaPBCM103kbS9+hTWlyKi4tRVVWFP/3pT2hvb8e7776LDRs24N1330V7ezvS09Nx6dIli+27R8lLJHQ6HV599VXMmDHD6KgSM2bMQFpaGvWwFxD98EX5+flwcHBAWloadu/ejbS0NDg4OGDr1q0G5SwNJS+R0H9KR0REYPTo0QaNu6NHj0Z4eLhFf0oLkX5ijbVr12LIkCHIzc3FRx99hNzcXAwZMgRr1641KGdpqKuESNw7GKGbm5vBNpVKRYMRClBYWBiAuzfaP/TQQ3jxxRe5bU888QRu3rwJjUbDlbM0ZlHzWrduHTfmVFhYGE6cONFt+W3btsHf3x+2trYYP348du/ePUCRmi999wjGGDffn15DQwM3AQeNKiEc//jHPwDcnfJPf4eE3oEDB6DRaAzKWRrek9fWrVuhVCqRmZmJU6dOITAwEDExMV2OO3Xs2DHMnTsXCxcuxLfffou4uDjExcV1ObcdIUJVWVnZp+VEZ0C+++xGaGgoe+mll7hlnU7Hhg0bxrKysoyWf+aZZ9j06dMN1oWFhbHf//73Jj2fWLtKBAYGcl0jpFIp++qrr9hnn33GvvrqKyaVSrltgYGBfIdKTOTn58edtxdeeMFgAo4XXniB2+bn58d3qH1KEBNwaDQalJaWGsyKLZVKERkZiZKSEqP7lJSUQKlUGqyLiYnBzp07jZZXq9VQq9XccktLCwBAq9VCq9X+wiMwH9999x3394gRI/D4449zy76+vrh06RJXTkzHLWb3zgS/Zs0aMMagUqkQERGBSZMmYcOGDVw5MZ1TU4+F1+TV2NgInU4HhUJhsF6hUKC8vNzoPrW1tUbL19bWGi2flZWF5cuXd1q/b98+7p4/sVm5ciXKy8u5QQj9/f0xZ84cbju1EQrPsGHDMHfuXPzqV79CWloaPvvsM4PtYjqnra2tJpUT/beN6enpBjW1lpYWeHt7Izo6Gs7OzjxG1n82bdqEV199FXV1dVAoFFi1apXB9tjYWJ4iI/fr9u3bWL9+PdavXw8AsLIyfOuK6Zzqr456wmvycnNzg0wmQ11dncH6uro6eHp6Gt3H09OzV+XlcrnBGO561tbWsLa2vs/Izc+0adPw5ZdfAgC++uor7Nq1i9vm5ORkUE5Mxy1mkZGR2L9/PwAgOTkZY8aMwYEDB/DEE0+goqICH3zwAVdOTOfU1GORMPZ/36HzJCwsDKGhoXjvvfcAAB0dHRgxYgRSU1OxbNmyTuUTExPR2tqK//znP9y6iIgITJgwAbm5uT0+X0tLC1xcXNDc3Cy6mpdEIumxDM+nm/SSJZ5TU9+jvF82KpVKJCUlISQkBKGhoVizZg1UKhWSk5MBAAsWLICXlxeysrIAAEuWLMHkyZOxatUqTJ8+Hfn5+fjmm2+4xktLxhjr9sUuthe5JaBz2jXe+3klJiYiOzsbGRkZCAoKQllZGfbs2cM1yldXVxv0Co+IiMCWLVuwYcMGBAYGYvv27di5cyfGjRvH1yGYFcYYpk2bZrBu2rRpFv0iFzrGGCIjIw3WRUZGWvw55f2ycaCJ+bLxXlqtFrt370ZsbKyo2kMsmaWcU1Pfo7zXvAgh5H5Q8iKECBIlL0KIIPH+beNA0zfxmdoRTqi0Wi1aW1vR0tIi6vYRS2Ip51T/3uypOd7iktetW7cAAN7e3jxHQgjpzq1bt+Di4tLldov7trGjowPXrl2Dk5OTSR0AhUp/G9RPP/0k6m9VLYmlnFPGGG7duoVhw4ZBKu26Zcvial5SqRTDhw/nO4wB4+zsLOoXuiWyhHPaXY1LjxrsCSGCRMmLECJIlLxESi6XIzMz0+iIGkSY6JwasrgGe0KIOFDNixAiSJS8CCGCRMmLECJIlLx4NmXKFLzyyit8h0GI4FDyEohDhw5BIpGgqamJ71BIL5jbh5O5xfNLUPIixMxpNBq+QzBLlLzMiFqtxuuvvw5vb2/I5XL4+flh48aNqKqqwtSpUwEArq6ukEgkeO6553p8vFu3buHZZ5+Fg4MDhg4ditWrV3f65N28eTNCQkLg5OQET09PzJs3D/X19dx2fY2vqKgIISEhsLe3R0REBCoqKvr68EXnueeew+HDh7F27VpIJBJIJBJUVlZi4cKF8PX1hZ2dHcaMGYO1a9d22i8uLg5vvvkmhg0bhjFjxgAAjh07hqCgINja2iIkJAQ7d+6ERCJBWVkZt++ZM2cwbdo0ODo6QqFQYP78+WhsbOwynqqqqoH6d/S9Pp+rm/TK5MmT2ZIlSxhjjD3zzDPM29ubFRQUsMrKSrZ//36Wn5/P2tvb2Y4dOxgAVlFRwWpqalhTU1OPj/273/2OjRw5ku3fv5+dPn2aPf3008zJyYl7PsYY27hxI9u9ezerrKxkJSUlLDw8nE2bNo3bfvDgQQaAhYWFsUOHDrGzZ8+yxx57jEVERPT1v0J0mpqaWHh4OEtJSWE1NTWspqaG3blzh2VkZLCTJ0+yixcvsk8++YTZ29uzrVu3cvslJSUxR0dHNn/+fHbmzBl25swZ1tzczAYPHsx++9vfsrNnz7Ldu3ez0aNHMwDs22+/ZYwxdvPmTebu7s7S09PZuXPn2KlTp1hUVBSbOnVql/G0t7fz8a/pE5S8eKZPXhUVFQwAKywsNFpOn0Ru3rxp0uO2tLQwa2trtm3bNm5dU1MTs7e3N0heP3fy5EkGgN26dcvgeffv38+V2bVrFwPA2traTIrFkt374dSVl156ic2ePZtbTkpKYgqFgqnVam7d+vXr2ZAhQwz+5x988IFB8nrjjTdYdHS0wWP/9NNP3IeeqfEIBV02momysjLIZDJMnjy5Tx7v4sWL0Gq1CA0N5da5uLhwlyB6paWlmDlzJkaMGAEnJyfu+aurqw3KTZgwgft76NChAGBweUlMt27dOgQHB8Pd3R2Ojo7YsGFDp//3+PHjYWNjwy1XVFRgwoQJsLW15dbde24B4LvvvsPBgwfh6OjI/fj7+wMAKisr+/GI+GFxQ+KYKzs7uwF/TpVKhZiYGMTExODTTz+Fu7s7qqurERMT06mR+N6RO/XjoHV0dAxovGKQn5+PtLQ0rFq1CuHh4XBycsLKlSvx9ddfG5RzcHDo9WPfvn0bM2fOxN/+9rdO2/QfOGJCyctMjB8/Hh0dHTh8+HCnOfoAcJ/COp3OpMd74IEHYG1tjZMnT2LEiBEAgObmZpw/fx6PP/44AKC8vBzXr1/H22+/zY0s+8033/TF4ZD/Y2NjY3DOjh49ioiICCxevJhbZ0qtaMyYMfjkk0+gVqu5G7NPnjxpUOaRRx7Bjh074OPjAysr42/tn8cjZHTZaCZ8fHyQlJSE559/Hjt37sSlS5dw6NAh/O///i8AYOTIkZBIJPjiiy/Q0NCA27dvd/t4Tk5OSEpKwmuvvYaDBw/i7NmzWLhwIaRSKVdzGjFiBGxsbPDee+/h4sWL+Pe//4033nij34/Vkvj4+ODrr79GVVUVGhsb8eCDD+Kbb77B3r17cf78efzlL3/plISMmTdvHjo6OvDCCy/g3Llz2Lt3L7KzswH8tyb80ksv4caNG5g7dy5OnjyJyspK7N27F8nJyVzC+nk8Qq49U/IyI+vXr0dCQgIWL14Mf39/pKSkQKVSAQC8vLywfPlyLFu2DAqFAqmpqT0+Xk5ODsLDwzFjxgxERkZi0qRJGDt2LNdu4u7ujk2bNmHbtm0ICAjA22+/zb0hSN9IS0uDTCZDQEAA3N3dERMTg/j4eCQmJiIsLAzXr183qIV1xdnZGf/5z39QVlaGoKAg/PnPf0ZGRgYAcOdz2LBhOHr0KHQ6HaKjozF+/Hi88sorGDRoEDec8s/j+Xlbm5DQkDgWRKVSwcvLC6tWrcLChQv5Dof8Qp9++imSk5PR3NzMS5sp36jNS8S+/fZblJeXIzQ0FM3NzVixYgUAYNasWTxHRu7Hxx9/jAceeABeXl747rvv8Prrr+OZZ56xyMQFUPISrOrqagQEBHS5/YcffgAAZGdno6KiAjY2NggODkZxcTHc3NwGKkzSh2pra5GRkYHa2loMHToUc+bMwZtvvsl3WLyhy0aBam9v7/bWju6+cSJEDCh5EUIEib5tJIQIEiUvQoggUfIihAgSJS9CiCBR8iL96rnnnuMGvrO2toZCoUBUVBTy8vJ6dWvKpk2bMGjQoP4LtAv6gQGJ+aHkRfrdU089hZqaGlRVVeHLL7/E1KlTsWTJEsyYMQPt7e18h0eEis/BxIj4JSUlsVmzZnVaX1RUxACwDz74gDHG2KpVq9i4ceOYvb09Gz58OFu0aFGnARHv/cnMzGSMMfbxxx+z4OBg5ujoyBQKBZs7dy6rq6vjnufGjRts3rx5zM3Njdna2jI/Pz+Wl5fHba+urmZz5sxhLi4uzNXVlf36179mly5dYowxlpmZ2el5Dx482C//J9J7VPMivHjiiScQGBiIgoICAIBUKsW7776Ls2fP4p///CcOHDiAP/7xjwCAiIgIrFmzBs7OzqipqUFNTQ3S0tIAAFqtFm+88Qa+++477Ny5E1VVVQbj+//lL3/BDz/8gC+//BLnzp3D+vXruTsMtFotYmJi4OTkhOLiYhw9ehSOjo546qmnoNFokJaWhmeeeYarOdbU1CAiImJg/1Gka3xnTyJuXdW8GGMsMTGRjR071ui2bdu2sSFDhnDLH330EXNxcenx+X4+jPXMmTNZcnKy0bKbN29mY8aMYR0dHdw6tVrN7Ozs2N69e3uMn/CLal6EN4wxbiyq/fv348knn4SXlxecnJwwf/58XL9+Ha2trd0+Rk/DWC9atAj5+fkICgrCH//4Rxw7dozb97vvvsOFCxfg5OTEDZs8ePBg3LlzR5TDJosNJS/Cm3PnzsHX1xdVVVWYMWMGJkyYgB07dqC0tBTr1q0D0P2chfphrJ2dnfHpp5/i5MmT+Ne//mWw37Rp03D58mUsXboU165dw5NPPsldct6+fRvBwcEoKysz+Dl//jzmzZvXz0dPfim6c5fw4sCBAzh9+jSWLl2K0tJSdHR0YNWqVdygefoRZPWMDV9s6jDW7u7uSEpKQlJSEh577DG89tpryM7OxiOPPIKtW7fCw8MDzs7ORuMU07DJYkM1L9Lv1Go1amtrcfXqVZw6dQpvvfUWZs2ahRkzZmDBggXw8/ODVqvlhqPevHkzcnNzDR7Dx8cHt2/fRlFRERobG9Ha2mrSMNYZGRn4/PPPceHCBZw9exZffPEFxo4dCwB49tln4ebmhlmzZqG4uJgbevsPf/gDrly5wj3v999/j4qKCjQ2NkKr1Q7MP430jO9GNyJuSUlJXDcDKysr5u7uziIjI1leXh7T6XRcuZycHDZ06FBmZ2fHYmJi2Mcff9xpnsoXX3yRDRkyxKCrxJYtW5iPjw+Ty+UsPDyc/fvf/+40l+HYsWOZnZ0dGzx4MJs1axa7ePEi95g1NTVswYIFzM3NjcnlcvbAAw+wlJQU1tzczBhjrL6+nkVFRTFHR0fqKmFmaEgcQogg0WUjIUSQKHkRQgSJkhchRJAoeRFCBImSFyFEkCh5EUIEiZIXIUSQKHkRQgSJkhchRJAoeRFCBImSFyFEkCh5EUIE6f8D5ug3aJmcq0sAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T12:56:35.864727Z", + "iopub.status.busy": "2024-03-26T12:56:35.864414Z", + "iopub.status.idle": "2024-03-26T12:56:36.144741Z", + "shell.execute_reply": "2024-03-26T12:56:36.143746Z" + }, + "papermill": { + "duration": 0.29892, + "end_time": "2024-03-26T12:56:36.146879", + "exception": false, + "start_time": "2024-03-26T12:56:35.847959", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABMrklEQVR4nO3dd3xT1fvA8U9GN52UtrS0lL0pCLQMkVUoiqj49QsCskSciFJRy5C9VEC+IMpPFERkiqgIiEAFZO9N2YWW0UXpnknu74/QQKGFpiS9SXver1depDf3Jk9D+uTcc895jkKSJAlBEAQro5Q7AEEQhNIQyUsQBKskkpcgCFZJJC9BEKySSF6CIFglkbwEQbBKInkJgmCVRPISBMEqqeUOoKzpdDpu3ryJs7MzCoVC7nAEQXiAJEmkp6fj6+uLUll8+6rCJa+bN2/i7+8vdxiCIDxGbGws1apVK/bxCpe8nJ2dAf0b4+LiInM0giA8KC0tDX9/f8PfanEqXPIqOFV0cXERyUsQLNjjunVEh70gCFZJJC9BEKySSF6CIFilCtfnVR5IkoRGo0Gr1codimCBVCoVarW63A8FEsnLyuTl5XHr1i2ysrLkDkWwYI6OjlStWhVbW1u5QzEbkbweI1ejxU6tkjsMQD/ANjo6GpVKha+vL7a2tuX+21UwjiRJ5OXlkZiYSHR0NHXq1HnkQE9rJpJXMY7HpjB1w1mqONvx7Wst5A4H0Le6dDod/v7+ODo6yh2OYKEcHBywsbHh2rVr5OXlYW9vL3dIZiGSVzEcbFQcvnYHlVJBXGoOPq6W8wEor9+kgulUhM9I+f8NS6mejzPBNTzQ6iRWHoyROxxBEB4gktcjDGhdHYCVB2PI1+pkjkYQhPuJ5PUIYY188KxkR0J6LlvPxssdjlXr2LEjH374odxhlBs7duxAoVCQkpIidyiyEcnrEWzVSvoG6ytQLNt3TeZoKgbxR1lOpcdDTqpJn1Ikr8foGxyAUgH7rtzmUkK63OEI5UheXp7cIZSdTaNgXnO48LfJnlIkr8fwdXMgtIE3AD/vt6yOe0mSyMrTyHJ7koXWc3Nz+fTTT/H398fOzo7atWvzww8/cPXqVTp16gSAu7s7CoWCwYMHP/b50tPT6d+/P05OTlStWpWvvvrqodPUZcuW0bJlS5ydnfHx8aFfv34kJCQYHi9o8UVGRtKyZUscHR1p27Yt58+fL/HvNXXqVLy8vHB2duaNN94gIiKCZs2aGR4fPHgwL730EtOmTcPX15d69eqVKDaATZs2UbduXRwcHOjUqRNXr14tcVyyu7oHotZD9h1wNV0tPTFUogRea12dLWfj+fXIdT4Oq4eTnWW8bdn5WhqON903mTHOTg7D0bZ078PAgQPZt28f8+bNIygoiOjoaJKSkvD39+fXX3/lP//5D+fPn8fFxQUHB4fHPl94eDh79uxh/fr1eHt7M378eI4ePVooceTn5zNlyhTq1atHQkIC4eHhDB48mE2bNhV6rrFjxzJ79myqVKnC22+/zeuvv86ePXseG8Py5cuZNm0a33zzDe3atWPVqlXMnj2bGjVqFNovMjISFxcXtm7dWuLYYmNjefnll3nvvfd48803OXz4MB999NFjY7IIOh38PVp/v8UQ8G5osqe2jL9CC/d0bU8CKzty9XYWfxy/Sb+QALlDsloXLlxgzZo1bN26ldDQUABq1qxpeNzDwwMALy8v3NzcHvt86enpLF26lBUrVtClSxcAlixZgq+vb6H9Xn/9dcP9mjVrMm/ePFq1akVGRgaVKlUyPDZt2jQ6dOgAQEREBD169CAnJ+exAz3nz5/P0KFDGTJkCADjx49ny5YtZGRkFNrPycmJ77//vtC0ncfF9u2331KrVi1mz54NQL169Th16hSff/75Y98f2Z1YCbdOINm5kNn2Yyo9/ogSE8mrBJRKBa+1rs7UjVH8tO8qfYP9LWJajoONirOTw2R77dI4fvw4KpXKkCCe1JUrV8jPzyc4ONiwzdXV1XBKVuDIkSNMnDiREydOcOfOHXQ6/dCXmJgYGja81xpo2rSp4X7VqlUBSEhIICDg0V9Y58+f59133y20LTg4mH/++afQtiZNmjw03/BxsUVFRRESElLomDZt2jwyHouQmwGRkwDYXHkAn/9wjgX9HWnk62qSpxfJq4ReaVGNL/8+z7m4dI7G3KFFdQ+5Q0KhUJT61E0uJTkNNLXMzEzCwsIICwtj+fLlVKlShZiYGMLCwh7qNLexsTHcL/iCKkgmpuDk5FTq2KzOnrmQEU+mUwAfXAkmX5FFUobpfifRYV9Cbo62vBCkPxURwyZKr0mTJuh0Onbu3Fnk4wWtkpKW+6lZsyY2NjYcOnTIsC01NZULFy4Yfj537hy3b99m5syZtG/fnvr16z/UIf6k6tWrVygG4KGfi1KS2Bo0aMDBgwcLbdu/f/+TB21OKbGwdz4AEem9ycOG9zrWpkPdKiZ7CZG8jDCgjX7E/aZTcdzOyJU5GusUGBjIoEGDeP311/n999+Jjo5mx44drFmzBoDq1aujUCjYsGEDiYmJD/UZPcjZ2ZlBgwbx8ccfs337ds6cOcPQoUNRKpWGllNAQAC2trbMnz+fK1eusH79eqZMmWLS3+v999/nhx9+YOnSpVy8eJGpU6dy8uTJx3YvlCS2t99+m4sXL/Lxxx9z/vx5VqxYwY8//mjS+E1u20TQ5HBC3YQ/85oTUsODD0PrmPQlRPIyQtNqbgRVcyVPq2PN4etyh2O1vv32W1555RXeffdd6tevz7Bhw8jMzATAz8+PSZMmERERgbe3N8OHD3/s882ZM4c2bdrw/PPPExoaSrt27WjQoIGhk71KlSr8+OOP/PLLLzRs2JCZM2cya9Ysk/5O/fv3Z/To0YwaNYqnnnqK6OhoBg8e/NiO/pLEFhAQwK+//srvv/9OUFAQCxcuZPr06SaN36RiD8LptehQMDqzL5Wd7JjXtzlqlWnTjUJ6kgE7VigtLQ1XV1dSU1NLtXrQL4dj+XjtSaq5O7Dz406olGXXcZ+Tk0N0dDQ1atQot2VOTCEzMxM/Pz9mz57N0KFDZYuja9eu+Pj4sGzZsjJ/bdk+Kzod/NAVbhxmlaYjo7Vvsuz1EJ6u41nipyjp36hoeRmpZ5Avrg42XL+Tzc4Lpu03EUrn2LFjrFy5ksuXL3P06FH69+8PwIsvvlhmMWRlZTFnzhzOnDnDuXPnmDBhAtu2bWPQoEFlFoNFOL0WbhwmU7JntqY373euY1TiMoZIXkayt1HRu6V+FV/RcW9+MTExVKpUqdhbTIx+1sOsWbMICgoiNDSUzMxMdu3ahaen6f5oGjVqVGwMy5cvR6FQsGnTJp555hlatGjBn3/+ya+//moYy1Yh5GWh2zoBgAWaF6ldsxYfdDFtP9f9rOs6u4XoH1KdRbui2XEhkZjbWQRUFlVNzcXX15fjx48/8vGAgACOHDli1jg2bdpEfn5+kY95e3vj4ODAtm3bzBqDxdv3Ncr0m1yXPPnd/iV+79vMrN0qInmVQqCnE8/UrcK/FxJZfvAao59tIHdI5ZZaraZ27dpyh0H16tXlDsGypd1E8+8c1MBMTT9mDQjGy9m8fW3itLGUCgoVrjkUS06+WIJMqNjSNo5Hrc3mkK4utTu+Rtva5unnup9IXqXUub4Xfm4O3MnKZ9OpW3KHIwiyybl2CJfzvwDwp8/7vN+lbpm8rkhepaRSKgwTtJftFx33QgUlSdxcHQ7ARkUH3h/Qp8yGD4nk9QR6t/THRqXgWEwKp2+YtkqkIFiDfX/+QM2sk2RJdvj0mk4VZ7sye22RvJ5AFWc7nm2srzzws2h9CRXMheuJ+B+ZCcCp6oNo0bRxmb6+SF5PqGC+4+/Hb5CaXfSldOHJBAYGMnfuXMPPCoWC33//XbZ4BMjM1bDzp0lUUyRyW+VJq/4TyzwGkbyeUMvq7tT3cSYnX8evR8R8x7Jw69Ytnn32WbnDqLAkSeKLtTt5NXctALZhk1DaOT3mKNMTyesJKRQK+t8dNvHz/mtPVNtdKBkfHx/s7Mqub6UokiSh0WhkjUEuaw7HUj9qPs6KbDI8g3Bu2U+WOETyMoFezf1wslVxJSmTvZdvl90LSxLkZcpzMzJJl2SRjJK6/7Tx6tWrKBQK1q1bR6dOnXB0dCQoKIh9+/YVOmb37t20b98eBwcH/P39GTFihKGSBZR8gY6//vqLFi1aYGdnx+7du42O3dpF3UpjxR8b6aPaAUClF74EpTxpRIywN4FKdmpefqoay/ZfY9m+a7QrgwF6AORnwXTfx+9nDmNugm3JTxVKskjGkxg7diyzZs2iTp06jB07lr59+3Lp0iXUajWXL1+me/fuTJ06lcWLF5OYmMjw4cMZPnw4S5YsAUq+QEdERASzZs2iZs2auLu7myR2a5GRq+G9n48wVfETSoWE1Og/KAJCHn+gmcje8lqwYAGBgYHY29sTEhLyUMXIB82dO5d69eoZvkFHjhxJTk5OGUVbvNfunjpujYrnVmq2zNFYloJFMmbNmkWXLl1o3LgxS5YsKXG11JIYNWoUPXr0oG7dukyaNIlr165x6dIlAGbMmEH//v358MMPqVOnDm3btmXevHn89NNPhs/O66+/zrPPPkvNmjVp3bo18+bN46+//nqoGOLkyZPp2rUrtWrVMiwWUlHM3nKeWnf+pa3qLJLaHkXXibLGI2vLa/Xq1YSHh7Nw4UJCQkKYO3cuYWFhnD9/Hi8vr4f2X7FiBRERESxevJi2bdty4cIFBg8ejEKhYM6cOTL8BvfU83EmuIYHB6OTWXkwlvCuZTDK2MZR3wKSg03JJ6OXdJGMJ1Hcwhn169fnxIkTnDx5kuXLlxv2kSQJnU5HdHQ0DRo0KPECHS1btjRZzNZmV9QNFqn176GizXvgJu8qWrImrzlz5jBs2DDDclELFy5k48aNLF68mIiIiIf237t3L+3ataNfP30HYWBgIH379uXAgQNlGndxBrSufjd5xfB+59rYmLhy5EMUCqNO3cqzRy2ckZGRwVtvvcWIESMeOi4gIMCoRTAeXECjoriTmUeH1N+pYROPzskL5dMj5Q5JvtPGvLw8jhw5UqjekVKpJDQ09KHO1gJt27blyJEjhlPLK1eusGnTJp577rliXyc3N5e0tLRCN3MJa+SDZyU7EtNz2XIm3myvY21KskiGOT311FOcPXuW2rVrP3SztbUtkwU6rN2ZS5f5QP0bAMou48HOWeaIZExeSUlJaLVavL29C2339vYmLi6uyGP69evH5MmTefrpp7GxsaFWrVp07NiRMWPGFPs6M2bMwNXV1XDz9zfdcuMPslUr6Rusf/5l+6+a7XWsTUkWyTCnTz/9lL179zJ8+HCOHz/OxYsX+eOPPwz18ctigQ5rV2nfLFwUWVy3qwPN5Bka8SDZO+yNsWPHDqZPn84333zD0aNHWbduHRs3bnzkB2306NGkpqYabrGxsWaNsW9wAEoF7L+SzMX4dLO+ljV53CIZ5tS0aVN27tzJhQsXaN++Pc2bN2f8+PGGVbXLYoEOq5YQRdO4dQCcaRoBytItOGxykkxyc3MllUol/fbbb4W2Dxw4UHrhhReKPObpp5+WRo0aVWjbsmXLJAcHB0mr1ZbodVNTUyVASk1NLVXcJTFs6SGp+qcbpPG/nzLp82ZnZ0tnz56VsrOzTfq8csjIyJBcXV2l77//Xu5QyiVTflZ0P/WSpAku0qZxXaTjMXeePLjHKOnfqGwtL1tbW1q0aEFkZKRhm06nIzIystilzLOyslA+MCBOpdJ/C0gWNLK9YL7jr0dvkJlbMUdhP8gSFskQSuHiVhSXI8mV1MyS+tOgqvErbpmLrFcbw8PDGTRoEC1btiQ4OJi5c+eSmZlpuPo4cOBA/Pz8mDFjBgA9e/Zkzpw5NG/enJCQEC5dusRnn31Gz549DUnMErSr5UkNTyeikzLZePIWvVuZr5/NmsyaNYvz588bvrh27dpFVFTUI+cpPm7RWcGMtPnwt74/eYm2Oy6+dbFVW05Pk6zJq0+fPiQmJjJ+/Hji4uJo1qwZmzdvNnTix8TEFGppjRs3DoVCwbhx47hx4wZVqlShZ8+eTJs2Ta5foUhKpYJujbz5v51XOH0zld6I5NW8efMiF8nIzs5+5AIbgowOL4GkC2Sq3ViQ8xKv+LvJHVEhsk8PKpimUZQdO3YU+lmtVjNhwgQmTJhQBpE9mZqe+vFA0UmZj9mzYnNwcLCIBTaEB2TfgR36VbmX2r9GeoYjzSwseVlOG7CcCaysT15Xb5s+eVlS/55gmZ74M7LzC8i+g65KA+bf0fdBi+RlLTJvw8ZRsH16qQ6vUUWfvG7cySZPozNJSAWjyLOyskzyfEL5VfAZuX/mQYklXYSD3wFwpcVYsrUKPJxsCfCwrPVJZT9ttFgx++DQIlDZQfPXjJ7HVaWSHU62KjLztMQkZ1Hbq9ITh6RSqXBzczOM/nZ0dCyTQZ6C9ZAkiaysLBISEnBzcyvdhawtn4FOA3W7s1vbGDhLUDVXi/usieRVnPo9ILA9XN0FWyfAf5cYdbhCoSDQ04kzN9O4mpRpkuQF+kJ8gJi+IjySm5ub4bNilMvb4cJfoFRDt6kc35YCQDN/yyv/I5JXcRQKCJsO//cMnFkHIW+DkbWLCpJXQaf9+hM3+fPETeb0DsLZvhTNefRJsWrVqnh5eRW7/LxQsdnY2JSuxaXVGIZG0GoYeNbhxPUdADQLcDNZfKYiktejVG2qP2U8tgw2R8AbkUZVjaxxt9M++m6n/dK9Vzly7Q7/nEvgxWZ+TxSaSqWyqLFtQjlw7CdIOAv2btDhE1Ky8gxfvEHVXOWNrQiiw/5xOn8GtpXg5lE49YtRhwbeHS5x9e4HQKPTXwG6fkcUKxQsTE4q/HN3vGSnMeDowfHYFABqeDrh5mgrX2zFEMnrcZy9ob1+RWC2TdTXby+hGp76qzMFyavg8nVssrhaKFiYf2dBVhJ41oWWrwMYkpelDZEoIJJXSbR+D1wDIP0m7J1f4sMKxnrdTM0hJ19rWLMi9o5IXoIFSb4CBxbq73ebBip9f6xIXuWBjT10naS/v3supN4o0WEeTra42Ou7Fa/dzkKioOUlThsFC7J1PGjzoFZnqNMV0J8lnLibvIJE8rJyjXqBf2vQZEPk5BIdolAoqHHfNKGCltfNlGy0OjFKXrAAV3dD1J+gUOqvrt8dyxWTnMWdrHxsVUoaVJW/ampRRPIqKYUCut8dbX9yFVx/eJJxUQyd9rczKchXGp0kVhgS5KfTwubR+vsthoBXA8NDBaeMDX1dsFNb5lVtkbyM4dcCgvrq7/89ukQLrxb0e0UnZhaabyauOAqyO7ES4k6Cnav+CuN9jsWkAJbb3wUieRmvy3j9sl+xB/SDVx/DcNr4wARtccVRkFVu+r3ujw4fg1PhhZJPXE8BoLkFDk4tIJKXsVx8od0H+vtbJ0L+oxe8vX+sl+6+llesaHkJcto9FzLiwaMmBL9V6KE8jY4zN/WrbAVVcyv72EpIJK/SaDsCXPwgNQb2L3jkrgWj7BPSc8nMvbdC9HXR8hLkkhID+77W3+86BdSFB6BG3UojT6PD3dGG6pUtq5LE/Z4oeWVkZJTZmogWxdYRutwtiLhrDqQXv0ajq6MNHk76D8fN+zrpxVgvQTbbJoImR194oH6Phx4+ft8QCUurJHE/o5NXdHQ0PXr0wMnJCVdXV9zd3XF3d8fNzQ13d8ubeW42Tf6r78DPy4B/Hr3GX+Ddb6/7+/fFWC9BFjEH4PSvgKLQ0Ij7Wfrg1AJGT8x+7bXXkCSJxYsX4+3tbdGZ2ayUSgibAYu7wbGfIXgYVA0qctdATyeO3r16UyA+PYdcjdZiL0ML5ZBOp79KDvDUAH3hgSJY+uDUAkYnrxMnTnDkyBHq1atnjnisS0AINP6P/pts8xgYvKHIb7KCfq/7SZK+ymrNKqap8yUIj3V6Ldw4oi800GlckbukZuVz5e5c3GYW3FkPpThtbNWqldlXnbYqoRNBbQ/XdsO5DUXuUnDF8UHiiqNQZvKy9H1doC804Oxd5G7H7w6RCKzsiLuT5VWSuJ/RLa/vv/+et99+mxs3btC4ceOHamQ3bVp0U7TccguANsNh1yx9+dw63UBtV2iXGg8kL3dHG+5k5YuxXkLZ2Tsf0m7oCwy0fq/Y3Y5bweDUAkYnr8TERC5fvmxYGBb0c/gkSUKhUKDVah9xdDn19Eh9wcI70XDg/6DdiEIPP9jyCvBw5E5WqrjiKJSNtJuwZ67+ftdJ+kIDRYhOymT5gWsANA+w/ItvRp82vv766zRv3px9+/Zx5coVoqOjC/1bIdlV0o+8B/j3S8hMKvRwJTs1VZzvtcb8767Ccl1ccRTKQuRkyM/SFxZo1KvIXa7dzqTvd/tJSM+lnrczvZ56skq/ZcHolte1a9dYv369WCj0QUH99K2uuJOwfRo8/1Whh2tUdiIxPRfAsITUddHyEsztxhH9HEbQFxYo4oJSbHIWfb/bT1xaDnW8KrF8WAgupVxjoSwZ3fLq3LkzJ06cMEcs1k2phO4z9PeP/AjxZws9HOh5b6Rywahl0WEvmJUk6a+Cg76ggF+Lh3aJTc7i1e/2czM1h1pVnFg+LATPSnYP7WeJjG559ezZk5EjR3Lq1CmaNGnyUIf9Cy+8YLLgrE7g09Cgp74+0t9jYMBvhm+6+/u9/N31ySs5M4/MXA1OdmIdFMEMzvwGsfv1hQQKujXucyMlm37f7+dGSjY1PZ1YOaw1Xs5F94dZIqP/at5++20AJk9+uCBfhe2wv1/XyXDhb7iyHS5ugbphQOGxXi4ONrg52pCSlU/snSzq+7jIFa1QXuXn6NcbBWj3ob6gwP0Pa3UMXnyQ2ORsqld2ZMWw1ni5WE/iglKcNup0umJvFT5xgX6Wfog+wfP3WNDq11Z88IpjNXcHQEwTEsxk/wJ94QAXP2j7/kMPrzkcy8WEDCo72bJyWGt8XK0rcYGRySs/Px+1Ws3p06fNFU/58MwocPSE2xfh0A/AvaKEoD+TLDh1FGO9BJNLj9cXDACkLhPIUdhxJzOPGynZXEpI50RsCvMiLwIwvHNtfN0c5Iy21Iw6bbSxsSEgIEC0sB7H3hU6j4UNI2HHDGjaGwdHD+p4Vbr7bWdnGC4hxnoJJvfPFMjL4KRUm16rHNFKm4vczc/NgX4hAWUcnOkYfdo4duxYxowZQ3JysjniKT+aDwSvhpCTAjs/B2Dx4Fb8+k5bfFzt8RenjYI53DqJdOxnACbmvYZWuvcnbqtW4upgQ1VXe+p5OzP1pcZWXRjA6A77r7/+mkuXLuHr60v16tVxcircl3P06FGTBWfVVGoImwbLesGh76HlUPyr1DW0uKqJsV6CqUkS0t+jUSCxXtuGwGadWPhcfRxsVDjYqFCrylftUaOT10svvWTSABYsWMCXX35JXFwcQUFBzJ8/n+Dg4GL3T0lJYezYsaxbt47k5GSqV6/O3Llzee6550wal0nU6gx1u8OFzbBlHPRfY3jo/j6vgqlVgvBEzm1EcXU3OZINc3T9WB5Wz6qGPhjL6OQ1YcIEk7346tWrCQ8PZ+HChYSEhDB37lzCwsI4f/48Xl5eD+2fl5dH165d8fLyYu3atfj5+XHt2jXc3NxMFpPJdZsKl7bBxb/hUiTU7gLcu9qYmaflTla+odqqIJSKJhdpyzgUwCJtD7q0aYmflXbEl1SpR0ceOXKEqKgoABo1akTz5s2Nfo45c+YwbNgwwyTvhQsXsnHjRhYvXkxERMRD+y9evJjk5GT27t1rGBwbGBhY2l+hbHjWgVbD4MC3+qETNTqASo29jQovZzsS0nOJTc4SyUt4Mge/Q3EnmgTJjZ+UvdjcsZbcEZmd0SfBCQkJdO7cmVatWjFixAhGjBhBixYt6NKlC4mJiSV+nry8PI4cOUJoaOi9YJRKQkND2bdvX5HHrF+/njZt2vDee+/h7e1N48aNmT59+iOvfubm5spfZ7/DJ+DgDolRcHSpYbNhgraYJiQ8icwkpJ1fAPClpjf92jekspVM8XkSRiev999/n/T0dM6cOUNycjLJycmcPn2atLQ0RowY8fgnuCspKQmtVou3d+GiaN7e3sTFxRV5zJUrV1i7di1arZZNmzbx2WefMXv2bKZOnVrs68yYMQNXV1fDzd/fv8QxmoyjB3S8W353+zTITgG4d8VRdNoLT2L7dBS5aZzWBfKPXShvtK8hd0RlwujktXnzZr755hsaNLi3NHjDhg1ZsGABf/31l0mDe5BOp8PLy4vvvvuOFi1a0KdPH8aOHcvChQuLPWb06NGkpqYabrJVgW35OnjWhazb+sKF3Gt5iYGqQqnFn0U6sgSAKfkDeKdTHZytoCKEKZRqetCDk7FBP4BVp9OV+Hk8PT1RqVTExxdeNiw+Ph4fH58ij6latSp169ZFpbo3NqVBgwbExcWRl5dX5DF2dna4uLgUuslCZQPdpunv718Ity/fmyIkThuF0pAk2DIWhaTjL20rYl2a81rr6nJHVWZKVRLngw8+4ObNm4ZtN27cYOTIkXTp0qXEz2Nra0uLFi2IjIw0bNPpdERGRtKmTZsij2nXrh2XLl0qlCQvXLhA1apVsbW1gg7vOl2hVhfQ5cPW8YbhEmIBWqFULm6Fy/+Qh5oZmn58GFoXexvrHXRqLKOT19dff01aWhqBgYHUqlWLWrVqUaNGDdLS0pg/f75RzxUeHs6iRYtYunQpUVFRvPPOO2RmZhquPg4cOJDRo0cb9n/nnXdITk7mgw8+4MKFC2zcuJHp06fz3nvF1+S2KAqFfuCqQgXnNlA78xig77DX6aTHHCwI99Hm68suAYs13bHxrMnLVlD91JSMHirh7+/P0aNH2bZtG+fOnQP0p273XzUsqT59+pCYmMj48eOJi4ujWbNmbN682dCJHxMTg1J5L7/6+/vz999/M3LkSJo2bYqfnx8ffPABn376qdGvLRuvBtByCBz6nip7J2GjjCBPqyMhPdcqZ/YLMjm8GG5f5LbkwgLNS3zerV65G0H/OApJkirUV35aWhqurq6kpqbK1/+VeRvmNYfcVGbYvMf/pbfjl7fb0NzfjXNx6RyLTeHYtTtcSEhnYJtAereU4QqpYLmykvWfn5wUxuQP5ZTPy6wf3q7czNIo6d9oqQapRkZGEhkZSUJCwkOd9IsXLy7NU1YsTpX1Y7+2jOUd7Qp+5ik+WXuS+LQcsvIKj1lbcSBGJC+hsJ1fQE4K56QAVms78mP3euUmcRnD6HbmpEmT6NatG5GRkSQlJXHnzp1CN6GEgt8Ej5q46e7wrvoPopMyycrT4mynpn0dT55trL/iqqtYDWPhcZIuwqFFAEzJ709wTS+eru0pc1DyMLrltXDhQn788UcGDBhgjngqDrUtdJ0Cq/vzls1mAkLfpV6DxtSuUgmlUsGO8wn8dToOnSSRnadla1Q8nepVqTBjeIRibBkHOg3btE+xR9eE3ypoqwtK0fLKy8ujbdu25oil4qnfAwLbo5by6Jn4HXW9nVEq9R/Egg+kTgc/77/GiJXHmL3lgpzRCnK7/A9c2IxOoWaapj91vCpZxeKw5mJ08nrjjTdYsWKFOWKpeBQKCJsOKODMOojZb3jobg5DJ0kkZ+kH4G46dUsMqaiotBr9xH7gXMCrREtVH1oXoaIx+rQxJyeH7777jm3bttG0adOHRtvPmTPHZMFVCFWbwlMD4OhPsHk0vBEJSiXKuy0vSQLt3YSVkJ7LsdgUWlSvuN+2FdbRpZBwFhzc+dPtNSCZGiJ5GefkyZM0a9YM4KGFOCrqufcT6zQOTq+Dm0fh1BoIetWwsLFOkgzJC2DLmTiRvCqanFT9hH6AjmM4H6X/sy1YvLiiMjp5bd++3RxxVGzO3tD+I4icBNsmQYOehpbXg8nr7zNxRDxbX3xRVCT/fqmf0O9ZF1oO4equPUDhtUArooo1JNeStX4X3AIg/SbsmVfotFFz31i6q7ezuBCfIVeUQllLvqKfyA/QbRoaVIYSStUr+GmjSF6WwsZev9o2wJ7/YZd1CwAJCrW8QN/6EiqIreP1E/lrdYE6XbmZkkO+VsJWraSqla1wbWoieVmShi9BQBvQZFPt6JdA4dPGwLt9HCJ5VRDRuyDqT/1E/rBpoFBw9XYmANU9HA3DaioqkbwsiWHoBFS+/BtBikvoJAnN3eQV1sgHpQLO3EwjPi1HzkgFc9NpDVUjaDlEP6EfDMmrog+TgFIkr3///ReNRvPQdo1Gw7///muSoCo0v6cgqC8An9n8jE57r+VVxdnO8KG9KPq9yrfjKyDuJNi5Qkd9EtPqJHZfTALutcIrMqOTV6dOnYpcLTs1NZVOnTqZJKgKr8t4dGoHWiov0CJzBxtO6vu/bNVKanpWAiA6SSSvcis3Hf6Zor/f4RNwqkxWnoa3lh1hy1l95eGO9R5eGrCiMTp5FbdA6u3btx9aPVsoJRdfbjd7F4BPVCtQ63IJDvSga0NvalbRv8eXEzPljFAwp91fQUY8eNSE4DdJSM+hz//tZ1tUPLZqJQv6PUW7CjoZ+34lHuf18ssvA/qBqIMHD8bO7t7SSlqtlpMnT4o5jybk0fUjUk8so1p+AnueicLzuV4A1Lx72nglSSSvciklBvZ+rb/fbSoXbucyZMkhbqRk4+Fky6KBLWhR3UPeGC1EiZOXq6sroG95OTs74+BwbzVeW1tbWrduzbBhw0wfYQWlsnPCtec0WDcMz2NfQ/vXwdmHmlX0p41XEsVpY7m0dQJocyGwPXtVwbz17V7SczTU8HTixyGtqF7BB6ber8TJa8kS/fJKgYGBjBo1SpwiloXGr8CBhXDjiL4P5MUFhtPGGynZ5ORrK9SCC+VezAH9BH0UbA34kHeWHEKjk2gV6M53A1riLlZVL8ToPq8JEyaIxFVWlEroPlN//9hyuHWCyk62ONurkSS4dlusOlRu6HSwOQKAU14vMGxLLhqdRM8gX5YNDRGJqwhGJ6/4+HgGDBiAr68varUalUpV6CaYmH8wNP4PIMHmMShAnDqWR6d+gZtHyVE6MiSmOwDvdarF//o0E63rYhg9MXvw4MHExMTw2WefUbVqVTFBuCyEToRzG+Habji3gVqe/pyITTF02l9KyODk9RR6NfcT/x/WKC8T3dYJKIH/5b7AHaUbn/dqTJ9WAXJHZtGMTl67d+9m165dhrI4QhlwC4A2w2HXLNgyjtqN9MUgryRmkqvRMuCHA9xKzcHXzYHWNSvLHKxgrNTIObhm3CJWV4XV6udZ0r8Vz9StIndYFs/o00Z/f38q2GppluHpkVDJG+5cpVPKbwBcScpgzeHr3ErVTxW6GJ8uZ4RCKURfPo/dgXkALLQdxIp3OojEVUJGJ6+5c+cSERHB1atXzRCOUCy7StBlPAB1LyykMqlcTsjg2+2XDLvE3smWKzqhlG7+OgZ78jitbsiI90dR30emtUStkNHJq0+fPuzYsYNatWrh7OyMh4dHoZtgRkH9wKcpqrx0RtqsJS1Hw83UexO0Y5PF1UdrcuHoDtplbQPA8+XZeLs6POYI4X5G93nNnTvXDGEIJVIwdOLH5+ir2s4yTVfOSwE8XduT3ZeSiBHJy3pIEoq7VSMOu4bRsqGYnWIso5PXoEGDzBGHUFKB7aDBC6ii1jNO/TPhdhP5OKweuy8liZaXFYn592fq5J4hS7LDq9d0ucOxSqWq53X58mXGjRtH3759SUhIAOCvv/7izJkzJg1OKEbXyWgVNrRXnebLoDjqeOvHfaXlaEjNypc5OOGx8rNx3KWvGrGzSj8CAmvLHJB1Mjp57dy5kyZNmnDgwAHWrVtHRoZ+oOSJEyeYMGGCyQMUiuBRA0UbfdWJjlfn4qiS8KykH4FdUN9csFzxW+bgqYnnpuRB3V5j5Q7HahmdvCIiIpg6dSpbt27F1vbelIXOnTuzf//+RxwpmJLymVHg6Am3L8Gh7/H30BenE/1eFi49Htcj8wHY5vsOtfzEsIjSMjp5nTp1il69ej203cvLi6SkJJMEJZSAvQt0Hqe/v2Mm9Vz0p4ui38uyabZNxl6XzTFdbRp2e13ucKya0cnLzc2NW7duPbT92LFj+Pn5mSQooYSeGghejSAnhd4ZywH9qHtBPjG3s/jvwr1su1vxtJBbJ1Cd0P8/fWM3lKeqi9kQT8Lo5PXqq6/y6aefEhcXh0KhQKfTsWfPHkaNGsXAgQPNEaNQHKUKuuuvVDWL/5VaihtsOnWL9BzRaS+XkWuOc+jqHd746XDhB6SCifUSf2jbUj2oY4Vf/edJGZ28pk+fTv369fH39ycjI4OGDRvyzDPP0LZtW8aNG1eqIBYsWEBgYCD29vaEhIRw8ODBEh23atUqFAoFL730Uqlet1yo2RHqPotS0jLNcTXpuRpWHoyRO6oKq9hKH+c2wLXd5Eg2fJ7/Kj2aVi3bwMoho5OXra0tixYt4vLly2zYsIGff/6Zc+fOsWzZslKVxFm9ejXh4eFMmDCBo0ePEhQURFhYmGEIRnGuXr3KqFGjaN++vdGvWe50mwpKNa21h2mvPMni3VfJ0+gef5xgcqnZRbR6Nbmw5TMAvtP2QOnuTzN/t7INrBwq9bqNAQEBPPfcc/Tu3Zs6deqUOoA5c+YwbNgwhgwZQsOGDVm4cCGOjo4sXry42GO0Wi39+/dn0qRJ1KxZs9SvXW541obgNwGYaLucxLRMIn49yY0UMdexrOmKqllw4P/gTjTJSg8Wal4grJGPKF1kAiUaYR8eHs6UKVNwcnIiPDz8kfvOmTOnxC+el5fHkSNHGD16tGGbUqkkNDSUffv2FXvc5MmT8fLyYujQoezatavEr1eudfgETqykVnYsfVX/8POxrvx9Jo4zk7vLHVnFlpmEducXqIDpuf8lC3s6iKoRJlGi5HXs2DHy8/MN94tj7LdJUlISWq0Wb2/vQtu9vb05d+5ckcfs3r2bH374gePHj5foNXJzc8nNzTX8nJaWZlSMVsPBXb846V8fM1K9lvXatqTliXLdsts+DVVeOqd0gfyq1XdxBNcQBQxMoUTJa/v27UXeL2vp6ekMGDCARYsW4elZsnXrZsyYwaRJk8wcmYVoOQQOLaJy0gWGq39nuqY/p2+k0tjPVe7IKoT769y5OthA/Fk48iMAU/IHIKGka0NvUdbZRErd52UKnp6eqFQq4uMLj4mJj4/Hx8fnof0vX77M1atX6dmzJ2q1GrVazU8//cT69etRq9Vcvnz5oWNGjx5Namqq4RYbG2u230d2KhsI0w+dGKzaTHVFHK8s3EtOvlbmwCqGzLx777OznQr+HgOSjs26YA5KDfg4rB4zXm4iY4TlS4laXgULzpbEunXrSryvra0tLVq0IDIy0jDcQafTERkZyfDhwx/av379+pw6darQtnHjxpGens7//vc//P39HzrGzs6u0AK55V6drlCrC7aXIxmjXsFb+eEcvnqHp+uIFZbN7XbGve6J9hyDK9vJV9gwLb8vHepW4b1OYgK2KZUoeRUsOAv6pvFvv/2Gq6srLVu2BODIkSOkpKQYleQKhIeHM2jQIFq2bElwcDBz584lMzOTIUOGADBw4ED8/PyYMWMG9vb2NG7cuNDxbm5uAA9tr9DCpsG3OwhTHaaN9gz/XqwpklcZSMrIA0CNhrdz9VfLf8jvTqzkzcLu9eQMrVwqUfIqWHAW4NNPP6V3794sXLjQMK5Lq9Xy7rvv4uJifAnbPn36kJiYyPjx44mLi6NZs2Zs3rzZ0IkfExODUinr2a318Wpwt//rez5T/8yo863guQZyR1XuFbS8XlNto7p0gzw7D77OeZEank408hX9jqamkIxcTaNKlSrs3r2bevUKf5OcP3+etm3bcvv2bZMGaGppaWm4urqSmppaqmRrNTJvo5vXDGVuGp/kD+OtDyfi5WyHs72N3JGVWysPxjBz3T522o3ETZHJtlqjeeNME15u7secPs3kDs9qlPRv1OgmjUajKXIYw7lz59DpxKhui+FUGWWHTwH4WL2GF2ZvpvvcXYbOe51OEh35JnY7I5cP1b/ipsjkAgH8mKMfGtG8urvMkZVPRpeBHjJkCEOHDuXy5csEBwcDcODAAWbOnGnopxIsRPCb3Pl3IVVyYnlX/QdfprzK+bh0gvzdGPLjIXZdTOTQ2FAqV6pAFzTMKekCA1RbAZiY9xp7L6egUEAbsZamWRidvGbNmoWPjw+zZ882lMapWrUqH3/8MR999JHJAxSegNqW5Haf4R75Jm+o/mKltgsX4vXJa+eFRAA2nrrFwDaB8sZZTnSOmY9aoWOr9in26vQXkF5vV4PaXpVkjqx8Mvq0UalU8sknn3Djxg1SUlJISUnhxo0bfPLJJ6WamC2YV62nexPr2go7RT4R6pWcjyu8MG12njh1NIlLkTTM2Ee+pGK6pj8ANT2d+DhMXGU0lye6jOfi4lK+O73LA4UC/75fIaHgedV+dDGFS3VnieT15LQa+Ftfi/4nbTeq1W7CimEh/DOqoxhNb0ZGnzYCrF27ljVr1hATE0NeXl6hx44ePWqSwAQT8mnC7bp98LywilcSvwbdYMND2aLT/skdXQqJUWQqXfifphcj63vRtpYYV2duRre85s2bx5AhQ/D29ubYsWMEBwdTuXJlrly5wrPPPmuOGAUTcOg+gXTJgYbSZS5F3is3lJWnkTGqciA7BbZPA+BPj0GkUQm1qJBaJoxOXt988w3fffcd8+fPx9bWlk8++YStW7cyYsQIUlNTzRGjYAJOHr58o3kRgEq7p+JADiBOG5/YrlmQdZs89zqMu94KQJR3LiNGJ6+YmBjattUvTe7g4EB6ur4DeMCAAaxcudK00QkmtUb9PLG6Kvgo7vCWegOAGOv1BKTbl9Hu+xaA1e5vobnbC5MvqtiWCaOTl4+PD8nJyYC+mmrBWo3R0dEYOVhfKGPeHm7M0PQF4C3VBny4LVpeT+DOHxGoJA07tU3ZnHevWoSbo+0jjhJMxejk1blzZ9avXw/oB6yOHDmSrl270qdPnyLXcxQsxystqrFJF8JBXT0cFHl8YrOaHecTxZdOaUT/i0fMFjSSkqma19hzST8trmO9KnRv/HA5J8H0jJ7bqNPp0Ol0qNX6JvKqVavYu3cvderU4a233iq0irYlqjBzG4ug00n8evQ68ef2MfzSMABezJ3Mf198iddaV5c5Oiui08J3HSDuFD9pujJeo59ZMvTpGnz2fEOZg7N+ZpnbqNFomDp1KnFxcYZtr776KvPmzeP999+3+MRV0SmVCv7b0h+32iGs1T4DwHibZYz7/RSnb4iLLSV2fDnEnSJL4cRXmv8A0LSaK6O6iQGpZcmo5KVWq/niiy/QaMTldWv235bV+D91f7IkO1ooL9JTuY/n5++WOyzrkJuOFDkFgNl5L3EHF1YOa8364U/jYCsGpJYlo/u8unTpws6dO80Ri1BG7NQqBoW14VtNTwA+tVmFHXmPOUoAYNccFJkJROu8+UkbBkAtL7HQiRyMHmH/7LPPEhERwalTp2jRogVOToX/41544QWTBSeYj51aySJtD/qq/6GaIok3VJsAccHlke5cQ7P3a9TAdE1/8lET8Wx9vJzt5Y6sQjI6eb377rtA0eszKhQKtFpx6d0a2NuoyMGOmfl9mWe7gHfVf5B0cxyevqLjvji6rRNQ6/LYo23EVl0LDo0NpYqzKCckF6NPGwuuNhZ1E4nLejx1t0Deel1bjulq46TIJWfLRHmDsmQx+1Ge/Q2dpGCq5jVAIRKXzERx+ArKz82ByI86sOH99kzOH6DfdvU3dDeKX1S4oklIz2HlwRhy8vJhcwQAq7QdiZKq84xY9Vp2JT5tzM7OJjIykueffx7Qr4d4/0rUKpWKKVOmYG8vzv+tRa0q+iJ5qZWb8UdqW15U7eXmmo/w+zASCeNXQC9vPlpzgl0Xk1CcWMWrN4+hUTsxJ6c3APNebSZvcELJk9fSpUvZuHGjIXl9/fXXNGrUCAcHB0Bfw97X15eRI0eaJ1LBbLo18uHzna8SpjyEX+oR3ho7kcMOT/NUdXcWDWwpd3iy2XUxCQdy6Hj9G1DArqqDSLroSu+W1cQUIAtQ4tPG5cuX8+abbxbatmLFCrZv38727dv58ssvWbNmjckDFMzP182em3jynbYHAGPUK0jPzGTr2fgKPXXI19Wet9Qb8FHcIU7hxdsX9Ws2hNQQNektQYmT16VLl2jS5N7kU3t7+0LrKQYHB3P27FnTRieUiaqu+tbzQs0LxEtuVFcmMFi1GYDcClwhobVnDm+p9NU3JuX2JRd9ayukpoecYQl3lTh5paSkFOrjSkxMJDAw0PCzTqcr9LhgPaq66vsps7DnS00fAIarf6cyqUz68ywxt7PkDE82/01ZjIMijwO6+vyl07e66vs4U83dUebIBDAieVWrVo3Tp08X+/jJkyepVq2aSYISypavm4Ph/q/a9pzSBeKiyCZcvZaVB2OYvilKxuhkcv0IbTK3ATA1Xz80Yvuojqwc1lreuASDEiev5557jvHjx5OTk/PQY9nZ2UyaNIkePXqYNDihbLg73ltFW0LJlLtDJ15V/UM9RQx/n40r7tDySZLg79EArNU+wympJgA1PJ1wdxId9ZaixFcbx4wZw5o1a6hXrx7Dhw+nbt26AJw/f56vv/4ajUbDmDFjzBaoYD4KhYIVb4SQmJGLSqlg+ArYpA3mOdVBxql/Zoi2gv2/nlkHsQfIUdjxRb7+NPqT7qJihKUpcfLy9vZm7969vPPOO0RERBiuQikUCrp27co333yDt7e32QIVzKttbf1qN1qdBBxjhqYvXZRHaa86zTPao0DFaFXrcrNgy3iUwB9OvUnIdmdElzq8/UwtuUMTHmDU3MYaNWqwefNmkpOTuXTpEgC1a9fGw0NcfSkvVHcXj4iVvFmifZa31X8yVr0cTd7HqG3L/wDkDd+N44W062Q7+LAw7zlAopGvi1hUwwKVanqQh4cHwcHBBAcHi8RVDk3rpV+q/mvNiyRKLtRS3uLor7Nljsr8dh45RZeknwH4NPU/RKdJqJQKAjzE1UVLJOY2Cg/pH1KdF4J8ycCROZr/AlDv3NdEXb4qb2BmFvfbOJwUuRzT1Wa9ri29W1Zj44inaVC1YpULtxYieQlF+vTZ+gRVc2W1thNRugBcFVnsX/IJlxMz5A7NLHJjjvJflb7I5uT8ARwe15UvXgmivo9IXJZKJC+hSH5uDvwx/GlOTXqW7x3fAGCAait/bN1Oala+zNGZmCRxbeWHKBUSf2jbckyqg2clUe7G0onkJTySk52akNCX2aptgVqho1nULIImbyE12/oTWEJ6DtvOxnNu+wrqZp8gR7Lh8/xX+b8BLeQOTSgBi0heCxYsIDAwEHt7e0JCQjh48GCx+y5atIj27dvj7u6Ou7s7oaGhj9xfeHJNq7kyXdOPfElFZ9VxnlGe4OT1FPK11j3v8bXvD/DuT/tw2T0ZgO+0Pegf1o6wRmLdRWsge/JavXo14eHhTJgwgaNHjxIUFERYWBgJCQlF7r9jxw769u3L9u3b2bdvH/7+/nTr1o0bN26UceQVh4+LPdFSVZZquwEwTv0zX/0dRZOJf3MpwXr7wC7EZzBYtRlfXRzxkhsLNS/QPMBN7rCEEpI9ec2ZM4dhw4YxZMgQGjZsyMKFC3F0dGTx4sVF7r98+XLeffddmjVrRv369fn+++/R6XRERkaWceQVh5ujLb+83YZ5ml4kS5Woq7xBw1vryMnX8d7yo1ZbNqcyqQxX/w7Al5o+ZGFPEz9XeYMSSkzW5JWXl8eRI0cIDQ01bFMqlYSGhrJv374SPUdWVhb5+fnFjjfLzc0lLS2t0E0wXqtAD3oEN2Tu3UVWw9W/4EIm5+PT+WrrBZmjM15mroZw9VpcFNmc0gXyq7Y943o0wNne5vEHCxZB1uSVlJSEVqt9aFqRt7d3oVW5H+XTTz/F19e3UAK834wZM3B1dTXc/P39nzjuimryi41Yoe3CRZ0fHooMQ6tl3j+X5A2sFO5EH+dV1T8ATM4fSGM/d95oX1PmqARjyH7a+CRmzpzJqlWr+O2334qtnT969GhSU1MNt9jY2DKOsvywUSnRoGaapj8Ag1WbCVTckjkq4605GEPC2nBUColN2mCuuzRj8eBWcoclGEnW5OXp6YlKpSI+Pr7Q9vj4eHx8Hn3FZ9asWcycOZMtW7bQtGnTYvezs7PDxcWl0E0ovQ51q7BD14wd2iBsFVpGq1fKHVKJ5eTrl+bb/PtSntKcIFdSs8n7HXZ83FEsY2aFZE1etra2tGjRolBne0Hne5s2bYo97osvvmDKlCls3ryZli0r7gIRcpj13yAApmr6o5GUhKkO00Z5RuaoHm/KhrPU/2wzO85cZ6x6OQBLtM8SULshdmqVzNEJpSH7aWN4eDiLFi1i6dKlREVF8c4775CZmcmQIUMAGDhwIKNHjzbs//nnn/PZZ5+xePFiAgMDiYuLIy4ujowM671kb00KWiiXpGos13YB4DP1z2w+dV3OsB7rh93RAOxcMZNaylskSi58rXmR9zrVljkyobRkT159+vRh1qxZjB8/nmbNmnH8+HE2b95s6MSPiYnh1q17/SrffvsteXl5vPLKK1StWtVwmzVrlly/QoVzcKw+ac3V/Ic0yZGGymtsXzVX3qCKkZKVxx/H9WMA3UjnA/U6AOZo/ksGjjjZGVUVSrAgCslaB+mUUlpaGq6urqSmpor+rydwLOYOvb7Zy1DVJj6z+ZlEyZUqo0+DvWW9p72+2cOxmBQAJqiXMkT9N1G6AHrkTWdEaD0+DK0rb4DCQ0r6Nyp7y0uwTu53F139SduNKzofqihSSd/2hcxRPawgcdVS3GCAaisAUzSvoUPJB13qyBiZ8KRE8hJKxcfVHlu1knzUTL87dML20EJuRp+TObKijVUvR63QsVXbgr26xnRr6I1CIaqjWjORvIRSsbdRsX1UR5a/EcI23VPs0TbCTpHP0cUjLG660DPKE3RWHSdfUjFd0w+Ab/o/JXNUwpMSyUsoNT83B9rWqkzTam5M0QxAKyl4XnWA0/v+ZsPJm6TlyF82R4WWcWp9aeel2m4k2wewZeQzqFXio2/txP+g8EQUCgXrhz/NOSmA1dqOAEibI3h/xRHeW35U1tgkSaKv6h/qKm+QLFVinqYXb3eoRV1vZ1njEkxDXCcWTGJvRGdemJlKT9V+miqj6aXczbqLz8gSy9K9V0nNzqeKTTbh6l8A+ErzCmlU4rkmolZXeSFaXoJJ+Lo5UNUvgAWaFwH4xGY1Djy8urq55Wt1TFh/hjlbL5C+ZSYeigwu6vxYoe3Ce51qUb2yU5nHJJiHSF6CybSo7s4SbXdidFXwUdzhbfWGMo8h5W59/UDFLQarNgMwVfMa9X3d+TisfpnHI5iPSF6CyYR3q0u7+tWYcfeK3puqDcREl22tr5SsPABGq1diq9CyQxvETl0QS0TViHJHJC/BZFzsbfh+YEv+0gVzQFcfB0UecetGP/5AE7qTlU8b5RnCVIfRSEqmavoTPeM5vFzK/2rfFY1IXoJJKZUKQhv4MCX/NXSSguD0bXD9cJm9/p3MbD67OzRiubYLTn6NxGDUckokL8HkZvcO4rRUk3W69gAc//4dfjtq/iKQkiTxz8qvaKi8RprkyFzNfxjeWUwBKq9E8hJMztXBhpMTu/FFfh+yJDuacYF/1i7kUkK6WV83KSmJUeo1APxP8zKzB3ema0PvxxwlWCuRvASzcLG34bm2zflW0xOACJuV9JizzayvGb9pBlUUqVzR+eDZeTid64vEVZ6J5CWYzcdh9Vik7cENqTJ+itu8odpkvhe7c5W60T8BMF3TnyHPiFI35Z1IXoLZONmpOTixJ5/nvwrAu+o/OHPuvFle69KKj7Alnz3aRkz95GPsbURp5/JOJC/BrFzsbViva8tRXW2cFLmc+fljk1edkK7tpXbiNrSSgimaAfi4OZj0+QXLJJKXYHaTX2zMlPwBALyi+pdt/2wx2XNrtVpO/vAeAKu1HflqRH+TPbdg2UTyEsxuYJtA2nbszu/atigVEh67JoKJWl+xO5YQpLxCuuTAHE1vGlS1rDLUgvmI5CWUiVHd6vFF/qvkSDa04CwLvplD1K20Uj/fgSu3aTv5T+z/nQrAAs2LfDW0q6nCFayASF5CmVAoFOydOYglvADA8/ELeXn+9lI/3+s/HqJP3jp8FHeI0VXBof17tK9TxVThClZAJC+hTJ2sPph4yY3qygQGKP4q9fO45CXwpkpftWKGph9vdWlkqhAFKyGSl1CmvhnSni/uDp14X/078bdijDpep5P4+p+LfGKzCgdFHgd09WnSdYAYGlEBieQllCmFQkHnPiM4qauBsyKbbQs+NOr4P0/eZNvWTfRS7UEnKfjZ9W3e7STmL1ZEInkJZe65pr6GoROvqv5h564dJT727M1UPrNZBsA6XXvGvtHXDBEK1kAkL6HMKRQKJo8YxkZtMCqFhGrLmBIPnagau4kWyotkSXasdhmMj6uo01VRieQlyKJBVRdmavqSK6l5WnWGtau+f+wxOVkZdL35DQDfanqy7MOXzBylYMlE8hJk8/ekgSzWPgtA86jZXI5LLnZfSZLY9N04/BS3uSFVpmnvcaKTvoITyUuQjaOtmvig4SRKLtRS3mL9osnF7nv6/HnC7qwA4PP8V+ncJLCMohQslUhegqwm/rc1szW9AXhds5r89KQi94v7bRxOilxOK+sxd8pUVEpR2rmiE8lLkN37H00iSheAqyKLrd98+NDjt6L20yVHX8gwqd0ElCrxsRVE8hIsgJ9HJWKDxwHQLWsjJ44eMDx2MS6NmJUfoFRI/KFtxzOdnpMrTMHCiOQlWISuPXqzVdsCtUJH8m8fG7b/b8EcQpTnyJFsaDbkK5TidFG4yyKS14IFCwgMDMTe3p6QkBAOHjz4yP1/+eUX6tevj729PU2aNGHTJjOWFxbKhEKhYJqmH3mSik6qE3z17QK0eTl8otAvY7ZM+SLVa9aTOUrBksievFavXk14eDgTJkzg6NGjBAUFERYWRkJCQpH77927l759+zJ06FCOHTvGSy+9xEsvvcTp06fLOHLB1K5KVflJ2w2AHrcWcGrtNAKUiSRI7rwy4kuZoxMsjUIydU1eI4WEhNCqVSu+/vprAHQ6Hf7+/rz//vtEREQ8tH+fPn3IzMxkw4YNhm2tW7emWbNmLFy48LGvl5aWhqurK6mpqbi4iMJ1liQwYiMuZLDDLhwPRYZh+86Gk+jQ+0P5AhPKVEn/RmVteeXl5XHkyBFCQ0MN25RKJaGhoezbt6/IY/bt21dof4CwsLBi98/NzSUtLa3QTbBcaVTiK80rhp/PK2vR9uXhMkYkWCpZk1dSUhJarRZv78Lr63l7exMXF1fkMXFxcUbtP2PGDFxdXQ03f39/0wQvmNwzdfXFBNMavcZZXXU0khJttxnYqNUyRyZYonL/qRg9ejTh4eGGn9PS0kQCs1CLB7UkMSOXqq4ObDy0hhybLJ5q9pTcYQkWStbk5enpiUqlIj4+vtD2+Ph4fHx8ijzGx8fHqP3t7Oyws7MzTcCCWalVSqq66pct69GqvszRCJZO1tNGW1tbWrRoQWRkpGGbTqcjMjKSNm3aFHlMmzZtCu0PsHXr1mL3FwShfJL9tDE8PJxBgwbRsmVLgoODmTt3LpmZmQwZMgSAgQMH4ufnx4wZMwD44IMP6NChA7Nnz6ZHjx6sWrWKw4cP891338n5awiCUMZkT159+vQhMTGR8ePHExcXR7Nmzdi8ebOhUz4mJgal8l4DsW3btqxYsYJx48YxZswY6tSpw++//07jxo3l+hUEQZCB7OO8ypoY5yUIls0qxnkJgiCUlkhegiBYJZG8BEGwSrJ32Je1gi4+MU1IECxTwd/m47rjK1zySk9PBxCj7AXBwqWnp+Pq6lrs4xXuaqNOp+PmzZs4OzujUJRtYbuCqUmxsbHiSudjiPeq5MrbeyVJEunp6fj6+hYaJvWgCtfyUiqVVKtWTdYYXFxcysWHrCyI96rkytN79agWVwHRYS8IglUSyUsQBKskklcZsrOzY8KECaLKRQmI96rkKup7VeE67AVBKB9Ey0sQBKskkpcgCFZJJC9BEKySSF6CIFglkbzMLDk5mf79++Pi4oKbmxtDhw4lIyPjkfu///771KtXDwcHBwICAhgxYgSpqallGHXZECull5wx79WiRYto37497u7uuLu7Exoa+tj31ipJgll1795dCgoKkvbv3y/t2rVLql27ttS3b99i9z916pT08ssvS+vXr5cuXbokRUZGSnXq1JH+85//lGHU5rdq1SrJ1tZWWrx4sXTmzBlp2LBhkpubmxQfH1/k/nv27JFUKpX0xRdfSGfPnpXGjRsn2djYSKdOnSrjyMuese9Vv379pAULFkjHjh2ToqKipMGDB0uurq7S9evXyzhy8xLJy4zOnj0rAdKhQ4cM2/766y9JoVBIN27cKPHzrFmzRrK1tZXy8/PNEaYsgoODpffee8/ws1arlXx9faUZM2YUuX/v3r2lHj16FNoWEhIivfXWW2aN0xIY+149SKPRSM7OztLSpUvNFaIsxGmjGe3btw83Nzdatmxp2BYaGopSqeTAgQMlfp6CcrjqcrL4almslF5elOa9elBWVhb5+fl4eHiYK0xZiORlRnFxcXh5eRXaplar8fDwKHaF7wclJSUxZcoU3nzzTXOEKIuyWCm9vCjNe/WgTz/9FF9f34eSv7UTyasUIiIiUCgUj7ydO3fuiV8nLS2NHj160LBhQyZOnPjkgQsVzsyZM1m1ahW//fYb9vb2codjUuXjPKSMffTRRwwePPiR+9SsWRMfHx8SEhIKbddoNCQnJxe7wneB9PR0unfvjrOzM7/99hs2NjZPGrbFKIuV0suL0rxXBWbNmsXMmTPZtm0bTZs2NWeY8pC70608K+iwP3z4sGHb33///dgO+9TUVKl169ZShw4dpMzMzLIItcwFBwdLw4cPN/ys1WolPz+/R3bYP//884W2tWnTpsJ02BvzXkmSJH3++eeSi4uLtG/fvrIIURYieZlZ9+7dpebNm0sHDhyQdu/eLdWpU6fQUInr169L9erVkw4cOCBJkj5xhYSESE2aNJEuXbok3bp1y3DTaDRy/Romt2rVKsnOzk768ccfpbNnz0pvvvmm5ObmJsXFxUmSJEkDBgyQIiIiDPvv2bNHUqvV0qxZs6SoqChpwoQJFWqohDHv1cyZMyVbW1tp7dq1hT4/6enpcv0KZiGSl5ndvn1b6tu3r1SpUiXJxcVFGjJkSKEPUXR0tARI27dvlyRJkrZv3y4BRd6io6Pl+SXMZP78+VJAQIBka2srBQcHS/v37zc81qFDB2nQoEGF9l+zZo1Ut25dydbWVmrUqJG0cePGMo5YPsa8V9WrVy/y8zNhwoSyD9yMREkcQRCskrjaKAiCVRLJSxAEqySSlyAIVkkkL0EQrJJIXoIgWCWRvARBsEoieQmCYJVE8hIEwSqJ5CVYhMGDBxdZnaN79+5yhyZYKFFVQrAY3bt3Z8mSJYW2FbcKdH5+/kOVNvLy8rC1tTX6dUt7nCAv0fISLIadnR0+Pj6Fbu7u7gAoFAq+/fZbXnjhBZycnJg2bRoTJ06kWbNmfP/999SoUcNQryomJoYXX3yRSpUq4eLiQu/evQuVlCnuOMG6iOQlWI2JEyfSq1cvTp06xeuvvw7ApUuX+PXXX1m3bh3Hjx9Hp9Px4osvkpyczM6dO9m6dStXrlyhT58+hZ7rweME6yNOGwWLsWHDBipVqlRo25gxYxgzZgwA/fr1Y8iQIYUez8vL46effqJKlSoAbN26lVOnThEdHY2/vz8AP/30E40aNeLQoUO0atWqyOME6yOSl2AxOnXqxLffflto2/2LRty/kEmB6tWrF0pAUVFR+Pv7GxIXQMOGDXFzcyMqKsqQvB48TrA+InkJFsPJyYnatWs/8vGSbCvpawnWTfR5CeVKgwYNiI2NJTY21rDt7NmzpKSk0LBhQxkjE0xNtLwEi5Gbm/vQcl5qtRpPT88SP0doaChNmjShf//+zJ07F41Gw7vvvkuHDh2KPO0UrJdoeQkWY/PmzVStWrXQ7emnnzbqORQKBX/88Qfu7u4888wzhIaGUrNmTVavXm2mqAW5iDLQgiBYJdHyEgTBKonkJQiCVRLJSxAEqySSlyAIVkkkL0EQrJJIXoIgWCWRvARBsEoieQmCYJVE8hIEwSqJ5CUIglUSyUsQBKskkpcgCFbp/wG7hDKJXXKHngAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.016118, + "end_time": "2024-03-26T12:56:36.178869", + "exception": false, + "start_time": "2024-03-26T12:56:36.162751", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 642.1685, + "end_time": "2024-03-26T12:56:38.916310", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/lct_gan/2/mlu-eval-load.ipynb", + "output_path": "eval/treatment/lct_gan/2/mlu-eval-load.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/treatment/lct_gan/2", + "path_prefix": "../../../../", + "random_seed": 2, + "single_model": "lct_gan" + }, + "start_time": "2024-03-26T12:45:56.747810", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/lct_gan/mlu-eval.ipynb b/treatment/lct_gan/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..d92383f779915f49bcfd2b91c46885c0c228ec3c --- /dev/null +++ b/treatment/lct_gan/mlu-eval.ipynb @@ -0,0 +1,2279 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:21.091940Z", + "iopub.status.busy": "2024-03-24T16:58:21.091560Z", + "iopub.status.idle": "2024-03-24T16:58:21.125274Z", + "shell.execute_reply": "2024-03-24T16:58:21.124389Z" + }, + "papermill": { + "duration": 0.048756, + "end_time": "2024-03-24T16:58:21.127396", + "exception": false, + "start_time": "2024-03-24T16:58:21.078640", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:21.153154Z", + "iopub.status.busy": "2024-03-24T16:58:21.152724Z", + "iopub.status.idle": "2024-03-24T16:58:21.159407Z", + "shell.execute_reply": "2024-03-24T16:58:21.158556Z" + }, + "papermill": { + "duration": 0.02187, + "end_time": "2024-03-24T16:58:21.161410", + "exception": false, + "start_time": "2024-03-24T16:58:21.139540", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:21.185026Z", + "iopub.status.busy": "2024-03-24T16:58:21.184745Z", + "iopub.status.idle": "2024-03-24T16:58:21.188767Z", + "shell.execute_reply": "2024-03-24T16:58:21.188008Z" + }, + "papermill": { + "duration": 0.018246, + "end_time": "2024-03-24T16:58:21.190701", + "exception": false, + "start_time": "2024-03-24T16:58:21.172455", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:21.214804Z", + "iopub.status.busy": "2024-03-24T16:58:21.214488Z", + "iopub.status.idle": "2024-03-24T16:58:21.218468Z", + "shell.execute_reply": "2024-03-24T16:58:21.217617Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018126, + "end_time": "2024-03-24T16:58:21.220375", + "exception": false, + "start_time": "2024-03-24T16:58:21.202249", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:21.244151Z", + "iopub.status.busy": "2024-03-24T16:58:21.243435Z", + "iopub.status.idle": "2024-03-24T16:58:21.249282Z", + "shell.execute_reply": "2024-03-24T16:58:21.248474Z" + }, + "papermill": { + "duration": 0.019829, + "end_time": "2024-03-24T16:58:21.251146", + "exception": false, + "start_time": "2024-03-24T16:58:21.231317", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "23897d82", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:21.276242Z", + "iopub.status.busy": "2024-03-24T16:58:21.275955Z", + "iopub.status.idle": "2024-03-24T16:58:21.281213Z", + "shell.execute_reply": "2024-03-24T16:58:21.280483Z" + }, + "papermill": { + "duration": 0.019941, + "end_time": "2024-03-24T16:58:21.283085", + "exception": false, + "start_time": "2024-03-24T16:58:21.263144", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"lct_gan\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 2\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/lct_gan/2\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011005, + "end_time": "2024-03-24T16:58:21.305163", + "exception": false, + "start_time": "2024-03-24T16:58:21.294158", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:21.329084Z", + "iopub.status.busy": "2024-03-24T16:58:21.328491Z", + "iopub.status.idle": "2024-03-24T16:58:21.337905Z", + "shell.execute_reply": "2024-03-24T16:58:21.337083Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023484, + "end_time": "2024-03-24T16:58:21.339846", + "exception": false, + "start_time": "2024-03-24T16:58:21.316362", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/lct_gan/2\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:21.363973Z", + "iopub.status.busy": "2024-03-24T16:58:21.363391Z", + "iopub.status.idle": "2024-03-24T16:58:23.430650Z", + "shell.execute_reply": "2024-03-24T16:58:23.429721Z" + }, + "papermill": { + "duration": 2.081592, + "end_time": "2024-03-24T16:58:23.432857", + "exception": false, + "start_time": "2024-03-24T16:58:21.351265", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:23.461726Z", + "iopub.status.busy": "2024-03-24T16:58:23.461209Z", + "iopub.status.idle": "2024-03-24T16:58:23.477666Z", + "shell.execute_reply": "2024-03-24T16:58:23.476963Z" + }, + "papermill": { + "duration": 0.033074, + "end_time": "2024-03-24T16:58:23.479701", + "exception": false, + "start_time": "2024-03-24T16:58:23.446627", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:23.504469Z", + "iopub.status.busy": "2024-03-24T16:58:23.503790Z", + "iopub.status.idle": "2024-03-24T16:58:23.512173Z", + "shell.execute_reply": "2024-03-24T16:58:23.511249Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.022969, + "end_time": "2024-03-24T16:58:23.514411", + "exception": false, + "start_time": "2024-03-24T16:58:23.491442", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:23.538311Z", + "iopub.status.busy": "2024-03-24T16:58:23.538052Z", + "iopub.status.idle": "2024-03-24T16:58:23.634368Z", + "shell.execute_reply": "2024-03-24T16:58:23.633426Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.111187, + "end_time": "2024-03-24T16:58:23.636770", + "exception": false, + "start_time": "2024-03-24T16:58:23.525583", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:23.662750Z", + "iopub.status.busy": "2024-03-24T16:58:23.662424Z", + "iopub.status.idle": "2024-03-24T16:58:28.318470Z", + "shell.execute_reply": "2024-03-24T16:58:28.317516Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.671873, + "end_time": "2024-03-24T16:58:28.320974", + "exception": false, + "start_time": "2024-03-24T16:58:23.649101", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-24 16:58:25.896764: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-24 16:58:25.896835: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-24 16:58:25.898441: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:28.346120Z", + "iopub.status.busy": "2024-03-24T16:58:28.345522Z", + "iopub.status.idle": "2024-03-24T16:58:28.351523Z", + "shell.execute_reply": "2024-03-24T16:58:28.350811Z" + }, + "papermill": { + "duration": 0.020523, + "end_time": "2024-03-24T16:58:28.353397", + "exception": false, + "start_time": "2024-03-24T16:58:28.332874", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:28.380101Z", + "iopub.status.busy": "2024-03-24T16:58:28.379399Z", + "iopub.status.idle": "2024-03-24T16:58:50.232327Z", + "shell.execute_reply": "2024-03-24T16:58:50.231133Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 21.869167, + "end_time": "2024-03-24T16:58:50.235121", + "exception": false, + "start_time": "2024-03-24T16:58:28.365954", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-24T16:58:50.263419Z", + "iopub.status.busy": "2024-03-24T16:58:50.263077Z", + "iopub.status.idle": "2024-03-24T16:58:50.269795Z", + "shell.execute_reply": "2024-03-24T16:58:50.268943Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.022981, + "end_time": "2024-03-24T16:58:50.271662", + "exception": false, + "start_time": "2024-03-24T16:58:50.248681", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 95,\n", + " 'realtabformer': (69, 281, Embedding(281, 768), True),\n", + " 'lct_gan': 75,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:50.298075Z", + "iopub.status.busy": "2024-03-24T16:58:50.297371Z", + "iopub.status.idle": "2024-03-24T16:58:50.302202Z", + "shell.execute_reply": "2024-03-24T16:58:50.301370Z" + }, + "papermill": { + "duration": 0.0201, + "end_time": "2024-03-24T16:58:50.304171", + "exception": false, + "start_time": "2024-03-24T16:58:50.284071", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:50.330132Z", + "iopub.status.busy": "2024-03-24T16:58:50.329838Z", + "iopub.status.idle": "2024-03-24T16:58:51.054174Z", + "shell.execute_reply": "2024-03-24T16:58:51.052937Z" + }, + "papermill": { + "duration": 0.740572, + "end_time": "2024-03-24T16:58:51.056917", + "exception": false, + "start_time": "2024-03-24T16:58:50.316345", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/aug_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_bs_test/lct_gan/all inf False\n", + "../../../../ml-utility-loss/bs_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_synth_test/lct_gan/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:51.091666Z", + "iopub.status.busy": "2024-03-24T16:58:51.090805Z", + "iopub.status.idle": "2024-03-24T16:58:51.428833Z", + "shell.execute_reply": "2024-03-24T16:58:51.427875Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.357023, + "end_time": "2024-03-24T16:58:51.431109", + "exception": false, + "start_time": "2024-03-24T16:58:51.074086", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'loss_balancer_beta': 0.75,\n", + " 'loss_balancer_r': 0.96,\n", + " 'tf_pma_low': 16,\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.75,\n", + " 'bias_weight_decay': 0.05,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 2,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.06,\n", + " 'n_warmup_steps': 80,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'lct_gan',\n", + " 'g_loss_mul': 0.2,\n", + " 'd_model': 512,\n", + " 'attn_activation': torch.nn.modules.activation.SELU,\n", + " 'tf_d_inner': 256,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 2048,\n", + " 'ada_n_layers': 6,\n", + " 'ada_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 7,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'single_model': True,\n", + " 'models': ['lct_gan'],\n", + " 'max_seconds': 3600,\n", + " 'Body': 'twin_encoder',\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.5, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "# params = {\n", + "# **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "# }\n", + "params = getattr(PARAMS, dataset_name).BEST_DICT[gp][gp_multiply][single_model]\n", + "if isinstance(params, (list, tuple)):\n", + " params = params[param_index]\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:51.460657Z", + "iopub.status.busy": "2024-03-24T16:58:51.460270Z", + "iopub.status.idle": "2024-03-24T16:58:51.567534Z", + "shell.execute_reply": "2024-03-24T16:58:51.566513Z" + }, + "papermill": { + "duration": 0.124274, + "end_time": "2024-03-24T16:58:51.569590", + "exception": false, + "start_time": "2024-03-24T16:58:51.445316", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_train/lct_gan/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/treatment [400, 0]\n", + "Caching in ../../../../treatment/_cache_aug_val/lct_gan/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/treatment [0, 200]\n", + "Caching in ../../../../treatment/_cache_bs_train/lct_gan/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/treatment [100, 0]\n", + "Caching in ../../../../treatment/_cache_bs_val/lct_gan/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/treatment [0, 50]\n", + "Caching in ../../../../treatment/_cache_synth/lct_gan/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/treatment [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-24T16:58:51.601417Z", + "iopub.status.busy": "2024-03-24T16:58:51.601104Z", + "iopub.status.idle": "2024-03-24T16:58:52.258323Z", + "shell.execute_reply": "2024-03-24T16:58:52.257597Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.676996, + "end_time": "2024-03-24T16:58:52.260402", + "exception": false, + "start_time": "2024-03-24T16:58:51.583406", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['lct_gan'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:52.289735Z", + "iopub.status.busy": "2024-03-24T16:58:52.289329Z", + "iopub.status.idle": "2024-03-24T16:58:52.293910Z", + "shell.execute_reply": "2024-03-24T16:58:52.293109Z" + }, + "papermill": { + "duration": 0.02153, + "end_time": "2024-03-24T16:58:52.295855", + "exception": false, + "start_time": "2024-03-24T16:58:52.274325", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:52.325079Z", + "iopub.status.busy": "2024-03-24T16:58:52.324403Z", + "iopub.status.idle": "2024-03-24T16:58:52.331488Z", + "shell.execute_reply": "2024-03-24T16:58:52.330641Z" + }, + "papermill": { + "duration": 0.023709, + "end_time": "2024-03-24T16:58:52.333547", + "exception": false, + "start_time": "2024-03-24T16:58:52.309838", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "29688705" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:52.361784Z", + "iopub.status.busy": "2024-03-24T16:58:52.361447Z", + "iopub.status.idle": "2024-03-24T16:58:52.458804Z", + "shell.execute_reply": "2024-03-24T16:58:52.457761Z" + }, + "papermill": { + "duration": 0.115145, + "end_time": "2024-03-24T16:58:52.462153", + "exception": false, + "start_time": "2024-03-24T16:58:52.347008", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 75] --\n", + "├─Adapter: 1-1 [2, 2648, 75] --\n", + "│ └─Sequential: 2-1 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 2048] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 2048] 155,648\n", + "│ │ │ └─LeakyHardtanh: 4-2 [2, 2648, 2048] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 2048] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 2048] 4,196,352\n", + "│ │ │ └─LeakyHardtanh: 4-4 [2, 2648, 2048] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 2048] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 2048] 4,196,352\n", + "│ │ │ └─LeakyHardtanh: 4-6 [2, 2648, 2048] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 2048] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 2048] 4,196,352\n", + "│ │ │ └─LeakyHardtanh: 4-8 [2, 2648, 2048] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 2048] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 2048] 4,196,352\n", + "│ │ │ └─LeakyHardtanh: 4-10 [2, 2648, 2048] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 512] 1,049,088\n", + "│ │ │ └─LeakyHardsigmoid: 4-12 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 75] (recursive)\n", + "│ └─Sequential: 2-2 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-7 [2, 661, 2048] (recursive)\n", + "│ │ │ └─Linear: 4-13 [2, 661, 2048] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-14 [2, 661, 2048] --\n", + "│ │ └─FeedForward: 3-8 [2, 661, 2048] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 2048] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-16 [2, 661, 2048] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 2048] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 2048] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-18 [2, 661, 2048] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 2048] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 2048] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-20 [2, 661, 2048] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 2048] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 2048] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-22 [2, 661, 2048] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-24 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 8192] --\n", + "│ └─Encoder: 2-3 [2, 16, 512] --\n", + "│ │ └─ModuleList: 3-14 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-25 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-6 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 2648, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 256] 131,328\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 256] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 131,584\n", + "│ │ │ └─EncoderLayer: 4-26 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-18 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 2648, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 256] 131,328\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 256] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 131,584\n", + "│ │ │ └─EncoderLayer: 4-27 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-30 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 2648, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 256] 131,328\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 256] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 131,584\n", + "│ │ │ └─EncoderLayer: 4-28 [2, 16, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-42 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 2648, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 256] 131,328\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 256] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 131,584\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 512] 8,192\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 16, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 16, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-54 [2, 16, 512] --\n", + "│ └─Encoder: 2-4 [2, 16, 512] (recursive)\n", + "│ │ └─ModuleList: 3-14 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-60 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 661, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 661, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-31 [2, 661, 256] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-72 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 661, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 661, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 661, 256] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-40 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-84 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 32, 661, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 32, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-42 [2, 661, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-43 [2, 661, 256] --\n", + "│ │ │ │ │ └─Linear: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-45 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-46 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 32, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-96 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 32, 661, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 32, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-48 [2, 661, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-49 [2, 661, 256] --\n", + "│ │ │ │ │ └─Linear: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-18 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-51 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-52 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 32, 16, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 32, 16, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-108 [2, 16, 512] --\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-15 [2, 128] --\n", + "│ │ │ └─Linear: 4-33 [2, 128] 1,048,704\n", + "│ │ │ └─LeakyHardsigmoid: 4-34 [2, 128] --\n", + "│ │ └─FeedForward: 3-16 [2, 128] --\n", + "│ │ │ └─Linear: 4-35 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-36 [2, 128] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 1] --\n", + "│ │ │ └─Linear: 4-45 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 29,688,705\n", + "Trainable params: 29,688,705\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 116.20\n", + "========================================================================================================================\n", + "Input size (MB): 1.99\n", + "Forward/backward pass size (MB): 1231.11\n", + "Params size (MB): 118.75\n", + "Estimated Total Size (MB): 1351.85\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T16:58:52.497718Z", + "iopub.status.busy": "2024-03-24T16:58:52.497372Z", + "iopub.status.idle": "2024-03-24T18:08:21.596264Z", + "shell.execute_reply": "2024-03-24T18:08:21.595395Z" + }, + "papermill": { + "duration": 4169.120934, + "end_time": "2024-03-24T18:08:21.598862", + "exception": false, + "start_time": "2024-03-24T16:58:52.477928", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.2\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.04985262780790322, 'avg_role_model_std_loss': 3.6183568816933374, 'avg_role_model_mean_pred_loss': 0.00821856617848477, 'avg_role_model_g_mag_loss': 0.08611974312061244, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.21120899681122424, 'n_size': 900, 'n_batch': 450, 'duration': 569.3217799663544, 'duration_batch': 1.265159511036343, 'duration_size': 0.6325797555181715, 'avg_pred_std': 0.11481869970826059}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.02072129439680187, 'avg_role_model_std_loss': 1.9086491901219171, 'avg_role_model_mean_pred_loss': 0.0010903308281523874, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.02072129439680187, 'n_size': 450, 'n_batch': 225, 'duration': 196.6758050918579, 'duration_batch': 0.8741146892971463, 'duration_size': 0.43705734464857315, 'avg_pred_std': 0.09171225249049408}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.013151365245423803, 'avg_role_model_std_loss': 0.6379314760027313, 'avg_role_model_mean_pred_loss': 0.0008179183341412702, 'avg_role_model_g_mag_loss': 0.19476993432928189, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.013397588028950243, 'n_size': 900, 'n_batch': 450, 'duration': 571.9026720523834, 'duration_batch': 1.2708948267830742, 'duration_size': 0.6354474133915371, 'avg_pred_std': 0.20376968434217915}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.009131540145187592, 'avg_role_model_std_loss': 0.9951137590999442, 'avg_role_model_mean_pred_loss': 0.00024677024046790486, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009131540145187592, 'n_size': 450, 'n_batch': 225, 'duration': 198.25886011123657, 'duration_batch': 0.8811504893832737, 'duration_size': 0.44057524469163684, 'avg_pred_std': 0.10169168976094321}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.008621372499677054, 'avg_role_model_std_loss': 0.8382067257209166, 'avg_role_model_mean_pred_loss': 0.00026993465244648865, 'avg_role_model_g_mag_loss': 0.17589655292686074, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008767833580902504, 'n_size': 900, 'n_batch': 450, 'duration': 570.2824792861938, 'duration_batch': 1.2672943984137641, 'duration_size': 0.6336471992068821, 'avg_pred_std': 0.1927573654050098}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.008410379777208779, 'avg_role_model_std_loss': 0.9130473620787616, 'avg_role_model_mean_pred_loss': 0.0003690813285744622, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008410379777208779, 'n_size': 450, 'n_batch': 225, 'duration': 195.47286939620972, 'duration_batch': 0.8687683084275988, 'duration_size': 0.4343841542137994, 'avg_pred_std': 0.11685342669800876}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0067645793513913645, 'avg_role_model_std_loss': 0.7113772220321888, 'avg_role_model_mean_pred_loss': 0.00023273553968767465, 'avg_role_model_g_mag_loss': 0.16243632634791236, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006880470157437446, 'n_size': 900, 'n_batch': 450, 'duration': 570.8296418190002, 'duration_batch': 1.268510315153334, 'duration_size': 0.634255157576667, 'avg_pred_std': 0.20007731687348698}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007127348931724993, 'avg_role_model_std_loss': 1.0058974532183806, 'avg_role_model_mean_pred_loss': 0.00017908157135019585, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007127348931724993, 'n_size': 450, 'n_batch': 225, 'duration': 198.4686758518219, 'duration_batch': 0.8820830037858751, 'duration_size': 0.44104150189293756, 'avg_pred_std': 0.1067870712840603}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0054238730589898565, 'avg_role_model_std_loss': 0.7306501947289327, 'avg_role_model_mean_pred_loss': 0.00017920905095154637, 'avg_role_model_g_mag_loss': 0.14884096833528018, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005523487775589094, 'n_size': 900, 'n_batch': 450, 'duration': 574.5667185783386, 'duration_batch': 1.2768149301740859, 'duration_size': 0.6384074650870429, 'avg_pred_std': 0.19242201213822732}\n", + "Time out: 3650.911562681198/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'lct_gan', 'n_size': 1050, 'n_batch': 525, 'role_model_metrics': {'avg_loss': 0.005483000135315382, 'avg_g_mag_loss': 0.12952380770728802, 'avg_g_cos_loss': 0.0, 'pred_duration': 11.623510837554932, 'grad_duration': 10.326574325561523, 'total_duration': 21.950085163116455, 'pred_std': 0.23299649357795715, 'std_loss': 0.0016777205746620893, 'mean_pred_loss': 0.00011530155461514369, 'pred_rmse': 0.0740472823381424, 'pred_mae': 0.0485844723880291, 'pred_mape': 6742932.5, 'grad_rmse': 0.16324253380298615, 'grad_mae': 0.10115493088960648, 'grad_mape': 1.8389217853546143}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.005483000135315382, 'avg_g_mag_loss': 0.12952380770728802, 'avg_g_cos_loss': 0.0, 'avg_pred_duration': 11.623510837554932, 'avg_grad_duration': 10.326574325561523, 'avg_total_duration': 21.950085163116455, 'avg_pred_std': 0.23299649357795715, 'avg_std_loss': 0.0016777205746620893, 'avg_mean_pred_loss': 0.00011530155461514369}, 'min_metrics': {'avg_loss': 0.005483000135315382, 'avg_g_mag_loss': 0.12952380770728802, 'avg_g_cos_loss': 0.0, 'pred_duration': 11.623510837554932, 'grad_duration': 10.326574325561523, 'total_duration': 21.950085163116455, 'pred_std': 0.23299649357795715, 'std_loss': 0.0016777205746620893, 'mean_pred_loss': 0.00011530155461514369, 'pred_rmse': 0.0740472823381424, 'pred_mae': 0.0485844723880291, 'pred_mape': 6742932.5, 'grad_rmse': 0.16324253380298615, 'grad_mae': 0.10115493088960648, 'grad_mape': 1.8389217853546143}, 'model_metrics': {'lct_gan': {'avg_loss': 0.005483000135315382, 'avg_g_mag_loss': 0.12952380770728802, 'avg_g_cos_loss': 0.0, 'pred_duration': 11.623510837554932, 'grad_duration': 10.326574325561523, 'total_duration': 21.950085163116455, 'pred_std': 0.23299649357795715, 'std_loss': 0.0016777205746620893, 'mean_pred_loss': 0.00011530155461514369, 'pred_rmse': 0.0740472823381424, 'pred_mae': 0.0485844723880291, 'pred_mape': 6742932.5, 'grad_rmse': 0.16324253380298615, 'grad_mae': 0.10115493088960648, 'grad_mape': 1.8389217853546143}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T18:08:21.632493Z", + "iopub.status.busy": "2024-03-24T18:08:21.632158Z", + "iopub.status.idle": "2024-03-24T18:08:21.636519Z", + "shell.execute_reply": "2024-03-24T18:08:21.635628Z" + }, + "papermill": { + "duration": 0.023634, + "end_time": "2024-03-24T18:08:21.638606", + "exception": false, + "start_time": "2024-03-24T18:08:21.614972", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T18:08:21.670342Z", + "iopub.status.busy": "2024-03-24T18:08:21.670029Z", + "iopub.status.idle": "2024-03-24T18:08:21.861324Z", + "shell.execute_reply": "2024-03-24T18:08:21.859735Z" + }, + "papermill": { + "duration": 0.211485, + "end_time": "2024-03-24T18:08:21.865314", + "exception": false, + "start_time": "2024-03-24T18:08:21.653829", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T18:08:21.911240Z", + "iopub.status.busy": "2024-03-24T18:08:21.910286Z", + "iopub.status.idle": "2024-03-24T18:08:22.182529Z", + "shell.execute_reply": "2024-03-24T18:08:22.181588Z" + }, + "papermill": { + "duration": 0.300297, + "end_time": "2024-03-24T18:08:22.184679", + "exception": false, + "start_time": "2024-03-24T18:08:21.884382", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAESCAYAAABU2qhcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvwUlEQVR4nO3de1RU19kG8OfMHRAG5D6I4A1BEUi8UEzURieiUQzWJsblCsYv0cZKW0Nt89FGTNusYkyazya6YmOWJukK0TStqTGGGIlojXgXRVEqikLkrmEGBpnr/v4YZmAUlBkGzlze31p7OZw5c+adCTw5+1z25hhjDIQQ4gIEfBdACCEWFEiEEJdBgUQIcRkUSIQQl0GBRAhxGRRIhBCXQYFECHEZIr4LcAaTyYTa2lr4+/uD4zi+yyGE3IUxhtbWVigUCggEve8HeUQg1dbWIjo6mu8yCCEPUFNTg2HDhvX6vEcEkr+/PwDzhw0ICOC5GkLI3dRqNaKjo61/q73xiECydNMCAgIokAhxYQ86pEIHtQkhLoMCiRDiMiiQCCEuwyOOIZH+MRqN0Ov1fJdB3JhYLIZQKOz3diiQvBhjDPX19WhpaeG7FOIBAgMDERER0a9rASmQvJgljMLCwuDr60sXlRKHMMbQ3t6OxsZGAEBkZKTD2/K6QKpq1uD89y14MiWK71J4ZTQarWEUHBzMdznEzfn4+AAAGhsbERYW5nD3zasC6cYtDR57sxhiIYcZcaEI9JXwXRJvLMeMfH19ea6EeArL75Jer3c4kLzqLFtMsB/iI/yhNzIUXqjnuxyXQN004izO+F3yqkACgAUpCgDAnnO1PFdCCLmb1wVSRpI5kEqu3UKjuoPnaggh3XldIEUP9cXDwwPBGLD3fB3f5RA3xHEcPv/8c77LcKpXX30VKSkpfJfhfYEEABnJ5r2kL85Tt424pw8++ACBgYFO297atWtRVFTktO05yisDaV5SJAQccLa6BTW32/kuh5ABo9Pp+rTekCFDXOLyD68MpDB/GdJGmb98OrjdhTGGdp1h0Ju9kycXFhbi0UcfRWBgIIKDgzF//nxcvXoVADB16lS8/PLLNus3NTVBLBbj8OHDAIC6ujrMmzcPPj4+GDFiBAoKChAbG4tNmzY59L2VlZVh5syZ8PHxQXBwMFauXIm2tjbr88XFxZgyZQr8/PwQGBiIRx55BDdu3AAAnDt3Do899hj8/f0REBCAiRMn4tSpU/d9v+LiYixfvhwqlQocx4HjOLz66qsAgNjYWPzpT39CVlYWAgICsHLlSgDAyy+/jLi4OPj6+mLkyJFYt26dze1Cd3fZnnvuOWRmZuLNN99EZGQkgoODsXr16gG/xcirrkPqLiNJge8qb+GLc7VY/dhovstxCXf0RozL+3rQ37f8j+nwlfT9V1Gj0SAnJwdJSUloa2tDXl4eFi5ciNLSUixduhQbN27Ehg0brKehd+3aBYVCgWnTpgEAsrKy0NzcjOLiYojFYuTk5FivMraXRqNBeno60tLScPLkSTQ2NuKFF15AdnY2PvjgAxgMBmRmZmLFihX45JNPoNPpcOLECWttS5cuxUMPPYR3330XQqEQpaWlEIvF933PqVOnYtOmTcjLy0NFRQUA8x6OxZtvvom8vDysX7/euszf3x8ffPABFAoFysrKsGLFCvj7++O3v/1tr+9z8OBBREZG4uDBg6isrMTixYuRkpKCFStWOPRd9YVDe0hbtmxBbGwsZDIZUlNTceLEiV7X3bZtG6ZNm4agoCAEBQVBqVTesz5jDHl5eYiMjISPjw+USiWuXLniSGl9NjcxEmIhh8v1rbjS0Dqg70Wca9GiRfjJT36C0aNHIyUlBdu3b0dZWRnKy8vx9NNPo7a2FkeOHLGuX1BQgCVLloDjOFy+fBkHDhzAtm3bkJqaiocffhjvv/8+7ty541AtBQUF6OjowEcffYTExETMnDkTmzdvxt///nc0NDRArVZDpVJh/vz5GDVqFBISErBs2TIMHz4cAFBdXQ2lUon4+HiMGTMGTz31FJKTk+/7nhKJBHK5HBzHISIiAhERETaBNHPmTPz617/GqFGjMGrUKADAK6+8gqlTpyI2NhYZGRlYu3YtPv300/u+T1BQEDZv3oz4+HjMnz8f8+bNG/DjTHbvIe3atQs5OTnYunUrUlNTsWnTJqSnp6OiogJhYWH3rF9cXIwlS5Zg6tSpkMlkeP311zF79mxcvHgRUVHm2zc2btyIt99+Gx9++CFGjBiBdevWIT09HeXl5ZDJZP3/lD2Q+4oxIy4UBy41Ys+5Wvx69tgBeR934iMWovyP6by8rz2uXLmCvLw8HD9+HM3NzTCZTADMf9yJiYmYPXs2Pv74Y0ybNg1VVVUoKSnB3/72NwBARUUFRCIRHn74Yev2Ro8ejaCgIIdqv3TpEpKTk+Hn52dd9sgjj8BkMqGiogLTp0/Hc889h/T0dDz++ONQKpV4+umnrfd75eTk4IUXXsDf//53KJVKPPXUU9YQcdSkSZPuWbZr1y68/fbbuHr1Ktra2mAwGB44uur48eNtrriOjIxEWVlZv2p7ELv3kN566y2sWLECy5cvx7hx47B161b4+vpi+/btPa7/8ccf4+c//zlSUlIQHx+P999/HyaTyZq0jDFs2rQJr7zyCp588kkkJSXho48+Qm1tba+nVrVaLdRqtU1zhOVs255ztXYfx/BEHMfBVyIa9GbvFb4ZGRm4ffs2tm3bhuPHj+P48eMAug7gLl26FJ999hn0ej0KCgowYcIETJgwwenfV1/t2LEDJSUlmDp1Knbt2oW4uDgcO3YMgPnYzcWLFzFv3jx8++23GDduHHbv3t2v9+sejgBQUlKCpUuX4oknnsDevXtx9uxZ/P73v3/gAe+7u44cx1nDf6DYFUg6nQ6nT5+GUqns2oBAAKVSiZKSkj5to729HXq9HkOHDgUAVFVVob6+3mabcrkcqampvW4zPz8fcrnc2hydcUSZEA6ZWIAbt9pRdlPl0DbI4Lp16xYqKirwyiuvYNasWUhISMAPP/xgs86TTz6Jjo4OFBYWoqCgAEuXLrU+N3bsWBgMBpw9e9a6rLKy8p5t9FVCQgLOnTsHjUZjXfbdd99BIBBg7Niuve6HHnoIubm5OHr0KBITE1FQUGB9Li4uDi+99BL279+Pn/zkJ9ixY8cD31cikcBoNPapxqNHjyImJga///3vMWnSJIwZM8Z6UN3V2BVIzc3NMBqNCA8Pt1keHh6O+vq+3Rv28ssvQ6FQWAPI8jp7tpmbmwuVSmVtNTU19nwMKz+pCMoE8/vuKaWzbe4gKCgIwcHBeO+991BZWYlvv/0WOTk5Nuv4+fkhMzMT69atw6VLl7BkyRLrc/Hx8VAqlVi5ciVOnDiBs2fPYuXKlfDx8XHoXqylS5dCJpNh2bJluHDhAg4ePIhf/OIXePbZZxEeHo6qqirk5uaipKQEN27cwP79+3HlyhUkJCTgzp07yM7ORnFxMW7cuIHvvvsOJ0+eREJCwgPfNzY2Fm1tbSgqKkJzczPa23u/fGXMmDGorq7Gzp07cfXqVbz99tv93gsbKIN62n/Dhg3YuXMndu/e3a9jQ1Kp1DrDSH9nGlnQ2W3be74OJhN121ydQCDAzp07cfr0aSQmJuKll17CG2+8cc96S5cuxblz5zBt2jTrAWSLjz76COHh4Zg+fToWLlxoPePkyO+kr68vvv76a9y+fRuTJ0/GT3/6U8yaNQubN2+2Pn/58mUsWrQIcXFxWLlyJVavXo2f/exnEAqFuHXrFrKyshAXF4enn34ac+fOxR/+8IcHvu/UqVPx4osvYvHixQgNDcXGjRt7XXfBggV46aWXkJ2djZSUFBw9ehTr1q2z+7MOCmYHrVbLhEIh2717t83yrKwstmDBgvu+9o033mByuZydPHnSZvnVq1cZAHb27Fmb5dOnT2e//OUv+1SXSqViAJhKperT+t116A0scX0hi3l5Lyu52mz3693VnTt3WHl5Obtz5w7fpfCupqaGAWAHDhzguxS3dr/fqb7+jdq1hySRSDBx4kSbU3+WA9RpaWm9vm7jxo3405/+hMLCwnvOAIwYMQIRERE221Sr1Th+/Ph9t+ksUpEQc8ZHAAC+oIskvcK3336LPXv2oKqqCkePHsUzzzyD2NhYTJ8+ne/SvJ7dXbacnBxs27YNH374IS5duoRVq1ZBo9Fg+fLlAMwXneXm5lrXf/3117Fu3Tps374dsbGxqK+vR319vfVKVo7jsGbNGrz22mvYs2cPysrKkJWVBYVCgczMTOd8ygewDEmyr6wOeuPAnkUg/NPr9fjd736H8ePHY+HChQgNDbVeJPnxxx9jyJAhPbbx48cPWo1z587ttY4///nPg1bHYLP7OqTFixejqakJeXl5qK+vR0pKCgoLC60HpaurqyEQdOXcu+++C51Oh5/+9Kc221m/fr31cvff/va30Gg0WLlyJVpaWvDoo4+isLBwwK5BulvayGCEDJGguU2HI5XNeGzsvddTEc+Rnp6O9PSer7dasGABUlNTe3zuQVdQO9P9Lta0nKH2RBxj7n8Bjlqthlwuh0qlcvgAd96/L+Cjkhv4ycNReOvpFOcW6II6OjpQVVWFESNGDFrwE892v9+pvv6NeuXNtT2xnG3bf7EBHfq+Xd9BCHEuCqRODw8PQlSgD9q0Bhy87NiNloSQ/qFA6iQQcJifbL6/iIYkIYQfFEjdWMbb/vZyI1o7aGppQgYbBVI34xUBGBnqB63BhG/KG/guhxCvQ4HUDcdx1oPb1G0jvfHEQf5dBQXSXSxDkhy50ozbmr6NR0zIYHP2IP+AeewyjuPQ0tLi1O3agwLpLqNCh2C8IgAGE8NXF2iaJEIGEwVSD6zdNm8bkoQxQKcZ/EaD/DttkH+tVou1a9ciKioKfn5+SE1NRXFxsfW1N27cQEZGBoKCguDn54fx48dj3759uH79Oh577DEA5iFeOI7Dc88959D30R9eO8j//cxPViD/q8s4cf026lUdiJB7yZXM+nbgz4rBf9/f1QISvwev14kG+e99kP/s7GyUl5dj586dUCgU2L17N+bMmYOysjKMGTMGq1evhk6nw+HDh+Hn54fy8nIMGTIE0dHR+Oc//4lFixahoqICAQEB8PHxceg76Q8KpB5EBfpgUkwQTt34AXvP1+KFaSP5Lol0s2jRIpuft2/fjtDQUOsg/2vWrMGRI0esAdTTIP8nT560jjzx/vvvY8yYMQ7V0n2Qf8vQsZs3b0ZGRgZef/11iMVim0H+AdgMwFZdXY3f/OY3iI+PB4A+1XH3IP/dt7Vjxw5UV1dDoTD/j2Xt2rUoLCzEjh078Oc//xnV1dVYtGiRdUjfkSO7frct98iFhYU5/fhUX1Eg9WJBigKnbvyAL855USCJfc17K3y8rx1okP+elZWVwWg0Ii4uzma5Vqu1TgL5y1/+EqtWrcL+/fuhVCqxaNEiJCUlOfR+A4GOIfXiiQnm2W3Pfa/C9WbNg1/gCTjO3HUa7EaD/DtlkP+2tjYIhUKcPn0apaWl1nbp0iX89a9/BQC88MILuHbtGp599lmUlZVh0qRJeOedd5z2WfuLAqkXIUOkeGR0CABg73kvO7jtwmiQf7OeBvl/6KGHYDQa0djYiNGjR9u07l276OhovPjii/jXv/6FX//619i2bZt1mwD6PHnAQKBAuo8MukjS5dAg/2Y9DfIfFxeHpUuXIisrC//6179QVVWFEydOID8/H19++SUAYM2aNfj6669RVVWFM2fO4ODBg9b3i4mJAcdx2Lt3L5qammzOFA6agRldd3D1Z0zt+2lp17Exv9vHYl7eyy7VOXfbfHPnMbW/+eYblpCQwKRSKUtKSmLFxcUMgM1Y7/v27WMA2PTp0+95fW1tLZs7dy6TSqUsJiaGFRQUsLCwMLZ169Y+vf/d73X+/Hn22GOPMZlMxoYOHcpWrFjBWltbGWOM1dfXs8zMTBYZGckkEgmLiYlheXl5zGg0Mq1Wy5555hkWHR3NJBIJUygULDs7u8//TV588UUWHBzMALD169czxhjT6XQsLy+PxcbGMrFYzCIjI9nChQvZ+fPnGWOMZWdns1GjRjGpVMpCQ0PZs88+y5qbu8aS/+Mf/8giIiIYx3Fs2bJlfarDwhljatMAbQ+w8qNT2F/egJ//eBR+OyfeqdvmEw3Q1uX7779HdHQ0Dhw4gFmzZvFdjtuiAdoGgaXb9sV5mt3WU9Ag/66LAukBlAnh8JUIUXP7DkprWvguhzgBDfLvuug6pAfwkQjx+Lhw/Lu0FnvO1eKh4Y5dr0JcBw3y77ookPogI0mBf5fW4svzdXhl3jgIBfafjSHuwd/fH/7+/nyXgaioKL5L4AV12fpgelwo5D5iNLZqcbzqFt/lOBUdFyPO4ozfJQqkPpCIBJib6Fmz21q6H+3t7TxXQjyF5XepP11b6rL10YJkBXaerMG+snr8YUEiJCL3znKhUIjAwEDrXe6+vr4OXRhICGMM7e3taGxsRGBgIIRCocPbokDqo9SRwQj1l6KpVYsjlU2YGR/Od0n9ZrmdwNGhNwjpLjAw0OYWFUdQIPWRUMBh3oRIfHD0OvaU1npEIHEch8jISISFhUGvp1lWiOPEYnG/9owsKJDssCBFgQ+OXsf+8gbc0RnhI+n/fwBXIBQKnfLLREh/ufeBkEH2UHQghgX5oF1nRNFlmiaJEGejQLIDx3Fdt5J4yNk2QlwJBZKdLBMAHKxogppmtyXEqSiQ7BQf4Y8xYUOgM5jw9YV6vsshxKNQINnJptt2nuZtI8SZKJAcYOm2fVfZjFttWp6rIcRzUCA5IDbED0nD5DCaGPaV0V4SIc5CgeSgjCQab5sQZ6NActD85EhwHHDy+g+obel53BpCiH0okBwUKffB5FjzQFk0TRIhzkGB1A8LaJokQpyKAqkf5iZGQCjgcOGmGteaeJjDihAPQ4HUD8FDpHi0c3bbL87R2TZC+suhQNqyZQtiY2Mhk8mQmpqKEydO9LruxYsXsWjRIsTGxoLjOGzatOmedV599VVwHGfT4uPdYw60rm7bTRoOlpB+sjuQdu3ahZycHKxfvx5nzpxBcnIy0tPTex3kq729HSNHjsSGDRvuO3jT+PHjUVdXZ21HjhyxtzRezB4fDolIgKtNGpTXqfkuhxC3ZncgvfXWW1ixYgWWL1+OcePGYevWrfD19cX27dt7XH/y5Ml444038Mwzz0Aqlfa6XZFIhIiICGsLCQmxtzRe+MvEmDk2DAB12wjpL7sCSafT4fTp01AqlV0bEAigVCpRUlLSr0KuXLkChUKBkSNHYunSpaiuru51Xa1WC7VabdP4tCCla0gS6rYR4ji7Aqm5uRlGoxHh4bbDt4aHh6O+3vE731NTU/HBBx+gsLAQ7777LqqqqjBt2jS0trb2uH5+fj7kcrm1RUdHO/zezjAzPgx+EiFuttzBmeofeK2FEHfmEmfZ5s6di6eeegpJSUlIT0/Hvn370NLSgk8//bTH9XNzc6FSqaytpqZmkCu2JRMLMXu8ZZok6rYR4ii7AikkJARCoRANDbbDtzY0NPR7toHuAgMDERcXh8rKyh6fl0qlCAgIsGl8s5xt23u+DgajiedqCHFPdgWSRCLBxIkTUVRUZF1mMplQVFSEtLQ0pxXV1taGq1evIjIy0mnbHGiPjglBkK8YzW1aHLt2m+9yCHFLdnfZcnJysG3bNnz44Ye4dOkSVq1aBY1Gg+XLlwMAsrKykJuba11fp9OhtLQUpaWl0Ol0uHnzJkpLS232ftauXYtDhw7h+vXrOHr0KBYuXAihUIglS5Y44SMODrFQgLkTzAG659xNnqshxE0xB7zzzjts+PDhTCKRsClTprBjx45Zn5sxYwZbtmyZ9eeqqioG4J42Y8YM6zqLFy9mkZGRTCKRsKioKLZ48WJWWVnZ53pUKhUDwFQqlSMfx2mOVjazmJf3sgnrC1mH3sBrLYS4kr7+jXKMuf95arVaDblcDpVKxevxJKOJYeqGIjSotdiWNQmPj3P/ySQJcYa+/o26xFk2TyEUcJhPA7cR4jAKJCezTABwoLwB7ToDz9UQ4l4okJwseZgcMcG+uKM34sClnu/vI4T0jALJyTiO6xpvu5S6bYTYgwJpAFjubTv030ao2ml2W0L6igJpAMSF+2NsuD/0RoavL9LstoT0FQXSALHsJdHZNkL6jgJpgFiOIx292ozG1g6eqyHEPVAgDZDhwb5Ijg6EiQFflVG3jZC+oEAaQDRNEiH2oUAaQPOTzLPbnr7xA77/oZ3vcghxeRRIAyg8QIYfjQgGQAO3EdIXFEgDzHIryRfUbSPkgSiQBtjcxAiIBBzK69SobKTZbQm5HwqkARbkJ8H0uFAAdHCbkAehQBoEGcnmkSRpmiRC7o8CaRA8Pi4CUpEAVc0aXKyl2W0J6Q0F0iAYIhVBmWAePZK6bYT0jgJpkHQ/22YyUbeNkJ5QIA2SH48Nhb9UhDpVB07T7LaE9IgCaZB0n92WBm4jpGcUSIPIMiTJvjKa3ZaQnlAgDaKpo4Ix1E+CWxodvrt6i+9yCHE5FEiDSCwU4IkJ5m4b3UpCyL0okAbZguQoAMDXF+rRoTfyXA0hroUCaZBNiglCpFyGVq0BxRVNfJdDiEuhQBpkAgGH+Umdt5Kcp24bId1RIPHA0m0rutQAjZZmtyXEggKJB4lRARgR4ocOvQnflDfwXQ4hLoMCiQfm2W3N3Ta6t42QLhRIPLFcJHn4v01oadfxXA0hroECiSejw/yREBkAg4nhqws0TRIhAAUSr6zTJNG9bYQAoEDileX0/7GqW2hU0+y2hFAg8Sh6qC8eHh4IxoC952maJEIokHhGs9sS0oUCiWdPJEVCwAGlNS2ovkWz2xLvRoHEszB/GdJGdc5uS7eSEC9HgeQCFtDstoQAoEByCXPGR0Is5HC5vhX/bWjluxxCeONQIG3ZsgWxsbGQyWRITU3FiRMnel334sWLWLRoEWJjY8FxHDZt2tTvbXoaua8YMzpnt6W9JOLN7A6kXbt2IScnB+vXr8eZM2eQnJyM9PR0NDY29rh+e3s7Ro4ciQ0bNiAiIsIp2/REGd3OttHstsRrMTtNmTKFrV692vqz0WhkCoWC5efnP/C1MTEx7P/+7/+cuk3GGFOpVAwAU6lUfVrfFWm0ehb/ylcs5uW9rLT6B77LIcSp+vo3atcekk6nw+nTp6FUKq3LBAIBlEolSkpKHApER7ap1WqhVqttmrvzlYigHEez2xLvZlcgNTc3w2g0Ijw83GZ5eHg46usdu0HUkW3m5+dDLpdbW3R0tEPv7WosQ5LsPU+z2xLv5JZn2XJzc6FSqaytpqaG75KcYsbYUATIRGhQa3Hi+m2+yyFk0NkVSCEhIRAKhWhosB3lsKGhodcD1gOxTalUioCAAJvmCaQiIeYkds5uS9024oXsCiSJRIKJEyeiqKjIusxkMqGoqAhpaWkOFTAQ23RnlrNtX5XVQU+z2xIvI7L3BTk5OVi2bBkmTZqEKVOmYNOmTdBoNFi+fDkAICsrC1FRUcjPzwdgPmhdXl5ufXzz5k2UlpZiyJAhGD16dJ+26U3SRgYjZIgEzW06HKlsxmNjw/guiZDB48gpvHfeeYcNHz6cSSQSNmXKFHbs2DHrczNmzGDLli2z/lxVVcUA3NNmzJjR520+iCec9u8u7/MyFvPyXvbSzrN8l0KIU/T1b5RjzP2vwlOr1ZDL5VCpVB5xPOn0jdtY9G4J/CRCnF73OGRiId8lEdIvff0bdcuzbJ7uoeggRAX6QKMz4uBl77lanRAKJBckEHCYn0zTJBHvQ4HkoixDkhRdbkRrh57naggZHBRILmpcZABGhvpBZzBh/0Wa3ZZ4BwokF8VxXNfAbTSSJPESFEguzBJIR64047aGZrclno8CyYWNDB2CxCjz7Lb7ymiaJOL5KJBcXEYSjbdNvAcFkoub39ltO3H9NupVNLst8WwUSC4uKtAHk2ODOme3pb0k4tkokNxABs1uS7wEBZIbeGJCJIQCDue/V+F6s4bvcggZMBRIbiBkiBRTLbPb0l4S8WAUSG5iAU2TRLwABZKbmD0+AhKhAFca21BBs9sSD0WB5CbkPmL8eKx5dts9pdRtI56JAsmNLEjpureNum3EE1EguZFZ8eHwlQhRc/sOzta08F0OIU5HgeRGfCRCPN45uy2dbSOeiALJzVjOtu09XwcjzW5LPAwFkpuZNiYUch8xmlq1OH7tFt/lEOJUFEhuRiISYG7n7LY0cBvxNBRIbsjSbdtXVg+dgWa3JZ6DAskNpY4MRqi/FKo7evznShPf5RDiNBRIbkgo4DBvAk2TRDwPBZKbslwk+U15A+7ojDxXQ4hzUCC5qYeiAxE91AftOiOKLtM0ScQzUCC5KY7jrONt071txFNQILkxy0iSxRVNUN2h2W2J+6NAcmPxEf4YEzYEOqMJ+y/W810OIf1GgeTGus9uS2fbiCegQHJzlm7b0au30Nym5bkaQvqHAsnNxYb4IWmYHEYTw1c0uy1xcxRIHoC6bcRTUCB5gPlJCnAccPL6D7jZcofvcghxGAWSB4iQyzA5digAYC/tJRE3RoHkISzdNhqShLgzCiQP8cSESIgEHC7cVONaUxvf5RDiEAokDzHUT4JHx4QAoIPbxH1RIHkQy71tX9DstsRNUSB5kNnjwyEVCXC1SYPyOjXf5RBiN4cCacuWLYiNjYVMJkNqaipOnDhx3/X/8Y9/ID4+HjKZDBMmTMC+fftsnn/uuefAcZxNmzNnjiOleTV/mRgz48MAULeNuCe7A2nXrl3IycnB+vXrcebMGSQnJyM9PR2NjY09rn/06FEsWbIEzz//PM6ePYvMzExkZmbiwoULNuvNmTMHdXV11vbJJ5849om8nOVWkr3n6mCiaZKIm+GYnQcbUlNTMXnyZGzevBkAYDKZEB0djV/84hf43//933vWX7x4MTQaDfbu3Wtd9qMf/QgpKSnYunUrAPMeUktLCz7//PM+1aDVaqHVdt23pVarER0dDZVKhYCAAHs+jsfp0Bsx6bUDaNMa8M9VaZgYM5TvkgiBWq2GXC5/4N+oXXtIOp0Op0+fhlKp7NqAQAClUomSkpIeX1NSUmKzPgCkp6ffs35xcTHCwsIwduxYrFq1Crdu9T7nWH5+PuRyubVFR0fb8zE8mkwsxOzO2W1p4DbibuwKpObmZhiNRoSHh9ssDw8PR319z+Px1NfXP3D9OXPm4KOPPkJRURFef/11HDp0CHPnzoXR2PNY0bm5uVCpVNZWU1Njz8fweBmd421/WVYHg5GmSSLuQ8R3AQDwzDPPWB9PmDABSUlJGDVqFIqLizFr1qx71pdKpZBKpYNZolt5dHQIgnzFaG7T4di129brkwhxdXbtIYWEhEAoFKKhwXZQ+YaGBkRERPT4moiICLvWB4CRI0ciJCQElZWV9pRHOomFAsy1TpN0k+dqCOk7uwJJIpFg4sSJKCoqsi4zmUwoKipCWlpaj69JS0uzWR8Avvnmm17XB4Dvv/8et27dQmRkpD3lkW4s97Z9daEeWgNNk0Tcg92n/XNycrBt2zZ8+OGHuHTpElatWgWNRoPly5cDALKyspCbm2td/1e/+hUKCwvxl7/8BZcvX8arr76KU6dOITs7GwDQ1taG3/zmNzh27BiuX7+OoqIiPPnkkxg9ejTS09Od9DG9z+TYoQgPkKK1w4BDFTS7LXEPdgfS4sWL8eabbyIvLw8pKSkoLS1FYWGh9cB1dXU16uq6Ri6cOnUqCgoK8N577yE5ORmfffYZPv/8cyQmJgIAhEIhzp8/jwULFiAuLg7PP/88Jk6ciP/85z90nKgfhAIO8y23kpynkSSJe7D7OiRX1NdrHLzNuZoWPLnlO/iIhTi9TglfiUucwyBeaECuQyLuJWmYHDHBvrijN+Kbcprdlrg+CiQP1n122y/OUbeNuD4KJA+3oPMiyUP/bYSqnWa3Ja6NAsnDxYX7Iz7CH3ojQ+FF2ksiro0CyQtk0DRJxE1QIHkBy3Gkkqu30NjawXM1hPSOAskLDA/2RUp0IEwM2EfXJBEXRoHkJWh2W+IOKJC8xLykSHAccKa6BTW32/kuh5AeUSB5ifAAGX40IhgAsJe6bcRFUSB5Ecs1SdRtI66KAsmLzBkfAZGAw6U6NSobW/kuh5B7UCB5kSA/CabHhQIA9tCtJMQFUSB5GcvZNprdlrgiCiQv8/i4cMjEAlQ1a3DhJs1uS1wLBZKX8ZOKMCu+c5okGm+buBgKJC9knd32PM1uS1wLBZIX+vHYUPhLRahTdeDUjR/4LocQKwokLyQTCzF7vHkaKuq2EVdCgeSlLBdJ7iurp9lticugQPJSj4wKRrCfBLc1Onx39Rbf5RACwBsD6dvXgGNbgWvFQFsj4KXX4oiEAjxhmd22lG4lIa7Bu+bFMWiB/7wFsG4zufoMBcISgNB4879hCUBoAuAXzF+dgyQjWYG/H7uB/Rfr0aFPhEws5Lsk4uW8L5AefQlovAQ0XQJuVwF3bgM3vjO37vxCu8IpLB4IG2cOLZ9AXkofCJNighApl6FO1YHiiibMSYzguyTi5bwrkGQBwKx1XT/r2oHm/wJNlztD6jLQWA60VAOaJqCqCag6bLsN/8jOvalx5qAKTQBCx5q37WYEAg4ZyQq8d/gavjhXS4FEeOddgXQ3iS+gSDG37rRtQHOFOaSsQXUZUH8PtNaZ27WDtq+RR3cGVbe9qdCxgMRvsD6NQzKSzIF04FID2rQGDJF6968E4Rf99vVEOgSImmhu3XWogKbuQXXJHFRt9YCqxtwqv+n2Ag4IHG67NxWWAITEAWLZoH6k3iRGBWBEiB+qmjU4UN6AzIei+C6JeDEKJHvI5ED0FHPrrv32Xd2+zsBqbwZabpjbf7/qWp8TAEEjuh1E7zygHjwGEEkG9SNxnLnb9nbRFew5V0uBRHjFMQ8Yg0KtVkMul0OlUiEgwIWO5Wia792baiwHOlp6Xl8gAoaOst2bCksAho4EhOIBK7OysRXKtw5DJOBw8vdKBPkNbigSz9fXv1EKpMHGGNDWYHsQvfGy+bG2l+FABGJzN88aVJ3HqYJiAYFzTtXP/et/cKlOjaF+EgT7SSD3EZubr7jrsY8YgTY/d60nEXnfJW2k7yiQ3A1jgPpmZzh125tqqgD0mp5fI5IBIWNs96ZC44HAGEBgX0B8cqIauf8qc7h8X4nQGk4BPmIE9hBiAdafu4IsQCaCSEhh5ukokDyFyWQ+WN6929d0yRxUhl5moRX7ms/w3X0NlXwYwHG9vlWjugNNbVqo2vVQ3TG3ljtdj7svV93Ro6Vdh1atod8Xu/tLRd3CStzj3llgt70xy3P+UhEEgt4/D3EdFEiezmQEfrh+b7ev+b+AUdfzayT+nd2++G4H08cB/hH3Dar7MZoY2joMaLmj6xZUXaGlvuvnls5lqjt6tGkNjn9+AAIO8JfZhlhf9878JEJwDn5mYj8KJG9lNAC3r93V7bsM3KoETL0EgExu3psKGWPeu+IE5mNTnKDbY2EPywS9LBf28Fhwz3IDOLTrTNDoGdr0DG06E9p0DK06htYOE1p1Jqi15tbaYYJK29k6DGg3ACYmgAkcjBDABEHnv1y3x7bLga4AEgk4mxC79/hY91CTwFcihFQkgMTShLaPKdzujwKJ2DLogNtXu+1NdZ79u30NYN4x/IgJHIzMElRct9Dq5THremyAEEYIoYcQhs6mZ13LTJzI3AQisM7HTGBpYvMZVKEIEIjBCc0/c0IROKEYnFACTiSGQCiGoPNfoUgCgcj8r1AkhlAshlAsgVgkMf8rFkMklkAskZqbSAKxRAKpRAKJRAqBSOLwXu9A6OvfKF2H5C1Ekq4D393pO4BbV8whdfuaubvHTOYbkE1G88F262PTXY9NnY+N3R7ftfyeZXdvw2g+TmazDcvjByzvvo0+hKoADALOCMD4wHUBdN+h6jvW2dD3txkoBiaAkRN2hqkIBs78r4nrClBjZ3CaOCGYQGwOUM4cnBAKO8NUDE4oAoRicAJziApE5iAVCEWQjZ+L0HEznFIzBZK3E8uAiAnm5s4YcyA8OwO3p+Umk+02TIauZtTDZNTDoNfBYNDDqNfBYNDBpNfBaNDDaNDDZNTBZNCbm1EHZjSAGfVgBj2YSQ8Y9WAmg7mLbdKDMxnAmfSAyQjOpIeAGSAwGcAxAwTMCAEzQGhtxs79NQNEzAgRDBBy93Z0RJwJIpgghb7zO4Ltv05SovWhQCLEBsd1dlEEA3oRqYUAgKSzuQJmMkKv10On00Kv00Gn08Kg00Fv0EGn73ys15nDU6/vDM7On436rvA06DrDtDNAbVpXeMJo/peZDAgOc97/zCiQCPEAnEAIiVQIidQ17pF0FF2RRghxGQ4F0pYtWxAbGwuZTIbU1FScOHHivuv/4x//QHx8PGQyGSZMmIB9+/bZPM8YQ15eHiIjI+Hj4wOlUokrV644UhohxI3ZHUi7du1CTk4O1q9fjzNnziA5ORnp6elobGzscf2jR49iyZIleP7553H27FlkZmYiMzMTFy5csK6zceNGvP3229i6dSuOHz8OPz8/pKeno6OjlyuRCSGeidlpypQpbPXq1dafjUYjUygULD8/v8f1n376aTZv3jybZampqexnP/sZY4wxk8nEIiIi2BtvvGF9vqWlhUmlUvbJJ5/0qSaVSsUAMJVKZe/HIYQMgr7+jdq1h6TT6XD69GkolUrrMoFAAKVSiZKSkh5fU1JSYrM+AKSnp1vXr6qqQn19vc06crkcqampvW5Tq9VCrVbbNEKI+7MrkJqbm2E0GhEeHm6zPDw8HPX19T2+pr6+/r7rW/61Z5v5+fmQy+XWFh0dbc/HIIS4KLc8y5abmwuVSmVtNTU1fJdECHECu65DCgkJgVAoRENDg83yhoYGRET0PGNFRETEfde3/NvQ0IDIyEibdVJSUnrcplQqhVQqtf7MOm/Ho64bIa7J8rfJHnTrrL0Hp6ZMmcKys7OtPxuNRhYVFXXfg9rz58+3WZaWlnbPQe0333zT5gCYPQe1a2pqLHcQUaNGzYVbTU3Nff+W7b5SOycnB8uWLcOkSZMwZcoUbNq0CRqNBsuXLwcAZGVlISoqCvn5+QCAX/3qV5gxYwb+8pe/YN68edi5cydOnTqF9957D4B5kPk1a9bgtddew5gxYzBixAisW7cOCoUCmZmZfapJoVCgpqYG/v7+DxwGQq1WIzo6GjU1NTQyQDf0vfSOvpue2fO9MMbQ2toKhUJx/432aRfkLu+88w4bPnw4k0gkbMqUKezYsWPW52bMmMGWLVtms/6nn37K4uLimEQiYePHj2dffvmlzfMmk4mtW7eOhYeHM6lUymbNmsUqKiocKe2B6BKBntH30jv6bno2EN+LR4yHZA8aO6ln9L30jr6bng3E9+KWZ9kIIZ7J6wJJKpVi/fr1NmfpCH0v90PfTc8G4nvxui4bIcR1ed0eEiHEdVEgEUJcBgUSIcRlUCARQlwGBRIhxGV4XSDZO/yuNzh8+DAyMjKgUCjAcRw+//xzvktyCfn5+Zg8eTL8/f0RFhaGzMxMVFRU8F0W7959910kJSUhICAAAQEBSEtLw1dffeWUbXtVINk7/K630Gg0SE5OxpYtW/guxaUcOnQIq1evxrFjx/DNN99Ar9dj9uzZ0Gg0fJfGq2HDhmHDhg04ffo0Tp06hZkzZ+LJJ5/ExYsX+79xp92E4gbsHX7XGwFgu3fv5rsMl9TY2MgAsEOHDvFdissJCgpi77//fr+34zV7SI4Mv0tIdyqVCgAwdOhQnitxHUajETt37oRGo0FaWlq/t+c1E0Xeb/jdy5cv81QVcRcmkwlr1qzBI488gsTERL7L4V1ZWRnS0tLQ0dGBIUOGYPfu3Rg3bly/t+s1gURIf6xevRoXLlzAkSNH+C7FJYwdOxalpaVQqVT47LPPsGzZMhw6dKjfoeQ1geTI8LuEAEB2djb27t2Lw4cPY9iwYXyX4xIkEglGjx4NAJg4cSJOnjyJv/71r/jb3/7Wr+16zTEkiUSCiRMnoqioyLrMZDKhqKjIKX1f4nkYY8jOzsbu3bvx7bffYsSIEXyX5LJMJhO0Wm2/t+M1e0jAg4ff9VZtbW2orKy0/lxVVYXS0lIMHToUw4cP57Eyfq1evRoFBQX497//DX9/f+u0XHK5HD4+PjxXx5/c3FzMnTsXw4cPR2trKwoKClBcXIyvv/66/xvv/wk/93K/4Xe91cGDB3sckP3uoYi9TU/fCQC2Y8cOvkvj1f/8z/+wmJgYJpFIWGhoKJs1axbbv3+/U7ZN4yERQlyG1xxDIoS4PgokQojLoEAihLgMCiRCiMugQCKEuAwKJEKIy6BAIoS4DAokQojLoEAihLgMCiRCiMugQCKEuIz/B77/150oaM/zAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T18:08:22.220017Z", + "iopub.status.busy": "2024-03-24T18:08:22.219099Z", + "iopub.status.idle": "2024-03-24T18:17:00.873662Z", + "shell.execute_reply": "2024-03-24T18:17:00.872785Z" + }, + "papermill": { + "duration": 518.675073, + "end_time": "2024-03-24T18:17:00.876287", + "exception": false, + "start_time": "2024-03-24T18:08:22.201214", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T18:17:00.911347Z", + "iopub.status.busy": "2024-03-24T18:17:00.910616Z", + "iopub.status.idle": "2024-03-24T18:17:00.931555Z", + "shell.execute_reply": "2024-03-24T18:17:00.930725Z" + }, + "papermill": { + "duration": 0.040351, + "end_time": "2024-03-24T18:17:00.933479", + "exception": false, + "start_time": "2024-03-24T18:17:00.893128", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
lct_gan0.00.1295240.00548310.2521290.1011551.8389220.1632430.00011511.49620.0485846742933.00.0740470.2329960.00167821.748329
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "lct_gan 0.0 0.129524 0.005483 10.252129 0.101155 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "lct_gan 1.838922 0.163243 0.000115 11.4962 0.048584 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "lct_gan 6742933.0 0.074047 0.232996 0.001678 21.748329 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T18:17:00.965839Z", + "iopub.status.busy": "2024-03-24T18:17:00.965557Z", + "iopub.status.idle": "2024-03-24T18:17:01.331164Z", + "shell.execute_reply": "2024-03-24T18:17:01.330160Z" + }, + "papermill": { + "duration": 0.383892, + "end_time": "2024-03-24T18:17:01.333235", + "exception": false, + "start_time": "2024-03-24T18:17:00.949343", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T18:17:01.367721Z", + "iopub.status.busy": "2024-03-24T18:17:01.367371Z", + "iopub.status.idle": "2024-03-24T18:25:47.074126Z", + "shell.execute_reply": "2024-03-24T18:25:47.073257Z" + }, + "papermill": { + "duration": 525.726879, + "end_time": "2024-03-24T18:25:47.076650", + "exception": false, + "start_time": "2024-03-24T18:17:01.349771", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/lct_gan/all inf False\n", + "Caching in ../../../../treatment/_cache_bs_test/lct_gan/all inf False\n", + "Caching in ../../../../treatment/_cache_synth_test/lct_gan/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T18:25:47.112239Z", + "iopub.status.busy": "2024-03-24T18:25:47.111898Z", + "iopub.status.idle": "2024-03-24T18:25:47.138135Z", + "shell.execute_reply": "2024-03-24T18:25:47.137414Z" + }, + "papermill": { + "duration": 0.045958, + "end_time": "2024-03-24T18:25:47.140157", + "exception": false, + "start_time": "2024-03-24T18:25:47.094199", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T18:25:47.172989Z", + "iopub.status.busy": "2024-03-24T18:25:47.172215Z", + "iopub.status.idle": "2024-03-24T18:25:47.177945Z", + "shell.execute_reply": "2024-03-24T18:25:47.177078Z" + }, + "papermill": { + "duration": 0.024067, + "end_time": "2024-03-24T18:25:47.179844", + "exception": false, + "start_time": "2024-03-24T18:25:47.155777", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'lct_gan': 0.42631170449976796}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T18:25:47.215753Z", + "iopub.status.busy": "2024-03-24T18:25:47.215222Z", + "iopub.status.idle": "2024-03-24T18:25:47.615806Z", + "shell.execute_reply": "2024-03-24T18:25:47.614856Z" + }, + "papermill": { + "duration": 0.421901, + "end_time": "2024-03-24T18:25:47.618051", + "exception": false, + "start_time": "2024-03-24T18:25:47.196150", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAE8CAYAAACRnDftAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABNiElEQVR4nO3de3yU1Z348c8z95lkZpKQOySE+/0mgiZWAbUgshbWrj9bW7koVltstdTdNbtbbLVtekNkty7abSF1LWrtKrbaqhRFqgKWq9wKcksCuUMyk5lkrs/z++NJBkIuTCaTTGZy3q/XvMI8c+aZM0Pmm3PO8z3nSIqiKAiCIMQZTawrIAiCEAkRvARBiEsieAmCEJdE8BIEIS6J4CUIQlwSwUsQhLgkgpcgCHFJBC9BEOKSCF6CIMQlEbyEqCstLUWSJM6ePRvrqggJTAQvYUDYvHkzzzzzTKyrIcQREbyEAUEEL6GnRPASBCEuieAl9Is///nPzJkzB6vVis1mY9asWWzevBmAuXPn8tZbb1FWVoYkSUiSREFBQdjnlmWZ733ve+Tm5mKxWJg3bx5Hjx6loKCA5cuXh8pdvHiRxx57jClTppCcnIzNZmPhwoUcPHiw3fm2b9+OJEn87ne/44c//CHDhg3DZDJxyy23cPLkyWh8HEIU6GJdASHxlZaWct999zFp0iSKi4tJSUlh//79vP3229xzzz38+7//Ow6Hg3PnzrFu3ToAkpOTwz5/cXExP/3pT7njjjtYsGABBw8eZMGCBXg8nnblTp8+zZYtW7jrrrsYMWIENTU1PP/888yZM4ejR4+Sm5vbrvyPf/xjNBoNjz32GA6Hg5/+9Kd85StfYffu3b3/UITeUwQhyjZt2qQAypkzZ5TGxkbFarUq1113ndLS0tKunCzLoX8vWrRIGT58eI9fq7q6WtHpdMqSJUvaHf/e976nAMqyZctCxzwejxIMBtuVO3PmjGI0GpUnn3wydOz9999XAGXChAmK1+sNHV+/fr0CKIcOHepxPYXoE91GoU9t3bqVpqYmHn/8cUwmU7vHJEnq9fm3bdtGIBDgG9/4Rrvj3/zmNzuUNRqNaDTqr3wwGOTChQskJyczbtw49u3b16H8ihUrMBgMofs33ngjoLbghNgTwUvoU6dOnQJg8uTJfXL+srIyAEaPHt3ueFpaGqmpqe2OybLMunXrGDNmDEajkfT0dDIyMvj0009xOBwdzp2fn9/uftv5GhoaovkWhAiJ4CUMGj/60Y9YvXo1N910Ey+++CLvvPMOW7duZdKkSciy3KG8Vqvt9DyKWDl9QBAD9kKfGjVqFACHDx/u0Dq6XKRdyOHDhwNw8uRJRowYETp+4cKFDi2k3//+98ybN49f//rX7Y43NjaSnp4e0esLsSNaXkKfmj9/PlarlZKSkg5X/y5vwSQlJXXadbuaW265BZ1Ox4YNG9od/8UvftGhrFar7dBqevXVVzl//nyPX1eIPdHyEvqUzWZj3bp1rFy5klmzZnHPPfeQmprKwYMHaW5u5je/+Q0AM2fO5JVXXmH16tXMmjWL5ORk7rjjjquePysri0ceeYS1a9fyhS98gdtuu42DBw/y5z//mfT09HYtun/4h3/gySefZMWKFRQVFXHo0CF++9vfMnLkyD57/0IfivHVTiEBXZ4q0eYPf/iDUlRUpJjNZsVmsymzZ89WXnrppdDjLpdLueeee5SUlBQF6FHaRCAQUL773e8q2dnZitlsVm6++Wbl2LFjypAhQ5SHHnooVM7j8Sjf+c53lJycHMVsNis33HCDsnPnTmXOnDnKnDlzQuXaUiVeffXVdq9z5swZBVA2bdrU049E6AOSoojRRyHxNDY2kpqayg9+8AP+/d//PdbVEfqAGPMS4l5LS0uHY22TvOfOndu/lRH6jRjzEgasuro6gsFgl48bDAbS0tJ45ZVXKC0t5fbbbyc5OZkPP/yQl156ifnz53PDDTf0Y42F/iSClzBgzZo1K5SE2pk5c+awfft2pk6dik6n46c//SlOpzM0iP+DH/ygH2sr9Dcx5iUMWB999FGnXcI2qampzJw5sx9rJAwkIngJghCXxIC9IAhxadCNecmyTGVlJVarNSqrGgiCEF2KotDU1ERubm5oFZDODLrgVVlZSV5eXqyrIQjCVVRUVDBs2LAuHx90wctqtQLqB2Oz2WJcG0EQruR0OsnLywt9V7sy6IJXW1fRZrOJ4CUIA9jVhnXEgL0gCHFJBC9BEOKSCF6CIMSlQTfmFQ5FUQgEAt3OqxM6p9Vq0el0Ig1F6HMieF3B5/NRVVVFc3NzrKsStywWCzk5Oe123hGEaItp8NqwYQMbNmzg7NmzAEyaNIk1a9awcOHCTsuXlpayYsWKdseMRmOH5YUjJcsyZ86cQavVkpubi8FgEC2IHlAUBZ/PR11dHWfOnGHMmDHdJhkKQm/ENHgNGzaMH//4x4wZMwZFUfjNb37D4sWL2b9/P5MmTer0OTabjePHj4fuRzO4+Hw+ZFkmLy8Pi8UStfMOJmazGb1eT1lZGT6fr8NejYPBxyfrGZttJT3ZGOuqJLSYBq8r1yj/4Q9/yIYNG9i1a1eXwUuSJLKzs/u0XqK10DuD+fNTFIVPzzs4WuXk3sLhGHWdb58m9N6A+S0LBoO8/PLLuN1uCgsLuyzncrkYPnw4eXl5LF68mCNHjnR7Xq/Xi9PpbHcThL4SlBXynAcYf/JXVH3wa/C6Yl2lhBXz4HXo0CGSk5MxGo089NBDvP7660ycOLHTsuPGjWPjxo288cYbvPjii8iyTFFREefOnevy/CUlJdjt9tBNzGsU+sLx6iYa3D50F09yrfwpWtlH4/lTKMf/FOuqJayYB69x48Zx4MABdu/ezde//nWWLVvG0aNHOy1bWFjI0qVLmT59OnPmzOG1114jIyOD559/vsvzFxcX43A4QreKioq+eiuDRkFBQWiNeAE8/iBvH66m9KMztJx4nyHJBprMw2j2y3iqT4CrNtZVTEgxT5UwGAyhnZRnzpzJ3/72N9avX99tQGqj1+uZMWMGJ0+e7LKM0WjEaBQDp0LfOVPvRlYU8vQOzL6LoDfgHv0P+E69S0NzNeaqgzDm87GuZsKJecvrSrIs4/V6wyobDAY5dOgQOTk5fVyrxOPz+WJdhYRxvkFdqnqstko9kDaK/Kw06i2jafIEoP4zEAsWR11Mg1dxcTE7duzg7NmzHDp0iOLiYrZv385XvvIVAJYuXUpxcXGo/JNPPsm7777L6dOn2bdvH1/96lcpKytj5cqVfV5XX0Du8hYIymGX9YdZtqfmzp3Lww8/zMMPP4zdbic9PZ3vfve7oe3tCwoKeOqpp1i6dCk2m42vfe1rAHz44YfceOONmM1m8vLy+Na3voXb7Q6dt7a2ljvuuAOz2cyIESP47W9/2+O6JbpKhxq8cvytQxIZ48m2m3GYhuH0AR4HuOtjV8EEFdNuY21tLUuXLqWqqgq73c7UqVN55513+Pzn1SZ2eXl5u8vuDQ0NPPDAA1RXV4c2X/j444+7HOCPpmff77prOiI9iSUzhobu/3LHKfzBzv/SDks1c9e1ly4abPzoDC2+jtOQvv35sT2u429+8xvuv/9+PvnkE/bs2cPXvvY18vPzeeCBBwD4+c9/zpo1a3jiiScAOHXqFLfddhs/+MEP2LhxI3V1daEAuGnTJgCWL19OZWUl77//Pnq9nm9961vU1ooxnDYef5ALLh/6YDMpkgMkLaSNIEtj5B+vLSDn7ERwVoDzHCRnxLq6CSWmwevXv/51t49v37693f1169axbt26PqxRfMvLy2PdunVIksS4ceM4dOgQ69atCwWvm2++me985zuh8itXruQrX/kKjz76KABjxozhP//zP5kzZw4bNmygvLycP//5z3zyySfMmjULUP/PJkyY0O/vbaCqdqizO4ZK9ei1GkhKB70ZHZA/xAKOPDV4Oc5D7ozYVjbBxHzAPl6smje6y8c0VyT5f+2mUV2WvXJCwH03jOhNtdq5/vrr2804KCwsZO3ataEJ5tdee2278gcPHuTTTz9t1xVUFCU0TerEiRPodLp224uNHz+elJSUqNU53tW71PHZYZoL6gH7Fak49tZljJ3n+7FWg4MIXmEy6MIfHuyrsr2VlJTU7r7L5eLBBx/kW9/6Voey+fn5nDhxor+qFrcm5drJsBqxH/sA/FwKVsAFl5ejNQYKGprJA/B7QD/4pkv1FRG8Esju3bvb3d+1axdjxoxBq+18iso111zD0aNHQ6kqVxo/fjyBQIC9e/eGuo3Hjx+nsbExqvWOZ2aDluGpZgg2qgeSL01da/EH2XO+BX2LnrxUwF0LKfkxqWciGnCpEkLkysvLWb16NcePH+ell17iv/7rv3jkkUe6LP+v//qvfPzxxzz88MMcOHCAzz77jDfeeIOHH34YUBOIb7vtNh588EF2797N3r17WblyJWazub/eUnxouQhyALR6sKSFDrdNzL4opRCQZXDXxaqGCUkErwSydOlSWlpamD17NqtWreKRRx4JpUR0ZurUqXzwwQecOHGCG2+8kRkzZrBmzRpyc3NDZTZt2kRubi5z5szhzjvv5Gtf+xqZmZn98XYGvMZmHx+drKei4ox6IDmz3aCmSa8l2aijWZ9Giz8ILhG8okl0GxOIXq/nmWeeYcOGDR0ea1sz7UqzZs3i3Xff7fKc2dnZvPnmm+2O3Xvvvb2qZ6Kodnr45MxFAv5T5JmB5KwOZVKTDDTr0/D4TmJ1ixSTaBItL0GIUIPbD8AQGtUDyR1bpClmPS36FDyBIDRf6MfaJT4RvAQhQo4WdYqVTWldZimpYxJqikWPR2fH4w+qVxv9Lf1ZxYQmuo0J4sqEXqHvNTT70ch+kpVmwAiWIR3KpFj0yBo9bql1Zd7mi2Af2qGc0HOi5SUIEVAUhYZmH+aAA6NeCwYL6DtehR0+JIkH54xk2pgC9UBLQ/9WNIGJ4CUIEfD4Zbx+GZO/EZNeC+a0TsvptRosBh1SW6us5WI/1jKxieAlCBFoaFbHu4ZITWglqdMuYzvmVPVnswhe0SKClyBEINtmYsUNBRTltn6FLJ23vAD2lzewrcxPY7NPdBujSAQvQYiARiORYjGQSuuVxm5aXjVOLyedOty+IHga+6eCg4AIXoIQKUW5NIbVxZgXgM2kw6uz4gu0pksEwlspWOieSJUQhAjsPn0BvE6mebyYDHowp3RZ1mrSE9QYaFH06gGPUyxMGAWi5SUIETh03sHhk2fxy7IauDRdby5rNaltBCetuV5esXdoNIjgJQg9FJQVXN4AxoBL3RHbZO+2vM2stricSmvwEuNeUSGC19UoCgR8sbmFuePMCy+8wJAhQzrsurRkyRIxiboPuDwBFAXMshu9VgKjrdvyyUa15eWWktSlcTyi5RUNMR3z2rBhAxs2bAiteDBp0iTWrFnDwoULu3zOq6++yne/+13Onj3LmDFj+MlPfsLtt9/ed5UM+uGva/vu/N258TugM1y12F133cW3vvUt/vCHP3DXXXcB6uYmb731VrcrRgiRcXrUCdkpmhYkJDB1H7wMOg1mg5ag3oo/qKDzOPqjmgkvpi2vYcOG8eMf/5i9e/eyZ88ebr75ZhYvXsyRI0c6Lf/xxx/z5S9/mfvvv5/9+/ezZMkSlixZwuHDh/u55gOL2WzmnnvuCe34A/Diiy+Sn5/P3LlzY1exBOVoUYOXXdOsHrhKywtgeVEBXyyaiFmvFWNeURLTltcdd9zR7v4Pf/hDNmzYwK5du5g0aVKH8uvXr+e2227jn//5nwF46qmn2Lp1K7/4xS947rnn+qaSWr3aAooFrT7sog888ACzZs3i/PnzDB06lNLSUpYvX95uQw4hOpo8AQCSaQ1eV2l5Aa1TiFLUO6LbGBUDJlUiGAzy6quv4na7KSws7LTMzp07Wb16dbtjCxYsYMuWLV2e1+v1thsLcjp7+IsjSWF13WJtxowZTJs2jRdeeIH58+dz5MgR3nrrrVhXKyE5PX5QFJKUZsBw1QH7kLYWms8FcrDbK5TC1cU8eB06dIjCwkI8Hg/Jycm8/vrrXW4iW11dTVZW+9Uqs7KyqK6u7vL8JSUlfP/7349qnQeqlStX8swzz3D+/HluvfVW8vLyrv4kocduGZ/J7FwjSXt16h83g/Wqzym/0Mz+8kZmNHrJTzGqXce2+Y5CRGJ+tXHcuHEcOHCA3bt38/Wvf51ly5Zx9OjRqJ2/uLgYh8MRulVUVETt3APNPffcw7lz5/if//kf7rvvvlhXJ2HptBpStR4MWg0YkkB79TZAsz/A6fpmLvhbW/FeVx/XMvHFPHgZDAZGjx7NzJkzKSkpYdq0aaxfv77TstnZ2dTU1LQ7VlNTQ3Z2dqflAYxGIzabrd0tUdntdr74xS+SnJzMkiVLYl2dxNY26B7GYD1AkqE1XYLWNb98Inj1VsyD15VkWe6Qr9SmsLCQbdu2tTu2devWLsfIBqPz58/zla98BaPRGOuqJCS3N8A7R6o5crpcPRDGYD1cyvVytQUv0fLqtZiOeRUXF7Nw4ULy8/Npampi8+bNbN++nXfeeQdQt/IaOnQoJSUlADzyyCPMmTOHtWvXsmjRIl5++WX27NnDL3/5y1i+jQGhoaGB7du3s337dv77v/871tVJWI0tfo5WOlHcNUyyE37LqzV4NWMmIF9AJ9Ilei2mwau2tpalS5dSVVWF3W5n6tSpvPPOO3z+858H1E1UNZpLjcOioiI2b97Mf/zHf/Bv//ZvjBkzhi1btjB58uRYvYUBY8aMGTQ0NPCTn/yEcePGxbo6CcvVmiYRyvEypYT1PINOg0Gnwae1qImqotvYazENXr/+9a+7fbyzTSXuuuuuUBa5cElX+zIK0eXyqgmq6qYbhN1tBHWCtk+bhC8gYxbdxl4bcGNegjCQubxBgNYcL8LuNoI6aK8YkgnKMnib+qJ6g0rM87wGIiXMCdFC5xL583N5AmjkACY8gLlHLa/F03PRei1Iuz8CX5M68V7MgIiYaHldRq9Xp+M0NzfHuCbxre3za/s8E4nbG8AQdGHUadTpWzpT2M/VaTVIxtaE1mBArKjaS6LldRmtVktKSgq1tbUAWCwWMTewBxRFobm5mdraWlJSUtBqE2/6S1PrOl4GvUadFtTT3w+tHnRGNXD5XKAPP/gJ7YngdYW2hNe2ACb0XEpKSreJw/FsaeFwPGX1WM7qwp/T2KrW6WHn6QuMbYAJVtRE16T0vqnoICCC1xUkSSInJ4fMzEz8fn+sqxN39Hp9Qra42ui1GvRSi9ri6sFgPUBAVjhd5ybFq2eC1SsSVXtJBK8uaLXahP4SCr3QtqRNDwbrAZJb17Jvkk0oNCGJXK9eEcFLEMJU5Wjh0DkHE2qqyJPoccurbX6jR2MmEFTQi5ZXr4irjYIQpromL0cqnTQ2XFAP9LDlpdVImPRa/FoL/qAsJmf3kgheghAmlydw2SKE9HjAHiDJqMWvNeMPKuBzR7mGg4sIXoIQJpc3gD7YjEGjhL0I4ZUsBh0+TWvLyy/yCXtDBC9BCJPLG8AYdGHQacBoBU3Pvz4WgxYMFoKyIrqNvSQG7AUhTKHgpdf0eLC+zYJJ2WjHp8KHf1H35gz6e7TRinCJaHkJQpjadsk2aDU9Hqxvo9VIaoa9prXdIFpfERPBSxDC4A/KeP0yhkBrtzGCwfoQSVLXvgfwiXGvSIngJQhh0Gs1PHzzaOaPMqutpwi7jfUuL28cOM+hutbZG+KKY8TEmJcghEmv1aBX3IAUcctLVtQpQskeHVgR3cZeEC0vQeiJHu4adCVLa5a9SzGq656JdImIieAlCGE4UdPEu5+Wc6HRoR6IcMDeotciSai5XrJIVO2NmAavkpISZs2ahdVqJTMzkyVLlnD8+PFun1NaWookSe1uJpNYE0noW+cbWjhdUYnbG1SvFuoi21pOo5Ew69uy7MUUod6IafD64IMPWLVqFbt27WLr1q34/X7mz5+P2939XyObzUZVVVXoVlZW1k81FgardgmqvbnSiJqoeil4iZZXpGI6YP/222+3u19aWkpmZiZ79+7lpptu6vJ5kiSFvdid1+ttt4mt0yn2yxN6LpTjZYhG8NLRoLXgDygiVaIXBtSYl8OhjiekpaV1W87lcjF8+HDy8vJYvHgxR44c6bJsSUkJdrs9dMvLy4tqnYXBweVR165XpwZFNt7VJsmoBUMSsiKmCPXGgAlesizz6KOPcsMNN3S7iey4cePYuHEjb7zxBi+++CKyLFNUVMS5c+c6LV9cXIzD4QjdKioq+uotCAlKlhXcvsu7jb0LXvMnZrPy5slk2Uzq9KCAL0o1HVwGTJ7XqlWrOHz4MB9++GG35QoLCyksLAzdLyoqYsKECTz//PM89dRTHcobjUaMxsgGVwUBwO0LoChgCrrR9yJBtY1GI4FkAK1O3UXI5wJd970NoaMB0fJ6+OGHefPNN3n//fcZNmxYj56r1+uZMWMGJ0+e7KPaCYNds0/daNaKW91NqpctL6B1ilCy+m+R6xWRmAYvRVF4+OGHef3113nvvfcYMWJEj88RDAY5dOgQOTk5fVBDQYAsm4lvzhvJjOzW1R96OWB/oXWK0Ke1AfWAuOIYkZh2G1etWsXmzZt54403sFqtVFdXA2C32zGbzQAsXbqUoUOHUlJSAsCTTz7J9ddfz+jRo2lsbORnP/sZZWVlrFy5MmbvQ0h8uoAbnQbQaC+1mCIkK3C6zo3FoxVThHohpsFrw4YNAMydO7fd8U2bNrF8+XIAysvL0Vy26FtDQwMPPPAA1dXVpKamMnPmTD7++GMmTpzYX9UWBqO2HYOM1p5vNHuFJKO6K1UzRmRFQSNaXhGJafBSFOWqZbZv397u/rp161i3bl0f1UgQOtpbdhHP+dNM8fixpfR+vMuk06KRJHwaCwFZwSByvSIyIAbsBWEgO1vfzLmqGrx+OSqD9RqNhNmgwa8x4Q+IKUKREsFLEK5Cza5visrUoDYWg+7SFmjiamNERPAShKsITQ2KQnZ9m7b5jT4xvzFiIngJQje8gSC+gIwh6EKvjVKOF63rehksKAoieEVowGTYC8JA5PYG1Y1mZTc6jRWM0ek2zp+YhWZ8Kvx166UpQjpDVM49WIiWlyB0w+UJoFV8mDRqln20Wl4ajQRaw6VdhPyi9dVTIngJQjda/MFLg/V6c3T3WBS7CPWK6DYKQjfGZVsZde0QlEPJUbvSCOouQh9+Vs+oOj9TrIhxrwiIlpcgXIXO70Lfi41mOyMrCmfq3dR4RLcxUiJ4CcLVeFs33YjSYD1AUusuQs2KQZ1pIlpePSaClyB04/3jtez/rJxmXyCqLS9zaBchc+suQmLMq6dE8BKEbpyqdVFXX4esELUEVbh8F6G2LHvR8uopEbwEoQuKouD2qlcb1QTV6HUboTXLXmMSuwhFKKLgdfr06WjXQxAGnGZfEEUOYJBbMER5wB4um98odhGKSETBa/To0cybN48XX3wRj8cT7ToJwoDg8gYwBN0YNBKSVg96S1TPL3YR6p2Igte+ffuYOnUqq1evJjs7mwcffJBPPvkk2nUThJhqNyHbZOv1IoRXmj8xm/vnTVJ3EQp4QQ5G9fyJLqLgNX36dNavX09lZSUbN26kqqqKz33uc0yePJmnn36aurq6aNdTEPqdy3PZUjhRHKxvo9FIata+1Po1FONePdKrAXudTsedd97Jq6++yk9+8hNOnjzJY489Rl5eHkuXLqWqqipa9RSEfufxB9WNZvtgsD5EksDQ2h0VwatHehW89uzZwze+8Q1ycnJ4+umneeyxxzh16hRbt26lsrKSxYsXd/v8kpISZs2ahdVqJTMzkyVLlnD8+PGrvu6rr77K+PHjMZlMTJkyhT/96U+9eRuC0KnrRg7hixOt5KVZ+iR41bu8bNl/nkN1rbsIiUUJeySi4PX0008zZcoUioqKqKys5IUXXqCsrIwf/OAHjBgxghtvvJHS0lL27dvX7Xk++OADVq1axa5du9i6dSt+v5/58+fjdnf9F+jjjz/my1/+Mvfffz/79+9nyZIlLFmyhMOHD0fyVgShW1qfE50meiuoXk6W1SlCdV51Qw4xaN8zkhLOLhhXGDNmDPfddx/Lly/vcr9En8/HSy+9xLJly8I+b11dHZmZmXzwwQfcdNNNnZa5++67cbvdvPnmm6Fj119/PdOnT+e555676ms4nU7sdjsOhwObLfrjGEKC2fUctDTAjK9ASn5UT+3yBvifHacZfeF9/iHzAtKoeTC88OpPTHDhfkcjWlVi69at5Ofnt9uSDNSkvoqKCvLz8zEYDD0KXAAOhzqHLC2t663Pd+7cyerVq9sdW7BgAVu2bOm0vNfrxev1hu47nc4e1UkYvP5w4DwTys9TkGZC3wctL7NebXH5tBYCcj16kWXfIxF1G0eNGkV9fX2H4xcvXoxo12sAWZZ59NFHueGGG5g8eXKX5aqrq8nKymp3LCsrK7Rh7ZVKSkqw2+2hW15eXkT1EwYXf1CmoqqG+qYW0GjAYI36a2g1EubWLHuxln3PRRS8uuppulwuTCZTRBVZtWoVhw8f5uWXX47o+V0pLi7G4XCEbhUVFVE9v5CY3N4AxqALrQQ6s00NYH1A3YjDQiAosux7qkfdxrbumiRJrFmzBovlUsZxMBhk9+7dTJ8+vceVePjhh3nzzTfZsWMHw4YN67ZsdnY2NTU17Y7V1NSQnZ3daXmj0YjRaOxxnYTBrckTwNCaoCqZUvrsdSwGHU5N2y5CYsC+J3oUvPbv3w+oLa9Dhw5hMFzaMMBgMDBt2jQee+yxsM+nKArf/OY3ef3119m+fXtYXc7CwkK2bdvGo48+Gjq2detWCgvFQKcQPU2eAKZAEwadtu9yvIAkg5YmowXFq4hUiR7qUfB6//33AVixYgXr16/v9dW6VatWsXnzZt544w2sVmto3Mput2M2mwFYunQpQ4cOpaSkBIBHHnmEOXPmsHbtWhYtWsTLL7/Mnj17+OUvf9mrugjC5Zo8fozBpktTg/rIgknZaMZa4eP3wd8CstxnXdREE9GntGnTpqikGWzYsAGHw8HcuXPJyckJ3V555ZVQmfLy8naZ+kVFRWzevJlf/vKXTJs2jd///vds2bKl20F+Qegpl1ftNhqjuEt2Z9QpQhY1014Rra+eCLvldeedd1JaWorNZuPOO+/stuxrr70W1jnDSTHbvn17h2N33XUXd911V1ivIQiR8AVkjEFX61I4fRe8ALWlpTerA/b+ZjAm9+3rJYiwg5fdbkdqnVVvt/fxf6YgxNjCydkELxhA7ptJ2W3qmrx8eLKOMXUBJttpHbTP7LPXSyRhB69NmzZ1+m9BSEj+ZrRKUG0V9WHLS1YUztY3kxqaIiS6jeGKaMyrpaWF5uZLH3JZWRnPPPMM7777btQqJggx5WndMciQDBptn72MxaCe262YUBC7CPVERMFr8eLFvPDCCwA0NjYye/Zs1q5dy+LFi9mwYUNUKygI/a2x2ce7+05wpt7dp1caQc3zAvBqTGqiqpgiFLaIV1K98cYbAfj9739PdnY2ZWVlvPDCC/znf/5nVCsoCP2tsdnPxQt1OD3+Ph+s12okTJfvIiS6jWGLKHg1Nzdjtapzvd59913uvPNONBoN119/PWVlZVGtoCD0N5dXTVDt6zSJNklGLX6NGX9QdBt7IuINOLZs2UJFRQXvvPMO8+fPB6C2tlYsMyPEPafHjzHgbE1QTenz11P3bzSL/Rt7KKLgtWbNGh577DEKCgq47rrrQlNz3n33XWbMmBHVCgpCfwtNDdJqwJzS56+XZNQhGdt2ERLBK1wRref1T//0T3zuc5+jqqqKadOmhY7fcsst/OM//mPUKicIseBq8ZETbMKos4A5tc9fb+HkbKQxFti5XR3zUpSo71SUiCIKXqCu7nDlSg6zZ8/udYUEIda8rgYkRcZg0PXJOl5XkiQJ9EnqHUWGgEfNuBe6FVHwcrvd/PjHP2bbtm3U1tYiy3K7x8WO2kI80/qbANAnpfbfJGmtDnRGdf9Gn1sErzBEFLxWrlzJBx98wL333ktOTk5o2pAgJIK7JyUha9OQbOn98nq1TR4+/KyecfVBJqWgBq+k/nnteBZR8Przn//MW2+9xQ033BDt+ghC7LU0opGkfhnvAnUVnLILzQzx6YCAGLQPU0Rt4tTU1G43yRCEuNbSoP7sp+BlMapThJoVozpFSCyLE5aIgtdTTz3FmjVr2s1vFIREcPi8g73Hz1Lj9PRLmgSApXUXIY9kJiiLdIlwRdRtXLt2LadOnSIrK4uCggL0en27x6+22awgDFS1zhYMzjp8Nl2/JKgC6LQajHoNfq0JX0BGJ4JXWCIKXkuWLIlyNQRhYHC5msiQ/Rj1hn5reQEkGXSt8xtFtzFcEQWvJ554Itr1EIQBwdt0EQC92Q5a/VVKR4/ZoKVZ0zpFSOwiFJaIk1gaGxv51a9+RXFxMRcvqv/h+/bt4/z581GrnCD0J0VR8LvU32WjbUi/vnaSQYfGaEFRECtLhCmi4PXpp58yduxYfvKTn/Dzn/+cxsZGQF27vri4OOzz7NixgzvuuIPc3FwkSWLLli3dlt++fTuSJHW4dbVbtiD0hNsXROdrRAKM1v69mn77lGyWzplEhtWoTs4OY3+HwS6i4LV69WqWL1/OZ5991m6H7Ntvv50dO3aEfR632820adN49tlne/T6x48fp6qqKnTLzBRrfgu952zxYw44MOg0aPs5SbTdFKFgAIK+fn39eBTRmNff/vY3nn/++Q7Hhw4d2qNW0MKFC1m4cGGPXz8zM5OUlJQeP08QuuMLyNgVp7qOlyUGeYw6gzrOFvSr6RI6sdN7dyJqeRmNRpxOZ4fjJ06cICMjo9eVuprp06eTk5PD5z//eT766KNuy3q9XpxOZ7ubIHSmYIiFm4ZqmJhjA0v/jnnVOj28tu8cxy4E1QMiXeKqIgpeX/jCF3jyySfx+/2A2uQtLy/nX//1X/niF78Y1QpeLicnh+eee47/+7//4//+7//Iy8tj7ty53eaVlZSUYLfbQ7e8vLw+q58Q53wuCPqRNP2zCOHlgopC2YVm6tt2ERLpElcVUfBau3YtLpeLjIwMWlpamDNnDqNHj8ZqtfLDH/4w2nUMGTduHA8++CAzZ86kqKiIjRs3UlRUxLp167p8TnFxMQ6HI3SrqKjos/oJca75gvrTlKKu8tCP2jbicIV2ERLpElcT0f+Q3W5n69atfPTRRxw8eBCXy8U111zDrbfeGu36XdXs2bP58MMPu3zcaDRiNIqxAwDcF+DMdvB7IO86SB8d6xoNKFv3HGNonZPhY4aT1M+v3bYFmk9jIig70Il0iavqcfCSZZnS0lJee+01zp49iyRJjBgxguzsbBRF6fflcQ4cOEBOTk6/vmZcammE/f8L/hb1fmM5TPknSB8T02oNFIGgjKuxBkeLH21y/y9Ho9dqMOg0+DRqlr1OdBuvqkfBS1EUvvCFL/CnP/2JadOmMWXKFBRF4dixYyxfvpzXXnvtqrlal3O5XJw8eTJ0/8yZMxw4cIC0tDTy8/MpLi7m/PnzoT0in3nmGUaMGMGkSZPweDz86le/4r333hOb3Ybj1Htq4ErOVK+k1f4djv8ZUoarV7kGOUeLH5PPgVYjYeyndbyulGS4tBGHWXQbr6pHwau0tJQdO3awbds25s2b1+6x9957jyVLlvDCCy+wdOnSsM63Z8+edudZvXo1AMuWLaO0tJSqqirKy8tDj/t8Pr7zne9w/vx5LBYLU6dO5S9/+UuHughXaKqBuuPquugTvqAGr6YademXyn2Qf32saxhzDc1+zIFGTHoNUj9faWxjMejwaE1i/8YwSYoSfirv/Pnzufnmm3n88cc7ffxHP/oRH3zwAe+8807UKhhtTqcTu92Ow+EYPNu0nXgXzu+FzPEwqXWDlMoDasvLnArXPTjoN3zYe6YW33s/Iz1Zz5glxWDs+7Xrr/Tmp5U0VJ7iNv82MjKy4fqH+r0OA0G439EeXW389NNPue2227p8fOHChRw8eLAnpxT6mhyE2iPqv7OnXjqeOVHtLrY0qONfg5yroR5QMBjNYEiOSR1un5zDvTdOJCPZKPZvDEOPgtfFixfJysrq8vGsrCwaGhp6XSkhii6eVq8uGpIgdcSl4zoDZIxX/11/IjZ1G0BaHLUA6K3pMWuFajSS+v8EEPCp04SELvUoeAWDQXS6rofJtFotgYD4wAeUC6fUnxnjO+6Ekz5W/Vl/YtBPBLYFG9FrJUwpXf9x7hc6I2haE1XFoH23eny1cfny5V3mTXm93qhUSogSRVFbXgBpIzs+nlqgJmN6nOCqBWuMv7gxVJQtgzYNJSM3ZnW44PLywYk6xtYHmJwmqVn2/bggYrzpUfBatmzZVcuEe6VR6ActDeBxqH/JU/I7Pq7Vq13J+s/UIDeIgxfuegCkpL6fm9sVSZIou9BMqlcHBMErWl7d6VHw2rRpU1/VQ+gLF8+oP+3Dus7lSi1Qg1djOQwv7LeqDSRK0I/UtmNQDINXUtsuQlgIyk60vqaY1SUe9NN2wEJMNJ5Vf6YWAGq3v4O2FpmjQr0yOQjtPXaKfWUXqHTJMUmRaGPUadUse20SvqAMXhG8uiOCV6JSFHC0Lsltz0OWFV7bd56DFY2hIHak0sFnLpO6tXzQD02Dc0Vad0MV3oCMzxS7K41tko3qRhy+gCy6jVfRv1Pnhf7jcahrQkkasGbz9+omyi82U+30MCozmWpHC+8eqSHJqGWEfRi6i59BYxnYh8a65v3O21iDATDH+kojkGTU4dFaRMsrDKLllaiclepPaxaypGPXaXW5l9kj0kg26hiRnozdrMftDXLc27oztGPwbZ4SCMrITXUAJA+J/QT/ZKMWny4Jf0DsInQ1InglKmdrILINpaKhWZ14rNcybVgKAFqNxPUj1Tl8Bx0WZEWBpspBl+/V0OzH5L+AViNhSc2OdXVINuqRjDYUEC2vqxDBK1GFglcuRyrVpa/HZ1sx6C79l4/NSsZi0FKr2GloCaqTgT2NMahs7DQ4HBgDLiwGLVJy7DdyuWH0EFbcPIWhKWYIeNVMe6FTIngloqBfXTUC8CblcKpW7X5MzG0/yVWn1TAx14Yi6Tjnb73K5qzq16rGWlO9GuT1SanqhYsYkyRJzbJv2/BWdB27JIJXInLXgSKDwUKFW09AVrCb9WRaO86MGJelBq0yv52ALKtdx0HEFriA1ajDlBq7zPpOtaVseMWGMV0RVxsTUVvKQ3IWVrOeSbk2rCZ9p6vcZliNpCUZ0EvD8AfPo3MOruA1xtIMQ+0wvJPpUzHg8Qf506Eqss55KUpTkES6RJdE8EpELnWFBJKzyLKZmD+p64FoSZL48ux8DD4b7N6jdjfl4KXJwYnOpXavscZ+sB7U5aDLLzaj9xvwBxUMotvYJdFtTERtX8jk8PKWDDqNuiihzghyQO12DgJebwtBlzqnMdzPqq9pNRIWgxaftjVdQlxx7JIIXolGlsGttrzqsFPj9CDLYaQ/SBL+pGx1CeJB0nU8fvIUfztzgZMOwBibBQg7k2TUiSlCYRDBK9G0XFQXsdPq2VWlsHl3OfvKr75A5O7TF/jjaYUqh2fQTBNy1Z9X86kGSKurTbJRh09rwStaXt2KafDasWMHd9xxB7m5uUiSFNbOQ9u3b+eaa67BaDQyevRoSktL+7yecaW1y6gkZVDpUNdXy025egqAzaynSZ9OQ7MPmgZHuoSnQU2TSE4fFuOatGcz69WWl8iy71ZMg5fb7WbatGk8++yzYZU/c+YMixYtYt68eRw4cIBHH32UlStXDugNP/pda/By6YfQ7Aui00idpkhcqWBIEi5DBs2+ID5nrZorlsC8gSBSa/fYltnJWmcxZDOp3UZvoHVNr0E26yFcMb3auHDhQhYuXBh2+eeee44RI0awdu1aACZMmMCHH37IunXrWLBgQV9VM760JqfWyHYAsuwmdNqr/40yG7TYU9LwV5txNntJd9Um9CTtuoZGTP5GjDoNlvSBFbysJj1aUzJSk0bN1/O5B9SY3EARV2NeO3fu5NZbb213bMGCBezcubPL53i9XpxOZ7tbwlKUUMurwq9m0w8Lo8vYJm9IEm5DOs6WxF8e52JVGQAGWzoYLDGuTXtjMpN5aN5YRg9rHYsTiaqdiqvgVV1d3WH3oqysLJxOJy0tLZ0+p6SkBLvdHrrl5eX1R1Vjw9uk7ootaSjzqF/IcMa72uSlmnEZMnB4/Ak/7tVUcxYA85CB9/sQSiY2qa1nPCJ4dSauglckiouLcTgcoVtFRUWsq9R3WpNT/cZUGjzqOEmWzRT204emmmk2ZuDxy7Q0JPbyOMP1jaQnG0jNKYh1VbpmbJ2L6nHEth4DVFxl2GdnZ1NTU9PuWE1NDTabDbO58xaG0WjscrejhNPaZdRYs/ji2GFcbPZhNoSfKW/UaRk7egwjlA/RtVxQVzToau37eKYoDNM2QKYVskdcvXwMbD1ag3Law/U6PzYRvDoVVy2vwsJCtm3b1u7Y1q1bKSwcnBtHdNCanKq1ZZM/xML0vJQen6JwYgHZGRnoNdKlTP1E465XA7NWD0mxXwanM84WP/V+Ix5/UIx5dSGmwcvlcnHgwAEOHDgAqKkQBw4coLxc3X6+uLi43VZqDz30EKdPn+Zf/uVf+Pvf/85///d/87vf/Y5vf/vbsaj+wONqndaT3MsdcNrm+SXooH1V+We0+IMo1pyOG/EOEDazHq/OquZ6DbI11sIV0/+5PXv2MGPGDGbMmAHA6tWrmTFjBmvWrAGgqqoqFMgARowYwVtvvcXWrVuZNm0aa9eu5Ve/+pVIkwA1L6vlIgCf1Ok4Xt2kTvWJgMuQQY3TQ/PFc9Gs4YCgKAqHj37KgYpG6nQDK7P+claTDq82Wc2yFwP2nYrpmNfcuXM7346rVWfZ83PnzmX//v19WKs45a4DRcGnNfFRuQdJquLrc0dFdKod1TpS693km8qwTItyPWOsvsmLsek8WgnSckfHujpdspnUlpc3IKsrqvo9oA//4stgMDDbzELPtV5pdGjUzTRSLQaMusiWtUnPVpM23Q216hcngVRWnUMvt5BsMaFLGVjTgi5nNemQNXqa5dYVVcW4VwcieCWK1mVs6pXWzHpb5FdYh2Wl49Ul42jxIzsSa4WJi+c/A8CcUQDagXux3WZSg5YTCwqKSJfohAheiaK15VUtq7lBPcnvulK2zYTXlElAVmisTZy8OG8giKf2DDCwu4wAySYdZoMWY1IqQVkR416dEMErEbROC1JQOOdT58D1JnhpNBJJ6Wrm+YXqsqhUcSAoq3eT3FKJWa/BljMwln3uilYj8dCcUdwweRQ6jUZcceyECF6JwOuEgBdfEC4oVjSSREYYK0l0Jy17OADu+sS54ni+4gw62YPdmoxki5NJ56bWLHsx5tXBwO30C+Frze9yamwoio4hyQb0Yawk0Z2cocNxAL6mCwQ8bnSmpChUNLZuTHPgzLZiyB4fP2v0t81vbGmMaTUGIhG8EkFrZn161jDuGzlCzcrupRS7jTHD80iRXOiaa8A0sLtZ4dA1nibNYoCcsbGuSlhO1rrYd9TFNbUuRusbY12dAUd0GxNB62C9ZM3Gbtb3aryrjSRJpOcMV8dbEiHT3ue+9D6GRJb/1t8kCaq8Zpr9ATXPy9cc6yoNKCJ4JYK2rc6Sejkt6ErWHPVnnC+P0+IL8of3P6Liohs5KfPShq4DXIpZj6zR4ZTNarpEy9X3IhhMRPCKd63TgjyBIG+dkdlz9mLUTq1Ys6hoaGb/kWM0eeJ3WegjlQ40F0/S2OxHkx4frS4Au1mPJIFbYyUQFMHrSiJ4xTt3PSgKTUE9Jy7KnKiJ3oYNkjUHR0sAj6uRs5W1UTtvf5JlhYNldaS2lKuJu+njYl2lsOm0GpKNOjw6mzqO2RK9P0yJQASveNc6WH9RSgFJ6lVmfQc6I9Y0dfJy5bmz0TtvPzpd70LXeBajJsiQ9MwBszN2uFIsBjz6FDx+WbS8riCCV7xrHe+qCbZNC4ru5N0hrfleTXUVUbmK2Z8URWFfWSNpzWfItJrQZo5XR8HjSIpZr7a8AkERvK4ggle8a6pCQaGyj4JXckYeFoMWi7eOsxfcUT13Xyu/2EzVxSbSPWVk202QMT7WVeqxdKuRpJQMdBpJBK8riOAVz2QZXDV4/DKN2jT0WokhSVFettmaTarFQLKvlpM18bV7867TF0jxlJNj1WJISgFbbqyr1GPT81L4p89NIcduFukSVxDBK541X4BggCa/RIvOTobViEYT5W5RchZpyUb0wRbOVdfGVdfxlglZTNeVqzsoZU2Kuy5jiFZ/Kb1DDNqHiOAVz1xq0mWzIR2dVhv1LiMAWj1JqdkkG3VMTG7GF+HqrLGQrvcxLekiBq0GsqfGujq9IluGICuKenVZAMT0oPjWujt2QcEovjFqdMTLPl+NZMth8tBapJwAtK4zNZA5mv3YLXqoPqyuuGEfBpa0WFcrYm8frsZ70s/nTD6GiOAVIlpe8awt892ahVYjYdL30WRjazYSEjgH/sKE1Q4PpR+f5f1jNchVB9WD2VNiW6le0kjg0qbQ7AuGFp0UBkjwevbZZykoKMBkMnHdddfxySefdFm2tLQUSZLa3UymQbi2d+tgvYICyX2cu2RXl4XGeY56p5sjlQNzVU9/UObtw1XIioLUcBpNS4O672TmhFhXrVeGJBto1qfS4gtCs2h5tYl58HrllVdYvXo1TzzxBPv27WPatGksWLCA2tquM7ptNhtVVVWhW1lZ4iyYF7aWixD0U+uW+d9Pm9hb1oeX0ZPSwZBEc4uHNz74G+8dq8XtDfTd60VAURS2HauhodmP1aSjyKSumErONNDF96bDaUlGWvSptPiD4HWJK46tYh68nn76aR544AFWrFjBxIkTee6557BYLGzcuLHL50iSRHZ2duiWldX1FlZerxen09nulhBaV0iow069O4C3L68CShKkDsds0DJCV09AVtjTl8EyAvvKGzhW1YRGklg4QofBUabWe+i1sa5arw1JNhDUGGiULeqgvWh9ATEOXj6fj71793LrrbeGjmk0Gm699VZ27tzZ5fNcLhfDhw8nLy+PxYsXc+TIkS7LlpSUYLfbQ7e8vLyovoeYcZ4HoDKoDkTnpJj79vVShiMhMSWpEYCDFY00Nvv69jXDdLLWxV8/U7/QN41NZ2hT61hX+lgwp8SuYlFiNeow6bW427qOYtwLiHHwqq+vJxgMdmg5ZWVlUV3d+RpS48aNY+PGjbzxxhu8+OKLyLJMUVER5851vlxxcXExDocjdKuoSJANJRzn8AdlqhgCQI69j8f9UtVpQhnyBUam6gjKCtuP13W772Z/aPYFePtwFYoCk4famT4kCDWtf8zyr49p3aJFkiQyrWrX0eUNiHSJVnGXKlFYWEhhYWHoflFRERMmTOD555/nqaee6lDeaDRiNMb3mEcHAS+462jyBGgyZDEk2dB3VxrbmFPVVkxLI3My3JQ5TJypd3PwnIPpeSl9+9rdsBh0LJiUzWe1Lm4Zn4l07A01PSJ9TFxm1HelIN2CqzkPo/N0YiwOGQUxbXmlp6ej1Wqpqalpd7ympobs7PCuoOn1embMmMHJkyf7oooDU1MVKAoNihm/Lolcex93GdukjwEgpfksN45JB2DHiToczf2/1tflOW1jsqzcPiUHTVMl1B5TD464qd/r1JdmDk9jzsyppFgM6mR8OX5mOvSVmAYvg8HAzJkz2bZtW+iYLMts27atXeuqO8FgkEOHDpGTk9NX1Rx4HOp4V43S2mVM6adUkSFq8OLCSaYPszEu28pNYzPUhNB+IssKH52s54WdZe0XSJRl+Oxd9d/ZUyA5s9/q1G/MqeqVU1l0HWEAdBtXr17NsmXLuPbaa5k9ezbPPPMMbrebFStWALB06VKGDh1KSUkJAE8++STXX389o0ePprGxkZ/97GeUlZWxcuXKWL6N/tU6WG8ckodN0fdfy8ueB3oz+FuQHOdYODkf6bL5grVOD8kmHRZD3/xaOZr9bD1WQ8VFNVXgRI2LmcNT1Qcr96vdKZ0RRs7tk9ePNQVoMWZg8JWja6oEa9dX2QeDmAevu+++m7q6OtasWUN1dTXTp0/n7bffDg3il5eXo9FcaiA2NDTwwAMPUF1dTWpqKjNnzuTjjz9m4sSJsXoL/UuWwaFedJg5dSozrdn9N2iu0cCQ0VB9COqOI7UO4gMEZYW3DlXh9gYYl21jfLaVoSnmqEwU9wdlDlY0suv0BfxBBb1W4taJWYzPbt3T0H0BTr+n/nvEHDAm9/o1B6I/flqFvwxuMPjIFuNeSEqsLxf1M6fTid1ux+FwYLPZYl2dnnOcg33/C3oT3PBo/6+UcOEUfPo7tQVW9M3Q/odOj583D1ZR4/SEipr0WoYPsZCbYmZ4moXUCJbrOXzewc5TF9SrbMCwVDO3Tsi6dK6gH/b/rzrPM7UApn0pflePuIpdpy9w/NDfKPLsYMzIUTDr/lhXqU+E+x2NectL6KEGdTaB0zSUZEWd99avUkeoLRuvC+o/g0x1gT+bSc+XZ+dR6fBw6JyDM/VuPP4gx6ubOF7dxKyCND7XOsjf4gtystZFikWP2aDFpNciKwr+gIyjxU9uijl09dTp8ePyBrCZ9Vw/Mo2JObZLXVVFgb+/pQYuvRnGL0rYwAUwNMXMXkMmTY0BFHcdUsAb97MHekMEr3jTcJaALPNWhZ4G1ynuu2FE36dJXE6jgazJUL4Lqj8NBS9Q85GGppgZmmJGlhUqHS2ca2ihsrGFvLRL43LVTg9/OVbT2dkBWDQ1h7FZ6vpVk3Lt2Ex6JuTY0F4eqRUFTm1Try5KGpj0j2CKw5Z0D2TbTQQNVpwk4fUFMDnOxc0elH1BBK94EvSD8zzOlgCNxqEkGXT9G7ja5EyDit1qF9JVB8kd94vUaCSGpVoYlmrp8JhOIzF8iIUmT4BmXxBfQEYjgVYrYTPp8QUupUHYzXrsQ+3tTyDLauA6t0e9P+62UBJtItNrNWRajTiMuTR5zmNqLBPBS4gTjeUgB6n3G/Ek2Rmf1jEw9AtLmprzVXcCKnbBhDt69PS8NAt5kdbd3wLH/qgGToCxC9RgOkgMTTVz1pSDw3OWjMYEmS0SoZhPzBZ6oP4zFBROK1kgSYzMSIpdXfJb8/BqjvZPzpGiQN1x+OR/1MCl1cHExTD0mr5/7QGkYEgSTcYcGpt9KE1V6myLQUoEr3ihKHDhM1zeAFW6PAw6Taddsn5jy1VbX0prcmhfXrRuLIcDv4XDr4HPDZYhMONeyBok6TGXyU0xM3FkHiPyhqnd54azsa5SzIhuY7xoqgKvi4secFhyGZue1H4AOxZG3wIXz6hXQCv3R7cVFPCqE6yrDoSWu0ajhbzZMPxzastrENJqJOaOywTNJDj3N/WKb0b87AIeTYPzNyAe1R1Xu4zBTBRJF9suYxtzKoy4EU69Dyf/oraIejNw7nOrX8b6z9QWhdy64KFGq26gMbwQTPZuTzFopI9Rg9eFk2oLTDP4OlEieMUDWYaaw0hITJ1ZiM6fysj0AZJFnnedOl2p7gQc+h2MWxR+dy7oV5NuG8vU1lvrhPOQpHR1MD5rMhhi2EUegGo0GbgaA+QmuTA7z0FKfqyr1O9E8IoHjWfVpFC9iSF5k7hxIHWZJAkmLAb5NXUg/egbalcvZ7q6a0/bfoP+FvA41FVAm2rUQNVU1XF1BGu2uohg+lg1eCVw0mlv7DrTgOTNwiBXkFdzRAQvYYCq+lT9mTlpYI71aHUw+Z/g7F/V/K+GstBMgKsyWtWuZspwdXpPgieaRsv4bBsfVoyh7uJJhtUcRRp1i7rZyCAyAL8JQjseB9Qdp8bp4ZAli6lOD5l9sblsb2k0MHKO2s2r3Ke2wpovqlcj2xiS1HEya466IoJtqHpftK56bGRGEn9JGobzYhKNTS5S64/H/RZvPSWC10B3bg+KHOSEL41DTjNDGlsGZvBqY06BUTerN1mGQIt6XGscmK3GOKXXapial0LVxXFUNh4g9dzf1LHBQfSHYPBdoognPjdU7ueC28dJ4ySMeg2TcuPoaptGo7a2DEkicPWB6Xkp1Nsm0eCVaKo7d2nWwSAhgtdAduavyAEff3cn0WjK45r8VAw68V8mqKwmPaOHZlBjnUjZxWaUs39VW7uDhPgmDFSO81B1gPONLfw9aTbJJj3X5KfGulbCAFM0Op2m9Bmk2pLBWaWONw4SIngNRH4PHPsjTR4fB33DaDLlcNPYDNHqEjpINuq4d84khl6zUF3n7PT2QbO+vfg2DDRBPxx5DVoaqGjWczq1iLFZVsZlW2NdM2GA0mokyL0GUvLxeD149r2s5gUmuAERvJ599lkKCgowmUxcd911fPLJJ92Wf/XVVxk/fjwmk4kpU6bwpz/9qZ9q2se8Ljj4spojpdUz7tYVzB6Ty+cnDu6NFoQwSBKNI25nfy0cO11O066NCd8Ci3nweuWVV1i9ejVPPPEE+/btY9q0aSxYsIDa2tpOy3/88cd8+ctf5v7772f//v0sWbKEJUuWcPjw4X6ueRQF/XBuL75dz1NZ9hlBrQGm/j+0tmyuHzlEdBeFsEjGZE7l/gMOkjh6qpyz7/yCluPvqVetE1DMN+C47rrrmDVrFr/4xS8Add/GvLw8vvnNb/L44493KH/33Xfjdrt58803Q8euv/56pk+fznPPPXfV14vpBhyKAkE/it+N3+3A56ihpb4MX+0JmlxuHC1+XLohJM24k+unDM6VAoTe8fiD/OXgKbTH/0SK5xwSYLUYMWWMZPjoCeisWWBORdGbkXSmAZkXFhcbcPh8Pvbu3UtxcXHomEaj4dZbb2Xnzp2dPmfnzp2sXr263bEFCxawZcuWTst7vV683ksLtjmdzrDr52j2c+zN9UhKEFBQFJBQWrcaU8ixmxieZgFFwRsIsr/8IgCSoqC0/gQZRYEsm5ERqeqGob5AkH3lje3rqUumMmU2Uu50rsvuuKyyIITDpNeyaOYYzuTfz/HDe9Cf343SXE9T2TFGampCwepETRONniBo9OqqHZIWNBoUtGg0ErML0tQTShInappoaPZd8UqXgt6sgrTQRjAn65q54G6/QGK9fQoX7RO5bVI2GdbobRgS0+BVX19PMBgM7dHYJisri7///e+dPqe6urrT8tXVne9jV1JSwve///2I6hdUFGRXLdJlU1wub6YqehOYW/9TA0EUT1O7Mu3K+mSQ1fXmtTo9Xl0yHsMQSM5EmzEGe+Zw5mcmMyTJ0G4jV0HoKUmSGJlpZeTN87joKuJ81Xl8NSeQUj3grgOPg0BQRg4GIRjs8HwZoPnSb6/ibkJ2Xxm8LuMOXmrBuZqQXe3LujVO6jVegnJ0O3kJn/ZcXFzcrqXmdDrJy8sL67nJRh3D5953qWUtSa3/lpAkCYtBD2Y9SBI6BUZN9CFJ6mMKhPYlk9CoG2WYTaC3oNXoKJQVDFqNCFRCn0pLNpI2ZiSMGdnu+EiPF09zE4GAD+QgSlBGlv0oclBdlch+aQpautuHNSB3sVqugnRZ2bRmH2Zf+0TZYUY7iimFFIs+mm8ttsErPT0drVZLTU37bbBqamrIzs7u9DnZ2dk9Km80GjEaI2uqGnQaho4cf/WCgBbIDHPmjgQYxRi8EENmkxGzKbzvxZAe5EanpUVYoQjE9CtkMBiYOXMm27ZtCx2TZZlt27ZRWFjY6XMKCwvblQfYunVrl+UFQUhMMe82rl69mmXLlnHttdcye/ZsnnnmGdxuNytWrABg6dKlDB06lJKSEgAeeeQR5syZw9q1a1m0aBEvv/wye/bs4Ze//GUs34YgCP0s5sHr7rvvpq6ujjVr1lBdXc306dN5++23Q4Py5eXlaC5bn7uoqIjNmzfzH//xH/zbv/0bY8aMYcuWLUyePDlWb0EQhBiIeZ5Xf4tpnpcgCFcV7ndUDBsLghCXRPASBCEuieAlCEJcivmAfX9rG+LryTQhQRD6T9t382rD8YMueDU1qVN4ws2yFwQhNpqamrDbu878HnRXG2VZprKyEqvVOiin5rRNj6qoqBBXW8MgPq+eicbnpSgKTU1N5ObmtkuTutKga3lpNBqGDRsW62rEnM1mE1/GHhCfV8/09vPqrsXVRgzYC4IQl0TwEgQhLongNcgYjUaeeOKJiFfaGGzE59Uz/fl5DboBe0EQEoNoeQmCEJdE8BIEIS6J4CUIQlwSwUsQhLgkglcCEjuQ90xPPq/S0tLQJittN5PJ1GX5RLNjxw7uuOMOcnNzkSSpyy0HL7d9+3auueYajEYjo0ePprS0NCp1EcErwYgdyHump58XqNnjVVVVoVtZWVk/1ji23G4306ZN49lnnw2r/JkzZ1i0aBHz5s3jwIEDPProo6xcuZJ33nmn95VRhIQye/ZsZdWqVaH7wWBQyc3NVUpKSjot///+3/9TFi1a1O7Yddddpzz44IN9Ws+Boqef16ZNmxS73d5PtRvYAOX111/vtsy//Mu/KJMmTWp37O6771YWLFjQ69cXLa8E0rYD+a233ho6Fs4O5JeXB3UH8q7KJ5JIPi8Al8vF8OHDycvLY/HixRw5cqQ/qhuX+vL3SwSvBNLdDuRd7Sje0x3IE0kkn9e4cePYuHEjb7zxBi+++CKyLFNUVMS5c+f6o8pxp6vfL6fTSUtLS6/OPehWlRCE3igsLGy3R2hRURETJkzg+eef56mnnophzQYf0fJKIP2xA3kiieTzupJer2fGjBmcPHmyL6oY97r6/bLZbJjN5l6dWwSvBCJ2IO+ZSD6vKwWDQQ4dOkROTk5fVTOu9envV6+H/IUB5eWXX1aMRqNSWlqqHD16VPna176mpKSkKNXV1YqiKMq9996rPP7446HyH330kaLT6ZSf//znyrFjx5QnnnhC0ev1yqFDh2L1FvpVTz+v73//+8o777yjnDp1Stm7d6/ypS99STGZTMqRI0di9Rb6VVNTk7J//35l//79CqA8/fTTyv79+5WysjJFURTl8ccfV+69995Q+dOnTysWi0X553/+Z+XYsWPKs88+q2i1WuXtt9/udV1E8EpA//Vf/6Xk5+crBoNBmT17trJr167QY3PmzFGWLVvWrvzvfvc7ZezYsYrBYFAmTZqkvPXWW/1c49jqyef16KOPhspmZWUpt99+u7Jv374Y1Do23n//fQXocGv7jJYtW6bMmTOnw3OmT5+uGAwGZeTIkcqmTZuiUhexJI4gCHFJjHkJghCXRPASBCEuieAlCEJcEsFLEIS4JIKXIAhxSQQvQRDikghegiDEJRG8BEGISyJ4CQmvtLSUlJSU0P3vfe97TJ8+PXR/+fLlLFmypN/rJfSOCF5C1CxfvhxJknjooYc6PLZq1SokSWL58uXtykc7aBQUFPDMM8+0O3b33Xdz4sSJLp+zfv36duuqz507l0cffTSq9RKiTwQvIary8vJ4+eWX2y005/F42Lx5M/n5+TGpk9lsJjMzs8vH7XZ7u5aZEB9E8BKi6pprriEvL4/XXnstdOy1114jPz+fGTNm9OrcnbWIlixZEmrNzZ07l7KyMr797W+HdvaBjt3GK13eAly+fDkffPAB69evD53jzJkzjB49mp///OftnnfgwAEkSRJrecWICF5C1N13331s2rQpdH/jxo2sWLGiz1/3tddeY9iwYTz55JOhnX16av369RQWFvLAAw+EzpGfn9/hPQFs2rSJm266idGjR0frLQg9IIKXEHVf/epX+fDDDykrK6OsrIyPPvqIr371q33+umlpaWi1WqxWK9nZ2RGtBmu32zEYDFgsltA5tFoty5cv5/jx46E9Hf1+P5s3b+a+++6L9tsQwiTWsBeiLiMjg0WLFlFaWoqiKCxatIj09PRYV6tXcnNzWbRoERs3bmT27Nn88Y9/xOv1ctddd8W6aoOWaHkJfeK+++6jtLSU3/zmN1FrnWg0Gq5cfs7v90fl3OFYuXJl6GLEpk2buPvuu7FYLP32+kJ7IngJfeK2227D5/Ph9/tZsGBBVM6ZkZHRbhwrGAx22NnbYDAQDAZ79TpdneP2228nKSmJDRs28Pbbb4suY4yJbqPQJ7RaLceOHQv9uysOh4MDBw60OzZkyBDy8vI6lL355ptZvXo1b731FqNGjeLpp5+msbGxXZmCggJ27NjBl770JYxGY0Td1YKCAnbv3s3Zs2dJTk4mLS0NjUYTGvsqLi5mzJgxg2KTkoFMtLyEPmOz2bDZbN2W2b59OzNmzGh3+/73v99p2fvuu49ly5axdOlS5syZw8iRI5k3b167Mk8++SRnz55l1KhRZGRkRFTvxx57DK1Wy8SJE8nIyKC8vDz02P3334/P5+uXq6dC98Qa9oLQA3/961+55ZZbqKio6LATtNC/RPAShDB4vV7q6upYtmwZ2dnZ/Pa3v411lQY90W0UhDC89NJLDB8+nMbGRn7605/GujoCouUlCEKcEi0vQRDikghegiDEJRG8BEGISyJ4CYIQl0TwEgQhLongJQhCXBLBSxCEuCSClyAIcen/Axb91ToXC2baAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T18:25:47.653714Z", + "iopub.status.busy": "2024-03-24T18:25:47.653370Z", + "iopub.status.idle": "2024-03-24T18:25:48.001264Z", + "shell.execute_reply": "2024-03-24T18:25:48.000309Z" + }, + "papermill": { + "duration": 0.36818, + "end_time": "2024-03-24T18:25:48.003512", + "exception": false, + "start_time": "2024-03-24T18:25:47.635332", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABL9klEQVR4nO2deXxU9bn/32f2mWRmkpB9Y5F9RxQNVkGlUqQWrq31elsBUVtb7auUa1u5C171emNvRaW3/rDWQvRartUWsVUrUhRQQCuriAiyZIHs20xmklnP+f1xkoGBJEySSWYm+b5fr/NK5sx3zjwzmfnk+T7f5/s8kqIoCgKBQJBgaGJtgEAgEPQGIV4CgSAhEeIlEAgSEiFeAoEgIRHiJRAIEhIhXgKBICER4iUQCBISIV4CgSAh0cXagIFGlmUqKyuxWq1IkhRrcwQCwQUoikJLSwu5ubloNF37V0NOvCorKykoKIi1GQKB4BJUVFSQn5/f5f1DTrysViugvjE2my3G1ggEggtxOp0UFBSEvqtdMeTEq2OqaLPZhHgJBHHMpcI6ImAvEAgSEiFeAoEgIRHiJRAIEpIhF/OKBEVRCAQCBIPBWJsypNHr9Wi12libIYhThHhdgM/no6qqitbW1libMuSRJIn8/HySk5NjbYogDompeK1bt45169ZRWloKwKRJk1i9ejULFizodHxJSQl33XVX2Dmj0YjH44mKPbIsc/r0abRaLbm5uRgMBpHIGiMURaGuro4zZ84wZswY4YEJLiKm4pWfn88TTzzBmDFjUBSFF198kUWLFnHgwAEmTZrU6WNsNhvHjh0L3Y6muPh8PmRZpqCgAIvFErXrCnpHRkYGpaWl+P3+hBKv3SfqGZttJT3ZGGtTBjUxFa9bbrkl7Pbjjz/OunXr+Oijj7oUL0mSyM7O7le7utuSIBg4EtHrVRSFT886+LzKyZ1FwzHqEkd0E424+ZYGg0FeeeUV3G43RUVFXY5zuVwMHz6cgoICFi1axJEjR7q9rtfrxel0hh0CQX8RlBUKnAcZf+IFqnb8DryuWJs0aIm5eB0+fJjk5GSMRiP33Xcfr7/+OhMnTux07Lhx41i/fj1vvPEGL7/8MrIsM3v2bM6cOdPl9YuLi7Hb7aFD7GsU9AfHqltocvvQNZ7gCvlTtLKP5rMnUY69HWvTBi1SrFuf+Xw+ysvLcTgc/PGPf+SFF15gx44dXQrY+fj9fiZMmMAdd9zBY4891ukYr9eL1+sN3e7YN+VwOC7aHuTxeDh9+jQjR47EZDL17YUNMHPnzmX69Ok888wzsTYlaiTK38PjD/KbHaeQZZn7kraj99TzXk0SyW1nmVFgxzz7e5CcGWszEwan04ndbu/0O3o+Mfe8DAYDo0ePZubMmRQXFzNt2jTWrl0b0WP1ej0zZszgxIkTXY4xGo2hfYxiPyNs374dSZJobm6OtSmDhtP1bmRFoUDvwOxrRKc34B79dRotI2lq9UHVoVibOCiJuXhdiCzLYZ5SdwSDQQ4fPkxOTk4/WyUQdM3ZpjYAxmqr1BNpl1GYlUa9ZTQtngDUfwmit3PUial4rVq1ip07d1JaWsrhw4dZtWoV27dv5zvf+Q4AS5YsYdWqVaHxjz76KO+++y6nTp1i//79fPe736WsrIx77rmn3231BeQuj0BQjnisP8KxfcHr9fLzn/+cgoICjEYjo0eP5ne/+x2lpaVcf/31AKSmpiJJEsuWLbvk9VpaWvjOd75DUlISOTk5PP3008ydO5cVK1aExvzv//4vV1xxBVarlezsbP7pn/6J2tra0P0dHt+2bdu44oorsFgszJ49OyztJVGpdKjileOvUE9kjCfbbsZhysfpAzwOcNfHzsBBSkxTJWpra1myZAlVVVXY7XamTp3Kli1b+OpXvwpAeXl5WNpCU1MT9957L9XV1aSmpjJz5kx2794dUXysrzz7ftdT05HpSSyekRe6/fzOk/iDnf+nzU81c9sV5xYN1u86TZvv4m1IP/nq2F7bumTJEvbs2cOvfvUrpk2bxunTp6mvr6egoIA//elPfPOb3+TYsWPYbDbMZvMlr7dy5Up27drFn//8Z7Kysli9ejX79+9n+vTpoTF+v5/HHnuMcePGUVtby8qVK1m2bBlvvx0esP7Xf/1X1qxZQ0ZGBvfddx/Lly9n165dvX6tscbjD9Lg8qEPtpIiOUDSQtpIsjRG/uGKEeSUTgRnBTjPQHJGrM0dVMRUvH73u991e//27dvDbj/99NM8/fTT/WhR4nP8+HFeffVVtm7dyrx58wAYNWpU6P60tDQAMjMzSUlJueT1WlpaePHFF9m4cSM33ngjABs2bCA3Nzds3PLly0O/jxo1il/96ldceeWVuFyusO09jz/+OHPmzAHgoYceYuHChXg8nrgOyHdHtUPd3ZEn1aPXaiApHfRmdEDhMAs4ClTxcpyF3BmxNXaQIfY2Rsj914/u8j7NBbmU37vusi7HXph3ufyakX0x6yIOHjyIVqsNCURfOXXqFH6/n1mzZoXO2e12xo0bFzZu3759/Md//AeHDh2iqakJWVanvuXl5WGe8dSpU0O/d8Qqa2trKSwsjIq9A029S43P5msa1BP2C1Jx7O1ljJ1nB9CqoYEQrwgx6CIPD/bX2EiIZBoYbdxuN/Pnz2f+/Pn8/ve/JyMjg/LycubPn4/P5wsbq9frQ793ZNB3CF0iMinXTobViP3oDvBzTqyABpeXz2sMjGhqpQDA7wF9YnqY8UjcrTYK+saUKVOQZZkdO3Z0er/BYACIuNzPqFGj0Ov1fPLJJ6FzDoeD48ePh25/8cUXNDQ08MQTT3Dttdcyfvz4sGD9YMZs0DI81UxKsFk9kXxu61qbP8jes22cbWsXbPfQeE8GCiFeg4wRI0awdOlSli9fzubNmzl9+jTbt2/n1VdfBWD48OFIksSbb75JXV0dLlf321esVitLly7lpz/9Ke+//z5Hjhzh7rvvRqPRhDynwsJCDAYD//M//8OpU6f485//3GXS8KCkrRHkAGj1YEkLne7YmN0opRCQZXDXxcrCQYkQr0HIunXr+Na3vsUPf/hDxo8fz7333ovb7QYgLy+PRx55hIceeoisrCweeOCBS17vqaeeoqioiK9//evMmzePa665hgkTJoSC7BkZGZSUlPDaa68xceJEnnjiCZ588sl+fY3xQHOrj10n6qmoOK2eSM4MC2qa9FqSjTpa9Wm0+YPgEuIVTWK+PWig6W7rQaJsR4k1brebvLw81qxZw913391vzxPvf48vqp389XA1l/v3M8d8GvIuh7Hzw8b8cd8ZWis+ZYHm72Tkj4LLl8TI2sQh0u1BImAvuCQHDhzgiy++YNasWTgcDh599FEAFi1aFGPLYkuT2w/AMJrVE53sX0wx62nQp+DxBqG1YQCtG/wI8RriXJjKcCGff/45AE8++STHjh3DYDAwc+ZMPvjgA9LT0wfKzLjE0aaupNqU9jJLSRcnoaZY9Hh0djyuoLra6G8D/cCvCA9GhHgNcXJzczl48GC39xcWFrJv376BMypBaGr1o5H9JCutgBEswy4ak2LRI2v0uKX2yrytjWDPu2icoOcI8Rri6HQ6Ro/uOgFX0DmKotDU6sMccGDUa8Fg6dSjGj4sie/PGYX5yAhoroC2JiFeUUKsNgoEvcDjl/H6ZUz+Zkx6LZjTOh2n12qwGHRIHV5ZW+MAWjm4EeIlEPSCplY13jVMakErSZ1OGcMwp6o/W4V4RQshXgJBL8i2mbjrmhHMzm3/Clk697wADpQ3sa3MT3OrT502CqKCEC+BoBdoNBIpFgOptK80duN51Ti9nHDqcPuC4GkeGAOHAEK8BILeoijnYlhdxLwAbCYdXp0VX6A9XSIQWaVgQfeI1UaBoBd8fKoBvE6mebyYDHowp3Q51mrSE9QYaFPaN2h7nKIwYRQQntcg4cKyzLEm3uyJNofPOvjsRCl+WVaFS9N1c1mrSfURnLTnenlF79BoIMRLEOLC2luCzgnKCi5vAGPApXbENtm7HW8zqx6XU2kXLxH3igpCvC6FokDAF5sjwj3zy5YtY8eOHaxduxZJkpAkiZMnT3L33XczcuRIzGYz48aNu6il3LJly1i8eDGPP/44ubm5oeqou3fvZvr06ZhMJq644go2b96MJElhmfifffYZCxYsIDk5maysLO68807q6+u7tKe0tDQqf454wOUJoChglt3otRIYu2+nl2xUPS+3lKSWxvEIzysaxDTmtW7dOtatWxf6YE+aNInVq1ezYMGCLh/z2muv8e///u+UlpYyZswYfvGLX3DzzTf3n5FBP3ywpv+u3x3X/jPoDJcctnbtWo4fP87kyZNDm6ZTU1PJz8/ntddeY9iwYezevZvvfe975OTk8O1vfzv02G3btmGz2di6dSug7ui/5ZZbuPnmm9m4cSNlZWUXTf+am5u54YYbuOeee3j66adpa2vj5z//Od/+9rd57733OrUnI2PwxHicHnVDdoqmDQkJTN2Ll0GnwWzQEtRb8QcVdB7HQJg56ImpeOXn5/PEE08wZswYFEXhxRdfZNGiRRw4cIBJkyZdNH737t3ccccdFBcX8/Wvf52NGzeyePFi9u/fz+TJk2PwCuIDu92OwWDAYrGQnX2ukucjjzwS+n3kyJHs2bOHV199NUy8kpKSeOGFF0IVVp977jkkSeK3v/0tJpOJiRMncvbsWe69997QY379618zY8YM/uu//it0bv369RQUFHD8+HHGjh3bqT2DBUebKl52Tat64hKeF8Cy2SMwNnqRPv9SxLyiREzF65Zbbgm7/fjjj7Nu3To++uijTsVr7dq1fO1rX+OnP/0pAI899hhbt27l17/+Nc8991z/GKnVqx5QLNDqLz2mG5599lnWr19PeXk5bW1t+Hy+sHZloJaN7hAugGPHjjF16tSw+lnnN98AOHToEO+//35YV6AOTp48ydixvW/blgi0eAIAJNMuXpfwvID2LUQp6g0xbYwKcZMqEQwGee2113C73RQVFXU6Zs+ePaxcuTLs3Pz589m8eXOX1/V6vWEduJ3OHn5wJCmiqVu88corr/Dggw+yZs0aioqKsFqt/PKXv+Tjjz8OG5eUlNTja7tcLm655RZ+8YtfXHTfUOhe7vT4QVFIUloBwyUD9iE6PDSfC+RgtyuUgksTc/E6fPgwRUVFeDwekpOTef3117usL1VdXU1WVlbYuaysLKqrq7u8fnFxcdj0abBiMBjCmmrs2rWL2bNn88Mf/jB07uTJk5e8zrhx43j55Zfxer0YjWoN9vObbwBcfvnl/OlPf2LEiBHodJ1/hC60ZzBx4/hMZuUaSdqnU/+5GayXfEx5QysHypuZ0eylMMWoTh079jsKekXMVxvHjRvHwYMH+fjjj/nBD37A0qVLQwXwosGqVatwOByho6KiImrXjidGjBjBxx9/TGlpKfX19YwZM4a9e/eyZcsWjh8/zr//+79fJEKd8U//9E/Issz3vvc9jh49ypYtW0L16Dsabtx///00NjZyxx138Mknn3Dy5Em2bNnCXXfdFRKsC+1J5PZmF6LTakjVejBoNWBIAu2lfYBWf4BT9a00+Nu9eG/3jU8Elybm4mUwGBg9ejQzZ86kuLiYadOmXbSk30F2djY1NTVh52pqaroNChuNRmw2W9gxGHnwwQfRarVMnDiRjIwM5s+fz6233srtt9/OVVddRUNDQ5gX1hU2m42//OUvHDx4kOnTp/Ov//qvrF69GiAUB8vNzWXXrl0Eg0FuuukmpkyZwooVK0hJSUGj0XRqT3l5ef+9+FjQEXSPIFgPkGRoT5egveaXT4hXX4n5tPFCZFkOi1GdT1FREdu2bQtbut+6dWuXMbKhxNixY9mzZ0/YuQ0bNrBhw4awc8XFxaHfS0pKOr3W7NmzOXToUOj273//e/R6fVhX6zFjxrBp06Ye2TMYcHsDfHiinnxXOZMgomA9nMv1cmEGPMLzigIxFa9Vq1axYMECCgsLaWlpYePGjWzfvp0tW7YAsGTJEvLy8kJfuB//+MfMmTOHNWvWsHDhQl555RX27t3L888/H8uXMeh46aWXGDVqFHl5eRw6dCiUwxWLbtzxRnObn88rnSjuGibZidzzahevVswE5AZ0Il2iz8RUvGpra1myZAlVVVXY7XamTp3Kli1b+OpXvwqozSE6piGgegQbN27k3/7t3/iXf/kXxowZw+bNm4d0jld/UF1dzerVq6muriYnJ4fbbruNxx9/PNZmxQWu9jSJUI6XKSWixxl0Ggw6DT6tRU1UFdPGPhNT8frd737X7f3bt2+/6Nxtt93Gbbfd1k8WCQB+9rOf8bOf/SzWZsQlLq+aoKo23SDiaSOoG7R92iR8ARmzmDb2mZgH7AWCRMLlVVdTk5TIs+s7SDLoUAzJBGUZvC39Yd6QIu4C9vHAEGsiHrfE49/B5QmgkQOY8ADmHnlei6bnovVakD7eBb4WdeN9e/qJoOcIz+s89Hp1O05ra2uMLRHAuRI9Wm38ZKK7vQEMQRdGnUbdvqUzXfpB7ei0GiRje0JrMCAqqvYR4Xmdh1arJSUlhdraWgAsFksoMVMwsMiyTF1dHRaLpcss/ljQ0l7Hy6DXqNuCevr50OpBZ1SFy+cCfeTiJwgnfj4VcUJHwmuHgAlih0ajobCwMK7+gSwpGo6nrB5LqS7yPY3t1Do97DnVwNgmmGBFTXRNSu8fQ4cAQrwuQJIkcnJyyMzMxO/3x9qcIY3BYAhLlYkH9FoNeqlN9bh6EKwHCMgKp+rcpHj1TLB6RaJqHxHi1QVarTauYi2COKKjpE0PgvUAye217FtkEwotSCLXq08I8RIIIqTK0cbhMw4m1FRRINFjz6tjf6NHYyYQVNALz6tPxJdPLhDEMXUtXo5UOmlualBP9NDz0mokTHotfq0Ff1AWm7P7iBAvgSBCXJ7AeUUI6XHAHiDJqMWvNeMPKuBzR9nCoYUQL4EgQlzeAPpgKwaNEnERwguxGHT4NO2el1/kE/YFIV4CQYS4vAGMQRcGnQaMVujFSqjFoAWDhaCsiGljHxEBe4EgQkLipdf0OFjfwfxJ2WjHp8KHf1N7cwb9fW60MlQRnpdAECEdXbINWk2Pg/UdaDWSmmGvafcbhPfVa4R4CQQR4A/KeP0yhkD7tLEXwfoQkqTWvgfwibhXbxHiJRBEgF6r4YEbRnPTZWbVe+rltLHe5eWNg2c5XNe+e0OsOPYaEfMSCCJEr9WgV9yA1GvPS1bULULJHh1YEdPGPiA8L4GgJ/Swa9CFWNqz7F2KUa1XJtIlek1Mxau4uJgrr7wSq9VKZmYmixcv5tixY90+pqSkBEmSwo7zW9MLBP3B8ZoW3v20nIZmh3qilwF7i16LJKHmeskiUbUvxFS8duzYwf33389HH33E1q1b8fv93HTTTbjd3f9BbTYbVVVVoaOsrGyALBYMVc42tXGqohK3N6iuFuqMvbqORiNh1ndk2YstQn0hpjGvd955J+x2SUkJmZmZ7Nu3j+uuu67Lx0mS1G2jWYEg2oQlqPZlpRE1UfWceAnPq7fEVczL4VBd8rS0tG7HuVwuhg8fTkFBAYsWLeLIkSNdjvV6vTidzrBDIOgp4TlefRUvnbo5O6CIVIk+EDfiJcsyK1as4Jprrum2D+O4ceNYv349b7zxBi+//DKyLDN79mzOnDnT6fji4mLsdnvoKCgo6K+XIBjEuDxq7Xp1a1Dv4l0dJBm1YEhCVsQWob4gKXHSouUHP/gBf/3rX/nwww/Jz8+P+HF+v58JEyZwxx138Nhjj110v9frxes91+jA6XRSUFCAw+HAZuvbh1AwNJBlhV+99yWX1b/H/PQGDGNvhMKr+3Q9jeyDD55ST1z7z6AzRMnaxMfpdGK32y/5HY2LPK8HHniAN998k507d/ZIuEDt+DNjxgxOnDjR6f1GoxGjsXfBVYEAwO0LoChgCrrR9yFBtQONRgLJAFqd2kXI5wJd96ESwcXEdNqoKAoPPPAAr7/+Ou+99x4jR47s8TWCwSCHDx8mJyenHywUCKDVpzaateJWm4H0Mk0iDEkCQ7L6u8j16hUx9bzuv/9+Nm7cyBtvvIHVaqW6uhoAu92O2WwGYMmSJeTl5VFcXAzAo48+ytVXX83o0aNpbm7ml7/8JWVlZdxzzz0xex2CwU2WzcSPrh9FcEd79Yc+BuwbXF4+PFHPyNoAU62IFcdeElPxWrduHQBz584NO79hwwaWLVsGQHl5eVgHmaamJu69916qq6tJTU1l5syZ7N69m4kTJw6U2YIhiC7gRqcBNNpzHlMvkRU4VefG4tGKLUJ9IKbiFclawfbt28NuP/300zz99NP9ZJFA0AUdHYOM1p43mr2AJKPalaoVI7KioBGeV6+Ii4C9QBDP7CtrxHP2FFM8fmwpfY93mXRaNJKET2MhICsYRK5Xr4ibPC+BIF4prW/lTFUNXr8clWC9RiNhNmjwa0z4A2KLUG8R4iUQXAI1u74lKluDOghl2YtGHL1GiJdAcAlCW4OikF3fQcf+Rp/Y39hrhHgJBN3gDQTxBWQMQRd6bZRyvGiv62WwoCgI8eolImAvEHSD2xtUG83KbnQaKxijM228aWIWmvGp8MFWtYNQwCe2CPUQ4XkJBN3g8gTQKj5MGjXLPlqel0YjgdZwrouQX3hfPUWIl0DQDW3+4Llgvd4c3R6LootQnxDTRoGgG8ZlW7nsimEoh5OjttIIahehD7+s57I6P1PEFqFeITwvgeAS6Pwu9H1oNNsZsqJwut5NjUdMG3uLEC+B4FJ425tuRClYD5DU3kWoVTGo2+SE59VjhHgJBN3w/rFaDnxZTqsvEFXPyxzqImRu7yIkYl49RYiXQNANJ2td1NXXIStELUEVzu8i1JFlLzyvniLESyDoAkVRcHvV1UY1QTV600Zoz7LXmEQXoV7SK/E6depUtO0QCOKOVl8QRQ5gkNvauwZFt+eB6CLUN3olXqNHj+b666/n5ZdfxuPxRNsmgSAucHkDGIJuDBoJSasHvSWq1xddhPpGr8Rr//79TJ06lZUrV5Kdnc33v/99/v73v0fbNoEgpoRtyDbZ+lyE8EJumpjN3ddPIstmgoAX5GBUrz/Y6ZV4TZ8+nbVr11JZWcn69eupqqriK1/5CpMnT+app56irq4u2nYKBAOOy3NeKZwoBus70GgkNWtfav8airhXj+hTwF6n03Hrrbfy2muv8Ytf/IITJ07w4IMPUlBQwJIlS6iqqoqWnQLBgOPxB9VGs/0QrA8hSWBon44K8eoRfRKvvXv38sMf/pCcnByeeuopHnzwQU6ePMnWrVuprKxk0aJF3T6+uLiYK6+8EqvVSmZmJosXL+bYsWOXfN7XXnuN8ePHYzKZmDJlCm+//XZfXoZA0ClXjRrGNydaKUiz9It41bu8bD5wlsN1AfWEKErYI3olXk899RRTpkxh9uzZVFZW8tJLL1FWVsZ//ud/MnLkSK699lpKSkrYv39/t9fZsWMH999/Px999BFbt27F7/dz00034XZ3/R9o9+7d3HHHHdx9990cOHCAxYsXs3jxYj777LPevBSBoFu0Pic6TfQqqJ6PLKtbhOq8akMOEbTvGZISSQufCxgzZgzLly9n2bJlXTZ79fl8/N///R9Lly6N+Lp1dXVkZmayY8cOrrvuuk7H3H777bjdbt58883Quauvvprp06fz3HPPXfI5Im0lLhAA8NFz0NYEM74DKYVRvbTLG+C3O08xuuF9vp7ZgHTZ9TC8KKrPkYhE+h3tVVWJrVu3UlhYGNZPEdSkvoqKCgoLCzEYDD0SLgCHQ91DlpbWdevzPXv2sHLlyrBz8+fPZ/PmzZ2O93q9eL3e0G2n09kjmwRDlz8fPMuE8rOMSDOh7wfPy6xXPS6f1kJArkcvsux7RK+mjZdddhn19fUXnW9sbGTkyJG9MkSWZVasWME111zD5MmTuxxXXV1NVlZW2LmsrKxQt+0LKS4uxm63h46CgoJe2ScYWviDMhVVNdS3tIFGAwZr1J9Dq5Ewt2fZi1r2PadX4tXVTNPlcmEymXplyP33389nn33GK6+80qvHd8WqVatwOByho6KiIqrXFwxO3N4AxqALrQQ6s00VsH5AbcRhIRAUWfY9pUfTxo7pmiRJrF69GovlXMZxMBjk448/Zvr06T024oEHHuDNN99k586d5Ofndzs2OzubmpqasHM1NTVkZ2d3Ot5oNGI0Gntsk2Bo0+IJYGhPUJVMKf32PBaDDqemo4uQCNj3hB6J14EDBwDV8zp8+DAGw7mGAQaDgWnTpvHggw9GfD1FUfjRj37E66+/zvbt2yOachYVFbFt2zZWrFgROrd161aKikSgUxA9WjwBTIEWDDpt/+V4AUkGLS1GC4pXEakSPaRH4vX+++8DcNddd7F27do+r9bdf//9bNy4kTfeeAOr1RqKW9ntdsxmMwBLliwhLy+P4uJiAH784x8zZ84c1qxZw8KFC3nllVfYu3cvzz//fJ9sEQjOp8XjxxhsObc1qJ+YPykbzVgr7H4f/G0gy/02RR1s9Opd2rBhQ1TSDNatW4fD4WDu3Lnk5OSEjj/84Q+hMeXl5WGZ+rNnz2bjxo08//zzTJs2jT/+8Y9s3ry52yC/QNBTXF512miMYpfszlC3CFnUTHtFeF89IWLP69Zbb6WkpASbzcatt97a7dhNmzZFdM1IUsy2b99+0bnbbruN2267LaLnEAh6gy8gYwy62kvh9J94AaqnpTerAXt/KxiT+/f5BgkRi5fdbkdq31Vvt/fzH1MgiDELJmcTbDCA3D+bsjuoa/Hy4Yk6xtQFmGynPWif2W/PN5iIWLw2bNjQ6e8CwaDE34pWCapeUT96XrKiUFrfSmpoi5CYNkZKr2JebW1ttLaee5PLysp45plnePfdd6NmmEAQUzztHYMMyaDR9tvTWAzqtd2KCQXRRagn9Eq8Fi1axEsvvQRAc3Mzs2bNYs2aNSxatIh169ZF1UCBYKBpbvXx7v7jnK539+tKI6h5XgBejUlNVBVbhCKm15VUr732WgD++Mc/kp2dTVlZGS+99BK/+tWvomqgQDDQNLf6aWyow+nx93uwXquRMJ3fRUhMGyOmV+LV2tqK1aru9Xr33Xe59dZb0Wg0XH311ZSVlUXVQIFgoHF51QTV/k6T6CDJqMWvMeMPimljT+h1A47NmzdTUVHBli1buOmmmwCora0VZWYECY/T48cYcLYnqKb0+/Op/RvNon9jD+mVeK1evZoHH3yQESNGcNVVV4W25rz77rvMmDEjqgYKBANNaGuQVgPmlH5/viSjDsnY0UVIiFek9Kqe17e+9S2+8pWvUFVVxbRp00Lnb7zxRv7hH/4hasYJBLHA1eYjJ9iCUWcBc2q/P9+CydlIYyywZ7sa81KUqHcqGoz0SrxAre5wYSWHWbNm9dkggSDWeF1NSIqMwaDrlzpeFyJJEuiT1BuKDAGPmnEv6JZeiZfb7eaJJ55g27Zt1NbWIsty2P2io7YgkdH6WwDQJ6UO3CZprQ50RrV/o88txCsCeiVe99xzDzt27ODOO+8kJycntG1IIBgM3D4pCVmbhmRLH5Dnq23x8OGX9YyrDzIpBVW8kgbmuROZXonXX//6V9566y2uueaaaNsjEMSetmY0kjQg8S5Qq+CUNbQyzKcDAiJoHyG98olTU1O7bZIhECQ0bU3qzwESL4tR3SLUqhjVLUKiLE5E9Eq8HnvsMVavXh22v1EgGAx8dtbBvmOl1Dg9A5ImAWBp7yLkkcwEZZEuESm9mjauWbOGkydPkpWVxYgRI9Dr9WH3X6rZrEAQr9Q62zA46/DZdAOSoAqg02ow6jX4tSZ8ARmdEK+I6JV4LV68OMpmCATxgcvVQobsx6g3DJjnBZBk0LXvbxTTxkjplXg9/PDD0bZDIIgLvC2NAOjNdtDqLzE6epgNWlo17VuERBehiOh1EktzczMvvPACq1atorFR/YPv37+fs2fPRs04gWAgURQFv0v9LBttwwb0uZMMOjRGC4qCqCwRIb0Sr08//ZSxY8fyi1/8gieffJLm5mZArV2/atWqiK+zc+dObrnlFnJzc5Ekic2bN3c7fvv27UiSdNHRVbdsgaAnuH1BdL5mJMBoHdjV9JunZLNkziQyrEZ1c3YE/R2GOr0Sr5UrV7Js2TK+/PLLsA7ZN998Mzt37oz4Om63m2nTpvHss8/26PmPHTtGVVVV6MjMFDW/BX3H2ebHHHBg0GnQDnCSaNgWoWAAgr4Bff5EpFcxr08++YTf/OY3F53Py8vrkRe0YMECFixY0OPnz8zMJCUlpcePEwi6wxeQsStOtY6XJQZ5jDqDGmcL+tV0CZ3o9N4dvfK8jEYjTqfzovPHjx8nIyOjz0ZdiunTp5OTk8NXv/pVdu3a1e1Yr9eL0+kMOwSCzhgxzMJ1eRom5tjAMrAxr1qnh037z3C0IaieEOkSl6RX4vWNb3yDRx99FL/fD6gub3l5OT//+c/55je/GVUDzycnJ4fnnnuOP/3pT/zpT3+ioKCAuXPndptXVlxcjN1uDx0FBQX9Zp8gwfG5IOhH0gxMEcLzCSoKZQ2t1Hd0ERLpEpekV+K1Zs0aXC4XGRkZtLW1MWfOHEaPHo3VauXxxx+Pto0hxo0bx/e//31mzpzJ7NmzWb9+PbNnz+bpp5/u8jGrVq3C4XCEjoqKin6zT5DgtDaoP00papWHAaSjEYcr1EVIpEtcil79hex2O1u3bmXXrl0cOnQIl8vF5Zdfzrx586Jt3yWZNWsWH374YZf3G41GjEYROwDA3QCnt4PfAwVXQfroWFsUV2zde5S8OifDxwwnaYCfu6MFmk9jIig70Il0iUvSY/GSZZmSkhI2bdpEaWkpkiQxcuRIsrOzURRlwMvjHDx4kJycnAF9zoSkrRkO/C/429TbzeUw5VuQPiamZsULgaCMq7kGR5sfbfLAl6PRazUYdBp8GjXLXiemjZekR+KlKArf+MY3ePvtt5k2bRpTpkxBURSOHj3KsmXL2LRp0yVztc7H5XJx4sSJ0O3Tp09z8OBB0tLSKCwsZNWqVZw9ezbUI/KZZ55h5MiRTJo0CY/HwwsvvMB7770nmt1Gwsn3VOFKzlRX0mq/gGN/hZTh6irXEMfR5sfkc6DVSBgHqI7XhSQZzjXiMItp4yXpkXiVlJSwc+dOtm3bxvXXXx9233vvvcfixYt56aWXWLJkSUTX27t3b9h1Vq5cCcDSpUspKSmhqqqK8vLy0P0+n49//ud/5uzZs1gsFqZOncrf/va3i2wRXEBLDdQdU+uiT/iGKl4tNWrpl8r9UHh1rC2MOU2tfsyBZkx6DdIArzR2YDHo8GhNon9jhEiKEnkq70033cQNN9zAQw891On9//Vf/8WOHTvYsmVL1AyMNk6nE7vdjsPhGDpt2o6/C2f3QeZ4mNTeIKXyoOp5mVPhqu8P+YYP+07X4nvvl6Qn6xmzeBUY+792/YW8+WklTZUn+Zp/GxkZ2XD1fQNuQzwQ6Xe0R6uNn376KV/72te6vH/BggUcOnSoJ5cU9DdyEGqPqL9nTz13PnOiOl1sa1LjX0McV1M9oGAwmsGQHBMbbp6cw53XTiQj2Sj6N0ZAj8SrsbGRrKysLu/Pysqiqampz0YJokjjKXV10ZAEqSPPndcZIGO8+nv98djYFke0OWoB0FvTY+aFajSS+ncCCPjUbUKCLumReAWDQXS6rsNkWq2WQEC84XFFw0n1Z8b4izvhpI9Vf9YfH/IbgW3BZvRaCVNK1/+cBwSdETTtiaoiaN8tPV5tXLZsWZd5U16vNypGCaKEoqieF0DaqIvvTx2hJmN6nOCqBWuMv7gxZHa2DNo0lIzcmNnQ4PKy43gdY+sDTE6T1Cz7ASyImGj0SLyWLl16yTGRrjQKBoC2JvA41P/kKYUX36/Vq1PJ+i9VkRvC4oW7HgApqf/35naFJEmUNbSS6tUBQfAKz6s7eiReGzZs6C87BP1B42n1pz2/61yu1BGqeDWXw/CiATMtnlCCfqSOjkExFK+kji5CWAjKTrS+lpjZkggMUDtgQUxoLlV/po4A1Gn/RXR4ZI4KdWVyCLLv6En2lzVQ6ZJjkiLRgVGnVbPstUn4gjJ4hXh1hxCvwYqigKO9JLe9AFlW2LT/LIcqmkMidqTSwZcuk9paPuiHlqFZkdbdVIU3IOMzxW6lsYNko9qIwxeQxbTxEgzs1nnBwOFxqDWhJA1Ys/miuoXyxlaqnR4uy0ym2tHGu0dqSDJqGWnPR9f4JTSXgT0v1pYPON7mGgyAOdYrjUCSUYdHaxGeVwQIz2uw4qxUf1qzkCUdH51Sy73MGplGslHHyPRk7GY9bm+QY972ztCOodc8JRCUkVvqAEgeFvsN/slGLT5dEv6A6CJ0KYR4DVac7UJky6OiqVXdeKzXMi0/BQCtRuLqUeoevkMOC7KiQEvlkMv3amr1Y/I3oNVIWFKzY20OyUY9ktGGAsLzugRCvAYrIfHK5UilWvp6fLYVg+7cn3xsVjIWg5ZaxU5TW1DdDOxpjoGxsaPJ4cAYcGExaJGSY9/I5ZrRw7jrhinkpZgh4FUz7QWdIsRrMBL0q1UjAG9SDidr1enHxNzwTa46rYaJuTYUSccZf/sqm7NqQE2NNS31qsjrk1LVhYsYI0mSmmXf0fBWTB27RIjXYMRdB4oMBgsVbj0BWcFu1pNpvXhnxLgsVbTK/HYCsqxOHYcQtkADVqMOU2rsMus7pSNlwysaxnSFWG0cjHSkPCRnYTXrmZRrw2rSd1rlNsNqJC3JgF7Kxx88i845tMRrjKUV8uwwvJPtUzHA4w/y9uEqss54mZ2mIIl0iS4R4jUYcakVEkjOIstm4qZJXQeiJUnijlmFGHw2+HivOt2Ug+c2Bw92XOr0Gmvsg/WgloMub2xF7zfgDyoYxLSxS8S0cTDS8YVMjixvyaDTqEUJdUaQA+q0cwjg9bYRdKl7GiN9r/obrUbCYtDi07anS4gVxy4R4jXYkGVwq55XHXZqnB5kOYL0B0nCn5StliAeIlPHYydO8snpBk44AGNsChB2RpJRJ7YIRYAQr8FGW6NaxE6r56MqhY0fl7O//NIFIj8+1cBfTilUOTxDZpuQq/6smk8VJ15XB8lGHT6tBa/wvLolpuK1c+dObrnlFnJzc5EkKaLOQ9u3b+fyyy/HaDQyevRoSkpK+t3OhKJ9yqgkZVDpUOur5aZcOgXAZtbTok+nqdUHLUMjXcLTpKZJJKfnx9iScGxmvep5iSz7bompeLndbqZNm8azzz4b0fjTp0+zcOFCrr/+eg4ePMiKFSu455574rrhx4DTLl4u/TBafUF0GqnTFIkLGTEsCZchg1ZfEJ+zVs0VG8R4A0Gk9umxLbOTWmcxxGZSp43eQHtNryG26yFSYrrauGDBAhYsWBDx+Oeee46RI0eyZs0aACZMmMCHH37I008/zfz58/vLzMSiPTm1RrYDkGU3odNe+n+U2aDFnpKGv9qMs9VLuqt2UG/SrmtqxuRvxqjTYEmPL/GymvRoTclILRo1X8/njquYXLyQUDGvPXv2MG/evLBz8+fPZ8+ePV0+xuv14nQ6w45Bi6KEPK8Kv5pNnx/BlLGDgmFJuA3pONsGf3mcxqoyAAy2dDBYYmxNOGMyk7nv+rGMzm+PxYlE1U5JKPGqrq6+qHtRVlYWTqeTtra2Th9TXFyM3W4PHQUFBQNhamzwtqhdsSUNZR71CxlJvKuDglQzLkMGDo9/0Me9WmpKATAPi7/PQyiZ2KR6z3iEeHVGQolXb1i1ahUOhyN0VFRUxNqk/qM9OdVvTKXJo8ZJsmymiB+el2qm1ZiBxy/T1jS4y+MM1zeTnmwgNWdErE3pGmP7XlSPI7Z2xCkJlWGfnZ1NTU1N2LmamhpsNhtmc+cehtFo7LLb0aCjfcqosWbxzbH5NLb6MBsiz5Q36rSMHT2GkcqH6Noa1IoGXdW+T2QUhXxtE2RaIXvkpcfHgK2f16Cc8nC1zo9NiFenJJTnVVRUxLZt28LObd26laKiodk44iLak1O1tmwKh1mYXpDS40sUTRxBdkYGeo10LlN/sOGuV4VZq4ek2JfB6Qxnm596vxGPPyhiXl0QU/FyuVwcPHiQgwcPAmoqxMGDBykvV9vPr1q1KqyV2n333cepU6f42c9+xhdffMH/+3//j1dffZWf/OQnsTA//nC1b+tJ7mMHnI59foM0aF9V/iVt/iCKNefiRrxxgs2sx6uzqrleQ6zGWqTE9C+3d+9eZsyYwYwZMwBYuXIlM2bMYPXq1QBUVVWFhAxg5MiRvPXWW2zdupVp06axZs0aXnjhBZEmAWpeVlsjAH+v03GsukXd6tMLXIYMapweWhvPRNPCuEBRFD77/FMOVjRTp4uvzPrzsZp0eLXJapa9CNh3SkxjXnPnzu28HVc7nWXPz507lwMHDvSjVQmKuw4UBZ/WxK5yD5JUxQ/mXtarS+2s1pFa76bQVIZlWpTtjDH1LV6MLWfRSpCWOzrW5nSJzaR6Xt6ArFZU9XtAH/niy1AgPn1mQc9pX2l0aNRmGqkWA0Zd78rapGerSZvuplr1izOIqKw6g15uI9liQpcSX9uCzsdq0iFr9LTK7RVVRdzrIoR4DRbay9jUK+2Z9bber7DmZ6Xj1SXjaPMjOwZXhYnGs18CYM4YAdr4XWy3mVTRcmJBQRHpEp0gxGuw0O55VctqblBP8rsuJNtmwmvKJCArNNcOnrw4byCIp/Y0EN9TRoBkkw6zQYsxKZWgrIi4VycI8RoMtG8LUlA441P3wPVFvDQaiaR0NfO8obosKibGA2X1bpLbKjHrNdhy4qPsc1doNRL3zbmMayZfhk6jESuOnSDEazDgdULAiy8IDYoVjSSREUElie5Iyx4OgLt+8Kw4nq04jU72YLcmI9kSZNO5qT3LXsS8LiJ+J/2CyGnP73JqbCiKjmHJBvQRVJLojpy84TgAX0sDAY8bnSkpCobGlmvTHDizrRiyxydOjf6O/Y1tzTE1Ix4R4jUYaM+sT8/KZ/mokWpWdh9JsdsYM7yAFMmFrrUGTPE9zYoEXfMp0iwGyBkba1Mi4kSti/2fu7i81sVofXOszYk7xLRxMNAerJes2djN+j7FuzqQJIn0nOFqvGUwZNr73Odex7De5b8NNJIEVV4zrf6Amufla421SXGFEK/BQEers6Q+bgu6EGuO+jPBy+O0+YL8+f1dVDS6kZMyzzV0jXNSzHpkjQ6nbFbTJdou3YtgKCHEK9Fp3xbkCQR567TM3tLGqF1asWZR0dTKgSNHafEkblnoI5UONI0naG71o0lPDK8LwG7WI0ng1lgJBIV4XYgQr0THXQ+KQktQz/FGmeM10WvYIFlzcLQF8LiaKa2sjdp1BxJZVjhUVkdqW7mauJs+LtYmRYxOqyHZqMOjs6lxzLbo/WMaDAjxSnTag/WNUgpIUp8y6y9CZ8Sapm5erjxTGr3rDiCn6l3omksxaoIMS8+Mm87YkZJiMeDRp+Dxy8LzugAhXolOe7yrJtixLSi6m3eHted7tdRVRGUVcyBRFIX9Zc2ktZ4m02pCmzlejYInEClmvep5BYJCvC5AiFei01KFgkJlP4lXckYBFoMWi7eO0gZ3VK/d35Q3tlLV2EK6p4xsuwkyxsfapB6TbjWSlJKBTiMJ8boAIV6JjCyDqwaPX6ZZm4ZeKzEsKcplm63ZpFoMJPtqOVGTWN2bPzrVQIqnnByrFkNSCthyY21Sj5lekMK3vjKFHLtZpEtcgBCvRKa1AYIBWvwSbTo7GVYjGk2Up0XJWaQlG9EH2zhTXZtQU8cbJ2QxXVeudlDKmpRwU8YQWv259A4RtA8hxCuRcalJl62GdHRabdSnjABo9SSlZpNs1DExuRVfL6uzxoJ0vY9pSY0YtBrInhprc/qEbBmGrCjq6rIAENuDEpv27tgjRlzGDy8b3euyz5dCsuUwOa8WKScA7XWm4hlHqx+7RQ/Vn6kVN+z5YEmLtVm95p3PqvGe8PMVk49hQrxCxIXn9eyzzzJixAhMJhNXXXUVf//737scW1JSgiRJYYfJNETL43Zkvluz0GokTPp+2mxszUZCAmf8Fyasdngo2V3K+0drkKsOqSezp8TWqD6ikcClTaHVFwwVnRTEgXj94Q9/YOXKlTz88MPs37+fadOmMX/+fGpru06KtNlsVFVVhY6yssFTcypi2oP1Cgok93Pukl0tC43zDPVON0cq47Oqpz8o885nVciKgtR0Ck1bk9p3MnNCrE3rE8OSDbTqU2nzBaFVeF4dxFy8nnrqKe69917uuusuJk6cyHPPPYfFYmH9+vVdPkaSJLKzs0NHVlb8doHpN9oaIein1i3zv5+2sK+sH5fRk9LBkERrm4c3dnzCe0drcXsD/fd8vUBRFLYdraGp1Y/VpGO2Sa2YSs400CV20+G0JCNt+lTa/EHwusSKYzsxFS+fz8e+ffuYN29e6JxGo2HevHns2bOny8e5XC6GDx9OQUEBixYt4siRI12O9Xq9OJ3OsGNQ0F4hoQ479e4A3v5cBZQkSB2O2aBlpK6egKywtz/FshfsL2/iaFULGkliwUgdBkeZanfeFbE2rc8MSzYQ1Bholi1q0F54X0CMxau+vp5gMHiR55SVlUV1dedlWMaNG8f69et54403ePnll5FlmdmzZ3PmTOcVP4uLi7Hb7aGjoKAg6q8jJjjPAlAZVAPROSnm/n2+lOFISExJagbgUEUzza2+/n3OCDlR6+KDL9Uv9HVj08lraY91pY8Fc0rsDIsSVqMOk16Lu2PqKOJeQBxMG3tKUVERS5YsYfr06cyZM4dNmzaRkZHBb37zm07Hr1q1CofDEToqKgZJQwnHGfxBmSqGAZBj7+dFi1R1m1CG3MCoVB1BWWH7sbpu+24OBK2+AO98VoWiwOQ8O9OHBaGm3RMvvDqmtkULSZLItKpTR5c3INIl2ompeKWnp6PVaqmpqQk7X1NTQ3Z2ZEFovV7PjBkzOHHiRKf3G41GbDZb2JHwBLzgrqPFE6DFkMWwZEP/rTR2YE5VvRhFZk6GG61G4nS9m0NnYhu8txh0zJ+UzbhsKzeOz0Qq/VBNj0gfk5AZ9V0xIt1CenYBRt0gKQ4ZBWIqXgaDgZkzZ7Jt27bQOVmW2bZtG0VFRRFdIxgMcvjwYXJycvrLzPijpQoUhSbFjF+XRK69n6eMHaSPASCltZRrx6QDsPN4HY7Wga/1dX5O25gsKzdPyUHTUgm1R9WTI68bcJv6k5nD05gzcyopFoO6GV9OnJ0O/UXMp40rV67kt7/9LS+++CJHjx7lBz/4AW63m7vuuguAJUuWsGrVqtD4Rx99lHfffZdTp06xf/9+vvvd71JWVsY999wTq5cw8DjUeFeN0j5lTBmgPLdhqnjRcILp+TbGZVu5bmyGmhA6QMiywq4T9by0pyy8QKIsw5fvqr9nT4HkzAGzacAwp6orp7KYOkIcZNjffvvt1NXVsXr1aqqrq5k+fTrvvPNOKIhfXl6ORnNOY5uamrj33nuprq4mNTWVmTNnsnv3biZOnBirlzDwtAfrjcMKsCn6gfO87AWgN4O/DclxhgWTC5HO2y9Y6/SQbNJhMfTPx8rR6mfr0RoqGtVUgeM1LmYOT1XvrDygTqd0Rhg1t1+eP9YoQJsxA4OvHF1LJViHYIrQeUhKrCOuA4zT6cRut+NwOBIz/iXLsOtpCPjgirvAmo2iKGEi0q8cfROqD0PeTBh7U+h0UFZ4aU8pbm+Acdk2xmdbyUsxR2WjuD8oc6iimY9ONeAPKui1EvMmZjE+u/3v526AfeshGIAxN0H+zD4/Zzzy50OV+I9v4xrDCbLHXw3jFsTapH4h0u9ozD0vQQ9pqVSFS2+CZPU/74AJF6jZ6tWHofZzGH1jqP+h2xfAqNPS3Orns7MOPjvrwKTXMnyYhdwUM8PTLKT2olzPZ2cd7DnZoK6yAfmpZuZNyDp3raAfjr6hClfqCMi7PFqvNO7ItBo5ZsigxXOU7ATYqtXfCPFKNJrUrVBOUx7JirrvbUBJHQnGZDXTu/5LyFQL/NlMeu6YVUClw8PhMw5O17vx+IMcq27hWHULV45I4yvtQf42X5ATtS5SLHrMBi0mvRZZUfAHZBxtfnJTzKHVU6fHj8sbwGbWc/WoNCbm2M6JtaLAF2+pG9T1Zhi/MHHL3kRAXoqZfYZMWpoDKO46pIA34XcP9AUhXolGUykBWeatCj1NrpMsv2Zk/6dJnI9GA1mTofwjqP40JF6geoB5KWbyUszIskKlo40zTW1UNrdRkHYuLlft9PC3ozWdXR2AhVNzGJul1q+alGvHZtIzIceG9nylVhQ4uU1dXZQ0MOkfwJSAYYAekG03ETRYcZKE1xfA5DiTMD0o+wMhXolE0A/OszjbAjQb80gy6AZWuDrImQYVH0PDSXDVQfLF/SI1Gon8VAv5qZaL7tNpJIYPs9DiCdDqC+ILyGgk0GolbCY9vsC5NAi7WY89zx5+AVlWhevMXvX2uK+FkmgHM3qthkyrEYcxlxbPWUzNZUK8BAlCcznIQer9RjxJdsanXSwMA4IlTc35qjsOFR/BhFt69PCCNAsFvbXd3wZH/6IKJ8DY+aqYDhHyUs2UmnJweErJaB4ku0V6SczzvAQ9oP5LFBROKVkgSYzKSIqdLYXtScQ1nw9MzpGiQN0x+PtvVeHS6mDiokEdoO+MEcOSaDHm0NzqQ2mpUndbDFGEeCUKigINX+LyBqjSFWDQaTqdkg0YtlzV+1Lak0P7M+OmuRwO/h4+2wQ+N1iGwYw7IWsI5fa1k5tiZuKoAkYW5KvT56bSWJsUM8S0MVFoqQKvi0YPOCy5jE1PCg9gx4LRN0LjaXUFtPJAdL2ggFfdYF11MFTuGo0WCmbB8K+ontcQRKuRmDsuEzST4Mwn6opvRuJ0AY8mQ/MTkIjUHVOnjMFMFEkX2yljB+ZUGHktnHwfTvxN9Yj6Ejj3udUvY/2Xqkchtxc81GjVBhrDi8Bk7/YSQ4b0Map4NZxQPTDN0JtECfFKBGQZaj5DQmLqzCJ0/lRGpSfH2iqVgqvU7Up1x+HwqzBuYeTTuaAfHGeguUz13to3nIdISleD8VmTwRDDKXIcUqPJwNUcIDfJhdl5BlIKY23SgCPEKxFoLlWTQvUmhhVM4tp4mjJJEkxYBPImNZD++RvqVC9nutq1p6PfoL8NPA61CmhLjSpULVUXV0ewZqtFBNPHquI1iJNO+8JHp5uQvFkY5AoKao4I8RLEKVWfqj8zJ8VnrEerg8nfgtIP1PyvprLQToBLYrSqU82U4er2nkGeaBotxmfb+LBiDHWNJ8iv+RzpshvVZiNDiDj8JgjC8Dig7hg1Tg+HLVlMdXrI7I/msn1Fo4FRc9RpXuV+1QtrbVRXIzswJKlxMmuOWhHBlqfeFt5VjxmVkcTfkvJxNibR3OIitf5Ywrd46ylCvOKdM3tR5CDHfWkcdpoZ1twWn+LVgTkFLrtBPWQZAm3qea0xPr3GBEWv1TC1IIWqxnFUNh8k9cwnamxwCP0jGHpLFImEzw2VB2hw+zhhnIRRr2FSbgKttmk0qrdlSBLC1Q9ML0ih3jaJJq9ES92Zc7sOhghCvOKZ0x8gB3x84U6i2VTA5YWpGHTiTyZQsZr0jM7LoMY6kbLGVpTSD1Rvd4ggvgnxiuMsVB3kbHMbXyTNItmk5/LC1FhbJYgzZo9OpyV9Bqm2ZHBWqfHGIYIQr3jE74Gjf6HF4+OQL58WUw7Xjc0QXpfgIpKNOu6cM4m8yxeodc5ObR8y9e3FtyHeCPrhyCZoa6KiVc+p1NmMzbIyLtsaa8sEcYpWI0Hu5ZBSiMfrwbP/FTUvcJATF+L17LPPMmLECEwmE1dddRV///vfux3/2muvMX78eEwmE1OmTOHtt98eIEv7Ga8LDr2i5khp9YybdxezxuTy1YlDu9GCIAIkieaRN3OgFo6eKqflo/WD3gOLuXj94Q9/YOXKlTz88MPs37+fadOmMX/+fGprazsdv3v3bu644w7uvvtuDhw4wOLFi1m8eDGfffbZAFseRYJ+OLMP30e/obLsS4JaA0z9NlpbNlePGiami4KIkIzJnMz9Og6S+PxkOaVbfk3bsffUVetBSMy7B1111VVceeWV/PrXvwbUprMFBQX86Ec/4qGHHrpo/O23347b7ebNN98Mnbv66quZPn06zz333CWfL6bdgxQFgn4Uvxu/24HPUUNbfRm+2uO0uNw42vy4dMNImnErV08ZmpUCBH3D4w/yt0Mn0R57mxTPGSTAajFiyhjF8NET0FmzwJyKojcj6UxxmReWEN2DfD4f+/btC2sqq9FomDdvHnv27On0MXv27GHlypVh5+bPn8/mzZs7He/1evF6zxVsczqdEdvnaPVz9M21SEoQUFAUkFBQ9V4hx25ieJoFFAVvIMiB8kYAJEVBaf8JMooCWTYjI1PVhqG+QJD95c3hduqSqUyZhZQ7nauyLy6rLBBEgkmvZeHMMZwuvJtjn+1Ff/ZjlNZ6WsqOMkpTExKr4zUtNHuCoNGrVTskLWg0KGjRaCRmjUhTLyhJHK9poanVd8EznRO9K0ekhRrBnKhrpcEdXiCx3j6FRvtEvjYpmwxr9BqGxFS86uvrCQaDoQazHWRlZfHFF190+pjq6upOx1dXV3c6vri4mEceeaRX9gUVBdlVi3TeFpfz3VRFbwJz+x81EETxtISNCRvrk0FW681rdXq8umQ8hmGQnIk2Ywz2zOHclJnMsCTDwLYyEww6JEliVKaVUTdcT6NrNmerzuKrOY6U6gF3HXgcBIIycjAIweBFj5cBWs99ehV3C7L7QvE6D3fwnAfnakF2hY91a5zUa7wE5ehO8gZ92vOqVavCPDWn00lBQUFEj0026hg+d/k5z1qS2n+XkCQJi0EPZj1IEjoFLpvoQ5LU+xQI9SWT0KiNMswm0FvQanQUyQoGrUYIlaBfSUs2kjZmFIwZFXZ+lMeLp7WFQMAHchAlKCPLfhQ5qFYlsp/bgpbu9mENyF1Uy1WQzhub1urD7AtPlM032lFMKaRY9NF8abEVr/T0dLRaLTU14W2wampqyM7O7vQx2dnZPRpvNBoxGnvnqhp0GvJGjb/0QEALZEa4c0cCjCIGL4ghZpMRsymy78WwHuRGp6X10qBeENOvkMFgYObMmWzbti10TpZltm3bRlFRUaePKSoqChsPsHXr1i7HCwSCwUnMp40rV65k6dKlXHHFFcyaNYtnnnkGt9vNXXfdBcCSJUvIy8ujuLgYgB//+MfMmTOHNWvWsHDhQl555RX27t3L888/H8uXIRAIBpiYi9ftt99OXV0dq1evprq6munTp/POO++EgvLl5eVozqvPPXv2bDZu3Mi//du/8S//8i+MGTOGzZs3M3ny5Fi9BIFAEANinuc10MQ0z0sgEFySSL+jImwsEAgSEiFeAoEgIRHiJRAIEpKYB+wHmo4QX0+2CQkEgoGj47t5qXD8kBOvlhZ1C0+kWfYCgSA2tLS0YLd3nfk95FYbZVmmsrISq9U6JLfmdGyPqqioEKutESDer54RjfdLURRaWlrIzc0NS5O6kCHneWk0GvLz82NtRsyx2Wziy9gDxPvVM/r6fnXncXUgAvYCgSAhEeIlEAgSEiFeQwyj0cjDDz/c60obQw3xfvWMgXy/hlzAXiAQDA6E5yUQCBISIV4CgSAhEeIlEAgSEiFeAoEgIRHiNQgRHch7Rk/er5KSklCTlY7DZDJ1OX6wsXPnTm655RZyc3ORJKnLloPns337di6//HKMRiOjR4+mpKQkKrYI8RpkiA7kPaOn7xeo2eNVVVWho6ysbAAtji1ut5tp06bx7LPPRjT+9OnTLFy4kOuvv56DBw+yYsUK7rnnHrZs2dJ3YxTBoGLWrFnK/fffH7odDAaV3Nxcpbi4uNPx3/72t5WFCxeGnbvqqquU73//+/1qZ7zQ0/drw4YNit1uHyDr4htAef3117sd87Of/UyZNGlS2Lnbb79dmT9/fp+fX3heg4iODuTz5s0LnYukA/n540HtQN7V+MFEb94vAJfLxfDhwykoKGDRokUcOXJkIMxNSPrz8yXEaxDRXQfyrjqK97QD+WCiN+/XuHHjWL9+PW+88QYvv/wysiwze/Zszpw5MxAmJxxdfb6cTidtbW19uvaQqyohEPSFoqKisB6hs2fPZsKECfzmN7/hsccei6FlQw/heQ0iBqID+WCiN+/Xhej1embMmMGJEyf6w8SEp6vPl81mw2w29+naQrwGEaIDec/ozft1IcFgkMOHD5OTk9NfZiY0/fr56nPIXxBXvPLKK4rRaFRKSkqUzz//XPne976npKSkKNXV1YqiKMqdd96pPPTQQ6Hxu3btUnQ6nfLkk08qR48eVR5++GFFr9crhw8fjtVLGFB6+n498sgjypYtW5STJ08q+/btU/7xH/9RMZlMypEjR2L1EgaUlpYW5cCBA8qBAwcUQHnqqaeUAwcOKGVlZYqiKMpDDz2k3HnnnaHxp06dUiwWi/LTn/5UOXr0qPLss88qWq1Weeedd/psixCvQcj//M//KIWFhYrBYFBmzZqlfPTRR6H75syZoyxdujRs/KuvvqqMHTtWMRgMyqRJk5S33nprgC2OLT15v1asWBEam5WVpdx8883K/v37Y2B1bHj//fcV4KKj4z1aunSpMmfOnIseM336dMVgMCijRo1SNmzYEBVbREkcgUCQkIiYl0AgSEiEeAkEgoREiJdAIEhIhHgJBIKERIiXQCBISIR4CQSChESIl0AgSEiEeAkGPSUlJaSkpIRu/8d//AfTp08P3V62bBmLFy8ecLsEfUOIlyBqLFu2DEmSuO+++y667/7770eSJJYtWxY2PtqiMWLECJ555pmwc7fffjvHjx/v8jFr164NK008d+5cVqxYEVW7BNFHiJcgqhQUFPDKK6+E1WryeDxs3LiRwsLCmNhkNpvJzMzs8n673R7mmQkSAyFegqhy+eWXU1BQwKZNm0LnNm3aRGFhITNmzOjTtTvziBYvXhzy5ubOnUtZWRk/+clPQs0x4OJp44Wc7wEuW7aMHTt2sHbt2tA1Tp8+zejRo3nyySfDHnfw4EEkSRLlcGKEEC9B1Fm+fDkbNmwI3V6/fj133XVXvz/vpk2byM/P59FHHw01x+gpa9eupaioiHvvvTd0jcLCwoteE8CGDRu47rrrGD16dLRegqAHCPESRJ3vfve7fPjhh5SVlVFWVsauXbv47ne/2+/Pm5aWhlarxWq1kp2d3auCina7HYPBgMViCV1Dq9WybNkyjh07FmqL5vf72bhxI8uXL4/2yxBEiCgDLYg6GRkZLFy4kJKSEhRFYeHChaSnp8farD6Rm5vLwoULWb9+PbNmzeIvf/kLXq+X2267LdamDVmE5yXoF5YvX05JSQkvvvhi1LwTjUbDhRWc/H5/VK4dCffcc09oMWLDhg3cfvvtWCyWAXt+QThCvAT9wte+9jV8Ph9+v5/58+dH5ZoZGRlhcaxgMHhRc1yDwUAwGOzT83R1jZtvvpmkpCTWrVvHO++8I6aMMUZMGwX9glar5ejRo6Hfu8LhcHDw4MGwc8OGDaOgoOCisTfccAMrV67krbfe4rLLLuOpp56iubk5bMyIESPYuXMn//iP/4jRaOzVdHXEiBF8/PHHlJaWkpycTFpaGhqNJhT7WrVqFWPGjBkSdf7jGeF5CfoNm82GzWbrdsz27duZMWNG2PHII490Onb58uUsXbqUJUuWMGfOHEaNGsX1118fNubRRx+ltLSUyy67jIyMjF7Z/eCDD6LVapk4cSIZGRmUl5eH7rv77rvx+XwDsnoq6B5RBlog6AEffPABN954IxUVFRc1UxUMLEK8BIII8Hq91NXVsXTpUrKzs/n9738fa5OGPGLaKBBEwP/93/8xfPhwmpub+e///u9YmyNAeF4CgSBBEZ6XQCBISIR4CQSChESIl0AgSEiEeAkEgoREiJdAIEhIhHgJBIKERIiXQCBISIR4CQSChESIl0AgSEj+P8A5G8fw2iaVAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T18:25:48.040153Z", + "iopub.status.busy": "2024-03-24T18:25:48.039867Z", + "iopub.status.idle": "2024-03-24T18:25:48.269518Z", + "shell.execute_reply": "2024-03-24T18:25:48.268636Z" + }, + "papermill": { + "duration": 0.250192, + "end_time": "2024-03-24T18:25:48.271589", + "exception": false, + "start_time": "2024-03-24T18:25:48.021397", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAArwElEQVR4nO3de1wU570/8A/LZXHl4gUEJCheGpWoEKEgRk+05ZJrteZCNBHcWM45GlqTramhPYWDtsFURWxC5SSV2JiLHBtrcxqLIJEcjRgVgkWieEQRIxdBhRVIloWd3x/+mHTdXVgMyzC7n/frxQt39pnZ7/DAx5lnd55xEgRBABGRzCikLoCI6G4wvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZcpG6gKFmMBhQX18PT09PODk5SV0OEd1BEATcunUL48ePh0Jh+fjK4cKrvr4eQUFBUpdBRP24cuUK7rnnHovPO1x4eXp6Arj9g/Hy8pK4GtvR6/UoLCxEXFwcXF1dpS6HBoGj9KlWq0VQUJD4t2qJw4VX76mil5eX3YeXSqWCl5eXXf+iOxJH69P+hnU4YE9EsjQswisnJwfBwcFwd3dHVFQUTpw4YbHtwoUL4eTkZPL16KOPDmHFRCQ1ycMrPz8fGo0G6enpKC8vR2hoKOLj43Ht2jWz7fft24eGhgbx68yZM3B2dsZTTz01xJUTkZQkD6+srCwkJydDrVYjJCQEubm5UKlUyMvLM9t+zJgx8Pf3F7+KioqgUqkYXkQORtIB+66uLpSVlSE1NVVcplAoEBMTg9LSUqu2sXPnTjzzzDMYOXKk2ed1Oh10Op34WKvVArg9+KnX679D9cNb777Z8z46GkfpU2v3T9LwamlpQU9PD/z8/IyW+/n54dy5c/2uf+LECZw5cwY7d+602CYzMxMZGRkmywsLC6FSqQZetMwUFRVJXQINMnvv087OTqvayfqjEjt37sSsWbMQGRlpsU1qaio0Go34uPczJHFxcXb1UYnOzk5UV1eLj9u/1uHgkZOIX/B9eIxQGrWdNm2aQwS3vdHr9SgqKkJsbKxdf1Si9+yoP5KGl4+PD5ydndHU1GS0vKmpCf7+/n2u29HRgT179mDDhg19tlMqlVAqlSbLXV1d7eoXoKamBlFRUSbLf2embVlZGebMmWP7osgm7O13907W7puk4eXm5obw8HAUFxdjyZIlAG5fe1hcXIyUlJQ+1927dy90Oh2ee+65Iah0+Js+fTrKysrEx9UNrdDsrUTWU7MwLWCUSVsiuZP8tFGj0SApKQkRERGIjIxEdnY2Ojo6oFarAQCJiYkIDAxEZmam0Xo7d+7EkiVLMHbsWCnKHnZUKpXR0ZTi8nUoj3yNGTNDETaRPyOyP5KHV0JCApqbm5GWlobGxkaEhYWhoKBAHMSvq6szubK8uroaR48eRWFhoRQlE9lcZ2enyZtW7V/rcKyyBqN9ThmNY06fPt0hxzCdHO2+jVqtFt7e3mhra7OrAfs7VVy+jiU7jmP/6rk88pKh8vJyhIeHW9XW3sYwrf0blfzIi4hM3TmGCVgex3TUMUyGF9EwdOcYJsBxzDtJfnkQEdHdYHgRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZkjy8cnJyEBwcDHd3d0RFReHEiRN9tm9tbcULL7yAgIAAKJVK3HvvvThw4MAQVUtEw4Wk00Dn5+dDo9EgNzcXUVFRyM7ORnx8PKqrqzFu3DiT9l1dXYiNjcW4cePw5z//GYGBgbh8+TJGjRo19MUTkaQkDa+srCwkJyeL92jMzc3Fxx9/jLy8PLzyyism7fPy8nDjxg0cO3ZMvKtucHDwUJZMZDOXWjrQoeu2+HxNc4f43cXF/J/uSKULJvmMtEl9w41k4dXV1YWysjKkpqaKyxQKBWJiYlBaWmp2nY8++gjR0dF44YUX8Ne//hW+vr5Yvnw51q9fD2dnZ7Pr6HQ66HQ68bFWqwUA6PV66PX6Qdyj4aW7u1v8bs/7aS9qr3cgNvszq9r+/M+VfT5f9OIDCB4r3wCz9vdVsvBqaWlBT0+PeHPZXn5+fiY32+x18eJFfPLJJ3j22Wdx4MABXLhwAWvWrIFer0d6errZdTIzM5GRkWGyvLCw0K5v1HmlHQBccPz4cVw9I3U11J/e/loxtQd+I8zfSlVvAG7ogDFKwNXMaHXT107YfcEZB4s/RZCHTcu1qc7OTqvayerWZwaDAePGjcObb74JZ2dnhIeH4+rVq9i8ebPF8EpNTYVGoxEfa7VaBAUFIS4uzq5vOnu67gZQeQpz585F6IQxUpdD/aiq12JL5XE8EfsA7htv/vdSr9ejqKgIsbGx4rDJndvYfeE45s+fb3EbctB7dtQfycLLx8cHzs7OaGpqMlre1NQEf39/s+sEBATA1dXV6BRxxowZaGxsRFdXF9zc3EzWUSqVUCqVJstdXV3N/gLISV9jJJdv6sTv7u6W/ydzpDGS4ax3DMvFxaXf30tLv7sD2cZwZm3tkoWXm5sbwsPDUVxcjCVLlgC4fWRVXFyMlJQUs+s88MADeP/992EwGKBQ3D5uPn/+PAICAswGlz271NKBRVtK+m3X3/gIABxet5ABRrIj6WmjRqNBUlISIiIiEBkZiezsbHR0dIjvPiYmJiIwMBCZmZkAgNWrV+ONN97A2rVr8dOf/hT/93//h1dffRU/+9nPpNwNSfQecWUnhGHqONMBjo6vdfhbSSkeWxiNkSNMjzwB4MK1dryYX9HnO1xEw5Wk4ZWQkIDm5makpaWhsbERYWFhKCgoEAfx6+rqxCMsAAgKCsLBgwfx0ksvYfbs2QgMDMTatWuxfv16qXZBclPHeWBmoLfJcr1ej0ZfYM7E0bI+hSCyRPIB+5SUFIuniSUlJSbLoqOjcfz4cRtXRUTDneSXBxER3Q2GFxHJEsOLiGSJ4UVEssTwIiJZkvzdRiICdD3fQOF+FZe01VC4m78wsbu7G/Xd9Th746zZWSUuaduhcL8KXc83AEw/PmNvGF5Ew0B9x2WMnPQ6ftn3XJwAgD8U/MHicyMnAfUdYQiHn8U29oLhRTQMjB85ER2XfortCWGYYuaKCeD2kddnRz/DA/MfMHvkVXOtHWvzKzB+0URblzssMLxkqr/TjP5OMQDHO80YzpTO7jB8E4hJXtMQMtZ8X+j1elxyuYQZY2aYvWrC8E0bDN80Q+nsbutyhwWGl0xZe5rR1ykG4FinGWRfGF4y1d9pRn+nGIDjnWaQfWF4yVR/pxn9nWIAjneaQfaFn/MiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZGlYhFdOTg6Cg4Ph7u6OqKgonDhh+WPju3btgpOTk9GXuzs/p0TkaCQPr/z8fGg0GqSnp6O8vByhoaGIj4/HtWvXLK7j5eWFhoYG8evy5ctDWDERDQeSh1dWVhaSk5OhVqsREhKC3NxcqFQq5OXlWVzHyckJ/v7+4lfvrdKIyHFIenlQV1cXysrKkJqaKi5TKBSIiYlBaWmpxfXa29sxceJEGAwGzJkzB6+++iruu+8+s211Oh10Op34WKvVArh9+Yxerx+kPRl63d3d4ndz+9G7rK997G8bNHSs6Yv++tRe+tPa2iUNr5aWFvT09JgcOfn5+eHcuXNm15k2bRry8vIwe/ZstLW1YcuWLZg3bx6qqqpwzz33mLTPzMxERkaGyfLCwkKoVKrB2REJXGkHABccPXoUl81P/wQAKCoq+s7bINsbSF9Y6lN76c/Ozk6r2snuwuzo6GhER0eLj+fNm4cZM2bgv/7rv7Bx40aT9qmpqdBoNOJjrVaLoKAgxMXFwcvLa0hqtoWqei22VB7H/Pnzcd940/3Q6/UoKipCbGysxQuz+9sGDR1r+qK/PrWX/uw9O+qPpOHl4+MDZ2dnNDU1GS1vamqCv7+/VdtwdXXF/fffjwsXLph9XqlUQqlUml3P0h+1HPROc+Pi4tLnfvS1n9Zug2xvIH1hqU/tpT+trV3SAXs3NzeEh4ejuLhYXGYwGFBcXGx0dNWXnp4eVFZWIiAgwFZlEtEwJPlpo0ajQVJSEiIiIhAZGYns7Gx0dHRArVYDABITExEYGIjMzEwAwIYNGzB37lxMnToVra2t2Lx5My5fvoyf/OQnUu4GEQ0xycMrISEBzc3NSEtLQ2NjI8LCwlBQUCAO4tfV1UGh+PYA8ebNm0hOTkZjYyNGjx6N8PBwHDt2DCEhIVLtAhFJQPLwAoCUlBSkpKSYfa6kpMTo8bZt27Bt27YhqIqIhjPJP6RKRHQ3GF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWBhxeFy9etEUdREQDMuDwmjp1KhYtWoR3330X33zzjS1qIiLq14DDq7y8HLNnz4ZGo4G/vz/+7d/+rc+7/RAR2cKAwyssLAzbt29HfX098vLy0NDQgPnz52PmzJnIyspCc3OzLeokIjJy1wP2Li4uWLp0Kfbu3YvXXnsNFy5cwLp16xAUFITExEQ0NDQMZp1EREbuOrxOnTqFNWvWICAgAFlZWVi3bh1qampQVFSE+vp6LF68eDDrJCIyMuD5vLKysvD222+juroajzzyCN555x088sgj4oSBkyZNwq5duxAcHDzYtRIRiQYcXjt27MDzzz+PlStXWpw3fty4cdi5c+d3Lo6IyJIBh1dRUREmTJhgNDUzAAiCgCtXrmDChAlwc3NDUlLSoBVJRHSnAY95TZkyBS0tLSbLb9y4gUmTJg1KUURE/RlweAmCYHZ5e3s73N3dv3NBRETWsPq0sfeu005OTkhLS4NKpRKf6+npweeff46wsLC7KiInJwebN29GY2MjQkND8frrryMyMrLf9fbs2YNly5Zh8eLF2L9//129NhHJk9Xh9cUXXwC4feRVWVkJNzc38Tk3NzeEhoZi3bp1Ay4gPz8fGo0Gubm5iIqKQnZ2NuLj41FdXY1x48ZZXK+2thbr1q3DggULBvyaRCR/VofX4cOHAQBqtRrbt2+Hl5fXoBSQlZWF5ORk8Sazubm5+Pjjj5GXl4dXXnnF7Do9PT149tlnkZGRgSNHjqC1tXVQaiEi+Rjwu41vv/32oL14V1cXysrKkJqaKi5TKBSIiYlBaWmpxfU2bNiAcePGYdWqVThy5Eifr6HT6aDT6cTHWq0WAKDX66HX67/jHkinu7tb/G5uP3qX9bWP/W2Dho41fdFfn9pLf1pbu1XhtXTpUuzatQteXl5YunRpn2337dtn1QsDQEtLC3p6esS7Y/fy8/PDuXPnzK5z9OhR7Ny5ExUVFVa9RmZmJjIyMkyWFxYWGo3byc2VdgBwwdGjR3HZw3K7oqKi77wNsr2B9IWlPrWX/uzs7LSqnVXh5e3tDScnJ/HfUrl16xZWrFiBt956Cz4+Platk5qaKr7ZANw+8goKCkJcXNygnfpKoapeiy2VxzF//nzcN950P/R6PYqKihAbGwtXV9e72gYNHWv6or8+tZf+7D076o9V4fXPp4qDedro4+MDZ2dnNDU1GS1vamqCv7+/SfuamhrU1tbi8ccfF5cZDAYAty8Ur66uxpQpU4zWUSqVUCqVJttydXW1+EctBy4uLuL3vvajr/20dhtkewPpC0t9ai/9aW3tkk4D7ebmhvDwcBQXF4vLDAYDiouLER0dbdJ++vTpqKysREVFhfj1ox/9CIsWLUJFRQWCgoKGsnwikpBVR17333+/eNrYn/Ly8gEVoNFokJSUhIiICERGRiI7OxsdHR3iu4+JiYkIDAxEZmYm3N3dMXPmTKP1R40aBQAmy4nIvlkVXkuWLLFZAQkJCWhubkZaWhoaGxsRFhaGgoICcRC/rq7O5DpKIiKrwis9Pd2mRaSkpCAlJcXscyUlJX2uu2vXrsEviIiGPR7SEJEsWXXkNWbMGJw/fx4+Pj4YPXp0n+NfN27cGLTiiIgssSq8tm3bBk9PT/Hf1g7eExHZilXh9c8TC65cudJWtRARWW3AY17Ozs64du2ayfLr16/D2dl5UIoiIurPoE1GqNPpjKbJISKyJatnlfj9738P4PZkhH/84x/h4fHtlZ89PT343//9X0yfPn3wKyQiMsPq8Nq2bRuA20deubm5RqeIbm5uCA4ORm5u7uBXSERkhtXhdenSJQDAokWLsG/fPowePdpmRRER9WfAkxH2zqhKRCSlAYfX888/3+fzeXl5d10MEZG1BhxeN2/eNHqs1+tx5swZtLa24gc/+MGgFUZE1JcBh9df/vIXk2UGgwGrV682mQiQiMhWBuXCbIVCAY1GI74jSURka4M2q0RNTY149xIiIlsb8GnjP9/MArj9ua+GhgZ8/PHHRtdAEhHZ0oDDq/fO2b0UCgV8fX2xdevWft+JJCIaLPycFxHJEmdSJSJZGhbhlZOTg+DgYLi7uyMqKgonTpyw2Hbfvn2IiIjAqFGjMHLkSISFhWH37t1DWC0RDQeSh1d+fj40Gg3S09NRXl6O0NBQxMfHm50zDLg9JfWvfvUrlJaW4h//+AfUajXUajUOHjw4xJUTkZQkD6+srCwkJydDrVYjJCQEubm5UKlUFi8zWrhwIX784x9jxowZmDJlCtauXYvZs2fj6NGjQ1w5EUlpwAP2lnz11VfYsGED3nzzTavX6erqQllZGVJTU8VlCoUCMTExKC0t7Xd9QRDwySefoLq6Gq+99prZNjqdDjqdTnys1WoB3L6sSa/XW13rcNP7mbru7m6z+9G7rK997G8bNHSs6Yv++tRe+tPa2gctvK5fv46dO3cOKLxaWlrQ09Mj3mC2l5+fH86dO2dxvba2NgQGBkKn08HZ2Rl/+MMfEBsba7ZtZmYmMjIyTJYXFhZCpVJZXetwc6UdAFxw9OhRXPaw3K6oqOg7b4NsbyB9YalP7aU/Ozs7rWo3aOE1lDw9PVFRUYH29nYUFxdDo9Fg8uTJWLhwoUnb1NRUow/WarVaBAUFIS4uDl5eXkNY9eCqqtdiS+VxzJ8/H/eNN90PvV6PoqIixMbGwtXV9a62QUPHmr7or0/tpT97z476I2l4+fj4wNnZGU1NTUbLm5qa4O/vb3E9hUKBqVOnAgDCwsJw9uxZZGZmmg0vpVIJpVJpstzV1dXiH7UcuLi4iN/72o++9tPabZDtDaQvLPWpvfSntbVLOmDv5uaG8PBwFBcXi8sMBgOKi4sRHR1t9XYMBoPRuBYR2T+rj7yWLl3a5/Otra13VYBGo0FSUhIiIiIQGRmJ7OxsdHR0QK1WAwASExMRGBiIzMxMALfHsCIiIjBlyhTodDocOHAAu3fvxo4dO+7q9YlInqwOL29v736fT0xMHHABCQkJaG5uRlpaGhobGxEWFoaCggJxEL+urg4KxbcHiB0dHVizZg2++uorjBgxAtOnT8e7776LhISEAb82EcmX1eH19ttv26yIlJQUpKSkmH2upKTE6PFvfvMb/OY3v7FZLUQkD7J8t5GAr/U9AIAzV9vMPt/xtQ6nmgH/yzcxcoTpGxYAcOFau83qI7I1q8PL2ulueAOOoVHz/4PnlX2VfbRywe4LJ/vd1kgl/w8j+bH6t3bXrl2YOHEi7r//fgiCYMuayApx993+KMmUcR4Y4eps8nx1Qxt+/udKbH1yFqYFWB6vHKl0wSSfkTark8hWrA6v1atX44MPPsClS5egVqvx3HPPYcyYMbasjfowZqQbnomcYPH53ktFpviOxMzAvt9sIen1NwwA9D8U4GjDAFaHV05ODrKysrBv3z7k5eUhNTUVjz76KFatWoW4uDg4OTnZsk4iu2bdMABgzVCAowwDDGgvlUolli1bhmXLluHy5cvYtWsX1qxZg+7ublRVVcHDQ8YXVBFJqL9hAMC6oQBHGga464hWKBRwcnKCIAjo6ekZzJqIHE5/wwAAhwLuNKDLg3Q6HT744APExsbi3nvvRWVlJd544w3U1dXxqIuIhpTVR15r1qzBnj17EBQUhOeffx4ffPABfHx8bFkbEZFFVodXbm4uJkyYgMmTJ+PTTz/Fp59+arbdvn37Bq04IiJLrA6vxMREvqNIRMPGgD6kSkQ0XEh+Aw4iorvB8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRydKwCK+cnBwEBwfD3d0dUVFROHHihMW2b731FhYsWIDRo0dj9OjRiImJ6bM9EdknycMrPz8fGo0G6enpKC8vR2hoKOLj43Ht2jWz7UtKSrBs2TIcPnwYpaWl4t2vr169OsSVE5GUJA+vrKwsJCcnQ61WIyQkBLm5uVCpVBbnwn/vvfewZs0ahIWFYfr06fjjH/8o3qiWiByHpFMudnV1oaysDKmpqeIyhUKBmJgYlJaWWrWNzs5O6PV6i1NS63Q6o7tpa7VaAIBer4der/8O1Q9vvXM/dXd32/V+OhJH6VNr903S8GppaUFPT494g9lefn5+OHfunFXbWL9+PcaPH4+YmBizz2dmZiIjI8NkeWFhIVQq1cCLlokr7QDgguPHj+PqGamrocHgKH3a2dlpVTtZT3a9adMm7NmzByUlJXB3dzfbJjU1FRqNRnys1WrFcTIvL6+hKnXIna67AVSewty5cxE6gTdKsQeO0qe9Z0f9kTS8fHx84OzsjKamJqPlTU1N8Pf373PdLVu2YNOmTTh06BBmz55tsZ1SqYRSaXqnFVdXV7i6ut5d4TLg4uIifrfn/XQkjtKn1u6bpAP2bm5uCA8PNxps7x18j46Otrje7373O2zcuBEFBQWIiIgYilKJaJiR/LRRo9EgKSkJERERiIyMRHZ2Njo6OqBWqwHcngQxMDAQmZmZAIDXXnsNaWlpeP/99xEcHIzGxkYAgIeHB+fRJ3IgkodXQkICmpubkZaWhsbGRoSFhaGgoEAcxK+rq4NC8e0B4o4dO9DV1YUnn3zSaDvp6en4z//8z6EsnYgkJHl4AUBKSgpSUlLMPldSUmL0uLa21vYFEdGwJ/mHVImI7gbDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLIkeXjl5OQgODgY7u7uiIqKwokTJyy2raqqwhNPPIHg4GA4OTkhOzt76AolomFF0vDKz8+HRqNBeno6ysvLERoaivj4eFy7ds1s+87OTkyePBmbNm3q96a0RGTfJA2vrKwsJCcnQ61WIyQkBLm5uVCpVMjLyzPb/vvf/z42b96MZ555xuxdsInIcUh267Ouri6UlZUhNTVVXKZQKBATE4PS0tJBex2dTgedTic+1mq1AAC9Xg+9Xj9orzPcdHd3i9/teT8diaP0qbX7Jll4tbS0oKenR7y5bC8/Pz+cO3du0F4nMzMTGRkZJssLCwuhUqkG7XWGmyvtAOCC48eP4+oZqauhweAofdrZ2WlVu2Fx01lbSk1NhUajER9rtVoEBQUhLi4OXl5eElZmW6frbgCVpzB37lyEThgjdTk0CBylT3vPjvojWXj5+PjA2dkZTU1NRsubmpoGdTBeqVSaHR9zdXWFq6vroL3OcOPi4iJ+t+f9dCSO0qfW7ptkA/Zubm4IDw9HcXGxuMxgMKC4uBjR0dFSlUVEMiHpaaNGo0FSUhIiIiIQGRmJ7OxsdHR0QK1WAwASExMRGBiIzMxMALcH+b/88kvx31evXkVFRQU8PDwwdepUyfaDiIaepOGVkJCA5uZmpKWlobGxEWFhYSgoKBAH8evq6qBQfHtwWF9fj/vvv198vGXLFmzZsgUPPvggSkpKhrp8IpKQ5AP2KSkpSElJMfvcnYEUHBwMQRCGoCoiGu4kvzyIiOhuMLyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpKlYRFeOTk5CA4Ohru7O6KionDixIk+2+/duxfTp0+Hu7s7Zs2ahQMHDgxRpUQ0XEgeXvn5+dBoNEhPT0d5eTlCQ0MRHx+Pa9eumW1/7NgxLFu2DKtWrcIXX3yBJUuWYMmSJThz5swQV05EUpI8vLKyspCcnAy1Wo2QkBDk5uZCpVIhLy/PbPvt27fjoYcewssvv4wZM2Zg48aNmDNnDt54440hrpyIpCTpfRu7urpQVlaG1NRUcZlCoUBMTAxKS0vNrlNaWgqNRmO0LD4+Hvv37zfbXqfTQafTiY+1Wi0AQK/XQ6/Xf8c9GD46OztRXV0tPj7f0AZd4wWcqXBDV5O3Udtp06ZBpVINdYk0AHf2J2C5T+2tP639u5Q0vFpaWtDT0yPeIbuXn58fzp07Z3adxsZGs+0bGxvNts/MzERGRobJ8sLCQrvq8JqaGvz85z83Wb7iT6Ztt27diilTpgxBVXS3LPUnYNqn9tafnZ2dVrWT/I7Ztpaammp0pKbVahEUFIS4uDh4eXlJWNng6uzsxPz588XH7V/rcPDIScQv+D48RiiN2trb/9T26M7+BCz3qb31Z+/ZUX8kDS8fHx84OzujqanJaHlTUxP8/f3NruPv7z+g9kqlEkql0mS5q6srXF1d77Ly4cfb2xuRkZHiY71ej1utN7Bg3ly72k9HcWd/Ao7Tp9bum6QD9m5ubggPD0dxcbG4zGAwoLi4GNHR0WbXiY6ONmoPAEVFRRbbE5F9kvy0UaPRICkpCREREYiMjER2djY6OjqgVqsBAImJiQgMDERmZiYAYO3atXjwwQexdetWPProo9izZw9OnTqFN998U8rdIKIhJnl4JSQkoLm5GWlpaWhsbERYWBgKCgrEQfm6ujooFN8eIM6bNw/vv/8+/uM//gO//OUv8b3vfQ/79+/HzJkzpdoFIpKAkyAIgtRFDCWtVgtvb2+0tbXZ1YD9nfR6PQ4cOIBHHnnErsdHHImj9Km1f6OSf0iViOhuMLyISJYYXkQkS5IP2A+13iE+az8IJ1d6vR6dnZ3QarV2PT7iSBylT3v/Nvsbjne48Lp16xYAICgoSOJKiKgvt27dgre3t8XnHe7dRoPBgPr6enh6esLJyUnqcmym9zKoK1eu2PW7qo7EUfpUEATcunUL48ePN/qY1J0c7shLoVDgnnvukbqMIePl5WXXv+iOyBH6tK8jrl4csCciWWJ4EZEsMbzslFKpRHp6utkZNUie2KfGHG7AnojsA4+8iEiWGF5EJEsMLyKSJYaXxBYuXIgXX3xR6jKIZIfhJRMlJSVwcnJCa2ur1KXQAAy3/5yGWz3fBcOLaJjr6uqSuoRhieE1jOh0Oqxfvx5BQUFQKpWYOnUqdu7cidraWixatAgAMHr0aDg5OWHlypX9bu/WrVt49tlnMXLkSAQEBGDbtm0m//Pu3r0bERER8PT0hL+/P5YvX45r166Jz/ce8RUXFyMiIgIqlQrz5s0zuSEqmVq5ciU+/fRTbN++HU5OTnByckJNTQ1WrVqFSZMmYcSIEZg2bRq2b99ust6SJUvw29/+FuPHj8e0adMAAMeOHUNYWBjc3d0RERGB/fv3w8nJCRUVFeK6Z86cwcMPPwwPDw/4+flhxYoVaGlpsVhPbW3tUP04Bp9AknrwwQeFtWvXCoIgCE8//bQQFBQk7Nu3T6ipqREOHTok7NmzR+ju7hY+/PBDAYBQXV0tNDQ0CK2trf1u+yc/+YkwceJE4dChQ0JlZaXw4x//WPD09BRfTxAEYefOncKBAweEmpoaobS0VIiOjhYefvhh8fnDhw8LAISoqCihpKREqKqqEhYsWCDMmzdvsH8Udqe1tVWIjo4WkpOThYaGBqGhoUH45ptvhLS0NOHkyZPCxYsXhXfffVdQqVRCfn6+uF5SUpLg4eEhrFixQjhz5oxw5swZoa2tTRgzZozw3HPPCVVVVcKBAweEe++9VwAgfPHFF4IgCMLNmzcFX19fITU1VTh79qxQXl4uxMbGCosWLbJYT3d3txQ/mkHB8JJYb3hVV1cLAISioiKz7XpD5ObNm1ZtV6vVCq6ursLevXvFZa2trYJKpTIKrzudPHlSACDcunXL6HUPHToktvn4448FAMLXX39tVS2O7J//c7LkhRdeEJ544gnxcVJSkuDn5yfodDpx2Y4dO4SxY8ca/czfeusto/DauHGjEBcXZ7TtK1euiP/pWVuPXPC0cZioqKiAs7MzHnzwwUHZ3sWLF6HX641uXOrt7S2egvQqKyvD448/jgkTJsDT01N8/bq6OqN2s2fPFv8dEBAAAEanl2S9nJwchIeHw9fXFx4eHnjzzTdNft6zZs2Cm5ub+Li6uhqzZ8+Gu7u7uOzOm9KePn0ahw8fhoeHh/g1ffp0AEBNTY0N90gaDjclznA1YsSIIX/Njo4OxMfHIz4+Hu+99x58fX1RV1eH+Ph4k0Hif565s3ceNIPBMKT12oM9e/Zg3bp12Lp1K6Kjo+Hp6YnNmzfj888/N2o3cuTIAW+7vb0djz/+OF577TWT53r/w7EnDK9hYtasWTAYDPj0008RExNj8nzv/8I9PT1WbW/y5MlwdXXFyZMnMWHCBABAW1sbzp8/j3/5l38BAJw7dw7Xr1/Hpk2bxJllT506NRi7Q/+fm5ubUZ999tlnmDdvHtasWSMus+aoaNq0aXj33Xeh0+nEC7NPnjxp1GbOnDn48MMPERwcDBcX83/ad9YjZzxtHCaCg4ORlJSE559/Hvv378elS5dQUlKC//7v/wYATJw4EU5OTvjb3/6G5uZmtLe397k9T09PJCUl4eWXX8bhw4dRVVWFVatWQaFQiEdOEyZMgJubG15//XVcvHgRH330ETZu3GjzfXUkwcHB+Pzzz1FbW4uWlhZ873vfw6lTp3Dw4EGcP38ev/71r01CyJzly5fDYDDgX//1X3H27FkcPHgQW7ZsAfDtkfALL7yAGzduYNmyZTh58iRqampw8OBBqNVqMbDurEfOR88Mr2Fkx44dePLJJ7FmzRpMnz4dycnJ6OjoAAAEBgYiIyMDr7zyCvz8/JCSktLv9rKyshAdHY3HHnsMMTExeOCBBzBjxgxx3MTX1xe7du3C3r17ERISgk2bNol/EDQ41q1bB2dnZ4SEhMDX1xfx8fFYunQpEhISEBUVhevXrxsdhVni5eWF//mf/0FFRQXCwsLwq1/9CmlpaQAg9uf48ePx2WefoaenB3FxcZg1axZefPFFjBo1SpxO+c567hxrkxNOieNAOjo6EBgYiK1bt2LVqlVSl0Pf0XvvvQe1Wo22tjZJxkylxjEvO/bFF1/g3LlziIyMRFtbGzZs2AAAWLx4scSV0d145513MHnyZAQGBuL06dNYv349nn76aYcMLoDhJVt1dXUICQmx+PyXX34JANiyZQuqq6vh5uaG8PBwHDlyBD4+PkNVJg2ixsZGpKWlobGxEQEBAXjqqafw29/+VuqyJMPTRpnq7u7u89KOvt5xIrIHDC8ikiW+20hEssTwIiJZYngRkSwxvIhIlhheZFMrV64UJ75zdXWFn58fYmNjkZeXN6BLU3bt2oVRo0bZrlALeicGpOGH4UU299BDD6GhoQG1tbX4+9//jkWLFmHt2rV47LHH0N3dLXV5JFdSTiZG9i8pKUlYvHixyfLi4mIBgPDWW28JgiAIW7duFWbOnCmoVCrhnnvuEVavXm0yIeI/f6WnpwuCIAjvvPOOEB4eLnh4eAh+fn7CsmXLhKamJvF1bty4ISxfvlzw8fER3N3dhalTpwp5eXni83V1dcJTTz0leHt7C6NHjxZ+9KMfCZcuXRIEQRDS09NNXvfw4cM2+TnRwPHIiyTxgx/8AKGhodi3bx8AQKFQ4Pe//z2qqqrwpz/9CZ988gl+8YtfAADmzZuH7OxseHl5oaGhAQ0NDVi3bh0AQK/XY+PGjTh9+jT279+P2tpao/n9f/3rX+PLL7/E3//+d5w9exY7duwQrzDQ6/WIj4+Hp6cnjhw5gs8++wweHh546KGH0NXVhXXr1uHpp58WjxwbGhowb968of1BkWVSpyfZN0tHXoIgCAkJCcKMGTPMPrd3715h7Nix4uO3335b8Pb27vf17pzG+vHHHxfUarXZtrt37xamTZsmGAwGcZlOpxNGjBghHDx4sN/6SVo88iLJCIIgzkV16NAh/PCHP0RgYCA8PT2xYsUKXL9+HZ2dnX1uo79prFevXo09e/YgLCwMv/jFL3Ds2DFx3dOnT+PChQvw9PQUp00eM2YMvvnmG7ucNtneMLxIMmfPnsWkSZNQW1uLxx57DLNnz8aHH36IsrIy5OTkAOj7noW901h7eXnhvffew8mTJ/GXv/zFaL2HH34Yly9fxksvvYT6+nr88Ic/FE8529vbER4ejoqKCqOv8+fPY/ny5Tbee/queOUuSeKTTz5BZWUlXnrpJZSVlcFgMGDr1q3ipHm9M8j2Mjd9sbXTWPv6+iIpKQlJSUlYsGABXn75ZWzZsgVz5sxBfn4+xo0bBy8vL7N12tO0yfaGR15kczqdDo2Njbh69SrKy8vx6quvYvHixXjssceQmJiIqVOnQq/Xi9NR7969G7m5uUbbCA4ORnt7O4qLi9HS0oLOzk6rprFOS0vDX//6V1y4cAFVVVX429/+hhkzZgAAnn32Wfj4+GDx4sU4cuSIOPX2z372M3z11Vfi6/7jH/9AdXU1WlpaoNfrh+aHRv2TetCN7FtSUpL4MQMXFxfB19dXiImJEfLy8oSenh6xXVZWlhAQECCMGDFCiI+PF9555x2T+1T++7//uzB27Fijj0q8//77QnBwsKBUKoXo6Gjho48+MrmX4YwZM4QRI0YIY8aMERYvXixcvHhR3GZDQ4OQmJgo+Pj4CEqlUpg8ebKQnJwstLW1CYIgCNeuXRNiY2MFDw8PflRimOGUOEQkSzxtJCJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRL/w/btaIgfbMWvQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T18:25:48.309255Z", + "iopub.status.busy": "2024-03-24T18:25:48.308388Z", + "iopub.status.idle": "2024-03-24T18:25:48.515858Z", + "shell.execute_reply": "2024-03-24T18:25:48.514936Z" + }, + "papermill": { + "duration": 0.228878, + "end_time": "2024-03-24T18:25:48.518215", + "exception": false, + "start_time": "2024-03-24T18:25:48.289337", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABYm0lEQVR4nO3deVwU5R/A8c/sLsslpyiHgnjifQdqnomiqWl2qJl3WqZZUp6Vd3lk5s+yTM20PDLLvPJIyTu88L5FUfAAUeRG2GN+fyyuroCyCCzg83695uXuzDOz30H2y8w8lyTLsowgCEIJpLB0AIIgCAVFJDhBEEoskeAEQSixRIITBKHEEglOEIQSSyQ4QRBKLJHgBEEosUSCEwShxFJZOoCiSK/Xc/PmTRwcHJAkydLhCILwGFmWSUpKwsvLC4Ui5+s0keCycfPmTby9vS0dhiAITxEVFUX58uVz3C4SXDYcHBwAww/P0dHRwtEIgvC4xMREvL29jd/VnIgEl40Ht6WOjo4iwQlCEfa0R0iikkEQhBJLJDhBEEoskeAEQSixxDM4odDJsoxWq0Wn01k6FKGIUiqVqFSqZ26mJRKcUKgyMjK4desWqamplg5FKOLs7Ozw9PRErVbn+RgiweUDWZZFg+Bc0Ov1REREoFQq8fLyQq1Wi5+bkIUsy2RkZBAbG0tERARVq1Z9YmPeJxEJ7hn8fjiKxfuu0LmuFyPaVrV0OEVeRkYGer0eb29v7OzsLB2OUITZ2tpiZWXFtWvXyMjIwMbGJk/HEZUMzyBdp+diTDKHr8ZZOpRiJa9/jYXnS378nojftGfwgq8LAEev3UOr01s4GkEQHicS3DOoVtYBBxsVKRk6zt1KsnQ4giA8RiS4Z6BQSDSuYLiKE7epJVvr1q356KOPLB1GibFr1y4kSSI+Pr5AP8eiCW7Pnj106dIFLy8vJEli3bp1Tyz/4Ify+BIdHW1Sbv78+fj6+mJjY0NAQACHDh0qsHNo7OsKwJFrIsEJhffFFXLHogkuJSWFevXqMX/+fLP2u3DhArdu3TIuZcuWNW5bvXo1wcHBTJw4kaNHj1KvXj2CgoK4fft2focPwAuZCe7w1XuIObSF50lGRoalQ3gqiya4jh07Mm3aNF599VWz9itbtiweHh7G5dHaljlz5jB48GAGDBhAzZo1WbBgAXZ2dixZsiS/wwegbnkn1EoFsUnpRMaJxqvmkmWZ1AytRZa8/kFKT09nzJgxeHt7Y21tTZUqVfjpp5+4evUqbdq0AcDFxQVJkujfv/9Tj5eUlETv3r2xt7fH09OTb775Jsst8a+//krjxo1xcHDAw8ODt956y+SP9oMrx5CQEBo3boydnR3NmjXjwoULuT6vadOmUbZsWRwcHHjnnXcYO3Ys9evXN27v378/3bp144svvsDLyws/P79cxQawefNmqlWrhq2tLW3atOHq1au5jutZFMt2cPXr1yc9PZ3atWszadIkXnzxRcDwFyUsLIxx48YZyyoUCgIDAwkNDS2QWGyslNQp70TYtXscvnqPCqXtC+RzSqo0jY6aE7ZZ5LPPTgnCTm3+V6Bv376EhoYyb9486tWrR0REBHfu3MHb25s///yT1157jQsXLuDo6Iitre1TjxccHMz+/fvZsGED7u7uTJgwgaNHj5okF41Gw9SpU/Hz8+P27dsEBwfTv39/Nm/ebHKsTz/9lK+//poyZcrw3nvvMXDgQPbv3//UGFasWMEXX3zB999/z4svvshvv/3G119/TcWKFU3KhYSE4OjoyPbt23MdW1RUFN27d2fYsGEMGTKEI0eO8PHHHz81pvxQrBKcp6cnCxYsoHHjxqSnp7N48WJat27NwYMHadiwIXfu3EGn0+Hu7m6yn7u7O+fPn8/xuOnp6aSnpxvfJyYmmhVXY18Xwq7d48jVOF5vlPPookLxd/HiRX7//Xe2b99OYGAgAJUqVTJud3U1PLIoW7Yszs7OTz1eUlISy5YtY+XKlbRt2xaAn3/+GS8vL5NyAwcONL6uVKkS8+bN44UXXiA5OZlSpUoZt33xxRe0atUKgLFjx9KpUyfu37//1Iay3377LYMGDWLAgAEATJgwgX/++Yfk5GSTcvb29ixevNik+9TTYvvhhx+oXLkyX3/9NQB+fn6cOnWKmTNnPvXn86yKVYLz8/MzXhYDNGvWjMuXL/PNN9/w66+/5vm406dPZ/LkyXne/4UKrvzIFVGTmge2VkrOTgmy2Geb6/jx4yiVSmMSeVZXrlxBo9Hg7+9vXOfk5GTyew4QFhbGpEmTOHHiBPfu3UOvN7S7jIyMpGbNmsZydevWNb729PQE4Pbt2/j4+DwxjgsXLvD++++brPP39+fff/81WVenTp0sfUOfFtu5c+cICAgw2adp06ZPjCe/FPtmIv7+/oSHhwPg5uaGUqkkJibGpExMTAweHh45HmPcuHEkJCQYl6ioKLNiaJTZVORybAp3k9OfUlp4lCRJ2KlVFlny0g82N7ec+S0lJYWgoCAcHR1ZsWIFhw8f5q+//gKyPui3srIyvn5wfg8STn6wtzd9BGNObJZQ7BPc8ePHjX+p1Go1jRo1IiQkxLhdr9cTEhLyxL8Y1tbWxuHJ8zJMuYu9mqplDbcJYdfu5eEshOKiTp066PV6du/ene32B1c3uR0KqlKlSlhZWXH48GHjuoSEBC5evGh8f/78ee7evcuMGTNo0aIF1atXz/dWAX5+fiYxAFneZyc3sdWoUSNLU60DBw48e9C5YNEEl5yczPHjxzl+/DgAERERHD9+nMjISMBwZdW3b19j+blz57J+/XrCw8M5ffo0H330Ef/++y/Dhg0zlgkODmbRokUsW7aMc+fOMXToUFJSUozPFgrKw/ZwIsGVZL6+vvTr14+BAweybt06IiIi2LVrF7///jsAFSpUQJIkNm3aRGxsbJZnWI9zcHCgX79+jBo1ip07d3LmzBkGDRqEQqEwXoH5+PigVqv59ttvuXLlChs2bGDq1Kn5el4ffPABP/30E8uWLePSpUtMmzaNkydPPvUqNzexvffee1y6dIlRo0Zx4cIFVq5cydKlS/M1/hzJFrRz504ZyLL069dPlmVZ7tevn9yqVStj+ZkzZ8qVK1eWbWxsZFdXV7l169byv//+m+W43377rezj4yOr1WrZ399fPnDggFlxJSQkyICckJCQ633+DIuSK4zZJHebv8+sz3qepKWlyWfPnpXT0tIsHYrZWrVqJX/44YeyLBvOY+TIkbKnp6esVqvlKlWqyEuWLDGWnTJliuzh4SFLkmT8XX6SxMRE+a233pLt7OxkDw8Pec6cObK/v788duxYY5mVK1fKvr6+srW1tdy0aVN5w4YNMiAfO3ZMluWH36V79+4Z9zl27JgMyBEREbk6xylTpshubm5yqVKl5IEDB8ojRoyQmzRpYtzer18/uWvXrln2e1pssizLGzdulKtUqSJbW1vLLVq0kJcsWZIl3sc96fclt99RSZZF69THJSYm4uTkREJCQq5vV6PiUmkxaydWSomTE4OwVZv/ALuku3//PhEREVSsWDHPw988D1JSUihXrhxff/01gwYNslgc7dq1w8PD45kq8J7Fk35fcvsdLVa1qEVZeRdbPBxtiE68z4nr8TSpVNrSIQnFxLFjxzh//jz+/v4kJCQwZcoUALp27VpoMaSmprJgwQKCgoJQKpWsWrWKHTt2mLR3K46KfSVDUSFJEo0zh086IpqLCJkiIyMpVapUjsuD582zZ8+mXr16BAYGkpKSwt69e3Fzc8u3OGrVqpVjDCtWrECSJDZv3kzLli1p1KgRGzdu5M8//zS29SuuxBVcPnrB15VNJ29x+KqoaBAMvLy8jJVoOW338fEhLCysQOPYvHkzGo0m223u7u7Y2tqyY8eOAo3BEkSCy0eNHxkAU6eXUSrEfAPPO5VKRZUqVSwdBhUqVLB0CBYhblHzUXUPR0pZq0hK13I+2rzuXoIg5D+R4PKRUiHRsMKD53DiNlUQLE0kuHz2ghjhVxCKDJHg8llj4wCYcWIATEGwMJHgnpVOa/K2vrczKoVETGI61++lWSgoQRBAJLhnE3UYFrwI5x8OOmirVlK7nBMg5mkQDHx9fZk7d67xfW7mHxHyh0hwz+LcBog9D5tGQlq8cfWD+VJFezghO7du3aJjx46WDuO5IBLcs2gzHlwrQ3I0/POZcbVxZBFR0SBkw8PDA2tra4vGIMsyWq326QWLOZHgnoWVLXT9zvD62K9w2TD66YO5Ui/GJBOfavlB/4o0WYaMFMssZlQC5WZimNx69Bb16tWrSJLE2rVradOmDXZ2dtSrVy/LHCL79u2jRYsW2Nra4u3tzYgRI0hJSTFuz+2kNFu2bKFRo0ZYW1uzb98+s2MvbkRPhmdVoRm8MBgOL4INH8L7oZQuVYpKZey5EptC2LV7tK3h/vTjPK80qfCl19PLFYTxN0Gdu0mCcjMxzLP49NNPmT17NlWrVuXTTz+lV69ehIeHo1KpuHz5Mh06dGDatGksWbKE2NhYhg8fzvDhw/n555+B3E9KM3bsWGbPnk2lSpVwcXHJl9iLMpHg8kPgRLi4DRIiIWQKvDyLFyq4ciU2hcNXRYIr7nI7Mcyz+OSTT+jUqRMAkydPplatWoSHh1O9enWmT59O7969jVeLVatWZd68ebRq1YoffvgBGxubXE9KM2XKFNq1a5dvcRd1IsHlB2sH6DIXlneHQz9CrVdp7Fue1UeiWHv0Ot6utnSrXw57a/HjzsLKznAlZanPzoXcTgzzLHKaLKZ69eqcOHGCkydPsmLFCmMZWZbR6/VERERQo0aNXE9K07hx43yLuTgQ37j8UqUtNHgbji2HDcNp9dYOHG1U3E5K59O/TjN983lea1iOt5tUoKq7g6WjLTokKde3iSXZkyaLSU5O5t1332XEiBFZ9vPx8TFO/BIUFMSKFSsoU6YMkZGRBAUFZZn45fFJY0o6keDyU/sv4NIOuBtO2aPfsHf056wJi2LFwUgi7qSwLPQay0Kv0aSSK32b+tKupjtWSlHPU9Q9OjHMg+n3HkwM07JlywL//IYNG3L27NkcRyU5deqUceIXb29vAI4cOVLgcRUH4tuVn2ydofM3htf/fYvTvZO806ISIcGt+HWQP+1ruqOQ4MCVON5fcZQXZ/zLN9svEp1w36JhC0+Wm4lhCtKYMWP477//GD58OMePH+fSpUusX7+e4cOHA4UzKU1xJRJcfqv+MtR+DWQ9rP8AtBkoFBItqpZhYd/G7BvzEh+8VAW3UtbcTkrnfyGXeHHmvwxdHsZ/4XdE/9Uias6cOTRt2pTOnTsTGBjIiy++SI0aNQplbom6deuye/duLl68SIsWLWjQoAETJkwwVnKUKVOGpUuXsmbNGmrWrMmMGTOYPXt2gcdVHIhJZ7KRl0lnTKTcgfn+kHoXWo+D1mOzFMnQ6tl2JppfD1zjUMTDBsGVy9jTp0kFujcqj6ONVZb9irOSNOlMUZkYpiTLj0lnxBVcQbB3g46zDK/3fAUxZ7IUUasUdKnnxe/vNmXbRy3p06QC9moll2NTmLTxLAFfhPDj7suFHLiQk2PHjrFq1SouX77M0aNH6d27N1C4E8MI5hMJrqDUfg38OoFeC+uHZRl15FF+Hg5M7Vabg58GMrVbbaq5lyJNo2PG1vOcuyVGBi4qspsY5ty5c0+cVEawLIsmuD179tClSxe8vLxyNcLC2rVradeuHWXKlMHR0ZGmTZuybds2kzKTJk1CkiSTpXr16gV4FjmQJOj0NVg7wc1jEPrdU3cpZa2iT5MKbPuoJZ3qeiLLMHvbhUIIVniaBg0aEBYWRnJyMnFxcWzfvp06derQuHFjjh8/nuMiWJZFE1xKSgr16tVj/vz5uSq/Z88e2rVrx+bNmwkLC6NNmzZ06dKFY8eOmZSrVasWt27dMi4W63Pn6AkdvjS83vkl3LmUq90kSeKT9n4oFRIh52+LTvtFmK2tLVWqVMlxESzLou3gOnbsaNawMY+OqQXw5Zdfsn79ejZu3EiDBg2M61UqFR4eHvkV5rOp3xtO/2noiL/hA+i/GRRP/7tS0c2eNxuXZ9WhKGZtvcDqd5sUSpOEwiDqtYTcyI/fk2L9DE6v15OUlISrq6vJ+kuXLuHl5UWlSpXo3bu3cXLdnKSnp5OYmGiy5BtJgi7/A3UpiAyFw4tzveuItlVRqxQcuhrHroux+ReThTxorZ+ammrhSITi4MHvyaO9PMxVrHsyzJ49m+TkZN58803juoCAAJYuXYqfnx+3bt1i8uTJtGjRgtOnT+PgkH0XqenTpzN58uSCC9TZBwInweZPYMckqBYELk+fp9LTyZZ+TSuwaG8EX229QKuqZVAU47lWlUolzs7OxmF87OzsSsxVqZB/ZFkmNTWV27dv4+zsjFKpzPOxikw7OEmS+Ouvv+jWrVuuyq9cuZLBgwezfv16AgMDcywXHx9PhQoVmDNnTo7tldLT00lPTze+T0xMxNvbO+/t4LKj18PSThD5H1RqDX3WGa7unuJeSgYtZ+0kKV3Lt70a0KWehYYWyieyLBMdHU18fLylQxGKOGdnZzw8PLL9I5jbdnDF8grut99+45133mHNmjVPTG5g+CFVq1aN8PDwHMtYW1sX/AirCoVhcMwfmsGVXYYBMhv2fepuLvZqBresxJztF5mz/SIdansU6/6rkiTh6elJ2bJl0Wg0lg5HKKKsrKye6crtgWKX4FatWsXAgQP57bffjONnPUlycjKXL1+mT58+hRDdU5SubBjmfPsE2PYZVGlnqGl9ioHNK7Lsv6tE3Enhj7Dr9PL3KYRgC5ZSqcyXX2BBeBKLXgokJyebtBeKiIjg+PHjxkqBcePG0bfvw6uclStX0rdvX77++msCAgKIjo4mOjqahIQEY5lPPvmE3bt3c/XqVf777z9effVVlEolvXr1KtRzy1GTYeDVENITDJPV5OIJQSlrFcPaGJoc/G/HJe5rdAUdpSCUCBZNcEeOHKFBgwbGJh7BwcHGjsRgmH3o0RrQhQsXotVqGTZsGJ6ensblww8/NJa5fv06vXr1ws/PjzfffJPSpUtz4MABypQpU7gnlxOlCrrOB4UVXNxiaEKSC72b+FDO2ZboxPv8GnqtgIMUhJKhyFQyFCXP3Nk+N3bNhF1fgl1pGHbI0H/1KX4/EsXoP07iYmfF7tFtSlxnfEHILdHZvqhrPhLK1jKMOLJ5VK526d6gHJXL2HMvVcPiPVcKOEBBKP5EgrMUlRq6zQdJCWfWwrlNT99FqeCT9oZ5ABbvi+BOcvpT9hCE55tIcJbk1QCafWB4/XcwpN176i4dantQt7wTqRk65u/MuemLIAgiwVle67FQuiokxxiajjyFJEmMCjJcxa04EMn1e6LbkyDkRCQ4S7OyNTQARoLjyyF8x1N3aV7FjWaVS5Oh0/O/HbkboUQQnkciwRUFPk0g4F3D640fQXrSE4s/ehX359HrhN9+cnlBeF6JBFdUvPS5oVN+QpShQ/5TNPBxoX1Nd/QyfP3PxYKPTxCKoWdKcMnJyQU3zNDzxroUvPKt4fXhxXB1/1N3+STID0mCLaejOXk9vmDjE4RiyOwEFxERQadOnbC3t8fJyQkXFxdcXFxwdnbGxcWlIGJ8flRq/bAD/obhkPHkCoRq7g682qAcAF+Joc0FIQuzO9u//fbbyLLMkiVLcHd3F+N55bf20+DSDoi7Yujp0H7aE4uPDKzGxhM32XvpDv+F36FZlaf3iBCE54XZCe7EiROEhYXh5+dXEPEINk7Q+RtY1QNC50PNV6F8oxyLe7va8Za/D8tCrzFz2wXWVS4t/ugIQiazb1FfeOEFoqKiCiIW4QG/DlDnTZD1hikHtU/usTD8parYWik5ERXPP2djCilIQcg/Caka3ll2hDVH8je3mH0Ft3jxYt577z1u3LhB7dq1s4yXXrdu3XwL7rnWYYZhoprYc7BnNrz0aY5FyzhYM7C5L/N3Xmb2tgsE1nBHWYyHNheeP8sPXmPHuRhuJaTxRmPvfDuu2QkuNjaWy5cvM2DAAOM6SZKQZRlJktDpxFhl+cK+NLz8FfwxAPbNgZqvgEedHIsPaVmZ5QciuXQ7mXXHbvBao/KFGKwg5J0sy/wRdh0AnT5/Bzcy+xZ14MCBNGjQgNDQUK5cuUJERITJv0I+qvUqVO8Meq3hVlWnzbGok60VQ1tXBuCbHRfJ0OoLK0pBeCZh1+4RcScFAH0+j95m9hXctWvX2LBhg5jUtjBIEnT6Gq7uhVsn4L950CI4x+L9mvqyZF8E1++lsepQJP2a+RZerIKQR2uOXDe+tvgV3EsvvcSJEyfyNQjhCRw8DM/jAHbNgNicey3YqpWMaFsVgG//DSclPecrPkEoClIztGw6edP4Pr+H3zX7Cq5Lly6MHDmSU6dOUadOnSyVDK+88kq+BSdkqtfLMLR5+A5DA+ABW0CR/YQtPV7wZtHeK1y7m8rP+yMY/lLVQg5WEHJvy6loUjJ0KCTQy/l/i2r2kOUKRc4XfSWlkqFQhiw3V3wUfN8EMpKhw0xo8l6ORdcfv8GHvx3HwUbF3tFtcLZTF2KggpB7PReGcuBKHG38yrDzQizerrbsHf3SU/crsCHL9Xp9jktJSG5FlrM3tJtseB0yGeIicizapa4X1T0cSLqv5YfdlwspQEEwT+TdVA5ciUOSMDYN0edz3ZhZCU6j0aBSqTh9+nT+RiHkTqOBUKE5aFJh44gcH1goFA+HU1q6/yoxifcLM0pByJU/jhoqF16s7Ia3ix2Q/7eoZiU4KysrfHx8xJWapSgU8Mo8UNlCxB44uizHoi9VL0ujCi6ka/XMCxGDYgpFi14v82dm27c3GpdHkqCKdB2VPn/nGTH7FvXTTz9l/PjxxMXF5WsgQi6VrgwvZQ5t/s/nkHAj22KSJDE68ypu9eEort1NKawIBeGpQq/c5UZ8Gg42KoJqeaDSJPGrega/aD6Bu/n3WMXsBPfdd9+xZ88evLy88PPzo2HDhiaLOfbs2UOXLl3w8vJCkiTWrVv31H127dpFw4YNsba2pkqVKixdujRLmfnz5+Pr64uNjQ0BAQEcOnTIrLiKvCZDoVxjSE+ETSNzvFUNqFSa1n5l0Opl5mwXg2IKRceDPqev1PPCxkqJx8Ev8JTiUKIDB898+xyzm4l069Yt3z48JSWFevXqMXDgQLp37/7U8g/GonvvvfdYsWIFISEhvPPOO3h6ehIUFATA6tWrCQ4OZsGCBQQEBDB37lyCgoK4cOECZcuWzbfYLUqhNMzj8GNLuLQNTq2Bum9mW/ST9n7suhDLhhM3ebdlZWp6FZFaYeG5lXhfw5bT0UBm5cKVXTifWwnAFOk9Fqvt8u/D5CICkP/6668nlhk9erRcq1Ytk3U9evSQg4KCjO/9/f3lYcOGGd/rdDrZy8tLnj59eq5jSUhIkAE5ISEh1/tYxK5ZsjzRUZZnVJDlpJgciw1bESZXGLNJHvDzocKLTRBysOLANbnCmE1y4Ne7ZP39JFn+po4sT3SUf/m0u1xn4tZcHSO339E8D1keFhbG8uXLWb58OceOHcu3hPskoaGhBAYGmqwLCgoiNDQUgIyMDMLCwkzKKBQKAgMDjWWyk56eXjyHXm/+kaEDfto92Dwqx2Ift/dDqZD49/xtjlwVz04Fy1oTZrg9faNxeaR/p0H8NbQO5Zih7ZXvPRnMTnC3b9/mpZde4oUXXmDEiBGMGDGCRo0a0bZtW2JjY/M3usdER0fj7u5uss7d3Z3ExETS0tK4c+cOOp0u2zLR0dE5Hnf69Ok4OTkZF2/v/BuupUApraDrfJCUcHYdnN2QbbGKbva82dgwusisrReQ8/u3SBByKfx2Esci41EqJN4oewMOLgDgbpuvSMHWss1EAD744AOSkpI4c+YMcXFxxMXFcfr0aRITExkxYkS+BldYxo0bR0JCgnEpVgN6etYzXMkB/P0xpGZ/hTaibVXUKgWHrsax62LB/iEShJysyWwa0q6qEy7bgwEZ6vcmw7cNADpLJ7itW7fy/fffU6NGDeO6mjVrMn/+fLZs2ZKvwT3Ow8ODmBjTEWtjYmJwdHTE1tYWNzc3lEpltmU8PDxyPK61tTWOjo4mS7HScjS4VYOU27BtfLZFPJ1s6de0AgBfbb2APp9HbRCEp9Hq9Kw9amjWNNpmHdy9BKXcIegLFJkDtOb3r2Weumo93sEeDI2A9fndz+IxTZs2JSQkxGTd9u3badq0KQBqtZpGjRqZlNHr9YSEhBjLlEhWNoZbVSQ4sQoubc+22NDWVShlreLsrUT+PnWrcGMUnnt7LsUSm5TOi3ZRVLz4k2Flpzlg68KDAajz+/FJnoZL+vDDD7l58+EQJzdu3GDkyJG0bdvWrGMlJydz/Phxjh8/DhiagRw/fpzIyEjAcOvYt29fY/n33nuPK1euMHr0aM6fP8/333/P77//zsiRI41lgoODWbRoEcuWLePcuXMMHTqUlJQUkxGISyRvf0P7OICNH8H9rBUlrvZqBreoBMCc7RfR6MSgmELhWXPkOlZo+dp6EZKsMwzoWqMzAMrMiZLyezw4s5uJREZGyvXr15etrKzkSpUqyZUqVZKtrKzkBg0ayFFRUWYda+fOnTKQZenXr58sy7Lcr18/uVWrVln2qV+/vqxWq+VKlSrJP//8c5bjfvvtt7KPj4+sVqtlf39/+cCBA2bFVWyaiTwuPVmW59Y1NB3Z+FG2RZLua+SGU/6RK4zZJK88eK2QAxSeV3eT0+Uq4/+WZ48fmNm0yVeWk24bt99OvC9XGLNJrjBmU66Ol9vvqNnDJWUmRXbs2MH58+cBqFGjRpbmG8VZkRwuKbeu7IZfMsfk67cJKrbIUmTJvgimbDqLh6MNu0a1xsYq+7HlBCG//Lw/glWbtvG39adYoYXui6HuG8btcSkZNJsRgkKSOD0pyPhMLie5/Y7mKcGVdMU6wQFs/BDCloJLRRj6HzzWMjxdq+Ol2bu5EZ/G+JerM6RlZcvEKZQIyela4lMzKO+Scw+EznN3MS1uJPUVV6BaR+i1yjAkfx7l9jtqdlctgJCQEEJCQrh9+3aWioUlS5bk5ZBCfmo3xVDRcC8Cdn4BQV+YbLZWKfkwsCqj/zjJ97su09PfB0ebrBVHgpAb/ZYc4tT1BHaNao2Xs22W7WduJtA0djX1ra4gqx2QOs95puRmDrMrGSZPnkz79u0JCQnhzp073Lt3z2QRigAbJ+g81/A6dD5EHc5SpHuDclQuY098qobFewyzoSWkakhI1RRioEJxl3hfQ9i1e2To9BzOoZfM4bDDfKxaA4DU4Utw9Cq0+My+gluwYAFLly6lT58+BRGPkF+qtYe6PeHkb4YpB9/bCypr42aVUsEn7f0YuuIoi/dF0LtJBV75bh/3UjS83aQCn3aqISaPFp7K/4sdxtdnbybStX450wJ6Pc3PTcFG0nDdJYDyDQo3b5h9BZeRkUGzZs0KIhYhv3WYDvZl4M4F2D0r6+baHtQt70Rqho7JG88Qk5hOhk7Pkv0RxsEIBeFxsizzzrLDDPnlCPc1Dx9Rnb6ZkLXwkZ+oknqCVNmaSwFfFNqt6QNmJ7h33nmHlStXFkQsQn6zczXMqwqw7xvD3KqPkKSHQ5tvPmXaV/fAlbuFEqJQ/CSkadhx7jb/nDXtMXTmZqJpQ934SNgxCYCZ2p6U8a5WiFEamH2Lev/+fRYuXMiOHTuoW7dull4Nc+bMybfghHxQsyvUeAXObTDcqg7eaeikn6l5FTeaVipN6GMJ7cg18TxVyF5yDvPtxqdquBGfZqhNlWVDbX5GMof11fhF145g13wc5y2XzL6CO3nyJPXr10ehUHD69GmOHTtmXB70SBCKmJdng60LRJ+C/f8z2SRJEqM6+BnfN/BxRpIgMi6V20lishohq9SMnOdkOXMzswfN8ZVw+V/0SmvGaIbgZGeNk23h19SbfQW3c+fOgohDKEgO7tBhBvz1LuyeCdU7Q9nqxs0NfVxoX9Odf87GULlMKdIydJyPTiLs6j061sm/4aOFkiElhys4MCS4IB9g2zgAwmsO58phL+pa4OoN8nAFJxRTdXtA1fagy4ANw0Fv+ld4arfa9G1agSEtK9HY1wUQt6lC9lLSn3AFdz0eNgXD/QTwrM9et54AeIsEJxQoSYLO34DaAa4fNg40+IC7ow1TutammrsDL/i6AojRf4VspWTkfAXneX0LXPgbFCroOp/I+AwAfESCEwqcU3loP9XwOmQqxF3JtlijCoYruDM3E0l7wvMW4fmUmkOCKy0l8pF2seFNi0/AozbX4lIBqCASnFAoGvUH3xagTYMNIyCbMfzKOdvi4WiDVi9zPCq+0EMUirbkHG5RZ9mvxE1KJNmpGrT4GDBUVkExuoLbs2cPWm3WDK7VatmzZ0++BCUUIEmCV+aByhau7oWjS7MpItEo8zlc2DVxmyqYSs2mkiFQEUZb7R50ssSWSp+BSo1eL3M9Lg0oRs/g2rRpk+2s9gkJCbRp0yZfghIKmGslaDvB8PqfCZCQtddC4wqiokHIXspjjy0cSeELK8MIvYt0ndiVbJi0KTrxPhk6PSqFhKeTTaHHCXlIcLIsI2XT3eLu3bvY29vnS1BCIQh4F8r7Q0aSYQTgx0bNalzBUNFw9No9MX+DYOLxK7jxqhW4S/GkOfjyjfZ1Y5etB7en5VxsUSkt8zQs1+3gHsw8L0kS/fv3x9r6YcdtnU7HyZMnRR/V4kShhK7fwYLmEL4dTq6Gej2Nm2t4OmCnVpJ4X8uKQ5G8HeCT7R824fnzaC1qc8Upeqp2AaDtPI/0n5O5djeVxPsazmY2+q1cppQlwgTMuIJ7MGeoLMs4ODiYzCPq4eHBkCFDWL58eUHGKuS3Mn7Qaozh9ZYxkPSwb6FKqTDWpn6+7jR/Zs6GJAgP2sHZcZ8ZVosAWKptj4NfK8pljgd39mYiYZmPNx78HllCrq/gfv75ZwB8fX355JNPxO1oSfHih3B2PUSfhM2fQI9fjZsmdK7JW4sPEpuUzs7zt3m9UXkLBioUFQ+aiYxSraa8dIfrshuztD3pD9TycuRGfBpnbiZyJLOCqqGP5RKc2TfGEydOFMmtJFFaGaYcVKgMHfLPrDNuquruwPe9GwKG0UXE6PYCGDrbN5bOM0C1DYBxmndIxVCJUMvLCYB/zkQTk5iOUiFR39vZUqGan+BiYmLo06cPXl5eqFQqlEqlySIUQ551oXnm1IubP4HUh7Xkdcs7YWOl4G5KBuG3ky0UoFCUaNNTmZl5a7pa25q9+rrGbbXLGeZHOBhh+B2q5eWIrdpyecHszvb9+/cnMjKSzz//HE9PT/HguaRoOQrObYTY87B1LHRfCBjmb2hUwYX94Xc5cOUuVd0dLByoUJC0Oj2rj0RR2l5Nh9rZD7TwetJyKituESM784W2t8m2B1dwD1jy+Rvk4Qpu3759rFixgqFDh9KtWze6du1qsuTF/Pnz8fX1xcbGhoCAAA4dOpRj2datWyNJUpalU6dOxjL9+/fPsr1Dhw55iu25obI23KpKCkON6sVtxk1NKpYG4MAV0ei3JItLyaD7D//x6V+n+Wj18ewfSdwI442MdQDMUL5LIqaPq9wdrXErpTa+f9DcyFLMTnDe3t75+ixm9erVBAcHM3HiRI4ePUq9evUICgri9u3b2ZZfu3Ytt27dMi6nT59GqVTyxhtvmJTr0KGDSblVq1blW8wlVvnG0OR9w+uNHxlGhACaVH6Q4MRzuJLsy83nOHnd8H9+X6MnXftYNz5tBqz/ACV6Nuiactm1ZZZjSJJEzUeu4ordFdzcuXMZO3YsV69ezZcA5syZw+DBgxkwYAA1a9ZkwYIF2NnZ5Tj9oKurKx4eHsZl+/bt2NnZZUlw1tbWJuVcXCz7gy422nxqmE816SZsN/R2EM/hSr5rd1P465hpU6AsCW7fHLh9hruyA5M0/Sjv8nCKwEcnKKrtZXgOV87ZFg8L9WB4wOwE16NHD3bt2kXlypVxcHDA1dXVZDFHRkYGYWFhBAYGPgxIoSAwMJDQ0NBcHeOnn36iZ8+eWWp2d+3aRdmyZfHz82Po0KHcvZvzHAPp6ekkJiaaLM8ttZ2hATAYJo++stv4HA7EXA0l1Xf/hqPTy7T2K8ODXHVf80iXrJgzsOcrACZp+hGHI96PTPSseiTBta3hjkKCzvUsP1iq2ZUMc+fOzbcPv3PnDjqdDnd3d5P17u7unD9//qn7Hzp0iNOnT/PTTz+ZrO/QoQPdu3enYsWKXL58mfHjx9OxY0dCQ0OzremdPn06kydPfraTKUl8m0PjQXDkJ9jwAbwfSpOKpTMrGuLo09TX0hEK+eja3RTWZl69fdi2Ksej4olP1ZCYpsHd0QZ0WsN8HnotGVU6sPF0UwCTSZ4fTXCNKrhw7PP2lLLJ07zy+crsCPr161cQceTJTz/9RJ06dfD39zdZ37Pnwy5HderUoW7dulSuXJldu3bRtm3bLMcZN24cwcHBxveJiYl4e3sXXODFQeAkQ0VD/DUImUqTGqNh+8PncKL2vOSYv9Nw9daqWhka+LjgaqcmPlVDXIphsEpCv4Obx8DaidiW0+H0OWytlMYmIUCWOXSd7Ap//oXs5KkH7OXLl/nss8/o1auXsTJgy5YtnDlzxqzjuLm5oVQqiYkxnX4sJiYGDw+PJ+6bkpLCb7/9xqBBg576OZUqVcLNzY3w8PBst1tbW+Po6GiyPPdsHKFL5gQ1BxdQTz4vnsOVQJF3U43d8D4MrAqAc2ZyupeaAXcuwc4vDYU7fEmS2g0Ae2sVjR6pIbVUZ/qnMTuq3bt3U6dOHQ4ePMjatWtJTjb8sp84cYKJEyeadSy1Wk2jRo0ICQkxrtPr9YSEhNC0adMn7rtmzRrS09N5++23n/o5169f5+7du3h6Wv6ZQLFSNRDqvQXIqDeNoImP4TmneA5Xcvy45zI6vUzLamWMXapc7Q3NPOKS0w2PKHTpUPklqN/bOOGMvbXhUY+tleHfF3yLZiWe2Qlu7NixTJs2je3bt6NWP2zv8tJLL3HgwAGzAwgODmbRokUsW7aMc+fOMXToUFJSUhgwYAAAffv2Zdy4cVn2++mnn+jWrRulS5c2WZ+cnMyoUaM4cOAAV69eJSQkhK5du1KlShWCgoLMju+5F/QF2JeFu5cYrlgLiPZwJcnpG4ZmIW/5P3wk42Jn+F6XD18BkaGgLmW4mpekhx3t1YanW5tGNOf91pWZ0b0uRZHZz+BOnTqV7cz2ZcuW5c6dO2YH0KNHD2JjY5kwYQLR0dHUr1+frVu3GiseIiMjUShM8/CFCxfYt28f//zzT5bjKZVKTp48ybJly4iPj8fLy4v27dszdepUkyGehFyyc4XOc2D12zS6/gu1pCocuKIWz+GKudQMLQevxJGQpgHA0ebhMzNXezXlpViaXPnWsCJwEjj7ALA9czb7Mg6G71LlMqUY3eHhFJRFjdkJztnZmVu3blGxYkWT9ceOHaNcuXJ5CmL48OEMHz482227du3Kss7Pzy/HBqe2trZs27Yt221CHtXoAjW7IZ1dx2z1QrqkTCX8drLotlWMBa8+wdYz0cb3No/0F3Wxs2K6ahFqfRr4NDPUqAOnriew/OA1AN5rValwA84js29Re/bsyZgxY4iOjkaSJPR6Pfv37+eTTz6hb9++BRGjUBS8/BXYulJDusZ7yo0mz+EORcRxKyHNgsEJ5no0uQHYqB4muEZxf9NCeZoMSW1oE6lQoNfLfLb+NLIMXet70ayyW2GHnCdmJ7gvv/yS6tWr4+3tTXJyMjVr1qRly5Y0a9aMzz77rCBiFIqCUmWh40wARqjWcu18GGC4ZXnzx1CG/BJmyeiEZ2Qc8SPxFvXPGRr0rrR7G0pXBuC3w1GciIrHwVrFpy/XsFSYZjM7wanVahYtWsTly5fZtGkTy5cv5/z58/z6669iuKSSrs4b3Cv/EmpJR7drX6LVaJi51dAg+9SNBG7Gi6u44srGSmGYl2PTSKw0SRzXV2KZ3jCARVxKBrO2Gf6fR7arRllHy3a/Mkeemxr7+Pjg4+OTn7EIRZ0kYdd9Hkn/e4HaUjg7fplM+O2HzXn2XIylp7/4nSiObK2UcPpPuLgFWWHF6PR3uZNmqDGdueU88akaang60rdpBQtHap5cJbjg4GCmTp2Kvb29SYv/7MyZMydfAhOKJmtXb5a5vMuQ+Lk0j1yAr+SL1rkS1++lseeSSHDFlU3GPdgyGoD7TYO5GOIN97UcvHKX1UeiAJjWrVaRbdCbk1wluGPHjqHRaIyvcyKaDTwf7tfuzb7d22muPMM3Nj+R9tp63lp8mH2X7qDV6Yvdl0AA6+1jIfUulK2FuvXHKP7dgV6GT/44AcCbjcub9FwoLnKV4Hbu3Jnta+H51KSyG8Ehg/lHMYYGnEV3dz2ONj4k3tdy4nqCxccAE57MZJQQoJNVGNKZtSApodt8lFbWONupiUvJICouDSdbK8YU4bZuTyL+1Apmq+/tjJdvdda6GtpHKUMm0dXX8KXZczHWkqEJ2bideJ8TUfHG9w8a9wI4kswkZeZoPM0+AK8GwMP+qACjO/hRulTxbCSfqyu4B5M+58batWvzHIxQPKhVCn5/tynoA+DnoxB1gPdTvuNX3mf3xVhGtqtm6RCFTGdvJvLyvL0AbBzenDrlnUwS3GeqFZQhHkpXhdZjjetd7dRcIYV65Z3o+ULxfa6aqyu4Ryd5dnR0JCQkhCNHjhi3h4WFERISgpOT0xOOIpQ4CoWhIajSGs/Y/byu3MPJ6/HEp2ZYOjIh04PkBrAts3HvgwTXUnGCN1W70SMZ/h+tHo7v1rW+F5XK2DO9e90sQyEVJ7m6gnsw6TPAmDFjePPNN1mwYIGx3ZtOp+P9998Xwww9j9yqQptxsGMSE62Ws/t+XfaF36FzXS9LR/Zcy9DquXQ7yWTdzczeJgmpGuxJ40srw63pBuvOdPNpYlK2T1PfEjGwqdnP4JYsWcInn3xi0qhXqVQSHByc4zwKQgnX9APwrI8DKUyz+pk9F7KfMEgoPF9uPkeneftM1j1oiJ2QpmGM6jfKS3eI0pdhtUN/C0RYOMxOcFqtNtvhxM+fP49er89mD6HEU6qg63z0koog5RGszm8Qs29Z2NW7KVnW3Yy/D4DNzQP0VW0HYIx2MJJ1qUKNrTCZ3ZNhwIABDBo0iMuXLxuHCj948CAzZswwjuEmPIc8aqN/cSSKfV8xUruIy1f7UaViBWKT0vn75E1eb+xNKWvLj9H/vEjL0GVZdzM+DTJSaXZmEgCrtG34T1+bl6xKbhdLs3/jZs+ejYeHB19//TW3bt0CwNPTk1GjRvHxxx/ne4BC8aFqPZrrB9dQXnOV+K1jYOhvLN57hR/3XOHk9QTm9Khv6RCfC6kZWpIzR971LW3H1bupAGj1MvLOL3G5H8Ut2ZUvM2elty3BCc7sW1SFQsHo0aO5ceMG8fHxxMfHc+PGDUaPHi062z/vVGrC6k9FJ0tUidkCF7YQm5wOwF/Hb3A+2jAd4/7wOxwUw54XiK2no6k5YRtnbhp+1i72D0fdrieFw4H5AHyqGUgShmn/rK1KbnPYZzozMUGL8LhaL7Rhse5lAOSNI1HcN3zRZBlmb7vAldhkei8+SI+FB9DpxXO6/PbectNhq0pnJjg1GmZZLUSS9YTat+VffUNjmZJ8BZenhyJ//PEHv//+O5GRkWRkmLZ5Onr0aL4EJhRPlcuUYrB9H9qlhVEp+RZd+J4/6APAjnO3TZJamkYnnssVsAfzKwxTrcdPcZ10a1cW2g2Guw8rBG1KcIIz+wpu3rx5DBgwAHd3d44dO4a/vz+lS5fmypUrdOzYsSBiFIoRSZJo4leesZrBALRK3sKLilPGMfx3XnjYlevxPpHCs1mXOXnzo1zt1dSQrvG+cj0A+6qOISrdzqRMSb6CMzvBff/99yxcuJBvv/0WtVrN6NGj2b59OyNGjCAhIaEgYhSKmdcbleeQXIPl+vYAzFAt5o06zlnKZVfTJ+TdR6uPZ1lX2lbBLKsfsZJ0bNW9wF6r5iZdtSBzsMsSyuwzi4yMpFmzZoBhgpekJENr6T59+rBq1ar8jU4olhpVcKG+tzPTM3pwXXbDWxHLy7cXZSknruDyR1xKBov3Xsl2W5OYldRRXCVBtuNzTX+ux6dlSXC26pL7mMDsBOfh4UFcnGFeTB8fH+NcqBEREaJxp2D0TouKpGDLOM07ANS6vprGkmkD8TSR4PJFl2/3Me3vc1nWV5ZuUOvi9wBM0fQlFhfCbyeToTU8f5v1el061fGkY22PQo23MJmd4F566SU2bNgAGBr9jhw5knbt2tGjRw9effXVfA9QKJ461PKgnLMte/V1+V3bCgmZmVaLsOZhpVTvxQfFH8V8cCObuTAU6JlptQilPoNdunr8qW8BYGwTp5Dg9Yblmd+7IV7Otln2LynMTnALFy7k008/BWDYsGEsWbKEGjVqMGXKFH744Yc8BTF//nx8fX2xsbEhICCAQ4cO5Vh26dKlSJJkstjYmE6CIcsyEyZMwNPTE1tbWwIDA7l06VKeYhPyRqVU0L+ZLwDTtL3R2JWlsuIWH6n+NJZJuq/lv8uiPVxB6Kv8h8aKi+is7BmvGQSYjgjiaGuFohiPEpJbZiU4rVbLtGnTiI5+OKdiz549mTdvHh988AFqtfoJe2dv9erVBAcHM3HiRI4ePUq9evUICgri9u2cO2w7Ojpy69Yt43Lt2jWT7bNmzWLevHksWLCAgwcPYm9vT1BQEPfv3zc7PiHvevh742ij4r7KkfT2swEYotxEXcXD50V/ZVPzJ+SOLMtodFn7f3tLMYxWrQbgXrPPuIlhDtNHhz1ysrXKsl9JZFaCU6lUzJo1C61Wm28BzJkzh8GDBzNgwABq1qzJggULsLOze+LIJJIk4eHhYVzc3d2N22RZZu7cuXz22Wd07dqVunXr8ssvv3Dz5k3WrVuXb3ELT+doY8Xa95uxekgTStXvyn2/biglmdnqhVhh+B06fUPUvOfVkF/DqDVx22NrZWaoFmMnpXNAXwNNg37GLaUf6dUgElwO2rZty+7du/PlwzMyMggLCyMwMPBhQAoFgYGBhIaG5rhfcnIyFSpUwNvbm65du3LmzBnjtoiICKKjo02O6eTkREBAQI7HTE9PJzEx0WQR8keVsg408DHM0WDzytdgV5pqRBrbZYXfTiZdKyobzPX24oNsPxtjrDB4oKdyJy8qz5AmqxmjGYyT3cOhxj0fedb2vCQ4s+uHO3bsyNixYzl16hSNGjXC3t7eZPsrr7yS62PduXMHnU5ncgUG4O7unu2QTAB+fn4sWbKEunXrkpCQwOzZs2nWrBlnzpyhfPnyxtvn7I756K31o6ZPn87kyZNzHbeQR/Zu0HEW/DmIYap1bNW/wAW9D6dvJHI5NplmlUtT3sXu6ccR2Bd+J8s6D+4yXrUCgNnaN7gme2CnVrHinQD0sszqzNnpwfAM7nlgdoJ7//33geznP5UkCZ2uYP8aN23alKZNH0423KxZM2rUqMGPP/7I1KlT83TMcePGmcz3mpiYiLe39zPHKmSj9mvs+vMHWktHmGW1kO4Zk3nth/8A6FTXk/lvNXzKAYTsyUyzWoKjlMYxfRV+1j3sVfRiFcMzuJBzD59r+7g+H39IzL5F1ev1OS7mJjc3NzeUSiUxMTEm62NiYvDwyF3bHCsrKxo0aEB4eDiAcT9zjmltbW0cOEAMIFDAJIkZyiEkynbUU1xhkHKzcdPfJ29ZMLCi70Z8GpM3nuFY5L0s27oq9hOoPEa6rGK0Zgj6bL7abqUePoNr4O1ckKEWGRbto6FWq2nUqBEhISHGdXq9npCQEJOrtCfR6XScOnUKT09PACpWrIiHh4fJMRMTEzl48GCujykUrDtSaaZq3wYgWPUHFSVDYnteriryquPcPfy8/yqvfv+fyXo3Ephk9QsA32pf5ZJcPtv9H+1U/+C5aEmX61vUtLQ0QkJC6Ny5M2C4rUtPTzduVyqVTJ06NUubtKcJDg6mX79+NG7cGH9/f+bOnUtKSopxdOC+fftSrlw5pk+fDsCUKVNo0qQJVapUIT4+nq+++opr167xzjuGFvOSJPHRRx8xbdo0qlatSsWKFfn888/x8vKiW7duZsUmFAxJgjW6VnRRhNJSeYqZVgvpkfE591LEbFxPkng/+9YLk6yW4iIlc0ZfgQW6Ljnu72Dz8Ov+YPCDki7XCW7ZsmX8/fffxgT33XffUatWLWxtDTUz58+fx8vLi5EjR5oVQI8ePYiNjWXChAlER0dTv359tm7daqwkiIyMRKF4eKF57949Bg8eTHR0NC4uLjRq1Ij//vuPmjVrGsuMHj2alJQUhgwZQnx8PM2bN2fr1q1mJ1+hYJRztiU2KZ3x2nfYphiNv+ICfZTb+SU9iDd/DKXnC9409HFBIUn4lBZXdU8SpDhEZ+VBtLKC0Zp30T7hK921fjmOR8XTxq9sIUZoWZKcy74yLVq0YPTo0XTpYvgL4eDgwIkTJ6hUqRIAy5cvZ/78+U9s3lFcJCYm4uTkREJCgngeVwCu3knhs3Wneb91Zbb+PIUpVstIka0JypjFdbmMSdlLX3TESllyR7swh+/Yv03eO5HMDutRlJESmK99ha+0PbPsc3VGp8IKr1Dl9jua69+c8PBw6tSpY3xvY2NjcmXl7+/P2bNn8xiu8DzxdbNn+TsBNKvixq+6dhzS+2EvpfOlajFg+vc2JT3/GpUXZ9pseix8brWcMlIC4Xov5mm7WyCqoi/XCS4+Pt7kmVtsbCy+vr7G93q93mS7IOTGisFN+VI1jPuyFS2Vp3hDadqIPEWMGQdAXKrp88nWiuO8rtyDXpYYrRlCOlm7Sa4a3CTLuudNrhNc+fLlOX36dI7bT548Sfny2dfeCEJOmlV246/P+7LGsS8An6uWU5aHzSBSxRUcYBjz7YFSpPJF5qz0P+s6cFSulqX8pS860rRy6UKLr6jKdYJ7+eWXmTBhQrYd1tPS0pg8eTKdOpXM+32hYEmSxH9lenBCXwlHKZUvrJbw4FY1+TlOcAmpGkIv30Wvl7mb/DDBjVWtopx0l2v6sszWvpFlv2961BPPLTPluhZ1/Pjx/P777/j5+TF8+HCqVTP81bhw4QLfffcdWq2W8ePHF1igQslWxsme0ZohbFR/SjtlGF10oWzUNyMl/fm9RX31+/1cuZPCV6/XRa0yJKwmirO8rTK08RyrHUwapi0D1g97kXrPSSPe3Mh1gnN3d+e///5j6NChjB071jhQoSRJtGvXju+//z5L/09ByK3qHo78IvswX9uNkVZ/MslqGfvTa5OS8Xxewen1MlfupAAw6o+TdK3vhQ3pzFAZhn5fqX2JUH2tLPvV8BS1/o8yqy9qxYoV2bp1K3FxccauUVWqVMHV1bVAghOeH7W8DF/M73Vd6aA8RA1FFJOslpGS3sLCkVnGzK2mg02sP36TT1Vr8FXEcFN2Zbr2rWz3e3ClJxjk6afh6uqKv78//v7+IrkJ+cLPwwEADSpGa95FJ0u8ogzFJWqHhSMrfJM2nOHHPaaTyDSQLjFIuQWA8ZpBxlnpHyWu3rIqudPpCMXKo/0kT8mVWKjrzFDVRvzPTIX2XcH2+eg7CbD0v6sm7x/MSq+QZP7UNWeXvoFx2/e9G9LQx4Uj1+JoUaUMgilxPSsUGVs/asGM7obG5HO1r3FZ74l9xh3Y9pmFIys8C/dczrJuuOovqipuECs7MlXTx2RbUC0PPJxs6FzXCye752OMN3OIBCcUGdU9HOnp78OG4S+SjprRmiHISHB8OYSHPP0AxdztpPt8udn02VtN6SrvKw2z2H2uGUg8hlt5/4qu9HzB22SeBSErkeCEIqdueWdGBfkRJvuxVNsegLjVQ/nrwHniUjKYF3Ip26nyirukx0YLUaFlltVCVJKezTp/tur9jdt+f7cpM16rW9ghFjsiwQlFUuUyhqHwv9L2IEpfBldNDImbPuOj1ceZs/0iPX4s/oM6PCo1Q0vbr027qQ1RbqK24ir35FJM1PQHoKKbPa82KGeBCIsnkeCEIqlDbU82Dm9OzQqejNUaxvrrp9rO/Ut7ALh+r2Rdwe04ZzpNZmXpBh+q1gIwRdOHWJwB+PfjVnzTo34hR1d8iQQnFFl1yjvRtUE59uvrsErbBoCZVguxoWQN6nAlNplP1pwwvleg5yurH7GWtPyrq89f+ubGbZIknrmZQyQ4oUizzuxT+aW2N9GyCxUVMYxU/WHhqPLX+yuOmkz/11+5jYaKcJJkWz59ZFb6RX0bWyjC4kskOKFIq1y2FABJ2PGpZiAA7yg3U1fK2pyiuLp6N8X42keKYVTmrPTTtW9xi4cjgrSrKbpCmkskOKFIa+jjbHwdom/EOl0zlJLMV1Y/ImuyjmxTHLnYGcZyk9AzQ7UIWymD/3Q1WaVrYywjJuTJG5HghCLt8WdOkzV9uSM74qe4zvdT3+ff8zE57Fn0nYiK55M1J9DoDANX9FLupJnyLGmymrHawciZX88f+zTij/fEjHB5IRKcUKzcw9HYZGKItI5Zy9YSk1g8r+SGrzrKH2HXuZOcjid3GadaCRiaxkTKD29Hg2p5UNZRTJiUFyLBCUXer4P8qeftjLerYQa3v/UBbNM1xkrSMcvqR95ddtDCEZpHp5eZvzOcqLgHTV1kvrRajIOURpi+Kkt1QcayrzUUo2Q/C5HghCKvRdUyrB/2Ir6l7TPXSHymGUCCbEddRQRNolfy+brT7DhbPG5XVx+O4qttF4zvX1Xso43yRLaz0n/9Zj1LhFhiFIkEN3/+fHx9fbGxsSEgIIBDhw7lWHbRokW0aNECFxcXXFxcCAwMzFK+f//+SJJksnTo0KGgT0MoYGM6VDe+jsWFqVpDx/ORqj/ZfzCUd345YqnQck2WZRbvfTgUUhnimZg5K/3/tN25LD/spbBu2IuFHl9JY/EEt3r1aoKDg5k4cSJHjx6lXr16BAUFcfv27WzL79q1i169erFz505CQ0Px9vamffv23Lhxw6Rchw4duHXrlnFZtWpVYZyOUIBql3PiwrQOOGeOmvGHriW7dXWxljKHEyLr1HpFzaaTt4wj9QJMtlqKs5TCab0vC3Wdjevn9qhPfTH0+DOzeIKbM2cOgwcPZsCAAdSsWZMFCxZgZ2fHkiVLsi2/YsUK3n//ferXr0/16tVZvHgxer2ekBDT0Sasra3x8PAwLi4uz894YiWZtUrJ0c/aZb6TGKd5h2TZhsaKi/RV/sN9TdGdw0Gvl/n1wDXj+46Kg7ysPIRGVjJaM8RkVvrOdT0tEWKJY9EEl5GRQVhYGIGBgcZ1CoWCwMBAQkNz15k6NTUVjUaTZWThXbt2UbZsWfz8/Bg6dCh3797N19gFy1EoJI58ZviduYkbM7S9ABitWk3ghF+Iiku1ZHg5+mDVMQ5FxAHgTBJTrH4G4AddF87KvsZyg5pXRCVmxcoXFv0p3rlzB51Ol2WyGnd3d6Kjo3N1jDFjxuDl5WWSJDt06MAvv/xCSEgIM2fOZPfu3XTs2BGdLvu/7unp6SQmJposQtHmVsqani94A7BC15aD+urYSYZJWTr+b4+Fo8sq4k4Kf5+6ZXz/udWvlJESuagvx3faV03K+pYWjXrzS7EesnzGjBn89ttv7Nq1Cxubh+2EevbsaXxdp04d6tatS+XKldm1axdt27bNcpzp06czefLkQolZyD/Tu9fBv6Irwb+fYLRmCFvVY2muPEMnzQ6gaFUqtZm9y/i6teIYryn3oZMlRmveJQPDM8WOtT3wKW1HT38fC0VZ8lj0Cs7NzQ2lUklMjGn1fkxMDB4eHk/cd/bs2cyYMYN//vmHunWfPPBfpUqVcHNzM84E9rhx48aRkJBgXKKiosw7EcEiJEmie2Y7sWuyB19nToL8qWo5t6KKZl9VB1L5MnNW+iW6jhyXqwAwMrAa/+vZgHEda4hJm/ORRX+SarWaRo0amVQQPKgwaNo0564ps2bNYurUqWzdupXGjZ8+wsL169e5e/cunp7ZP7i1trbG0dHRZBGKDydbwxXQEl1Hjusr4yilcWbhIHzHbmLvpVgLR2dqnGolXlIcV/XuxoTcoZYHHwZWFVP+FQCL/0SDg4NZtGgRy5Yt49y5cwwdOpSUlBQGDBgAQN++fRk3bpyx/MyZM/n8889ZsmQJvr6+REdHEx0dTXJyMgDJycmMGjWKAwcOcPXqVUJCQujatStVqlQhKCgo2xiE4u3ExPZMfqUWehSM0rxLhqwkUHmMror99PnpEDq9bLHY4lIyjP1lmyrO8JbqXwDGaIZwH2sAFvRpZLH4SjqLJ7gePXowe/ZsJkyYQP369Tl+/Dhbt241VjxERkZy69bDh7M//PADGRkZvP7663h6ehqX2bNnA6BUKjl58iSvvPIK1apVY9CgQTRq1Ii9e/dibW1tkXMUCl6/Zr4AXJLLM0/bHYBJVr/gRgJrjljukUOneXsZuPQIttxnpmohAMu1bTko1wBgR3Ari8X2PJBkWbbcn7ciKjExEScnJxISEsTtajGy9uh1gn8/gQotG9SfU1NxjU26AM68OM+kF0RhydDqqfaZYbLmz1W/Mki1hRtyaYLSZ5KMHY0quPDn0GaFHldJkNvvqMWv4AQhv7zaoBx1yjmhRcUozRC0soLOyoNc2fObReI5fNXQ5q2hdJEByq0AjNe8QzJ2vBXgw5p3xRBIBU0kOKHEkCSJjR8Y5i84I1fkx8yuT1Otfub6Y135CoNapcCaDOOs9H/oWrJbb+g8/+WrdVCIOU0LnEhwQolz+cuXAZin7U643ouyUjwHFrzHyevxhfL56Vodp28kYGulZIRqLVUUN4mVnZiqeRswVIoIhUMkOKHEeTDbezpqw/BDssTryj18/f33pGsLvq+q32db6fztPnbv2s67yk0AfKYZSAKlmNergbFZi1DwRIITSqS9ow3zGRyVqxkHkPzSajF7T0cU6OdeiTU0V1Khpc2FKagkPZt0AWzTvwBAF9GJvlCJBCeUSN6udpyYYLgV/Er7JpH6MpST7nLrjzEF+rmxSYY5W99TbqSm4hpxcikmZQ6x/stAfzGvaSETCU4osZzsrHi3ZSXSsGGMdggAfVQ7+OfvPwpsxJHY5HSqStf5QPUXkDlJDk7sHd2GltXKFMhnCjkTCU4o0Ua0rQpAqL4WK7UvAVDt4Hjazdqarz0cZFlm+uZzjFgZxiyrhVhLWnboGrBebxiV11tM+2cRIsEJJZq9tco4dtx07Vvckl3xVcTwsWoNlcdvzrfPOXLtHj/uucIA5RYaKMJJfGRW+n9Gtsy3zxHMIxKcUOK5lbJm/9iXSMKO8ZpBAAxUbqGBdIn5O7MfYcYcer3MGwtCqSBFM0r1OwBfaN8mBlfOTelANXeHZ/4MIW9EghOeC+Wcbflfz/rs1Ddgra45SklmltVC/rft9DMfe9HeK0jomWm1CBtJwz5dLVbrWjP/rYbYqpX5EL2QVyLBCc+NV+p5ATBF04dY2ZGqihsMV/3FB6uOkXhfk6djpmt1TN9ynt7KEJoozpEqWzNWOxiQCKrl/tT9hYIlEpzw3JAkiYjpLxOPAxM0huG43ldu4PLJ/5i55Xyejvlr6DW8uMNYlWHWtlnaHlyXy3JwfFsxr0IRIP4HhOeKJEnsHtWaLfoANuv8UUl6ZlktZPXBK1yKSTLrWHq9zLS/zzLdajGlpPsc0Vdjma49X71eF3dHm6cfQChwIsEJz50Kpe35KLAqEzX9iZftqa24yhDlJvotyXnC8exciEniNcVeWilPki5bMUYzmHIu9rzR2LuAIhfMJRKc8Fz6KLAasTgzWdMXgA9Va7FNzP08Dvc1OsYs/YcJmbPSz9W+xmW5HPvGvFQg8Qp5IxKc8NzaPao1f+mbs1NXD2tJy1dWP3Lmelyu9p284QzDUn/ASUrlpL4iC3WdCP+iYwFHLJhLJDjhuVWhtD1HP2/PeM07JMm2NFSE8+cPE566n1anJzFsDUHKI2hkJWM0Q1j/QStRqVAEif8R4bnmaq9mcp/2TNe+BcAo1WoOhh154j4Jd6OZbLUUgO91XTknV6B2OaeCDlXIA5HghOde+1oerNK1IVRXE1spA926D4iOT8u2rF4vs+fbd3CTErmgL8932m6cnSJmayuqRIITBCD8y86M0Q4mTVbTTHmWn+ZNzFJGlmU+nfkVryr3Z85KP4SzX3TBTq2yQMRCbogEJwgYRgHe+HkfZmvfBGCE7hf++PeASZkPl+7mw/vfA7BY9zKjB/UWs9AXceJ/RxAyOdlZsdGmC0f1VXCQ0nDdOQYyZ9W8r9HR9PJcPKR7XNF7MEf7Bi9WcbNwxMLTFIkEN3/+fHx9fbGxsSEgIIBDh57c4HLNmjVUr14dGxsb6tSpw+bNpsPeyLLMhAkT8PT0xNbWlsDAQC5dulSQpyCUEIc+78AozbukyypeUh7n8ymfA7B42RJ6qXYCMFYzmAszXrVkmEIuWTzBrV69muDgYCZOnMjRo0epV68eQUFB3L59O9vy//33H7169WLQoEEcO3aMbt260a1bN06ffjgqxKxZs5g3bx4LFizg4MGD2NvbExQUxP379wvrtIRi7J8vBvM/bXcAPtYvIexYGF0jZwKwTNuOpZM/smB0gllkC/P395eHDRtmfK/T6WQvLy95+vTp2ZZ/88035U6dOpmsCwgIkN99911ZlmVZr9fLHh4e8ldffWXcHh8fL1tbW8urVq3KVUwJCQkyICckJJh7OkIJEXk7Xj79eR1ZnuhoXKImVJJ3nQy3dGiCnPvvqEWv4DIyMggLCyMwMNC4TqFQEBgYSGhoaLb7hIaGmpQHCAoKMpaPiIggOjrapIyTkxMBAQE5HjM9PZ3ExESTRXi+eZdxYoHLxybrFjp9SKs6lS0UkZAXFk1wd+7cQafT4e5uOm6Wu7s70dHR2e4THR39xPIP/jXnmNOnT8fJycm4eHuLztICfDuyH3Mzb1V/07Vh8sgPLByRYC6LP4MrCsaNG0dCQoJxiYqKsnRIQhHR85Pv+b3RKt6c+IeY8q8YsmgLRTc3N5RKJTExMSbrY2Ji8PDwyHYfDw+PJ5Z/8G9MTAyenp4mZerXr5/tMa2trbG2ts7raQglmIezLW92ednSYQh5ZNErOLVaTaNGjQgJCTGu0+v1hISE0LRp02z3adq0qUl5gO3btxvLV6xYEQ8PD5MyiYmJHDx4MMdjCoJQQhVSpUeOfvvtN9na2lpeunSpfPbsWXnIkCGys7OzHB0dLcuyLPfp00ceO3assfz+/ftllUolz549Wz537pw8ceJE2crKSj516pSxzIwZM2RnZ2d5/fr18smTJ+WuXbvKFStWlNPS0nIVk6hFFYSiLbffUYt3ouvRowexsbFMmDCB6Oho6tevz9atW42VBJGRkSgUDy80mzVrxsqVK/nss88YP348VatWZd26ddSuXdtYZvTo0aSkpDBkyBDi4+Np3rw5W7duxcZGDCMtCM8TSZbl/Jveu4RITEzEycmJhIQEHB0dLR2OIAiPye13VNSiCoJQYokEJwhCiSUSnCAIJZbFKxmKogePJUWXLUEomh58N59WhSASXDaSkgwTAIsuW4JQtCUlJeHklPN8GKIWNRt6vZ6bN2/i4OCQL91zEhMT8fb2JioqqkTWyorzK76K67nJskxSUhJeXl4mzcgeJ67gsqFQKChfvny+H9fR0bFY/RKZS5xf8VUcz+1JV24PiEoGQRBKLJHgBEEosUSCKwTW1tZMnDixxI5YIs6v+CrJ5waikkEQhBJMXMEJglBiiQQnCEKJJRKcIAgllkhwgiCUWCLBFZC4uDh69+6No6Mjzs7ODBo0iOTk5FztK8syHTt2RJIk1q1bV7CB5oG55xYXF8cHH3yAn58ftra2+Pj4MGLECBISEgox6pzNnz8fX19fbGxsCAgI4NChQ08sv2bNGqpXr46NjQ116tRh8+bNhRRp3phzfosWLaJFixa4uLjg4uJCYGDgU38eRVqBjiv8HOvQoYNcr149+cCBA/LevXvlKlWqyL169crVvnPmzJE7duwoA/Jff/1VsIHmgbnndurUKbl79+7yhg0b5PDwcDkkJESuWrWq/NprrxVi1Nn77bffZLVaLS9ZskQ+c+aMPHjwYNnZ2VmOiYnJtvz+/ftlpVIpz5o1Sz579qz82WefZRkyvygx9/zeeustef78+fKxY8fkc+fOyf3795ednJzk69evF3Lk+UMkuAJw9uxZGZAPHz5sXLdlyxZZkiT5xo0bT9z32LFjcrly5eRbt24VyQT3LOf2qN9//11Wq9WyRqMpiDBzzd/fXx42bJjxvU6nk728vOTp06dnW/7NN9+UO3XqZLIuICBAfvfddws0zrwy9/wep9VqZQcHB3nZsmUFFWKBEreoBSA0NBRnZ2caN25sXBcYGIhCoeDgwYM57peamspbb73F/Pnzc5w20dLyem6PezDUtEplue7QGRkZhIWFERgYaFynUCgIDAwkNDQ0231CQ0NNygMEBQXlWN6S8nJ+j0tNTUWj0eDq6lpQYRYokeAKQHR0NGXLljVZp1KpcHV1JTo6Osf9Ro4cSbNmzejatWtBh5hneT23R925c4epU6cyZMiQgggx1+7cuYNOpzNOcPSAu7t7jucSHR1tVnlLysv5PW7MmDF4eXllSerFhUhwZhg7diySJD1xOX/+fJ6OvWHDBv7991/mzp2bv0HnUkGe26MSExPp1KkTNWvWZNKkSc8euFBgZsyYwW+//cZff/1VbGekE8MlmeHjjz+mf//+TyxTqVIlPDw8uH37tsl6rVZLXFxcjree//77L5cvX8bZ2dlk/WuvvUaLFi3YtWvXM0T+dAV5bg8kJSXRoUMHHBwc+Ouvv7CysnrWsJ+Jm5sbSqWSmJgYk/UxMTE5nouHh4dZ5S0pL+f3wOzZs5kxYwY7duygbt26BRlmwbL0Q8CS6MGD+CNHjhjXbdu27YkP4m/duiWfOnXKZAHk//3vf/KVK1cKK/Snysu5ybJhot4mTZrIrVq1klNSUgoj1Fzx9/eXhw8fbnyv0+nkcuXKPbGSoXPnzibrmjZtWqQrGcw5P1mW5ZkzZ8qOjo5yaGhoYYRYoESCKyAdOnSQGzRoIB88eFDet2+fXLVqVZOmFNevX5f9/PzkgwcP5ngMimAtqiybf24JCQlyQECAXKdOHTk8PFy+deuWcdFqtZY6DVmWDc0orK2t5aVLl8pnz56VhwwZIjs7O8vR0dGyLMtynz595LFjxxrL79+/X1apVPLs2bPlc+fOyRMnTizyzUTMOb8ZM2bIarVa/uOPP0z+n5KSkix1Cs9EJLgCcvfuXblXr15yqVKlZEdHR3nAgAEmvyQREREyIO/cuTPHYxTVBGfuue3cuVMGsl0iIiIscxKP+Pbbb2UfHx9ZrVbL/v7+8oEDB4zbWrVqJffr18+k/O+//y5Xq1ZNVqvVcq1ateS///67kCM2jznnV6FChWz/nyZOnFj4gecDMVySIAgllqhFFQShxBIJThCEEkskOEEQSiyR4ARBKLFEghMEocQSCU4QhBJLJDhBEEoskeAEQSixRIITio3+/ftnO8pJhw4dLB2aUESJ0USEYqVDhw78/PPPJutympVdo9FkGbEkIyMDtVpt9ufmdT/BssQVnFCsWFtb4+HhYbK4uLgAIEkSP/zwA6+88gr29vZ88cUXTJo0ifr167N48WIqVqxoHNcsMjKSrl27UqpUKRwdHXnzzTdNhhXKaT+heBEJTihRJk2axKuvvsqpU6cYOHAgAOHh4fz555+sXbuW48ePo9fr6dq1K3FxcezevZvt27dz5coVevToYXKsx/cTih9xiyoUK5s2baJUqVIm68aPH8/48eMBeOuttxgwYIDJ9oyMDH755RfKlCkDwPbt2zl16hQRERF4e3sD8Msvv1CrVi0OHz7MCy+8kO1+QvEjEpxQrLRp04YffvjBZN2jE6I8OhnOAxUqVDBJUufOncPb29uY3ABq1qyJs7Mz586dMya4x/cTih+R4IRixd7enipVqjxxe27W5fazhOJNPIMTnjs1atQgKiqKqKgo47qzZ88SHx9PzZo1LRiZkN/EFZxQrKSnp2eZ8k6lUuHm5pbrYwQGBlKnTh169+7N3Llz0Wq1vP/++7Rq1SrbW1yh+BJXcEKxsnXrVjw9PU2W5s2bm3UMSZJYv349Li4utGzZksDAQCpVqsTq1asLKGrBUsSQ5YIglFjiCk4QhBJLJDhBEEoskeAEQSixRIITBKHEEglOEIQSSyQ4QRBKLJHgBEEosUSCEwShxBIJThCEEkskOEEQSiyR4ARBKLFEghMEocT6P8LGyHC+/RmsAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019279, + "end_time": "2024-03-24T18:25:48.556087", + "exception": false, + "start_time": "2024-03-24T18:25:48.536808", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 5251.625501, + "end_time": "2024-03-24T18:25:51.296482", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/lct_gan/2/mlu-eval.ipynb", + "output_path": "eval/treatment/lct_gan/2/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/treatment/lct_gan/2", + "path_prefix": "../../../../", + "random_seed": 2, + "single_model": "lct_gan" + }, + "start_time": "2024-03-24T16:58:19.670981", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/lct_gan/model.pt b/treatment/lct_gan/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..e821082fd754c609e919aeee95bd3dce40b471b3 --- /dev/null +++ b/treatment/lct_gan/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:656ff7263d065cfa0dafe254e41bde1662678162fdad7bf6eb863ac2bfc6c503 +size 118806677 diff --git a/treatment/lct_gan/params.json b/treatment/lct_gan/params.json new file mode 100644 index 0000000000000000000000000000000000000000..967f243bb558dcb3de3b7eb6086d2307e79bb439 --- /dev/null +++ b/treatment/lct_gan/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 1, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "torch", "grad_clip": 0.75, "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "loss_balancer_beta": 0.75, "loss_balancer_r": 0.96, "tf_pma_low_exp_2": 4, "dataset_size_exp_2": 11, "batch_size_exp_2": 1, "epochs": 100, "lr_mul": 0.06, "n_warmup_steps": 80, "Optim": "diffgrad", "fixed_role_model": "lct_gan", "mse_mag_target": 0.5, "g_loss_mul": 0.2, "d_model_exp_2": 9, "attn_activation": "selu", "tf_d_inner_exp_2": 8, "tf_n_layers_enc": 4, "tf_n_head_exp_2": 5, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds_exp_2": 5, "ada_d_hid_exp_2": 11, "ada_n_layers": 6, "ada_activation": "leakyhardtanh", "ada_activation_final": "leakyhardsigmoid", "head_d_hid_exp_2": 7, "head_n_layers": 7, "head_n_head_exp_2": 6, "head_activation": "leakyhardsigmoid", "head_activation_final": "leakyhardsigmoid", "mse_mag": true, "mse_mag_multiply": true, "models": ["lct_gan"], "max_seconds": 3600} \ No newline at end of file diff --git a/treatment/realtabformer/eval.csv b/treatment/realtabformer/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..01f5b63f40e2c33a3b55fed40def65a44952b3a9 --- /dev/null +++ b/treatment/realtabformer/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +realtabformer,0.0,0.09333333015442084,0.0033314223646012425,3.379798650741577,0.5676931142807007,11.912954330444336,0.8492485880851746,2.4299186407006346e-05,17.730737924575806,0.04105054959654808,2922347.75,0.05771847441792488,0.20964039862155914,0.0004608653543982655,21.110536575317383 diff --git a/treatment/realtabformer/history.csv b/treatment/realtabformer/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..1b6695d9e0ce94fb14810f1088c614ca5a907753 --- /dev/null +++ b/treatment/realtabformer/history.csv @@ -0,0 +1,5 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.09495536066487066,8.93220916427853,0.0419871234360975,1.3698969186252603,0.0,0.0,0.0,0.0,0.09889276345333473,900,450,658.6492667198181,1.4636650371551514,0.7318325185775757,0.12809311661881004,0.01607477423283967,5.248249570016049,0.0017426262642197404,0.0,0.0,0.0,0.0,0.0,0.01607477423283967,450,225,186.72706723213196,0.8298980765872531,0.41494903829362656,0.07684096147841084 +1,0.011189657653926462,1.6530850822855043,0.0004210917485972683,1.341028964128718,0.0,0.0,0.0,0.0,0.011419157582775041,900,450,666.2451884746552,1.4805448632770115,0.7402724316385058,0.19066866852136274,0.01324691695151491,2.477538332505353,0.0012280172330747016,0.0,0.0,0.0,0.0,0.0,0.01324691695151491,450,225,199.9650740623474,0.8887336624993218,0.4443668312496609,0.10047116354194967 +2,0.007910739361629894,1.2925524352938216,0.0002544926988752502,1.0837882141060295,0.0,0.0,0.0,0.0,0.008088903618937845,900,450,677.3252453804016,1.505167211956448,0.752583605978224,0.20559132802500749,0.009221325316331483,3.315624680383264,0.0006904058667853433,0.0,0.0,0.0,0.0,0.0,0.009221325316331483,450,225,199.83344054222107,0.8881486246320937,0.4440743123160468,0.08771737627721059 +3,0.006322342480156698,0.9110217978435494,0.00014503141735743212,0.9366931309483294,0.0,0.0,0.0,0.0,0.006472822059344278,900,450,680.0380289554596,1.5111956199010212,0.7555978099505106,0.18851194770978813,0.01230627063448872,2.257654165002171,0.0008331807685687578,0.0,0.0,0.0,0.0,0.0,0.01230627063448872,450,225,201.17853093147278,0.894126804139879,0.4470634020699395,0.0902620027468543 diff --git a/treatment/realtabformer/mlu-eval-load.ipynb b/treatment/realtabformer/mlu-eval-load.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..ab423f17f47a1688ee12897ddf62c3ea98fe910e --- /dev/null +++ b/treatment/realtabformer/mlu-eval-load.ipynb @@ -0,0 +1,2004 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:15:36.454330Z", + "iopub.status.busy": "2024-03-26T15:15:36.453907Z", + "iopub.status.idle": "2024-03-26T15:15:36.491298Z", + "shell.execute_reply": "2024-03-26T15:15:36.490346Z" + }, + "papermill": { + "duration": 0.052026, + "end_time": "2024-03-26T15:15:36.493580", + "exception": false, + "start_time": "2024-03-26T15:15:36.441554", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:15:36.517807Z", + "iopub.status.busy": "2024-03-26T15:15:36.517407Z", + "iopub.status.idle": "2024-03-26T15:15:36.524538Z", + "shell.execute_reply": "2024-03-26T15:15:36.523674Z" + }, + "papermill": { + "duration": 0.021767, + "end_time": "2024-03-26T15:15:36.526582", + "exception": false, + "start_time": "2024-03-26T15:15:36.504815", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:15:36.547898Z", + "iopub.status.busy": "2024-03-26T15:15:36.547607Z", + "iopub.status.idle": "2024-03-26T15:15:36.551778Z", + "shell.execute_reply": "2024-03-26T15:15:36.550914Z" + }, + "papermill": { + "duration": 0.017211, + "end_time": "2024-03-26T15:15:36.553719", + "exception": false, + "start_time": "2024-03-26T15:15:36.536508", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:15:36.575308Z", + "iopub.status.busy": "2024-03-26T15:15:36.575014Z", + "iopub.status.idle": "2024-03-26T15:15:36.579286Z", + "shell.execute_reply": "2024-03-26T15:15:36.578300Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.017161, + "end_time": "2024-03-26T15:15:36.581357", + "exception": false, + "start_time": "2024-03-26T15:15:36.564196", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:15:36.602531Z", + "iopub.status.busy": "2024-03-26T15:15:36.602258Z", + "iopub.status.idle": "2024-03-26T15:15:36.608155Z", + "shell.execute_reply": "2024-03-26T15:15:36.607325Z" + }, + "papermill": { + "duration": 0.018847, + "end_time": "2024-03-26T15:15:36.610144", + "exception": false, + "start_time": "2024-03-26T15:15:36.591297", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "8b47d4ae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:15:36.633755Z", + "iopub.status.busy": "2024-03-26T15:15:36.632880Z", + "iopub.status.idle": "2024-03-26T15:15:36.638798Z", + "shell.execute_reply": "2024-03-26T15:15:36.637980Z" + }, + "papermill": { + "duration": 0.019872, + "end_time": "2024-03-26T15:15:36.640680", + "exception": false, + "start_time": "2024-03-26T15:15:36.620808", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"realtabformer\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 4\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/realtabformer/4\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.010271, + "end_time": "2024-03-26T15:15:36.661396", + "exception": false, + "start_time": "2024-03-26T15:15:36.651125", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:15:36.682544Z", + "iopub.status.busy": "2024-03-26T15:15:36.682252Z", + "iopub.status.idle": "2024-03-26T15:15:36.691798Z", + "shell.execute_reply": "2024-03-26T15:15:36.690961Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.022643, + "end_time": "2024-03-26T15:15:36.693807", + "exception": false, + "start_time": "2024-03-26T15:15:36.671164", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/realtabformer/4\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:15:36.715748Z", + "iopub.status.busy": "2024-03-26T15:15:36.715444Z", + "iopub.status.idle": "2024-03-26T15:15:38.774903Z", + "shell.execute_reply": "2024-03-26T15:15:38.773773Z" + }, + "papermill": { + "duration": 2.072797, + "end_time": "2024-03-26T15:15:38.777226", + "exception": false, + "start_time": "2024-03-26T15:15:36.704429", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:15:38.802356Z", + "iopub.status.busy": "2024-03-26T15:15:38.801460Z", + "iopub.status.idle": "2024-03-26T15:15:38.817842Z", + "shell.execute_reply": "2024-03-26T15:15:38.816977Z" + }, + "papermill": { + "duration": 0.030974, + "end_time": "2024-03-26T15:15:38.819776", + "exception": false, + "start_time": "2024-03-26T15:15:38.788802", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:15:38.841897Z", + "iopub.status.busy": "2024-03-26T15:15:38.841612Z", + "iopub.status.idle": "2024-03-26T15:15:38.849509Z", + "shell.execute_reply": "2024-03-26T15:15:38.848627Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021208, + "end_time": "2024-03-26T15:15:38.851491", + "exception": false, + "start_time": "2024-03-26T15:15:38.830283", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:15:38.873334Z", + "iopub.status.busy": "2024-03-26T15:15:38.872980Z", + "iopub.status.idle": "2024-03-26T15:15:38.976852Z", + "shell.execute_reply": "2024-03-26T15:15:38.975750Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.117624, + "end_time": "2024-03-26T15:15:38.979575", + "exception": false, + "start_time": "2024-03-26T15:15:38.861951", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:15:39.006565Z", + "iopub.status.busy": "2024-03-26T15:15:39.006205Z", + "iopub.status.idle": "2024-03-26T15:15:43.842724Z", + "shell.execute_reply": "2024-03-26T15:15:43.841729Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.852692, + "end_time": "2024-03-26T15:15:43.845266", + "exception": false, + "start_time": "2024-03-26T15:15:38.992574", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-26 15:15:41.301646: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-26 15:15:41.301709: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-26 15:15:41.303429: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:15:43.868691Z", + "iopub.status.busy": "2024-03-26T15:15:43.867584Z", + "iopub.status.idle": "2024-03-26T15:15:43.874214Z", + "shell.execute_reply": "2024-03-26T15:15:43.873310Z" + }, + "papermill": { + "duration": 0.020437, + "end_time": "2024-03-26T15:15:43.876301", + "exception": false, + "start_time": "2024-03-26T15:15:43.855864", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:15:43.900029Z", + "iopub.status.busy": "2024-03-26T15:15:43.899669Z", + "iopub.status.idle": "2024-03-26T15:16:06.232616Z", + "shell.execute_reply": "2024-03-26T15:16:06.231537Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 22.347675, + "end_time": "2024-03-26T15:16:06.235160", + "exception": false, + "start_time": "2024-03-26T15:15:43.887485", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-26T15:16:06.263738Z", + "iopub.status.busy": "2024-03-26T15:16:06.262722Z", + "iopub.status.idle": "2024-03-26T15:16:06.270187Z", + "shell.execute_reply": "2024-03-26T15:16:06.269325Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.02369, + "end_time": "2024-03-26T15:16:06.272186", + "exception": false, + "start_time": "2024-03-26T15:16:06.248496", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 95,\n", + " 'realtabformer': (69, 281, Embedding(281, 768), True),\n", + " 'lct_gan': 75,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:16:06.296278Z", + "iopub.status.busy": "2024-03-26T15:16:06.295996Z", + "iopub.status.idle": "2024-03-26T15:16:06.363276Z", + "shell.execute_reply": "2024-03-26T15:16:06.362448Z" + }, + "papermill": { + "duration": 0.081651, + "end_time": "2024-03-26T15:16:06.365304", + "exception": false, + "start_time": "2024-03-26T15:16:06.283653", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/realtabformer/all inf False\n", + "../../../../ml-utility-loss/aug_test/treatment 0\n", + "Caching in ../../../../treatment/_cache_bs_test/realtabformer/all inf False\n", + "../../../../ml-utility-loss/bs_test/treatment 0\n", + "Caching in ../../../../treatment/_cache_synth_test/realtabformer/all inf False\n", + "../../../../ml-utility-loss/synthetics/treatment 600\n", + "600\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:16:06.390814Z", + "iopub.status.busy": "2024-03-26T15:16:06.390529Z", + "iopub.status.idle": "2024-03-26T15:16:06.718870Z", + "shell.execute_reply": "2024-03-26T15:16:06.717906Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.343898, + "end_time": "2024-03-26T15:16:06.721238", + "exception": false, + "start_time": "2024-03-26T15:16:06.377340", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 8,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.75,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'loss_balancer_beta': 0.7999999999999999,\n", + " 'loss_balancer_r': 0.95,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 2,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.09,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'realtabformer',\n", + " 'g_loss_mul': 0.1,\n", + " 'd_model': 512,\n", + " 'attn_activation': torch.nn.modules.activation.SELU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 8,\n", + " 'ada_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['realtabformer'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.2, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "# params = {\n", + "# **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "# }\n", + "params = getattr(PARAMS, dataset_name).BEST_DICT[gp][gp_multiply][single_model]\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).DEFAULTS,\n", + " **params,\n", + "}\n", + "if isinstance(params, (list, tuple)):\n", + " params = params[param_index]\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-26T15:16:06.747210Z", + "iopub.status.busy": "2024-03-26T15:16:06.746626Z", + "iopub.status.idle": "2024-03-26T15:16:07.306085Z", + "shell.execute_reply": "2024-03-26T15:16:07.305137Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.574877, + "end_time": "2024-03-26T15:16:07.308172", + "exception": false, + "start_time": "2024-03-26T15:16:06.733295", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n", + "[*] Embedding True True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['realtabformer'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:16:07.335819Z", + "iopub.status.busy": "2024-03-26T15:16:07.334952Z", + "iopub.status.idle": "2024-03-26T15:16:07.339549Z", + "shell.execute_reply": "2024-03-26T15:16:07.338691Z" + }, + "papermill": { + "duration": 0.0204, + "end_time": "2024-03-26T15:16:07.341604", + "exception": false, + "start_time": "2024-03-26T15:16:07.321204", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:16:07.365634Z", + "iopub.status.busy": "2024-03-26T15:16:07.365348Z", + "iopub.status.idle": "2024-03-26T15:16:07.372228Z", + "shell.execute_reply": "2024-03-26T15:16:07.371406Z" + }, + "papermill": { + "duration": 0.021136, + "end_time": "2024-03-26T15:16:07.374185", + "exception": false, + "start_time": "2024-03-26T15:16:07.353049", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "19911750" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:16:07.399795Z", + "iopub.status.busy": "2024-03-26T15:16:07.399042Z", + "iopub.status.idle": "2024-03-26T15:16:07.530423Z", + "shell.execute_reply": "2024-03-26T15:16:07.529470Z" + }, + "papermill": { + "duration": 0.146385, + "end_time": "2024-03-26T15:16:07.532716", + "exception": false, + "start_time": "2024-03-26T15:16:07.386331", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 52992] --\n", + "├─Adapter: 1-1 [2, 2648, 52992] --\n", + "│ └─Embedding: 2-1 [2, 2648, 69, 768] (215,808)\n", + "│ └─TensorInductionPoint: 2-2 [69, 1] 69\n", + "│ └─Sequential: 2-3 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 1024] 787,456\n", + "│ │ │ └─LeakyHardtanh: 4-2 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─LeakyHardtanh: 4-4 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─LeakyHardtanh: 4-6 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─LeakyHardtanh: 4-8 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─LeakyHardtanh: 4-10 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─LeakyHardtanh: 4-12 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─LeakyHardtanh: 4-14 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-15 [2, 2648, 512] 524,800\n", + "│ │ │ └─LeakyHardsigmoid: 4-16 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 52992] (recursive)\n", + "│ └─Embedding: 2-4 [2, 661, 69, 768] (recursive)\n", + "│ └─TensorInductionPoint: 2-5 [69, 1] (recursive)\n", + "│ └─Sequential: 2-6 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-9 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 1024] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-18 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 1024] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-20 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 1024] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-22 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 1024] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-24 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 661, 1024] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-26 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 661, 1024] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-28 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 661, 1024] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-30 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-32 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-7 [2, 8, 512] --\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-6 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-18 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-30 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 8, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-42 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 262,656\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 8, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 8, 512] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 8, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 8, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 8, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 8, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 8, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-54 [2, 8, 512] --\n", + "│ └─Encoder: 2-8 [2, 8, 512] (recursive)\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-60 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-31 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-72 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-39 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-40 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-84 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-42 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-43 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-40 [2, 8, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-45 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-46 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-96 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-48 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-49 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-18 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-51 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-52 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 8, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 8, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-108 [2, 8, 512] --\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-9 [2, 1] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 524,416\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-52 [2, 128] --\n", + "│ │ └─FeedForward: 3-25 [2, 128] --\n", + "│ │ │ └─Linear: 4-53 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-54 [2, 128] --\n", + "│ │ └─FeedForward: 3-26 [2, 1] --\n", + "│ │ │ └─Linear: 4-55 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-56 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 20,127,558\n", + "Trainable params: 19,911,750\n", + "Non-trainable params: 215,808\n", + "Total mult-adds (M): 78.72\n", + "========================================================================================================================\n", + "Input size (MB): 1.83\n", + "Forward/backward pass size (MB): 3938.39\n", + "Params size (MB): 80.51\n", + "Estimated Total Size (MB): 4020.73\n", + "========================================================================================================================" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = test_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "90576617", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:16:07.562069Z", + "iopub.status.busy": "2024-03-26T15:16:07.561705Z", + "iopub.status.idle": "2024-03-26T15:16:07.657354Z", + "shell.execute_reply": "2024-03-26T15:16:07.656292Z" + }, + "papermill": { + "duration": 0.113242, + "end_time": "2024-03-26T15:16:07.659921", + "exception": false, + "start_time": "2024-03-26T15:16:07.546679", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torch\n", + "model.load_state_dict(torch.load(\"model.pt\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:16:07.690529Z", + "iopub.status.busy": "2024-03-26T15:16:07.690196Z", + "iopub.status.idle": "2024-03-26T15:20:59.748945Z", + "shell.execute_reply": "2024-03-26T15:20:59.748086Z" + }, + "papermill": { + "duration": 292.076848, + "end_time": "2024-03-26T15:20:59.751531", + "exception": false, + "start_time": "2024-03-26T15:16:07.674683", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:20:59.782400Z", + "iopub.status.busy": "2024-03-26T15:20:59.782070Z", + "iopub.status.idle": "2024-03-26T15:20:59.804592Z", + "shell.execute_reply": "2024-03-26T15:20:59.803697Z" + }, + "papermill": { + "duration": 0.04057, + "end_time": "2024-03-26T15:20:59.806756", + "exception": false, + "start_time": "2024-03-26T15:20:59.766186", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
realtabformer0.00.0933330.0033313.3797990.56769311.9129540.8492490.00002417.7307380.0410512922347.750.0577180.209640.00046121.110537
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "realtabformer 0.0 0.093333 0.003331 3.379799 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss pred_duration \\\n", + "realtabformer 0.567693 11.912954 0.849249 0.000024 17.730738 \n", + "\n", + " pred_mae pred_mape pred_rmse pred_std std_loss \\\n", + "realtabformer 0.041051 2922347.75 0.057718 0.20964 0.000461 \n", + "\n", + " total_duration \n", + "realtabformer 21.110537 " + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:20:59.835711Z", + "iopub.status.busy": "2024-03-26T15:20:59.835416Z", + "iopub.status.idle": "2024-03-26T15:21:00.320144Z", + "shell.execute_reply": "2024-03-26T15:21:00.319249Z" + }, + "papermill": { + "duration": 0.50122, + "end_time": "2024-03-26T15:21:00.322241", + "exception": false, + "start_time": "2024-03-26T15:20:59.821021", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:21:00.352113Z", + "iopub.status.busy": "2024-03-26T15:21:00.351379Z", + "iopub.status.idle": "2024-03-26T15:26:42.292136Z", + "shell.execute_reply": "2024-03-26T15:26:42.291065Z" + }, + "papermill": { + "duration": 341.958336, + "end_time": "2024-03-26T15:26:42.294709", + "exception": false, + "start_time": "2024-03-26T15:21:00.336373", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/realtabformer/all inf False\n", + "Caching in ../../../../treatment/_cache_bs_test/realtabformer/all inf False\n", + "Caching in ../../../../treatment/_cache_synth_test/realtabformer/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:26:42.325245Z", + "iopub.status.busy": "2024-03-26T15:26:42.324878Z", + "iopub.status.idle": "2024-03-26T15:26:42.345699Z", + "shell.execute_reply": "2024-03-26T15:26:42.344916Z" + }, + "papermill": { + "duration": 0.038399, + "end_time": "2024-03-26T15:26:42.347815", + "exception": false, + "start_time": "2024-03-26T15:26:42.309416", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:26:42.376245Z", + "iopub.status.busy": "2024-03-26T15:26:42.375959Z", + "iopub.status.idle": "2024-03-26T15:26:42.381553Z", + "shell.execute_reply": "2024-03-26T15:26:42.380694Z" + }, + "papermill": { + "duration": 0.021985, + "end_time": "2024-03-26T15:26:42.383508", + "exception": false, + "start_time": "2024-03-26T15:26:42.361523", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'realtabformer': 0.47284586196532474}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:26:42.411529Z", + "iopub.status.busy": "2024-03-26T15:26:42.411249Z", + "iopub.status.idle": "2024-03-26T15:26:42.853590Z", + "shell.execute_reply": "2024-03-26T15:26:42.852621Z" + }, + "papermill": { + "duration": 0.458888, + "end_time": "2024-03-26T15:26:42.855799", + "exception": false, + "start_time": "2024-03-26T15:26:42.396911", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABEOklEQVR4nO3dd3hc1Zn48e+dXqQZ9WZLli25gLsBG9NsSqgBHEINiW3AmGRhCXG8G5zntwanObsbHAhhvWlIkJCwIaEkIfTebMAd27hhybKK1dvMaOr9/XE1Y8sqlkYjzUh6P8+jZzR37tz76krz6pxzT1FUVVURQog40sU7ACGEkEQkhIg7SURCiLiTRCSEiDtJREKIuJNEJISIO0lEQoi4k0QkhIg7SURCiLiTRCR69eCDD6IoSkyPWVpaiqIofPrpp/3a/7//+7+ZNGkSer2eOXPmxDQWkTgkEYkB+clPfsLzzz8/LOd69dVX+fd//3fOPfdcSkpK+MlPfjIs5xXDzxDvAMTI8pOf/ITrr7+eJUuWDPm53nzzTXQ6Hb/73e8wmUxDfj4RP1IiGiVcLle8Q4i52tparFZrzJKQqqp4PJ6YHGsw3G53vENIOJKIRqBw282ePXv42te+RmpqKueddx4Af/jDHzjjjDOwWq2kpaVx8803U1FR0eX97733HjfccAMFBQWYzWby8/P5zne+c8oPqaIouFwunnjiCRRFQVEUli9fDkB5eTn/8i//wtSpU7FaraSnp3PDDTdQVlbW47Hcbjd33XUX6enpOBwOli5dSlNTU5dzlZSU4HK5IucqLS0FIBAI8MMf/pCioiLMZjOFhYV8//vfx+v1djlHYWEhX/7yl3nllVc488wzsVqt/OpXv+Ltt99GURT+/Oc/s27dOsaNG0dycjLXX389LS0teL1e7rvvPrKyskhKSuK2227rduz+XuvFixczY8YMtmzZwgUXXIDNZuP73/9+n9d5LJKq2Qh2ww03MHnyZH7yk5+gqio//vGP+Y//+A9uvPFGVqxYQV1dHY8++igXXHAB27ZtIyUlBYBnnnkGt9vNt771LdLT0/n444959NFHOXr0KM8880yv5/v973/PihUrmD9/PitXrgSgqKgIgE8++YQPP/yQm2++mfHjx1NWVsbGjRtZvHgxe/bswWazdTnWPffcQ0pKCg8++CD79u1j48aNlJeXR5LE73//e37961/z8ccf89vf/haAc845B4AVK1bwxBNPcP311/Pd736XzZs3s379evbu3ctzzz3X5Tz79u3jlltu4a677uLOO+9k6tSpkdfWr1+P1Wrl/vvv5+DBgzz66KMYjUZ0Oh1NTU08+OCDbNq0idLSUiZOnMjatWsj7+3vtQZoaGjgiiuu4Oabb+brX/862dnZA/xNjwGqGHEeeOABFVBvueWWyLaysjJVr9erP/7xj7vsu2vXLtVgMHTZ7na7ux1z/fr1qqIoanl5ebfznMhut6vLli3r9v6ejvnRRx+pgPrkk09GtpWUlKiAesYZZ6g+ny+y/b/+679UQH3hhRci25YtW6ba7fYux9y+fbsKqCtWrOiyffXq1Sqgvvnmm5FtEyZMUAH15Zdf7rLvW2+9pQLqjBkzusRwyy23qIqiqFdccUWX/RcuXKhOmDAh8nwg13rRokUqoP7v//5vt+sjjpOq2Qj2zW9+M/L9s88+SygU4sYbb6S+vj7ylZOTw+TJk3nrrbci+1qt1sj3LpeL+vp6zjnnHFRVZdu2bVHFcuIx/X4/DQ0NFBcXk5KSwtatW7vtv3LlSoxGY+T5t771LQwGA//85z/7PE/49VWrVnXZ/t3vfheAF198scv2iRMnctlll/V4rKVLl3aJYcGCBaiqyu23395lvwULFlBRUUEgEAAGdq0BzGYzt912W58/11gnVbMRbOLEiZHvDxw4gKqqTJ48ucd9T/zAHTlyhLVr1/K3v/2tS7sMQEtLS1SxeDwe1q9fT0lJCZWVlagnTPzZ0zFPjjMpKYnc3Nxe25TCysvL0el0FBcXd9mek5NDSkoK5eXlXbafeI1OVlBQ0OW50+kEID8/v9v2UChES0sL6enpA7rWAOPGjZO7fqcgiWgEO7EUEgqFUBSFl156Cb1e323fpKQkAILBIF/60pdobGzke9/7HtOmTcNut1NZWcny5csJhUJRxfKv//qvlJSUcN9997Fw4UKcTieKonDzzTdHfcy+9Lej5YnX6GQ9Xae+toeTa3+vdX9iEBpJRKNEUVERqqoyceJEpkyZ0ut+u3btYv/+/TzxxBMsXbo0sv21117r13l6SwB/+ctfWLZsGQ899FBkW0dHB83NzT3uf+DAAS688MLI8/b2dqqrq7nyyiv7PP+ECRMIhUIcOHCA0047LbL92LFjNDc3M2HChH79HIPR32st+k/aiEaJ6667Dr1ez7p167pUi0D7T97Q0AAc/29/4j6qqvLII4/06zx2u73H5KLX67ud99FHHyUYDPZ4nF//+tf4/f7I840bNxIIBLjiiiv6PH84UT388MNdtm/YsAGAq6666lQ/wqD191qL/pMS0ShRVFTEj370I9asWUNZWRlLliwhOTmZw4cP89xzz7Fy5UpWr17NtGnTKCoqYvXq1VRWVuJwOPjrX//ara2oN2eccQavv/46GzZsIC8vj4kTJ7JgwQK+/OUv8/vf/x6n08npp5/ORx99xOuvv056enqPx/H5fFx88cXceOON7Nu3j//5n//hvPPO45prrunz/LNnz2bZsmX8+te/prm5mUWLFvHxxx/zxBNPsGTJki6lrKHS32st+k8S0Shy//33M2XKFH7+85+zbt06QGt4vfTSSyMfcKPRyN///nfuvfde1q9fj8Vi4Stf+Qr33HMPs2fPPuU5NmzYwMqVK/l//+//4fF4WLZsGQsWLOCRRx5Br9fz1FNP0dHRwbnnnsvrr7/e6x2rX/7ylzz11FOsXbsWv9/PLbfcwi9+8Yt+tf389re/ZdKkSZSWlvLcc8+Rk5PDmjVreOCBBwZwtQanP9da9J+inly2FEKIYSZtREKIuJNEJISIO0lEQoi4k0QkhIg7SURCiLiTRCSEiLsR3Y8oFApRVVVFcnJyzCd5F0IMnqqqtLW1kZeXh07Xe7lnRCeiqqqqbiOlhRCJp6KigvHjx/f6+ohORMnJyYD2QzocjjhHI4Q4WWtrK/n5+ZHPam/imogefPDBSPf4sKlTp/L555/36/3h6pjD4ZBEJEQCO1XTSdxLRNOnT+f111+PPDcY4h6SEGKYxf1TbzAYyMnJiXcYQog4ivvt+wMHDpCXl8ekSZO49dZbOXLkSK/7er1eWltbu3wJIUa+uI6+f+mll2hvb2fq1KlUV1ezbt06Kisr+eyzz3ps3OqpTQm0OZF7ayNSVZVAINDrBF2id3q9HoPBIF0jRNRaW1txOp19fkYhwaYBCU/1uWHDBu64445ur3u93i4L3YVb5Hv7IX0+H9XV1bKy5iDYbDZyc3Nl8ncRlf4mori3EZ0oJSWFKVOmcPDgwR5fN5vNmM3mfh0rFApx+PBh9Ho9eXl5mEwm+c8+AKqq4vP5qKur4/Dhw0yePLnPDmlCDEZCJaL29nYOHTrEN77xjUEfy+fzEQqFyM/P77bKqOgfq9WK0WikvLwcn8+HxWKJd0hilIrrv7jVq1fzzjvvUFZWxocffshXvvIV9Ho9t9xyS8zOIf/FB2esXr8Wj59AMPbLIImexbVEdPToUW655RYaGhrIzMzkvPPOY9OmTWRmZsYzLCF45bMaGt0+rpyRS7LFQLs3QH6alKyHSlwT0dNPPx3P0wvRo3ZvgMpmD4oCbV4/f916lGSLgTvOmyjtjENkbJa7RURhYWG3NcLGuvIGFwDZDgtTspMx6hXaOgI0uHxxjmz0kkQkxEkqmzwAFKTZMOp15Di1JaOrmzviGdaoJoloFPD55D91LB1r0/qqZTu0u4S5Tu2xplUS0VAZk4nIFwj1+nXynZK+9vX3Y99oLF68mHvuuYd77rkHp9NJRkYG//Ef/xFZ3riwsJAf/vCHLF26FIfDwcqVKwF4//33Of/887FareTn53Pvvfficrkix62treXqq6/GarUyceJEnnrqqajiG838wRCN7Vpiz3aYOx+1RHRMEtGQSah+RMPlsbd67jAJMDHDzpK54yLPf/3uIfzBnjufj0+1csOZxydme/yDw3h8XYeSfOdLU6KK8YknnuCOO+7g448/5tNPP2XlypUUFBRw5513AvCzn/2MtWvXRlY3PXToEJdffjk/+tGPePzxx6mrq4sks5KSEgCWL19OVVUVb731FkajkXvvvZfa2tqo4hut6tu9hFQVm0lPkln7eGQmaQmpyeUjFFLR6aTBOtbGZCIaCfLz8/n5z3+OoihMnTqVXbt28fOf/zySiC666CK++93vRvZfsWIFt956K/fddx8AkydP5he/+AWLFi1i48aNHDlyhJdeeomPP/6Ys846C4Df/e53nHbaacP+syUyq1HP2ZPS0euU4/NdWQ0Y9Qr+oEqLx0+qXYa7xNqYTER3X1jc62sn/7NbeUFRr/uefCf39nMnDiasLs4+++wut4oXLlzIQw89FBm8e+aZZ3bZf8eOHezcubNLdUtV1chQl/3792MwGDjjjDMir0+bNo2UlJSYxTwapNhMLCxK77JNURQWT83CYtRjM+vjFNnoNiYTkcnQ/6axodp3sOx2e5fn7e3t3HXXXdx7773d9i0oKGD//v3DFdqoNGOcM94hjGpjMhGNBJs3b+7yfNOmTUyePBm9vuf/yPPmzWPPnj0UF/dc2ps2bRqBQIAtW7ZEqmb79u2jubk5pnGPdEeb3CSbjTisMv3JcBqTd81GgiNHjrBq1Sr27dvHn/70Jx599FG+/e1v97r/9773PT788EPuuecetm/fzoEDB3jhhRe45557AG0u8Msvv5y77rqLzZs3s2XLFlasWIHVah2uHynhhUIqz26t5PEPDtPaEejyWoc/yIFjbXxeI5PxDQVJRAlq6dKleDwe5s+fz9133823v/3tyG36nsyaNYt33nmH/fv3c/755zN37lzWrl1LXl5eZJ+SkhLy8vJYtGgR1113HStXriQrK2s4fpwRoa0jQDCkYtApOCxdKwtNbh//2FnN+wfq4xTd6CZVswRlNBp5+OGH2bhxY7fXysrKenzPWWedxauvvtrrMXNycvjHP/7RZVssplwZLZrcWv+hFJuxW7XMaTUC2ji0QDCEQS//w2NJrqYQnVo7/AA4OpPOiaxGPSaDDlXVSk4itiQRCdEpnGAclu6JSFGOV9daPP5hjWsskKpZAnr77bfjHcKY1NZZIkq29PyxcFiN1Lf7IiUnETtSIhKiU/hOWXIPJSI4XmVr9UjVLNakRCREpzn5KYxPtZKV3PMCDcmdY8/avVIiijVJREJ0mpKdzJTs7uvphRVlJpFiM5Jqk7FmsSaJSIh+SrWbZMDrEJE2IiEAty9ARaNb7ojFiSQiIYCjTR7+suUor3xW0+s+qqqy/1gbW8qbop70TvRMqmZCcOpb96D1JXp97zG8/hAT0m1kJPVv1WFxalIiEgJo92rzPCX1kYjg+J0zl1du4ceSJCIhAHdnYrGZ+k5E9sgtfElEsTS2EpGqQsA3/F9qz3Ne9+TJJ58kPT0dr9fbZfuSJUtkgOoQCicW+ylmYAzPY90u481iamy1EQX98N5Dw3/e878Lhv7d9r3hhhu49957+dvf/sYNN9wAaKtvvPjii32OrBeD4+5c9MB+ihJROBG5fJKIYmlslYhGAKvVyte+9rXIyhsAf/jDHygoKGDx4sXxC2yUCyeWcNWrN+E2JBmBH1tjq0SkN2qlk3icdwDuvPNOzjrrLCorKxk3bhylpaUsX75cpi4dIqqqcn5xJi5fIFLi6Y3NpFXdTl42SgzO2EpEitLvKlI8zZ07l9mzZ/Pkk09y6aWXsnv3bl588cV4hzVqKYrCzPH9mxw/12nl6tl5OKxj66Mz1ORqJqgVK1bw8MMPU1lZySWXXEJ+fv6p3ySGnN1soDgrKd5hjDrSRpSgvva1r3H06FF+85vfcPvtt8c7nFGttcOvDe9wy/COeJFElKCcTidf/epXSUpKYsmSJfEOZ1Q7XOfiL1uO8u6Bun7tf+BYG1uPNEk7UQwlTCL66U9/iqIokSWTBVRWVnLrrbdiNstQgqHk6mcforB3D9Tzzr66yGT7YvASoo3ok08+4Ve/+hWzZs2KdygJoampibfffpu3336b//mf/4l3OKOeq7Nkc6pe1WF2k55Wjz/S90gMXtxLRO3t7dx666385je/ITU1Nd7hJIS5c+eyfPly/vM//5OpU6fGO5xRz93Zh+hUt+7DrJ238N3SqTFm4l4iuvvuu7nqqqu45JJL+NGPftTnvl6vt8vQh9bW0bnqZm/rlomh4fKGS0QnVc0CPqjaCkEfjDsDTPbO/bSPjZSIYieuiejpp59m69atfPLJJ/3af/369axbt26IoxJjzfE2ohM+DqEg7HoGmo9oz4/tgTNvA4MZu5SIYi5uVbOKigq+/e1v89RTT2GxWPr1njVr1tDS0hL5qqioOOV71AEMOBXdjfbrp6pqpGTTpURUvUNLQnojGK3gaYLyD4ETq2ZSIoqVuJWItmzZQm1tLfPmzYtsCwaDvPvuu/zyl7/E6/Wi13ctKpvN5n7fQTIatWEVbrcbq9Uau8DHGLfbDRy/nqNNSIWLpmXh9gWON1arKhzZpH0/6UKwOGDXX7Rq2oRzpWo2BOKWiC6++GJ27drVZdttt93GtGnT+N73vtctCQ2UXq8nJSWF2tpaAGw2m4zVGgBVVXG73dTW1pKSkjLo30ei0ut6GN7RXA4dLWAwQ+4s0BnAmgKeZqjfx7jU07hmTl6fszmKgYnblUxOTmbGjBldttntdtLT07ttj1ZOTg5AJBmJgUtJSYlcxzHj2B7tMeu04wOWs6dD2QdQf4CknJkkZcowj1ga1SldURRyc3PJysrC75fu+wNlNBpHbUkorK3DT7PbT7LFQIrNpFXLGg9pL2ae0HUirUhLRE2HtYZs3ei+LsMtoRLRUK35rtfrR/0HSkTncL2LN/bWUpSVxDWz88DdAN52rTrmPGGgcXKu1mjt90BrJfs6UnH5Apye68BilL+twUqoRCTEcAs3OFvDyaTxsPaYkt91HimdDlInQO3n0HKUd8r9uLxBxqdaJRHFQNx7VgsRT56Tb903lWmPqYXdd3aM0x5bqyKJq8Mn65vFgiQiMaZ5/Foishj1WvtQa6X2grOH+Z/CiajlKBaDrsv7xeBIIhJjWpfOjB3NWhuQTg9J2d13TsrW2o78HhxKOyCJKFYkEYkxLZxIrEY9tFZrG5OyQN9D86neAPYMABzBJu390qkxJiQRiTHN4wsvrKiHtiptY3Je729IytJ28WuJqENKRDEhd83EmHZOUQZuXxCH1Xi8ROTI7f0Ndi0R2QONgFTNYkUSkRjTZozrHN6hquDq7IGf1EdP8qRMALL1rVw7PQ+ndXSOwRtukoiEAG1sWcCnNVTb0nrfz64lIlugjUmpphGxPNVIIG1EYsxy+wIcaXDT6PKBq17baEvre/iGyR6ZIA1X/ybbF6cmiUiMWVXNHfx161Fe3V1zvFrWWeLpkz2DkKryRUUFW480jfo5m4aDJCIxZoVvvVtN+uOlm87G6D5Z01BV2Lr3EO/sq8MflEQ0WJKIxJjVpQ9R+wBKRLY0dDqwB1u7HEdETxKRGLPCc05bDYBbux0fvivWJ2saCgrJqpaIpC/R4EkiEmNWOIE41BZQQ9qMjGbHqd/YeVctKdQKqiq9q2NAEpEYs8LjzOyBFm2DPQP6M52wxQmKDpMSwhR0SdUsBiQRiTErnEBsIW0AK9Z+LvCp04M1BaNewRJokUQUA9KhUYxZZ05Io8Xjx9m2V9vQ30QEYE3DoK/A6m+hQ6pmgyaJSIxZU3OStW+2DbBE1LlvVrKZCzOM2PKcp95f9EkSkRAebSQ9lpT+v8eagt1kwG72gk3Gmw2WJCIxJnkDQY61eLEZQmR427SNAykRWTpLQR3NMY9tLJLGajEmNbp8/HXrUV7Zsk/bYDBrq3T0l8WJPxjiWG0tu6tahibIMUQSkRiTwn1/nOoJpaGBrARsceILhviiup4P91UNQYRjiyQiMSaF+xAl49I2DKRaBmAwY7B0jsL3tMjA10GSRCTGpHDfn6RQuESUMuBjGGzae4z+VrwBWVZoMCQRiTEpsp5ZSBsvNuASEaC3pqDXKZiDbTLMY5AkEYkxKTK8IxjFHbMwixODTsEcaKMjIIloMCQRiTGpwx9EUYOYg1F0ZgyzpmDU6zAH2qVENEjSj0iMSTPGOSmweUku02nrlZmSBn4QSwoGvYLF3ybjzQZJEpEYk4qzkkB/DKoMWo/qgdy6D7M4GZdiJVsBU7o95jGOJZKIxNjladYeo6mWAZgdOCydwzt0AeTjFD1pIxJjTjCkUt7gornxGCpq9InIYAKjRfs+PExERCWuiWjjxo3MmjULh8OBw+Fg4cKFvPTSS/EMSYwBLl+AZ7dWsmn3IW1DtIkI8Ohs1LV5Ka+uiVF0Y1NcE9H48eP56U9/ypYtW/j000+56KKLuPbaa9m9e3c8wxKjXHj+oGS1HQVlUImoOWjlYF07+8srYxXemBTXSu3VV1/d5fmPf/xjNm7cyKZNm5g+fXq3/b1eL16vN/K8tbV1yGMUo4/HHwQ1hD3UBiRH1as6zGDTRuGHOuRvcTASpo0oGAzy9NNP43K5WLhwYY/7rF+/HqfTGfnKz88f5ijFaOD2BTEF3Zh0qjbtqzn6ic2MncM8kEQ0KFEloi+++CJmAezatYukpCTMZjPf/OY3ee655zj99NN73HfNmjW0tLREvioqKmIWhxg7PP4glkALRr2izSuki/7/scnemcS8bTLwdRCi+g0UFxdz4YUX8oc//IGOjo5BBTB16lS2b9/O5s2b+da3vsWyZcvYs2dPj/uazeZIw3b4S4iB6vAFsQRaMeh0g2ofAjDZU7THQLsMfB2EqBLR1q1bmTVrFqtWrSInJ4e77rqLjz/+OKoATCYTxcXFnHHGGaxfv57Zs2fzyCOPRHUsIfrDHU5E+sE1VAMYrA70OkVbVsgbiFGEY09UiWjOnDk88sgjVFVV8fjjj1NdXc15553HjBkz2LBhA3V1dVEHFAqFujRICxFrU7KTmZ2h4rQaBzZPdU/MDgw6BZ0aoKOjPSbxjUWDaqw2GAxcd911PPPMM/znf/4nBw8eZPXq1eTn57N06VKqq6v7fP+aNWt49913KSsrY9euXaxZs4a3336bW2+9dTBhCdGngnQbxUl+rVf0IEtE6A1MyMlkWk4yqXr5BxqtQSWiTz/9lH/5l38hNzeXDRs2sHr1ag4dOsRrr71GVVUV1157bZ/vr62tZenSpUydOpWLL76YTz75hFdeeYUvfelLgwlLiL6p6vGVOwabiID09HRSbSYsQdegjzVWRdWPaMOGDZSUlLBv3z6uvPJKnnzySa688kp0nXcfJk6cSGlpKYWFhX0e53e/+100pxdiUI4eqyfV7cFqNqCzxGBNMrMD2o6BV27hRyuqRLRx40Zuv/12li9fTm5ubo/7ZGVlSaIRCScUUnnl072cfqyFeVMKMesH36e3VbXibfOiNDSQMS4GQY5BUf0WXnvtNQoKCiIloDBVVamoqKCgoACTycSyZctiEqQQseINhDAFtJKLMWnw1TKACreB1rp2bJYaMmbF5JBjTlRtREVFRdTX13fb3tjYyMSJEwcdlBBDxeMPYvG3ane6bGkxOabBGh7mISPwoxVVIuqtB2l7ezsWi2VQAQkxlNy+QMz6EIUZw72rO2ShxWgNqGq2atUqABRFYe3atdhstshrwWCQzZs3M2fOnJgGKEQsdUSGdwy+V3VYuHc1vjbtjlw0sz2OcQNKRNu2bQO0EtGuXbswmUyR10wmE7Nnz2b16tWxjVCIGPL4QlqJyBK7EpG5cwR+MBAAvxtMMm3sQA0oEb311lsA3HbbbTzyyCMy1kuMOB53O4aQF6PePPhe1Z2sFhN+vRUl6EHtaEWRRDRgUd01KykpiXUcQgyLCXYfgTQbFrtDm+o1BiwGHT69HWPQg8/dgtnRc5cW0bt+J6LrrruO0tJSHA4H1113XZ/7Pvvss4MOTIihkG3wQIoVnNkxO6ZBr+O0ieOwtXjR+2W8WTT6nYicTidKZyOc0xmD3qhCxEMMh3acaFxWNviPgiSiqPQ7EZ1YHZOqmRipGuuPYfYGMJudsZ0n2ZysPcpqHlGJqh+Rx+PB7XZHnpeXl/Pwww/z6quvxiwwIYbCrkPl7KxsoVmNYmXXPtT7TdS2ddDW0hjT444VUSWia6+9lieffBKA5uZm5s+fz0MPPcS1117Lxo0bYxqgELGiqir6jmYATEnpMT327oYQh+pcNDU1xPS4Y0XUMzSef/75APzlL38hJyeH8vJynnzySX7xi1/ENEAhYsXv86IPaFN1WByxTUT6E4d5yNzVAxZVInK73SQna3XiV199leuuuw6dTsfZZ59NeXl5TAMUIlY62rTSSkhvxmixnWLvgTGGOzX6fRAY3DzuY1HUk+c///zzVFRU8Morr3DppZcC2kRn0slRJCpfOBFZUiN3gGPFajET0FkIhFRpsI5CVIlo7dq1rF69msLCQhYsWBBZh+zVV19l7ty5MQ1QiFjxtWsNyeogFlTsjcWox6e3EwiGJBFFIao7mNdffz3nnXce1dXVzJ49O7L94osv5itf+UrMghMilgKdiUiJ0dCOE1mNerwGO4Fgs8zUGIWou1Lk5OSQk5PTZdv8+fMHHZAQQyVV144hzYZ3XOynUQyXiPw+KRFFI6pE5HK5+OlPf8obb7xBbW0toVDXheViuRKsELGSHGonOcUK48fH/NgOq4E5xQWk1ByVRBSFqBLRihUreOedd/jGN75Bbm5uzBv+hIi5UPB4lSnGwzsAzAY9heNyoNUkiSgKUSWil156iRdffJFzzz031vEIMTQ8zbR3+EFvxKSzEptx9yeRYR5Ri+quWWpqKmlpsZnvV4hh4Wnii/p2NlerHG32DMkpKt0Gats68Lqah+T4o1lUieiHP/wha9eu7TLeTIiE5mkiEFTpMDqwmWI63DXivSNeDtW5tM9FQFZ9HYiofiMPPfQQhw4dIjs7m8LCQoxGY5fXt27dGpPghIgV1d2APxjCY3BiNemH5Bxmi5mAzow/3JfIYB6S84xGUSWiJUuWxDgMIYaW39VISIUOgxP7ECUia7hTY6hDaxi3ZwzJeUajqBLRAw88EOs4hBhS/vbw8I4UDPqoWiROyWLU49HbCQQ90CGdGgci6t9Ic3Mzv/3tb1mzZg2NjVqP1a1bt1JZWRmz4ISIiVCQQGcDsmKL/a37MKtRj88gwzyiEVWJaOfOnVxyySU4nU7Kysq48847SUtL49lnn+XIkSORuYqESAieZgLBIEHFiNk6dIOyLUY9Xr0df0AGvg5UVCWiVatWsXz5cg4cONBlZdcrr7ySd999N2bBCRETniZsJgP5eXlMHz90861bTXp8+iQpEUUhqhLRJ598wq9+9atu28eNG0dNTc2ggxIipjxNWI16JmSOg7yhS0Q5TgtnnzaBrMPbZODrAEWViMxmM62t3S/0/v37yczMHHRQQsTUEK3ccTKHxYgjPw+qjVIiGqCoqmbXXHMNP/jBD/D7/QAoisKRI0f43ve+x1e/+tV+H2f9+vWcddZZJCcnk5WVxZIlS9i3b180IQnRO08jLm+ABjUJbyA4tOcKD/MIeCHgG9pzjSJRJaKHHnqI9vZ2MjMz8Xg8LFq0iOLiYpKTk/nxj3/c7+O888473H333WzatInXXnsNv9/PpZdeisvliiYsIXrmaeJwg4vn9ro50jB0owFUVeVQk59j7hBBVRqsByKqqpnT6eS1117jgw8+YMeOHbS3tzNv3jwuueSSAR3n5Zdf7vK8tLSUrKwstmzZwgUXXNBtf6/Xi9d7vOt8T9VDIboIBaGjBX8wRIfBgc08NMM7QKsZvPxZDac1qCzMUbF6W8Ee20n6R6sB/1ZCoRClpaU8++yzlJWVoSgKEydOJCcnB1VVBzUlSEtLC0CvA2rXr1/PunXroj6+GIM6WkBV8YYM+PU2bMah6VUdpk2QlkQg1CologEYUNVMVVWuueYaVqxYQWVlJTNnzmT69OmUl5ezfPnyQU0TGwqFuO+++zj33HOZMWNGj/usWbOGlpaWyFdFRUXU5xNjhLuRoKri0iWDogzZOLMwa2dfokBQqmYDMaASUWlpKe+++y5vvPEGF154YZfX3nzzTZYsWcKTTz7J0qVLBxzI3XffzWeffcb777/f6z5msxmzWQYSigHwNGrVMmMqBp2C2TA0wzvCrCadNmWs9CUakAH9Vv70pz/x/e9/v1sSArjooou4//77eeqppwYcxD333MM//vEP3nrrLcYPwTSeYgyLjLpPwWrSD/lsoseHeUiJaCAGlIh27tzJ5Zdf3uvrV1xxBTt27Oj38VRV5Z577uG5557jzTffZOLEiQMJR4hTczfgD6i4jakkDWFDdZjVZDihRCQ3U/prQL+ZxsZGsrOze309Ozubpqamfh/v7rvv5o9//CMvvPACycnJkV7ZTqcTq9U6kNCE6Jm7AatJz+yiSegdQ9erOsxm6lzNQ0pEAzKgRBQMBjEYen+LXq8nEAj0+3gbN24EYPHixV22l5SUsHz58oGEJkR3Pjf43FiNemZPngSGIZmpuouJGXaSZxWRu9sCfg8EtXmyRd8GlIhUVWX58uW9Nhif2Menv8cTYsh4tOlpsDiGJQkBZCSZybBnwmGrloS8bWCT+d1PZUCJaNmyZafcJ5o7ZkIMCbc2GVozyXhbO0i1mTAN8V0zABQFzA7t/JKI+mVAiaikpGSo4hAi9lz1AGyp17Fz8xG+MncchRn2IT1lMKRS3uDC5NYzDhVF2on6ZRj+PQgRJ26tataCNhmafRjumgG8sL2KrceCcgt/ACQRidHL3UBIVWlCGxFvNw9tr2oAvU7pHOYhnRoHQhKRGJ2CAeho1jozGlPR6xSsQzzOLMxuPvEWvvQl6g9JRGJ08jRpg10x4ddZsQ1Dr+owq1GP1yAlooGQRCRGJ7fWUN1hTAFFGZZe1WF2s0GqZgMkiUiMTu21ALgM2vSww9VQDccn0fcHVfC5tDmRRJ+G77cjxHBy1QGQkjWOC8yZOK3DWCIyGQjozHhDnVVBb+uQz5c90kkiEqNTZyJKzczjjNThTQKTMu0kWwzk7MsEtV2rnkki6pMkIjH6BHzgada+t2cN++kzksxkJJmhJg2a26WdqB+kjUiMPp2lIUx2jrTBsdYObdHD4RZe0UMS0SlJIhKjTzgRJWXx951V/HHzEVo8/mE7fTCkcqiuncPtOlSkd3V/SNVMjD6dichnTsMX0EpCyZbhm4pDVVX+tr2K7DY/+U4Vg3RqPCUpEYnRpzMRuUzaqHeLUT88o+47GfQ6TAadTJA2AJKIxOiiqpE+RG067U5VsmX4C/5JnZ0afQHp1NgfkojE6OJt02ZGVHQ067RR9/FIRHazAZ/Bji8Ykk6N/SCJSIwubdq859jTafNpHQodw9g+FJZk1uPXWfGFFK2U1tEy7DGMJJKIxOjS3pmIknJo69DulCXFqUSEomgLO4IkolOQu2ZidGk7pj0m5zLd5iQz2cL41OFfESY8yNal2AEvdDQPewwjiSQiMbq0VWuPydnkO23kp9niEsaEdDuXz9CTe2wcNDUe7+kteiSJSIwe3jatYVhRIKn39feGQ5rdRJrdBIEsaEKqZqcgbURi9AhXy2zpeFUdB2vbqG3tiG9MlhTtUapmfZJEJEaPSLUsh/p2H3/fUc3fd1bHLZwv6trZ0wSBUEiqZqcgVTMxeoRv3Sfn0uLW7pg5rfFbZfX1vcfo8PiYoIYw6DwQ8IKh58VJxzopEYnRQVWh9aj2vSOPZo8PgJQ4JiK72UBQZ6JD6Uw+0k7UK0lEYnRwN4K/A/QGSMqmtXO0vdMWv0QUvoXv0SdpG6R61itJRGJ0CJeGkvNAp6c5Aapmkb5Eus5EJA3WvZJEJEaHlkrt0TlOe9pZIop31QygXQknIqma9UYSkRgdWjsTkWM83kAQt08bZOpIgBJRq9rZqVKqZr2Su2Zi5PN7wKWtY4YjD52icPXsXFo7AliGaXXXnoRH/TepSaAAnsa4xZLo4loievfdd7n66qvJy8tDURSef/75eIYjRqpwtcyWDiYbRr2O4qxk5hXEd+WMrGQLl8/I4ZyZU7QNnmaZDqQXcU1ELpeL2bNn89hjj8UzDDHSNZdrj87x8Y3jJFaTntNyHeRkZmp389SQtBP1Iq5VsyuuuIIrrrginiGI0aCpTHtMnQBoPZpDqkpeihWbKQFaHxQFrGnazJHuRrClxTuihJMAv6X+83q9eL3eyPPW1jE6Kbm7EQ69Cc1HwGSHcWfCuHnaH/xY43NHpoYlRUtEn5Q1UtXcwZUzc5makxzH4KC6xUNtq5dCxYGTWmkn6sWIumu2fv16nE5n5Cs/Pz/eIQ2/1mrYUgr1B7QhA+5GOPAqfP6i1rt4rGk+oj3aM8CchKqqNLi0XtVpdlMcA9NsP9LMm5/XcszfOSeSWxJRT0ZUIlqzZg0tLS2Rr4qKiniHNLz8Hvjsr1oCcuTBvKVQfAkoOqjZBUc2xTvC4ReplhUC4PIF8fpDKAqkxrFXdVi4+0AznSUzKRH1aERVzcxmM2bzGB40ePB1bc4dWxpVE66lrD6AUV/EtHyV5CNvQNl7kDEF7OnxjnR4qOrxRNRZLWvqLA05rUYM+vj/nw3Pl90UTkTuhjhGk7hGVCIa09pqoOYzVFXlPf3ZbNlWG3lpk5LCnamFWFrLtGQ1+6b4xTmc3I3gaQKdPtJQnUjVMgCHVfuINQQ7OzV62yHgA0NixJco4vovo729ne3bt7N9+3YADh8+zPbt2zly5Eg8w0pMX7wDwA7/OLY0WVEUmJKdTEGajXOnZGI5/XKtitb4BTSPkSprw0HtMaUgMr1Go0u7mZFuT4ySc3iF2Sa/AdXYmYykVNRNXEtEn376KRdeeGHk+apVqwBYtmwZpaWlcYoqAbVWQ+MX1Lb7+VB/Oooerp6dR1Gm1jgLaHfMcmdB1XYoex/m3BLXkIdFOBGlF0c21bclVoko3LvaFwjhT07D5HeDux4cuXGOLLHENREtXrz4+AdJ9K5yC6qq8pk/F6/FycJJ6RRlagMplRNu2bdmn0Xdzg+ZqB5G13YMkuM7b/OQ8nugpXPEfXpRZPOXTs+mts3LuDis3NETo15HssVAW0eAdkMqaRw93t1AREgbUaLzuaB2D4qicPb5l2JusXNWYfcOcaqq8tyedtIDuVhbasir3ALTroxDwMOk4ZDWU9meAdbjQzlS7SZSE6Q0FHbxadmYDDqS29qhadfxcXEiIv63FUTfqndo45McudgyJ3BucQZ6XfeOi4qicGZhKjXJM6hs9uCv3qV19hutavdqj5lT4xtHP0zMsDMuxYrRkaVtcNXFN6AEJIkokakq1OzCHwxB3rxT7n5ajgNzegEthnSqG9u1vkWjkd8DTYe177NOj2zeV9PGp2WNNLR7e3ljnNkztUdvm/YziAhJRImsrZpAez1bK9t5riIJty/Q5+46ncI5xRkcSzqNmtYO/JXbR2dv6/r9WikxKVOrmnXaXdXCewfqqWqO8xJCJ/H4guw82synR11gcWobpVTUhSSiRFbzGfXtPmpNE2jxK1gMp55bZ1KGHTKn4Vf11NZUQmvVMAQ6zI7t0R5PKA2pqkpN5xpmWY7EuHUf1uEP8sbeWjZ90YAaTpySiLqQRJSoQkGo3UNtWwd19snMzk9B10Pb0MkURWFeUS4NtolUt3gIVG0f+liHk6dJ602tKF0SUYPLh9cfwmTQkZmUWInIaTWiUxT8QRW3qfNGQ7skohNJIkpUjV/gam+jOWCi3TaeaTmOfr91anYyoeyZZDssWqNu0D+EgQ6z6h3aY2ohWFMim6uatTaXHIelXwl7OOl0Cimd495adeFEVBPHiBKPJKJEVbOL2rYO6m3FFGc7sJr6P+WpTqdw7aKzyc/LwxDyQ93nQxjoMAoFoXqn9n3unC4vhRNRboplmIPqn3AiqtedUCKS2RojJBElIr+HYN0B6tt91NknMyPPOeBDKDod5MzUnoQ/vCNd3T6tX5XJDhmTu7xU2dlAPS4lMToyniw85KTWZwGjBUIBaSc6gSSiRFT3OY3tHlr1qRgcOeSnRffhUrNn0OTx88XBPagjfR4cVYWKzdr3eXO0ga6dXN4AHf4gOkUhx5mYJaKMZK2TZb3LB8mdwztG442EKEkiSkQ1n5FmN1E0cwHnFGd0GcYxEF5DMp+0ODnW6qV2/ycxDnKYNZdrMxDoDdqMlCewmw3cdcEkbp6fj7kfdxbjIdyAXt/uQ03K0Ta2STtRmAzxSDSeJmg5il6no/C0s8DS/0bqk1mMepxFZxLaVUHt/o/JnvUl0I3Q/z3lH2mPObPBZOv2skGv0xrnE1SqzcRX540nM9mM0tLZNtRWHd+gEsgI/ascxY7t1h5TJgwqCYVNm34GAb2V9rYWGir2DPp4cdH4hXbLXqeH/PldXgqF1BExcFqnUyhIt2k3HcJVM1edNtumkESUUFSVUM1n7K5qYXcwXxvaMUhOuwXL+FkAVOz+aNDHG3aqCl+8rX2fN6/LLXuAL+rbKfmgjK1HmoY9tKhZHNrPoarHZxAY4yQRJZK2aprrj9HshQ9b09HFaFWOwpnnANBRs5+WphHWaF2zE9qOaTMaTljY7eXPa9po8fhxefse/pIImt0+3j9Qz/sH6rXJ3OD45P9jnCSiRFLzGbVtHTRaC5k2vudR9tHIzB6HKb0AVVU5uPPDmBxzWPhc2rJJABPO027bn6DDH+RwnQsg7ssG9UeHP8QnZY3sqmxBDS8GKYkIkESUOEJBvFW7aHb7o+471Jfxpy/EatQxwbtvZHSkU1U48Br4OyApC8af1W2Xz2vaCIRU0pNMCTesoyeZyWYMOoUOf5BmU2c7UVuNNof1GCeJKFE0HKKusRmf3kpSbnHMJ/fKLprLrEnjyTD6RkZP6+od2vAURQdTr+h2t09VVbZ3tgvNGp8SdReH4aTXKZE7e5Uek9ZWpIagZYzMMd4HSUQJQq3aRm2blzr7FGaOTz31GwZKb0A3vnNOo4rNhGLQED5k2mvh4Gva9xMv0NZwO8mhOhdNbj9mo47Tcwd/d3G4hIeg1LR6Ia1zituGQ3GMKDFIIkoEnmaaK/fhDYRoTjmd4qykoTlP3jxUnZ6aynJeeOcjfIEETEYdrbDzzxAMQNokKDi72y6qqrLpC20ljFnjUjAZRs6fca5T6yVf2ew5Pul/w8HROW/UAIyc3+BoVr0Ds17Bll1E8YQCjEO1MKDJRiBrJlXNHuxVH/H+gQSbxN3ngp3/p81gaM+A06/Rpvs4iaIoXDkzl2k5yZxZOASlxyE0PtWKTlFodPloseSBzgAdLWN+HmtJRPEWDED1DmwmA7PnX8gFkzNO/Z5BME48l0nZTpK9NZQf+IzPKluG9Hz91tEK257SPpDmJJh5Axh7H2OXZjdxxcxcLMbEHNLRG4tRT67Tgs2kp9VHZKls6vfHM6y4kyEe8XbsM60kYE6GjMlD3+hqcZBSfDbj3W/jbt7MW3vHYzHqo64OtnsDlNW7aHT58AZCBILa5GRWo55Uu4m8FCtO6ynWoG+phN3PaqugmpNhzte6dVwEbcrVo01uJmcn/q36vnx5di5Wo177XXunalWzY7thwjk9lgDHAklE8aSqeA9/RFW9i/TZ5+PQDdN/94KFjK/eSYe/jvqWXfxjp47zJ2cyr+DUd59aPH70OoUkc+dSyu1eXttzrNf9Fxalc/akdACCIRVfIHR8bqVQCCq3wBdvaV0KbOkw68Yek1Bbh5+/7aiittXLJaeFmDk+tt0bhpPNdMLHLnMqHHhFW/21rbrHhvmxQBJRPNXvp6q6ksr2EJ825nDDcJ3XZEMpvphi/z+gYRdvePN57wAUZybh7JzAS1VVVBXavAHq271UNLopb3DT6PKxYGIa5xRrVcj0JDMT0m2k2U1YjXoMeh2+QAi3L0BDu4+cEwailje4+NuOKvKcVqbaXRQ1f4DdewwFBTKnwLQvR5aODguFVPbWtPL+gXrcviBWk568BJ38bKBCIZWOkB5bxhRtHu6aXZKIxDALhfDsf5Pa1g5qkueyYPIw/wHmzESp3Uuxcgib7yP2ZX81koQAfvPeF7i83Ts+6hQFt+/49iSzgevmje/XKY+1erF11GGv247bfZhdgNFkIThxMabkeUwLGQi3ClU0ujlY186h2nbaOrThGxnJZq6ZldclzpHqi7p2Xt97jBynlWvyZx5PRBMXaROnjTGSiOKlZgeVlUfxKRaUggVRT34WNUWB065G2VLCOFoZ538bAjeBwUwgGIokIb1OIdVuItdhoTDDxvhU28AbiDtaoeEAC5t2MtdcSXPQTxMmvtBNoMxxFj5PEuyvpyg7GSvasfdUt7KnqhUAq0nPmRNSmVuQGrNhL/HmsBpxeYOU1bvwTJuI1Z6hNdRXbetxTN1oJ4koHvwemve8SV2bl6OpC1k8bVx8egabbDDjetjxR63BeOuTMP069LY07rxgEnpFwWTQDezDr6rQ0azNPthaDS1HtEGrnSwmEznjZpJTcA5FlnSONLqpbe2gye0n2Xz8zzHHYcGk1zE+1crEDDuGoerSECcZSWayHRaOtXaw/WgLCwvOhr3/gKOfwLgztEG+Y4gkojgI7H+N8upaPMYUsqacHenkFhfJ2TD7Fq0ToasePv0dSu4cknJnQVJ2z3dxVFW70+dtBXcjeBq7Pp68aoiiaG0fGVO1ebQ7JzYzAcVZST3esZudnxL7nzXBnFmYyos7q9lxtJkzFk7DZH0fPM1w5EOYtDje4Q0rSUTD7dhuju3/FLcvRNX4C7l+ak68I4LkHDjzdtj3T224QeUW7ctgBmsqGCzamKiQH3xu8LX3PXBWp9cGqjrGaZOApU3sNnJedN4csBpp8fjZUtHKwqKL4bO/QsXHkD2jyyq2o50kouHUWg37/kmu00Jd2jwumDE3cTrkmZO0W+eNh7V2ioZD2uyBvc2rrChacrGmarfdrWlgS9MerSldJrcXPdPpFM6fnME/dlbzSVkj0xZMIDVtkjYj5e7n4IzloB/5DfP9IYlouLRWE9rxNLpgAF16EXNmXpOY80enTdS+QkGtquZtg0CHllh0Ru2OjjkZTEmSbGKgOCuJwgwb5Q1uqlo7SJ12FXz6uHbtdz8P07+iLRgwyiXEJ+Gxxx6jsLAQi8XCggUL+Pjjj+MdUuyoKv6j2zj82v9yoLIO1ZEH05ckZhI6kU6vtR9lFEPODMg6TfveOR4sTklCMaIoCpecls2SOeOYnufUSqbh5NNwsHPsXXu8wxxycf80/N///R+rVq3igQceYOvWrcyePZvLLruM2toEG5A5UKpKoO4gR9/6DZ+9+Sdqml18EczmaP413TrtibEt2WKkMON4G1q1kkFZ3lWoeqM2g+Mnv4Ejm0b1RPuKGuclEBYsWMBZZ53FL3/5SwBCoRD5+fn867/+K/fff3+f721tbcXpdNLS0oLDEac5aVQVgj7wu8Hbjqe5hqqjh/EeO0h7axPBkEpIMdCYeRanL7iUoqyRPU5KDK1AMMRTm4/Q6PKRY2jnTPd7pKtNWIx6TCYT+vRJKCn5YM8ES4pWgtKbEnaMWn8/o3GtfPp8PrZs2cKaNWsi23Q6HZdccgkffdR9xQmv14vXe/y/Qmtra/9O1HgYDr3RZc4Xl9fP7soW4MTlaFRQVVRgnNNCfqp2W93jD7Cjorlzl1DkOCqgD/nJdRjJT+1ca8sfpLZz36DOSHvaaWRPv4AzJyXu4n8icahAUWYS7d5magJJ/MN4GZmuA+Q17sDqr8NZ3czpuQcj++842kxIBVVvRFWMqIoOFQUUhWSLkWm5Tm2WS2BLeTOBUNc5qFLsJqZGO//VrJtisuQVxDkR1dfXEwwGyc7O7rI9Ozubzz/vPp3p+vXrWbdu3cBPFPBC+0nrjPsCBF29T4Ghmn3QuTKE4g+i9lJPDwK+gF6bV8Zkx5yahqLoMWdMImfCZHJSk0fENKYiMRj1Os6bnMFZE1Mpq3dT3uCivn02e13TMHrqmG5rgswOrb+Wt5UOf5CQCviDQEeXY4VCRnAf72YRam8kFOpaAVJDRrB3fV+/qbGbWG9ENcevWbOGVatWRZ63traSn59/6jem5MPsm08ovioYgyoFU7wonb2GFUUBRYcCKDpFGyFtNoKiYAipTJ7pR1VACe/TeSzFYMZstYNVKz3pgO5zCgoxMGaDnqk5yV1WJ/EHiwiGVDihy0fB9FbUgA814NU6kqohFFRQQ5j0CiSZtBK8GmJcYUe3iSDNRgXsUbZZxrBvWFwTUUZGBnq9nmPHuk4jcezYMXJyunf0M5vNmM1RXDSTXbslfeImYFw/+4sZgIyRO+uEGCWMeh0ndzvLTet/1SgvLcYBxVBc75qZTCbOOOMM3njjjci2UCjEG2+8wcKFY2/gnxBjVdyrZqtWrWLZsmWceeaZzJ8/n4cffhiXy8Vtt90W79CEEMMk7onopptuoq6ujrVr11JTU8OcOXN4+eWXuzVgCyFGr7j3IxqMhOhHJIToVX8/o3HvWS2EEJKIhBBxJ4lICBF3cW+sHoxw81a/h3oIIYZV+LN5qqboEZ2I2traAPrXu1oIETdtbW04nb33Ch7Rd81CoRBVVVUkJ4/N8VzhIS4VFRVy17Af5HoN3GCvmaqqtLW1kZeXh66PObhGdIlIp9Mxfnz/1tQazRwOh3ywBkCu18AN5pr1VRIKk8ZqIUTcSSISQsSdJKIRzGw288ADD0Q3I8EYJNdr4Ibrmo3oxmohxOggJSIhRNxJIhJCxJ0kIiFE3EkiEkLEnSSiBDbQFXCfeeYZpk2bhsViYebMmfzzn/8cpkgTx0CuWWlpKYqidPmyWCzDGG18vfvuu1x99dXk5eWhKArPP//8Kd/z9ttvM2/ePMxmM8XFxZSWlsYkFklECWqgK+B++OGH3HLLLdxxxx1s27aNJUuWsGTJEj777LNhjjx+olk12OFwUF1dHfkqLy8fxojjy+VyMXv2bB577LF+7X/48GGuuuoqLrzwQrZv3859993HihUreOWVVwYfjCoS0vz589W777478jwYDKp5eXnq+vXre9z/xhtvVK+66qou2xYsWKDeddddQxpnIhnoNSspKVGdTucwRZfYAPW5557rc59///d/V6dPn95l20033aRedtllgz6/lIgSUHgF3EsuuSSyra8VcAE++uijLvsDXHbZZb3uP9pEc80A2tvbmTBhAvn5+Vx77bXs3r17OMIdkYbyb0wSUQLqawXcmpqaHt9TU1MzoP1Hm2iu2dSpU3n88cd54YUX+MMf/kAoFOKcc87h6NGjwxHyiNPb31hraysej2dQxx7Ro++FGIyFCxd2WT/vnHPO4bTTTuNXv/oVP/zhD+MY2dgjJaIENNAVcAFycnIGtP9oE801O5nRaGTu3LkcPHhwKEIc8Xr7G3M4HFg7l1yPliSiBBTNCrgLFy7ssj/Aa6+9NmZWzI3FqsHBYJBdu3aRm5s7VGGOaEP6Nzbo5m4xJJ5++mnVbDarpaWl6p49e9SVK1eqKSkpak1NjaqqqvqNb3xDvf/++yP7f/DBB6rBYFB/9rOfqXv37lUfeOAB1Wg0qrt27YrXjzDsBnrN1q1bp77yyivqoUOH1C1btqg333yzarFY1N27d8frRxhWbW1t6rZt29Rt27apgLphwwZ127Ztanl5uaqqqnr//fer3/jGNyL7f/HFF6rNZlP/7d/+Td27d6/62GOPqXq9Xn355ZcHHYskogT26KOPqgUFBarJZFLnz5+vbtq0KfLaokWL1GXLlnXZ/89//rM6ZcoU1WQyqdOnT1dffPHFYY44/gZyze67777IvtnZ2eqVV16pbt26NQ5Rx8dbb72lAt2+wtdo2bJl6qJFi7q9Z86cOarJZFInTZqklpSUxCQWmQZECBF30kYkhIg7SURCiLiTRCSEiDtJREKIuJNEJISIO0lEQoi4k0QkhIg7SURCiLiTRCRGlNLSUlJSUiLPH3zwQebMmRN5vnz5cpYsWTLscYnBkUQkerR8+XIUReGb3/xmt9fuvvtuFEVh+fLlXfaPdQIoLCzk4Ycf7rLtpptuYv/+/b2+55FHHukyj/LixYu57777YhqXiD1JRKJX+fn5PP30010mvero6OCPf/wjBQUFcYnJarWSlZXV6+tOp7NLiUmMDJKIRK/mzZtHfn4+zz77bGTbs88+S0FBAXPnzh3UsXsqqSxZsiRSylq8eDHl5eV85zvfiaywAd2rZic7sWS2fPly3nnnHR555JHIMQ4fPkxxcTE/+9nPurxv+/btKIoicxHFiSQi0afbb7+dkpKSyPPHH3+c2267bcjP++yzzzJ+/Hh+8IMfRFbYGKhHHnmEhQsXcuedd0aOUVBQ0O1nAigpKeGCCy6guLg4Vj+CGABJRKJPX//613n//fcpLy+nvLycDz74gK9//etDft60tDT0ej3Jycnk5ORENdOk0+nEZDJhs9kix9Dr9Sxfvpx9+/ZF1jzz+/388Y9/5Pbbb4/1jyH6SeasFn3KzMzkqquuorS0FFVVueqqq8jIyIh3WIOSl5fHVVddxeOPP878+fP5+9//jtfr5YYbboh3aGOWlIjEKd1+++2UlpbyxBNPxKzUoNPpOHkqLL/fH5Nj98eKFSsiDfElJSXcdNNN2Gy2YTu/6EoSkTilyy+/HJ/Ph9/v57LLLovJMTMzM7u0+wSDwW6r0ppMJoLB4KDO09sxrrzySux2Oxs3buTll1+WalmcSdVMnJJer2fv3r2R73vT0tLC9u3bu2xLT08nPz+/274XXXQRq1at4sUXX6SoqIgNGzbQ3NzcZZ/CwkLeffddbr75Zsxmc1RVwsLCQjZv3kxZWRlJSUmkpaWh0+kibUVr1qxh8uTJY2aRgUQlJSLRLw6HA4fD0ec+b7/9NnPnzu3ytW7duh73vf3221m2bBlLly5l0aJFTJo0iQsvvLDLPj/4wQ8oKyujqKiIzMzMqOJevXo1er2e008/nczMTI4cORJ57Y477sDn8w3LXUDRN5mzWoxZ7733HhdffDEVFRXdVjAVw0sSkRhzvF4vdXV1LFu2jJycHJ566ql4hzTmSdVMjDl/+tOfmDBhAs3NzfzXf/1XvMMRSIlICJEApEQkhIg7SURCiLiTRCSEiDtJREKIuJNEJISIO0lEQoi4k0QkhIg7SURCiLj7/9S8nsLi2kUkAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:26:42.887382Z", + "iopub.status.busy": "2024-03-26T15:26:42.886821Z", + "iopub.status.idle": "2024-03-26T15:26:43.266183Z", + "shell.execute_reply": "2024-03-26T15:26:43.265216Z" + }, + "papermill": { + "duration": 0.397589, + "end_time": "2024-03-26T15:26:43.268335", + "exception": false, + "start_time": "2024-03-26T15:26:42.870746", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABBVklEQVR4nO3deXxU9b34/9eZfcnMZCULCfsuuwjivlUrFqVeK7WtgmCXR7GPi1yqcO/3YrG22M1Kq9f22gL111puXdtqFa0KKKhlFxBZAlkgCdlnMpPJrOf3x5kZCFnITCY5k8zn+XjkMcmZM2femcy889k/kizLMoIgCCrSqB2AIAiCSESCIKhOJCJBEFQnEpEgCKoTiUgQBNWJRCQIgupEIhIEQXUiEQmCoDqd2gH0RjgcpqqqCpvNhiRJaocjCMIFZFmmpaWFoqIiNJquyz0DOhFVVVVRUlKidhiCIFxEZWUlxcXFXd6vaiL6wQ9+wNq1a9sdGz9+PJ9//nmPHm+z2QDll7Tb7UmPTxCE3nG5XJSUlMQ+q11RvUR0ySWX8M9//jP2s07X85Ci1TG73S4SkSCksIs1naieiHQ6HQUFBWqHIQiCilTvNTt+/DhFRUWMGjWKr3/961RUVKgdkiAI/UzVEtGcOXPYtGkT48ePp7q6mrVr13L11Vdz6NChTuuUPp8Pn88X+9nlcvVnuIIg9BEpldYjam5uZvjw4Tz55JMsXbq0w/2dNW4DOJ1O0UYkCCnI5XLhcDgu+hlVvWp2vszMTMaNG8eJEyc6vX/16tU4nc7YV2VlZT9HKAhCX0ipROR2uyktLaWwsLDT+41GY6yHTPSUCcLgoWoiWrlyJdu2baOsrIydO3fy5S9/Ga1Wyz333KNmWIIg9DNVG6tPnz7NPffcQ0NDA3l5eVx11VV8/PHH5OXlqRmWIOD0BrAatOi0KVVpGLRUTUSbN29W8+kFoUtbDtXQ2Opn3uRCbCYdbl+QkmyL2mENWiLdC8IF3L4gZ5q9tAVCtPgCbNpZxpbDNaRQB/OgIxKRIFygvMEDQL7dxLh8G3qtREtbkAaPX+XIBi+RiAThAmeavAAMy7ag12oocJgBqG5uUzOsQU0kIkG4wNkWZfR+vt0EQKFDua1xiUTUV0QiEoTzBEJhGt1KFSzfbozcKonorEhEfUYkIkE4T73bR1iWsRi0ZBiVTuW8DCUhNXn8hMOiwbovqL4MiCCkErNey+WjctBqpHPrXZl16LUSgZCM0xsgy2pQOcrBRyQiQThPpsXA3NE57Y5JksR144dg0muxGLUqRTa4iUQkCD0weahD7RAGNdFGJAjnOd3UirM1IAYv9jNRIhKEiHBY5pW9ZwiFZZZcNRKHWR+7ry0QorKxlZAsM6FArPqQbCIRCUJES1uQUFhGp5Gwm9p/NJpa/bz+aTU2k04koj4gqmaCENHUqowfyrToO+w6ES0duX1BgqFwv8c22IlEJAgRrrYAAPbzqmRRZr0Wg06DLCslJyG5RCIShIhogrGbOiYiSTpXXXN6A/0aVzoQiUgQIloiJSKbqfOm02hJKVpyEpJHJCJBiHBFSkS2TkpEcF4i8oqqWbKJXjNBiJhekklxlpkhNmOn99sic8/cPlEiSjaRiAQhYly+jXH5HTf2jBqdl0GmRU+WRcw1SzaRiAShh7KsBjHhtY+INiJBAFr9QSobW0WPmEpEIhIE4HSTl5f2nGbLoZouz5FlmWNnW9hT3oQ/KAY1JpOomgkCF++6B2Us0T+PnMUXCDM8x0JuRueN2kL8RIlIEAC3LwRARjeJCM71nHl8ogs/mUQiEgSgNZJYLIbuE5E11oUvElEyiUQkCJxLLNaLrMAYXcfaLeabJZVIRIIAtPqVqpn1IiWiaCLy+EUiSiaRiASBc4klWvXqSrQNSczATy7RayakPVmWuXpMHh5/MFbi6YrFoFTdvJESlJAcIhEJaU+SJKYU92xx/EKHmfnTirCbxUcnmcSrKQhxsBp1jBmSoXYYg45oIxLSnqstoEzvaBXTO9SSMonoiSeeQJIkli9frnYoQpo5VefhpT2n2X68rkfnHz/bwt6KJtFOlEQpUTXbtWsXv/3tb5k6daraoQhpyNPDMURR24/X4/IGKLCbMBvMfRla2lC9ROR2u/n617/Oc889R1ZWltrhCGnIEynZXGxUdZQ10nPWKkpESaN6Ilq2bBm33XYbN910k9qhCGmqNTKG6GJd91HmWCISY4mSRdWq2ebNm9m7dy+7du3q0fk+nw+fzxf72eVy9VVoQhrx+KIloguqZkE/VO2FkB+GXgoGa+Q85WMjSkTJo1qJqLKykn//93/nT3/6EyaTqUePWbduHQ6HI/ZVUlLSx1EK6eBcG9F5/5fDITj4IpS+D2U7YO//B0Hln6BVlIiSTrVEtGfPHmpra5k5cyY6nQ6dTse2bdv41a9+hU6nIxTq+N9m9erVOJ3O2FdlZaUKkQuDiSzLsZJNuxJR9QForgCtHvRm8DZB+U7g/KqZKBEli2pVsxtvvJGDBw+2O3b//fczYcIEHnnkEbTajj0YRqMRo1EsRiUkT1iGGyYModUfPNdYLctQ8bHy/ajrwWSHgy8p1bThV4qqWR9QLRHZbDYmT57c7pjVaiUnJ6fDcUHoK1pNJ9M7msuhzQk6IxROBY0OzJngbYb6owzNmsjt04u6Xc1RiI/qvWaCkHLOfqbcDpmoVM0kCfIvUY7VHyfDqGN0XgZDbD1r2xQuLqVS+tatW9UOQUgzLW0BmlsD2Ew6Mi0GpVrWWKrcmTf+3InZo5VG66ZTSkO2pmeDH4WeESUiIa2dqo9O76hXDrQ2gM+tVMcc5/XK2gqVRuugH1xnOFqjTPNoC4h2omRIqRKRIPS3aIOzWR8p4TSeUm4zS5RqWZRGA1nDofZzcJ5mW3kAjy9EcZYZk16UjnpLlIiEtOa9sOu+qUy5zRrR8WT7UOXWVRVLXG1+sb9ZMohEJKQ1b6RqZdJrlfYh1xnlDkcng2Wjich5GpNO0+7xQu+IRCSktXaDGduaIeBVGqIz8juenJGvtB0FvNglNyASUbKIRCSktWgiMeu14KpWDmYMAW0nzadaHVhzAbCHmpTHi0GNSSESkZDWvP7oxopaaKlSDtqKun5AxhDllICSiESvWXKIXjMhrV0xOpdWfwi7WX+uRGQv7PoBViURWYONgKiaJYtIREJamzw0Mr1DlsFTq3yfUdD1AzLyAMjXurjjkiIcZn3X5wo9JhKRIIAytyzoVxqqLdldn2dVEpEl2MKoLAPoDP0U4OAm2oiEtNXqD1LR0Eqjxw+eyMhqS3b30zcM1tgCaXh6tti+cHEiEQlpq6q5jZf3nubtwzXnqmWREk+3rLmEZZmTlZXsrWhCluW+DTQNiEQkpK1o17vZoD1Xuok0RnfLnI0sw94jpWw7WkcgJBJRb4lEJKStdmOI3HGUiCzZaDRgDbnaXUdInEhEQtqKrjlt1gGtSnd8tFesW+ZsJCRsspKIxFii3hOJSEhb0QRil50gh5UVGY32iz8w0quWEXaBLIvR1UkgEpGQtqLzzKxBp3LAmqusxngxJgdIGgxSGEPII6pmSSASkZC2ognEElYmsGLu4U7DGi2YM9FrJUxBp0hESSAGNAppa9bwbJzeAI6WI8qBniYiAHM2Om0l5oCTNlE16zWRiIS0Nb7ApnyzL84SUeTcITYj1+fqsRQ5Ln6+0C2RiATBq8ykx5TZ88eYM7EadFiNPrCI+Wa9JRKRkJZ8wRBnnT4sujC5vhblYDwlIlOkFNTWnPTY0pForBbSUqPHz8t7T7Nlz1HlgM6o7NLRUyYHgVCYs7W1HK5y9k2QaUQkIiEtRcf+OOTzSkM96bqPMjnwh8KcrK5n59GqPogwvYhEJKSl6BgiGx7lQDzVMgCdEZ0pMgvf6xQTX3tJJCIhLUXH/mSEoyWizLivobMoj9EHXPiCYluh3hCJSEhLsf3Mwsp8sbhLRIDWnIlWI2EMtYhpHr0kEpGQlmLTO0IJ9JhFmRzoNBLGYAttQZGIekMkIiEttQVCSHIIYyiBwYxR5kz0Wg3GoFuUiHpJjCMS0tLkoQ6GWXzYyjTKfmWGjPgvYspEp5UwBVrEfLNeEolISEtjhmSA9ixU6ZQR1fF03UeZHAzNNJMvgSHHmvQY04lIREL68jYrt4lUywCMduymyPQOTRDxcUqcqm1Ezz77LFOnTsVut2O325k7dy5vvvmmmiEJaSAUlilv8NDceBYZOfFEpDOA3qR8H50mIiRE1URUXFzME088wZ49e9i9ezc33HADd9xxB4cPH1YzLGGQ8/iDvLL3DB8fLlUOJJqIAK/GQl2Lj/LqmiRFl55UTUTz589n3rx5jB07lnHjxvGjH/2IjIwMPv74YzXDEga56PpBNtmNhNSrRNQcMnOizs2x8jPJCi8tpUylNhQK8eKLL+LxeJg7d26n5/h8Pnw+X+xnl8vVX+EJg4g3EAI5jDXcAtgSGlUdpbMos/DDbeK92BuqJ6KDBw8yd+5c2trayMjI4NVXX2XSpEmdnrtu3TrWrl0b93OEQiECgUBvQxUGCbfHix0POlMGbVobyEZoa+v0XIPBgEbTdcVBH5nmgUhEvSLJCczWO3nyJKNGjUpKAH6/n4qKCpxOJy+99BK/+93v2LZtW6fJqLMSUUlJCU6nE7u94+4LsixTU1NDc3NzUmIVBgd/MIzf78coe9HrdGC0dXmuRqNh5MiRGAyd73HfWrGfA+/8EZdpKDd/bTlSIsMABjGXy4XD4ejyMxqVUIlozJgxXHvttSxdupS77roLk8mUcKAGg4ExY8YAcOmll7Jr1y7Wr1/Pb3/72w7nGo1GjEZjj68dTUJDhgzBYrGIN4kAgMcXwN/mxSp7MRiNXVbNwuEwVVVVVFdXM2zYsE7fPwar8lhD0I0vGMak1/Zh5INXQolo7969bNy4kRUrVvDggw+ycOFCli5dyuzZs3sdUDgcblfqSVQoFIoloZycnF5fTxg8/LIWTciPSdJhMFugm3+keXl5VFVVEQwG0es7LgmrM9vRaiRlWyFfUCSiBCXUazZ9+nTWr19PVVUVGzZsoLq6mquuuorJkyfz5JNPUldX16PrrF69mu3bt1NWVsbBgwdZvXo1W7du5etf/3oiYbUTbROyWCy9vpYwuBj1Gsw6Ga1GUrYG6ka0ShYKdTGFw2hHp5HQyEHa2tzJDjVt9Kr7XqfTceedd/Liiy/yk5/8hBMnTrBy5UpKSkq47777qK6u7vbxtbW13HfffYwfP54bb7yRXbt2sWXLFr7whS/0Jqx2RHVMuJBRp8WkldFKEkjdJ6KLvn+0OoYX5DGhwEaWtvcl+XTVq16z3bt3s2HDBjZv3ozVamXlypUsXbqU06dPs3btWu644w7+9a9/dfn43//+9715ekFIjCxDOFLC0fS+4zgnJwdaghDy9Ppa6SqhEtGTTz7JlClTuOKKK6iqquL555+nvLycxx9/nJEjR3L11VezadMm9u7dm+x4hQSVlZUhSRL79+/vl+vs2LGDKVOmoNfrWbBgQa+eM9n8wSChcBgZLlo16xFjpDfIJ7rwE5XQv4Nnn32WJUuWsHjxYgoLCzs9Z8iQIaLEk8K2bt3K9ddfT1NTE5mZmUm//ooVK5g+fTpvvvkmGRkJLLHRR2RZxulpwxwIYTEZk1J1d8lmfC0+pIYGcocmIcg0lFAieueddxg2bFiHgV6yLFNZWcmwYcMwGAwsWrQoKUGmO7/f3+U4llRVWlrKd77zHYqLixO+Rl/83rIMGlmplkla7XnHZUKhEDpd/B+JylYdrjo3FlMNuVOTFmpaSahqNnr0aOrr6zscb2xsZOTIkb0OKt1dd911PPjggyxfvpzc3FxuueUWDh06xK233kpGRgb5+fnce++97f4Gb731FldddRWZmZnk5OTwpS99idLS0k6vX1ZWxvXXXw9AVlYWkiSxePHiuK7z+eefc8UVV2AymZg8eTLbtm2LXVuSJBoaGliyZAmSJLFp0yYAtm3bxuzZszEajRQWFrJq1SqCwWC3v/fWrVuRJIktW7YwY8YMzGYzN9xwA7W1tbz55ptMnDgRu93O1772NVpbW2PXCofDrFu3jpEjR2I2m5k2bRovvfQSYVlGkkN8sPMjNNZc3nzzTS699FKMRiMffvhhQn8vnTk6zUPMwE9UQomoq8HYbre7V4Mb+4s/GO7yKxgK9/jcQA/OTdQf/vAHDAYDO3bs4IknnuCGG25gxowZ7N69m7feeouzZ89y9913x873eDysWLGC3bt38+6776LRaPjyl79MONwxhpKSEl5++WUAjh49SnV1NevXr4/rOt///vf5j//4D/bt28fcuXOZP38+DQ0NlJSUUF1djd1u56mnnqK6upqFCxdy5swZ5s2bx2WXXcaBAwd49tln+f3vf8/jjz/e5e/9m9/8Jnb8Bz/4AU8//TQ7d+6ksrKSu+++m6eeeooXXniBN954g7fffptf//rXsfPXrVvH888/z29+8xsOHz7MQw89xDe+8Q22btuKRg4pk12BVatW8cQTT3DkyBGmTk2sOKO3Rnd9FRstJiqucuiKFSsApUtzzZo17cbohEIhPvnkE6ZPn57UAPvCM++f6PK+kblWFsw4V9H/3+2lBEKdJ97iLDNfmVUS+3nDjlMd1i5+6AvjEopx7Nix/PSnPwXg8ccfZ8aMGfz4xz8+91wbNlBSUsKxY8cYN24c//Zv/9bu8Rs2bCAvL4/PPvuMyZMnt7tPq9WSnZ0NKG1557cR9fQ6Dz74YOzcZ599lrfeeovf//73PPzwwxQUFCBJEg6Hg4KCAgD+53/+h5KSEp5++mkkSWLChAlUVVXxyCOPsGbNmlg1//zfG4gNAXn88ce58sorAVi6dCmrV6+mtLQ0NtXorrvu4v333+eRRx7B5/Px4x//mH/+85+xCdSjRo3iww8/5Ln/fY7ZzzxJtGXoscce6/VwkejoavwtSt1PDBmJW1yJaN++fYBSIjp48GC7+rvBYGDatGmsXLkyuRGmqUsvvTT2/YEDB3j//fc7bfQtLS1l3LhxHD9+nDVr1vDJJ59QX18fK8FUVFR0SETd6el1zl8hQafTMWvWLI4cOdLldY8cOcLcuXPbNQ5feeWVuN1uTp8+zbBhwzr83uc7v7SSn5+PxWJpN98xPz8/NlTkxIkTtLa2dkgwfr+fadOnI8nnSnezZs3q+sXoIWNkBn4oGIRAKxjEsrHxiisRvf/++wDcf//9rF+/vttJbKls2fVjurxPc8E/s29dM7rLcy/8x7fkyuS1j1mt597Mbreb+fPn85Of/KTDedFey/nz5zN8+HCee+45ioqKCIfDTJ48Gb/fH9fzJus6iTr/9z7f+dMrJEnqMN1CkqRY0nS7lRHOb7zxBkOHtu/GCskSkhyO/e26er54mE0GAlozUsiL3OZCEokobgn1mm3cuDHZcfQrg67nTWN9dW48Zs6cycsvv8yIESM67dVpaGjg6NGjPPfcc1x99dUAF2147WzqQjzX+fjjj7nmmmsACAaD7NmzhwcffLDL55s4cSIvv/wysizHSkU7duzAZrP1qmetM5MmTcJoNFJRUcG1117b7r6A34fsqUebjPFDESadBr/Wij7kxd/qxGjvfEiL0LUeJ6I777yTTZs2YbfbufPOO7s995VXXul1YMI5y5Yt47nnnuOee+7h4YcfJjs7mxMnTrB582Z+97vfkZWVRU5ODv/7v/9LYWEhFRUVrFq1qttrDh8+HEmSeP3115k3bx5mszmu6zzzzDOMHTuWiRMn8stf/pKmpiaWLFnS5fN997vf5amnnuJ73/seDz74IEePHuXRRx9lxYoV3a73kwibzcbKlSt56KGHCIfDXHXVVTidTnbs2IHdYmLRV76EVpu8RKTTapg4cigWpw9tQMw3S0SP3wEOhyP2n8zhcHT7JSRXUVERO3bsIBQKcfPNNzNlyhSWL19OZmYmGo0GjUbD5s2b2bNnD5MnT+ahhx7iZz/7WbfXHDp0KGvXrmXVqlXk5+fz4IMPxnWdJ554gieeeIJp06bx4Ycf8re//Y3c3Nxun+8f//gH//rXv5g2bRrf+c53WLp0Kf/v//2/Xr02XfnhD3/If//3f7Nu3TomTpzIF7/4Rd544w1GDo90LiSxRAQwdEg+WRYDOpGIEpLQwmiportFl9ra2jh16hQjR44cEEMKhP4R9DQhBb1oTDakbhZEi+rx+6j8Izi5FQomw8T5yQt4gOvpwmgJlYm9Xm+7wWPl5eU89dRTvP3224lcThD6jdfnx+sPEZSTWyKqDxiobWmjxdmY1Oumi4QS0R133MHzzz8PQHNzM7Nnz+YXv/gFd9xxB88++2xSAxSEZJFlGcLKSG5NEtuIAA43hCmt89DU1JDU66aLhBLR3r17Y70qL730EgUFBZSXl/P888/zq1/9KqkBCkKyyLIcG0MkaZO7b4T2/GkeA7e1QzUJJaLW1lZsNqV+/fbbb3PnnXei0Wi4/PLLKS8vT2qAgpAscigyr03SIEnJ7anTRwc1BvwQ7HxHEKFrCf01xowZw2uvvUZlZSVbtmzh5ptvBpQVFwfqIEdh8AtHqmWyRpv0lTvNJiNBjYlgWBbbTycgoUS0Zs0aVq5cyYgRI5gzZ05suP/bb7/NjBkzkhqgICSLHB28meSuewCTXotfa1UmTYtEFLeEKsp33XUXV111FdXV1UybNi12/MYbb+TLX/5y0oIThGSSw9GqWfITkVmvxaezEgw1i5UaE5Bwi11BQUFsZnVUMrYTEoS+oiOEpNMg98Eic9ESUcAvSkSJSCgReTwennjiCd59911qa2s7rFVz8uTJpAQnCMmkJYxWqwF98hOR3axj+phhZNacFokoAQklogceeIBt27Zx7733UlhYKLbsEVKfLIPcd21ERp2WEUMLwGUQiSgBCSWiN998kzfeeCO2UJWQXNdddx3Tp0/nqaeeUjsUIPXiSUg4RCisLFomoendhn5diU4ZEYkobgn9PbKysmIr/Ampqb/WDxowwkF8wTAePx2W+E2WM606alva8Hma++T6g1lCieiHP/wha9asaTffTEiOxYsXs23bNtavX48kSUiSRGlpKUuXLo0tBD9+/PjYGtPnP27BggX86Ec/oqioiPHjxwOwc+dOpk+fjslkYtasWbz22msd9iXrbmH+zuIpKyvrr5cjeeSQsq+ipEXTR00JH1T4KK3zKJ+LoNj1NR4JVc1+8YtfUFpaSn5+PiNGjOiwWl7KbqwoyxAK9P/zavU9Xsd4/fr1HDt2jMmTJ/PYY48BSgm0uLiYF198kZycHHbu3Mm3vvUtCgsL2y2g/+6772K323nnnXcAZebz/PnzmTdvHi+88ALl5eUsX7683fM1Nzdzww038MADD/DLX/4Sr9fLI488wt133817773XaTx5eXlJeFH6lxwOIiMTlrQkeVB1jNFkJKgxKiUuXwvojH3zRINQQoko1Xbu7LFQAD74Rf8/79X/Abqe9dQ4HA4MBgMWi6Xd8Ii1a9fGvh85ciQfffQRf/nLX9olIqvVyu9+97vY6ou/+c1vkCSJ5557DpPJxKRJkzhz5gzf/OY3Y495+umnL7owf2fxDDRyKAgyyJJW2fO+D5ijgxrDbcpYImvX6zMJ7SWUiB599NFkxyFcxDPPPMOGDRuoqKjA6/Xi9/s77JgyZcqUdhsaHD16lKlTp7ZbR+fCsV49WZh/MJD7cHpHlEmvxau1Egx5oU0MaoxHwgMam5ubeemllygtLeX73/8+2dnZ7N27l/z8/A4LlqcMrV4pnajxvL2wefNmVq5cyS9+8Qvmzp2LzWbjZz/7GZ988km78xJZCL4nC/MPeLKMHI7s7qpJ7qz785n1Wpw6K8FQreg5i1NCf5VPP/2Um266CYfDQVlZGd/85jfJzs7mlVdeoaKiIrZWUcqRpB5XkdRkMBjaLWq/Y8cOrrjiCr773e/GjnW1i+v5xo8fzx//+Ed8Ph9Go9JesWvXrnbnXGxh/s7iGXDCoUi1TEJK8vrY5zPptfi0VgJBMfE1Xgn9VVasWMHixYs5fvx4u2L/vHnz2L59e9KCS1cjRozgk08+oaysjPr6esaOHcvu3bvZsmULx44d47//+787JJTOfO1rXyMcDvOtb32LI0eOsGXLFn7+858DxKony5Yto7GxkXvuuYddu3ZRWlrKli1buP/++2PJ58J4Ots9NqWFg2gkMOj1mPXJH8wYZTZo8WszxMTXBCSUiHbt2sW3v/3tDseHDh1KTU1Nr4NKdytXrkSr1TJp0iTy8vK45ZZbuPPOO1m4cCFz5syhoaGhXemoK3a7nb///e/s37+f6dOn81//9V+sWbMGIPYP5GIL83cWT0VFRd/98n1BDqGRJIwGA2ZD31XNChwmLp84nJJsi5j4Gi85AXl5efLevXtlWZbljIwMubS0VJZlWX777bfl4uLiHl/nxz/+sTxr1iw5IyNDzsvLk++44w75888/7/HjnU6nDMhOp7PDfV6vV/7ss89kr9fb4+ulgz/+8Y+yXq+XW1tb1Q6l/3ibZdlZJcveju+Tiz403veRu06W3/uxLG//RdzPNRh19xk9X0Ilottvv53HHnuMQEAZkyNJEhUVFTzyyCMd9k7vzrZt21i2bBkff/wx77zzDoFAgJtvvhmPx5NIWEInnn/+eT788ENOnTrFa6+9FhsjZDab1Q6t/4RDhGSZIFrCfb2Ma3SaR9AHQTG6vacSHtB41113kZeXh9fr5dprr6Wmpoa5c+fyox/9qMfXeeutt9r9vGnTJoYMGcKePXtiu4gKvVNTU8OaNWuoqamhsLCQr3zlK3H9jQaFcBB/MIxbDmPThjH1UTuRLMucbAqQ0Rom1yyh9bWALqdPnmuwSSgRORwO3nnnHXbs2MGBAwdwu93MnDmTm266qVfBOJ1OgC7nsfl8Pny+c0PnXS5RD7+Yhx9+mIcffljtMNQTmXUvRwYz9tX0DlBqBm8dqmFig8zcAhmzzwVWkYh6Iu5EFA6H2bRpE6+88gplZWVIksTIkSMpKChot695Itddvnw5V155JZMnT+70nHXr1rUbYSwIFxXpug8DMho0fbxijbJAWgbBsEv0nMUhrjYiWZa5/fbbeeCBBzhz5gxTpkzhkksuoby8nMWLF/dqmdhly5Zx6NAhNm/e3OU5q1evxul0xr4qKyt7FLOQxsJBZCCMFiTiLhHF+/4xR8YSBUNiLFE84ioRbdq0ie3bt/Puu+9y/fXXt7vvvffeY8GCBTz//PPcd999cQXx4IMP8vrrr7N9+3aKi4u7PM9oNMYG5l1MdCJua2trejXMCu3JochkV+WtHm+BPbqciraHGzKaDRplyVgxligucSWiP//5z/znf/5nhyQEcMMNN7Bq1Sr+9Kc/9TgRybLM9773PV599VW2bt3KyJEj4wmnW1qtlszMTGprawGwWCxiJcl05Gsl5A/SFpII48cXx+oc4XCYuro6LBZLl6POL2TWa2nQiRJRvOJKRJ9++ik//elPu7z/1ltvjWun12XLlvHCCy/w17/+FZvNFhsM6XA4klKKic4WjyYjIQ353YRDQVplI5JWj9MYX7OoRqNh2LBhPf4nZjbozltEX3Sm9FRcf5XGxkby8/O7vD8/P5+mpqYeX+/ZZ58FlKVIz7dx40YWL14cT2idkiSJwsJChgwZEhvzJKSZvc/T1uqhrPAWNBl5jMy3xfVwg8EQG2HeExZDZDcPUSKKS1yJKBQKdVtE1Wq1BIPBHl+vvxqStVptj+v4wiDibwVfAyYtTB8/pl8mPI/MtWKbOprCwyYIeJU1sHq5+kI6iCsRybLM4sWLu2ww9sVTAReEvuZtVG5N9n5bdSE3w0iuNQ9OmZUk5GsBi1jf/WLiSkSLFi266Dnx9pgJQp9pbQCgGRs+VxtZFgMGXd8tAxIjSWC0K88vElGPxJWINm7c2FdxCELyeZQNAPbUa/j0kwq+PGMoI3LjXzwuHqGwTHmDB0OrlqHISKKdqEf64d+DIKikVamaObEDYI2zxyxRf91fxd6zIdGFHweRiITBq7WBsCzThNJTZjX2fYeFViNFpnmIQY3xEIlIGJxCQWhrJhAK49VnodVIfbo64/msxvO78MVYop4QiUgYnLxNIMv4MBDQmLEY+m73jguZ9Vp8OlEiiodIRMLg1Ko0VLfpM0GSyOin9iFQ2qJE1Sw+IhEJg5Nbmdbj0WUB/ddQDecW0Q+EZPB7lKVIhG71319HEPqTpw6AzCFDucaYh8PcjyUig46gxogvHKkK+lxgzuq35x+IRCISBqdIIsrKK+LSrP5NAqPyrNhMOgqO5oHsVqpnIhF1SyQiYfAJ+sHbrHxvHdLvT5+bYSQ3wwg12dDsFu1EPSDaiITBJ1IawmClogXOutqUTQ/7W3RHD5GILkokImHwiSaijCH8/dMqXvikAqe3/5aBCYVlSuvcnHJrkBGjq3tCVM2EwSeSiPzGbPxBpSRkM/XfUhyyLPO3/VXktwQoccjoxKDGixIlImHwiSQij0GZ9W7Sa/tn1n2ETqvBoNOIBdLiIBKRMLjIcmwMUYtG6amymfq/4J8RGdToD4pBjT0hEpEwuPhalJURJQ3NGmXWvRqJyGrU4ddZ8YfCYlBjD4hEJAwuLcoGDFhzaPErAwrt/dg+FJVh1BLQmPGHJaWU1ubs9xgGEpGIhMHFHUlEGQW0tCk9ZRkqlYiQJDyaSBe+SETdEr1mwuDScla5tRVyicVBns1EcVb/b7AZnWTrkayAD9qa+z2GgUQkImFwaalWbm35lDgslGRbVAljeI6VL07WUnh2KDQ1nhvpLXRKJCJh8PC1KA3DkgQZXe+/1x+yrQayrQYIDoEmRNXsIkQbkTB4RKtllhx8soYTtS3UutrUjcmUqdyKqlm3RCISBo9YtayAerefvx+o5u+fVqsWzsk6N581QTAcFlWzixBVM2HwiHbd2wpxtio9Zg6zerus/vPIWdq8fobLYXQaLwR9oOt8c9J0J0pEwuAgy+A6rXxvL6LZ6wcgU8VEZDXqCGkMtEmR5CPaibokEpEwOLQ2QqANtDrIyMcVmW3vsKiXiKJd+F5thnJAVM+6JBKRMDhES0O2ItBoaU6BqllsLJEmkohEg3WXRCISBgfnGeXWMVT5MVIiUrtqBuCWoolIVM26IhKRMDi4IonIXowvGKLVr0wytadAicglRwZViqpZl0SvmTDwBbzgUfYxw16ERpKYP60QV1sQUz/t7tqZ6Kz/JjkDJMDbqFosqU7VEtH27duZP38+RUVFSJLEa6+9pmY4wkAVrZZZcsBgQa/VMGaIjZnD1N05Y4jNxBcnF3DFlHHKAW+zWA6kC6omIo/Hw7Rp03jmmWfUDEMY6JrLlVtHsbpxXMBs0DKx0E5BXp7SmyeHRTtRF1Stmt16663ceuutaoYgDAZNZcpt1nBAGdEclmWKMs1YDCnQ+iBJYM5WVo5sbQRLttoRpZwU+Cv1nM/nw+fzxX52udJ0UfLWRih9D5orwGCFobNg6EzlDZ9u/K2xpWHJVBLRrrJGqprbmDelkPEFNhWDg2qnl1qXjxGSHQe1op2oCwOq12zdunU4HI7YV0lJidoh9T9XNezZBPXHlSkDrY1w/G34/A1ldHG6aa5Qbq25YMxAlmUaPMqo6myrQcXAFPsrmnnv81rOBiJrIrWKRNSZAZWIVq9ejdPpjH1VVlaqHVL/Cnjh0MtKArIXwcz7YMxNIGmg5iBUfKx2hP0vVi0bAYDHH8IXCCNJkKXiqOqo6PCBZiIlM1Ei6tSAqpoZjUaMxjSeNHjin8qaO5ZsqobfQVl9EL12NBNKZGwV70LZB5A7Dqw5akfaP2T5XCKKVMuaIqUhh1mPTqv+/9noetlN0UTU2qBiNKlrQCWitNZSAzWHkGWZD7SXs2dfbeyuj6VMvpk1ApOrTElW0xaqF2d/am0EbxNotLGG6lSqlgHYzcpHrCEUGdToc0PQD7rUiC9VqPovw+12s3//fvbv3w/AqVOn2L9/PxUVFWqGlZpObgPgQGAoe5rMSBKMy7cxLNvClePyME36olJFazwJzWlSZW04odxmDostr9HoUTozcqypUXKO7jDbFNAh6yPJSJSKOlC1RLR7926uv/762M8rVqwAYNGiRWzatEmlqFKQqxoaT1LrDrBTOwlJC/OnFTE6T2mcBZQes8KpULUfyj6E6feoGnK/iCainDGxQ/UtqVUiio6u9gfDBGzZGAKt0FoP9kKVI0stqiai66677twHSejamT3IssyhQCE+k4O5o3IYnadMpJTO67J35V9G3ac7GSmfQtNyFmzqrtvcpwJecEZm3OeMjh3+wqR8alt8DFVh547O6LUabCYdLW1B3Lossjl9briBECPaiFKd3wO1nyFJEpdffTNGp5XLRnQcECfLMq9+5iYnWIjZWUPRmT0wYZ4KAfeThlJlpLI1F8znpnJkWQ1kpUhpKOrGifkYdBpsLW5oOnhuXpwQo363gtC96gPK/CR7IZa84Vw5JhetpuPARUmSmDUiixrbZM40ewlUH1QG+w1WtUeU27zx6sbRAyNzrQzNNKO3D1EOeOrUDSgFiUSUymQZag4SCIWhaOZFT59YYMeYMwynLofqRrcytmgwCnih6ZTy/ZBJscNHa1rYXdZIg9vXxQNVZs1Tbn0tyu8gxIhElMpaqgm669l7xs2rlRm0+oPdnq7RSFwxJpezGROpcbUROLN/cI62rj+mlBIz8pSqWcThKicfHK+nqlnlLYQu4PWH+PR0M7tPe8DkUA6KUlE7IhGlsppD1Lv91BqG4wxImHQXX1tnVK4V8iYQkLXU1pwBV1U/BNrPzn6m3J5XGpJlmZrIHmZD7KnRdR/VFgjx7pFaPj7ZgBxNnCIRtSMSUaoKh6D2M2pb2qizjmVaSSaaTtqGLiRJEjNHF9JgGUm100uwan/fx9qfvE3KaGpJapeIGjx+fIEwBp2GvIzUSkQOsx6NJBEIybQaIh0NbpGIzicSUapqPInH3UJz0IDbUsyEAnuPHzo+30Y4fwr5dpPSqBsK9GGg/az6gHKbNQLMmbHDVc1Km0uB3dSjhN2fNBqJzMi8N5cmmohqVIwo9YhElKpqDlLb0ka9ZQxj8u2YDT1f8lSjkbjj2sspKSpCFw5A3ed9GGg/Coeg+lPl+8Lp7e6KJqLCTFM/B9Uz0URUrzmvRCRWa4wRiSgVBbyE6o5T7/ZTZx3L5CJH3JeQNBoomKL8EP3wDnR1R5VxVQYr5I5td9eZSAP10MzUGMh4oeiUk1q/CfQmCAdFO9F5RCJKRXWf0+j24tJmobMXUJKd2IdLzp9MkzfAyROfIQ/0dXBkGSo/Ub4vmq5MdI3w+IK0BUJoJIkCR2qWiHJtyiDLeo8fbJHpHYOxIyFBIhGloppDZFsNjJ4yhyvG5LabxhEPn87GLqeDsy4ftcd2JTnIftZcrqxAoNUpK1Kex2rU8e1rRvHV2SUYe9CzqIZoA3q924+cUaAcbBHtRFFiikeq8TaB8zRajYYREy8DU88bqS9k0mtxjJ5F+GAltcf+Rf7UL4BmgP7vKf9IuS2YBgZLh7t1Wo3SOJ+isiwG/m1mMXk2I5Iz0jbUUq1uUClkgL4rB7Gzh5XbzOG9SkJREy65lKDWjLvFSUPlZ72+nioaTypd9hotlMxud1c4LA+IidMajcSwHIvS6RCtmnnqlNU2BZGIUoosE645xOEqJ4dDJcrUjl5yWE2YiqcCUHn4o15fr9/JMpzcqnxfNLNdlz3AyXo3G3eUsbeiqd9DS5jJrvwesnxuBYE0JxJRKmmpprn+LM0+2OnKQZOkXTlGTLkCgLaaYzibBlijdc2n0HJWWdFw+NwOd39e04LTG8Dj6376SypobvXz4fF6PjxeryzmBucW/09zIhGlkppD1La00WgewYTizmfZJyIvfyiGnGHIssyJT3cm5Zr9wu9Rtk0CGH6V0m1/nrZAiFN1HgDVtw3qibZAmF1ljRw840SObgYpEhEgElHqCIfwVR2kuTWQ8Nih7hRPmotZr2G47+jAGEgny3D8HQi0QcYQKL6swymf17QQDMvkZBhSblpHZ/JsRnQaibZAiGZDpJ2opUZZwzrNiUSUKhpKqWtsxq81k1E4JumLe+WPnsHUUcXk6v0DY6R19QFleoqkgfG3dujtk2WZ/ZF2oanFmQkPcehPWo0U69k74zUobUVyGJxpssZ4N0QiShFy1T5qW3zUWccxpTjr4g+Il1aHpjiyplHlJ4ST0BDeZ9y1cOId5fuR1yh7uF2gtM5DU2sAo17DpMLe9y72l+gUlBqXD7IjS9w2lKoYUWoQiSgVeJtpPnMUXzBMc+YkxgzJ6JvnKZqJrNFSc6acv277CH8wBZNRmws+/QuEgpA9CoZd3uEUWZb5+KSyE8bUoZkYdAPnbVzoUEbJn2n2nlv0v+HE4Fw3Kg4D5y84mFUfwKiVsOSPZszwYej7amNAg4XgkClUNXuxVn3Eh8dTbBF3vwc+/T9lBUNrLky6XVnu4wKSJDFvSiETCmzMGtEHpcc+VJxlRiNJNHr8OE1FoNFBmzPt17EWiUhtoSBUH8Bi0DFt9vVcMzb34o/pBf3IKxmV78Dmq6H8+CEOnXH26fP1WJsL9v1J+UAaM2DKV0Df9Ry7bKuBW6cUYtKn5pSOrpj0WgodJiwGLS4/sa2yqT+mZliqE1M81Hb2kFISMNogd2zfN7qa7GSOuZzi1q20Nn/C+0eKMem1CVcH3b4gZfUeGj1+fMEwwZCyOJlZryXLaqAo04zDfJE96J1n4PAryi6oRhtM/1qHgYugLLl6uqmVsfmp31XfnS9NK8Ss1yp/a994pWp29jAMv6LTEmA6EIlITbKM79RHVNV7yJl2NXZNP/13HzaX4upPaQvUUe88yOufarh6bB4zh12898npDaDVSGQYI1spu32889nZLs+fOzqHy0flABAKy/iD4XNrK4XDcGYPnHxfGVJgyYGpd3eahFraAvztQBW1Lh83TQwzpTi5wxv6k8Vw3scubzwc36Ls/tpS3WnDfDoQiUhN9ceoqj7DGXeY3Y0FfKW/ntdgQRpzI2MCr0PDQd71lfDBcRiTl4EjsoCXLMvIMrT4gtS7fVQ2tlLe0Eqjx8+ckdlcMUapQuZkGBmeYyHbasCs16LTavAHw7T6gzS4/RScNxG1vMHD3w5UUeQwM97qYXTzDqy+s0hIkDcOJnwptnV0VDgsc6TGxYfH62n1hzAbtBSl6OJn8QqHZdrCWiy545R1uGsOikQk9LNwGO+x96h1tVFjm8Gcsf38BiyYglR7hDFSKRb/RxzN/7dYEgJ47oOTeHwdBz5qJIlW/7njGUYdd84s7tFTnnX5sLTVYa3bT2vrKQ4CeoOJ0MjrMNhmMiGsI9oqVNnYyok6N6W1blralOkbuTYjt08tahfnQHWyzs0/j5ylwGHm9pIp5xLRyGuVhdPSjEhEaqk5wJkzp/FLJqRhcxJe/CxhkgQT5yPt2chQXAwNbIXgQtAZCYbCsSSk1UhkWQ0U2k2MyLVQnGWJv4G4zQUNx5nb9CkzjGdoDgVowsBJzXDK7Jfh92bAsXpG59swo1z7s2oXn1W5ADAbtMwansWMYVlJm/aiNrtZj8cXoqzeg3fCSMzWXKWhvmpfp3PqBjuRiNQQ8NL82XvUtfg4nTWX6yYMVWdksMECk++CAy8oDcZ7n4dL7kRryeab14xCK0kYdJr4PvyyDG3NyuqDrmpwViiTViNMBgMFQ6dQMOwKRptyqGhspdbVRlNrAJvx3NuxwG7CoNVQnGVmZK4VXV8NaVBJboaRfLuJs6429p92MnfY5XDkdTi9C4ZeqkzyTSMiEakgeOwdyqtr8eozGTLu8tggN1XY8mHaPcogQk897P49UuF0MgqnQkZ+5704sqz09Plc0NoI3sb2txfuGiJJSttH7nhlHe3IwmYGYMyQjE577KaVZCb/d00xs0Zk8can1Rw43cylcydgMH8I3mao2AmjrlM7vH4lElF/O3uYs8d20+oPU1V8PXeNL1A7IrAVwKwlcPQfynSDM3uUL50RzFmgMylzosIB8LeC3939xFmNVpmoah+qLAKWPbLDzHkh0jlg1uP0BthT6WLu6Bvh0MtQ+S/In9xuF9vBTiSi/uSqhqP/oNBhoi57JtdMnpE6A/KMGUrXeeMppZ2ioVRZPbCrdZUlSUku5iyl292cDZZs5dac2W5xe6FzGo3E1WNzef3TanaVNTJhznCyskcpK1IefhUuXQzagd8w3xMiEfUXVzXhA5vRhIJockYzfcrtqbl+dPZI5SscUqpqvhYItimJRaNXenSMNjBkiGSTBGOGZDAi10J5QytVrjayJtwGuzcor/3h1+CSLysbBgxyKfFJeOaZZxgxYgQmk4k5c+bwr3/9S+2QkkeWCZzex6l3fsPxM3XI9iK4ZEFqJqHzabRK+1HuGCiYDEMmKt87isHkEEkoSSRJ4qaJ+SyYPpRLihxKyTSafBpORObeudUOs8+p/mn4v//7P1asWMGjjz7K3r17mTZtGrfccgu1tSk2ITNeskyw7gSn33+OQ+/9mZpmDydD+Zwuub3DoD0hvdlMekbknmtDq5ZyKSu6DVmrV1Zw3PUcVHw8qBfal2SVt0CYM2cOl112GU8//TQA4XCYkpISvve977Fq1apuH+tyuXA4HDidTux2ldakkWUI+SHQCj433uYaqk6fwnf2BG5XE6GwTFjS0Zh3GZPm3MzoIQN7npTQt4KhMH/6pIJGj58CnZtZrR+QIzdh0msxGAxoc0YhZZaANQ9MmUoJSmtI2TlqPf2Mqlr59Pv97Nmzh9WrV8eOaTQabrrpJj76qOOOEz6fD5/v3H8Fl8vVsydqPAWl77Zb88XjC3D4jBM4fzsaGWQZGRjqMFGSpXSrewNBDlQ2R04Jx64jA9pwgEK7npKsyF5bgRC1kXNDGj3u7InkX3INs0al7uZ/QuqQgdF5Gbh9zdQEM3hdfwt5nuMUNR7AHKjDUd3MpMITsfMPnG4mLIOs1SNLemRJg4wEkoTNpGdCoUNZ5RLYU95MMNx+DapMq4Hxia5/NXVhUra8ApUTUX19PaFQiPz8/HbH8/Pz+fzzjsuZrlu3jrVr18b/REEfuC/YZ9wfJOTpegkM2eiHyM4QUiCE3EU9PQT4g1plXRmDFWNWNpKkxZg7ioLhYynIsg2IZUyF1KDXarhqbC6XjcyirL6V8gYP9e5pHPFMQO+t4xJLE+S1KeO1fC7aAiHCMhAIAW3trhUO66H13DCLsLuRcLh9BUgO68Ha/nE9JidvYb0B1Ry/evVqVqxYEfvZ5XJRUlJy8QdmlsC0r55XfJXQh2SGjfMhRUYNS5IEkgYJkDSSMkPaqAdJQheWGTslgCyBFD0nci1JZ8RotoJZKT1pgI5rCgpCfIw6LeMLbO12JwmERhMKy3DekI9hl7iQg37koE8ZSCqHkZBBDmPQSpBhUErwcpihI9o6LARp1EtgTbDNMoljw1RNRLm5uWi1Ws6ebb+MxNmzZyko6DjQz2g0YjQm8KIZrEqX9PmHgKE9HC+mA3IH7qoTwiCh12q4cNhZYXbPq0ZF2UkOKIlU7TUzGAxceumlvPvuu7Fj4XCYd999l7lz02/inyCkK9WrZitWrGDRokXMmjWL2bNn89RTT+HxeLj//vvVDk0QhH6ieiJauHAhdXV1rFmzhpqaGqZPn85bb73VoQFbEITBS/VxRL2REuOIBEHoUk8/o6qPrBYEQRCJSBAE1YlEJAiC6lRvrO6NaPNWj6d6CILQr6KfzYs1RQ/oRNTS0gLQs9HVgiCopqWlBYej61HBA7rXLBwOU1VVhc2WnvO5olNcKisrRa9hD4jXK369fc1kWaalpYWioiI03azBNaBLRBqNhuLinu2pNZjZ7XbxwYqDeL3i15vXrLuSUJRorBYEQXUiEQmCoDqRiAYwo9HIo48+mtiKBGlIvF7x66/XbEA3VguCMDiIEpEgCKoTiUgQBNWJRCQIgupEIhIEQXUiEaWweHfAffHFF5kwYQImk4kpU6bwj3/8o58iTR3xvGabNm1CkqR2XyaTqR+jVdf27duZP38+RUVFSJLEa6+9dtHHbN26lZkzZ2I0GhkzZgybNm1KSiwiEaWoeHfA3blzJ/fccw9Lly5l3759LFiwgAULFnDo0KF+jlw9iewabLfbqa6ujn2Vl5f3Y8Tq8ng8TJs2jWeeeaZH5586dYrbbruN66+/nv3797N8+XIeeOABtmzZ0vtgZCElzZ49W162bFns51AoJBcVFcnr1q3r9Py7775bvu2229odmzNnjvztb3+7T+NMJfG+Zhs3bpQdDkc/RZfaAPnVV1/t9pyHH35YvuSSS9odW7hwoXzLLbf0+vlFiSgFRXfAvemmm2LHutsBF+Cjjz5qdz7ALbfc0uX5g00irxmA2+1m+PDhlJSUcMcdd3D48OH+CHdA6sv3mEhEKai7HXBramo6fUxNTU1c5w82ibxm48ePZ8OGDfz1r3/lj3/8I+FwmCuuuILTp0/3R8gDTlfvMZfLhdfr7dW1B/Tse0Hojblz57bbP++KK65g4sSJ/Pa3v+WHP/yhipGlH1EiSkHx7oALUFBQENf5g00ir9mF9Ho9M2bM4MSJE30R4oDX1XvMbrdjjmy5niiRiFJQIjvgzp07t935AO+8807a7JibjF2DQ6EQBw8epLCwsK/CHND69D3W6+ZuoU9s3rxZNhqN8qZNm+TPPvtM/ta3viVnZmbKNTU1sizL8r333iuvWrUqdv6OHTtknU4n//znP5ePHDkiP/roo7Jer5cPHjyo1q/Q7+J9zdauXStv2bJFLi0tlffs2SN/9atflU0mk3z48GG1foV+1dLSIu/bt0/et2+fDMhPPvmkvG/fPrm8vFyWZVletWqVfO+998bOP3nypGyxWOTvf//78pEjR+RnnnlG1mq18ltvvdXrWEQiSmG//vWv5WHDhskGg0GePXu2/PHHH8fuu/baa+VFixa1O/8vf/mLPG7cONlgMMiXXHKJ/MYbb/RzxOqL5zVbvnx57Nz8/Hx53rx58t69e1WIWh3vv/++DHT4ir5GixYtkq+99toOj5k+fbpsMBjkUaNGyRs3bkxKLGIZEEEQVCfaiARBUJ1IRIIgqE4kIkEQVCcSkSAIqhOJSBAE1YlEJAiC6kQiEgRBdSIRCQPKpk2byMzMjP38gx/8gOnTp8d+Xrx4MQsWLOj3uITeEYlI6NTixYuRJInvfOc7He5btmwZkiSxePHiducnOwGMGDGCp556qt2xhQsXcuzYsS4fs379+nbLl1533XUsX748qXEJyScSkdClkpISNm/e3G6tmba2Nl544QWGDRumSkxms5khQ4Z0eb/D4WhXYhIGBpGIhC7NnDmTkpISXnnlldixV155hWHDhjFjxoxeXbuzksqCBQtipazrrruO8vJyHnroodjC9tCxanah80tmixcvZtu2baxfvz52jVOnTjFmzBh+/vOft3vc/v37kSRJLAGiEpGIhG4tWbKEjRs3xn7esGED999/f58/7yuvvEJxcTGPPfZYbGH7eK1fv565c+fyzW9+M3aNYcOGdfidADZu3Mg111zDmDFjkvUrCHEQiUjo1je+8Q0+/PBDysvLKS8vZ8eOHXzjG9/o8+fNzs5Gq9Vis9koKChIaIE3h8OBwWDAYrHErqHValm8eDFHjx6NbTUUCAR44YUXWLJkSbJ/DaGHxFKxQrfy8vK47bbb2LRpE7Isc9ttt5Gbm6t2WL1SVFTEbbfdxoYNG5g9ezZ///vf8fl8fOUrX1E7tLQlSkTCRS1ZsoRNmzbxhz/8IWmlBo1Gw4Ur0AQCgaRcuyceeOCBWEP8xo0bWbhwIRaLpd+eX2hPJCLhor74xS/i9/sJBALccsstSblmXl5eu3afUCjUYTNIg8FAKBTq1fN0dY158+ZhtVp59tlneeutt0S1TGWiaiZclFar5ciRI7Hvu+J0Otm/f3+7Yzk5OZSUlHQ494YbbmDFihW88cYbjB49mieffJLm5uZ254wYMYLt27fz1a9+FaPRmFCVcMSIEXzyySeUlZWRkZFBdnY2Go0m1la0evVqxo4dmzZre6cqUSISesRut2O327s9Z+vWrcyYMaPd19q1azs9d8mSJSxatIj77ruPa6+9llGjRnH99de3O+exxx6jrKyM0aNHk5eXl1DcK1euRKvVMmnSJPLy8qioqIjdt3TpUvx+f7/0AgrdE0vFCmnrgw8+4MYbb6SysrLDxoFC/xKJSEg7Pp+Puro6Fi1aREFBAX/605/UDintiaqZkHb+/Oc/M3z4cJqbm/npT3+qdjgCokQkCEIKECUiQRBUJxKRIAiqE4lIEATViUQkCILqRCISBEF1IhEJgqA6kYgEQVCdSESCIKhOJCJBEFT3/wNB7T19ZSho9gAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:26:43.302416Z", + "iopub.status.busy": "2024-03-26T15:26:43.302072Z", + "iopub.status.idle": "2024-03-26T15:26:43.558991Z", + "shell.execute_reply": "2024-03-26T15:26:43.557981Z" + }, + "papermill": { + "duration": 0.27676, + "end_time": "2024-03-26T15:26:43.561234", + "exception": false, + "start_time": "2024-03-26T15:26:43.284474", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+cElEQVR4nO3de1wU9f4/8Ndy2QUUVERAEVkvpXa8oBiEpnjhUpTpsToezSQsO6F0cQ9qdDpyzL6SilwqkzRRq+OlxNtRUlYERYU0TI8akKiIJncVFHBvfH5/8Ns5rCywKDC7s+/n48FDZ+Yzs+9hZt98ZuYzn4+IMcZACCEmxoLvAAgh5FFQ8iKEmCRKXoQQk0TJixBikih5EUJMEiUvQohJouRFCDFJlLwIISbJiu8AOlt9fT1u3boFe3t7iEQivsMhhDyEMYZ79+6hT58+sLBovn5ldsnr1q1bcHd35zsMQkgrbty4gb59+za73OySl729PYCGX4yDgwPP0XQclUqF1NRUBAYGwtramu9wSDswl2NaXV0Nd3d37rvaHLNLXtpLRQcHB8EnLzs7Ozg4OAj6RDcn5nZMW7utYxQ37NetWwepVAobGxv4+Pjg9OnTzZadOHEiRCJRk58XXnihEyMmhPCN9+S1c+dOyGQyREVF4ezZsxg5ciSCgoJQVlamt/zu3btRXFzM/Vy8eBGWlpZ49dVXOzlyQgifeE9esbGxmD9/PkJDQ/HUU08hMTERdnZ2SEpK0lve0dERrq6u3I9cLoednR0lL0LMDK/3vJRKJXJychAZGcnNs7CwgL+/P7KysgzaxqZNm/DXv/4VXbp00btcoVBAoVBw09XV1QAa7h+oVKrHiN64afdNyPtobszlmBq6f7wmr4qKCmg0Gri4uOjMd3FxQV5eXqvrnz59GhcvXsSmTZuaLRMdHY3ly5c3mZ+amgo7O7u2B21i5HI53yGQdib0Y1pbW2tQOZN+2rhp0yYMHz4c3t7ezZaJjIyETCbjprWPYQMDAwX/tFEulyMgIMAsnkyZA3M5ptqro9bwmrycnJxgaWmJ0tJSnfmlpaVwdXVtcd2amhrs2LEDn3zySYvlJBIJJBJJk/nW1taCPgG0zGU/haa2trbJ1cf9OgVOXbiCHk7O6Gr7v3N6yJAhgrqKMPR85TV5icVieHl5IS0tDdOnTwfQ8PpOWloawsPDW1z3xx9/hEKhwJw5czohUkI6V15eHry8vPQuW/3QdE5ODkaPHt3xQRkZ3i8bZTIZQkJCMGbMGHh7eyM+Ph41NTUIDQ0FAMydOxdubm6Ijo7WWW/Tpk2YPn06evbsyUfYhHSoIUOGICcnR2defvFdyH68gNhXh2Nw7+46Zc0R78lr5syZKC8vx7Jly1BSUgJPT08cOnSIu4lfVFTU5OXM/Px8nDhxAqmpqXyETEiHs7Oza1KbsrheCUlmHYYOGwlPD/qjzXvyAoDw8PBmLxMzMjKazBs8eDBoxDZCzBvvjVQJIeRRUPIihJgkSl6EEJNEyYsQYpIoeRFCTBIlL0KISaLkRQgxSZS8CCEmiZIXIcQkUfIihJgkSl6EEJNEyYsQYpIoeRFCTJJR9CpBHt/DPW/+r9fNX3R63QSE1/MmMU+UvASiuZ43H+51EzDfnjeJsFDyEoiHe95srtdNbVlCTB0lL4F4uOdN6nWTCB3dsCeEmCRKXoQQk8R78lq3bh2kUilsbGzg4+OD06dPt1j+7t27WLhwIXr37g2JRIInn3wSKSkpnRQtIcRY8HrPa+fOnZDJZEhMTISPjw/i4+MRFBSE/Px8ODs7NymvVCoREBAAZ2dn7Nq1C25ubrh+/Tq6d+/e+cETQnjFa/KKjY3F/PnzuTEaExMTcfDgQSQlJeHDDz9sUj4pKQm3b9/GqVOnuFF1pVJpZ4ZMCDESvCUvpVKJnJwcREZGcvMsLCzg7++PrKwsvevs378fvr6+WLhwIfbt24devXph9uzZWLp0KSwtLTsrdEI6xLWKGtQo1M0uv1Jew/1rZaX/q9tFYoX+Tl06JD5jw1vyqqiogEaj4QaX1XJxcdFpKd7Y1atXcfToUbz22mtISUlBQUEBFixYAJVKhaioKL3rKBQKKBQKbrq6uhoAoFKpoFKp2mlvjI9areb+FfJ+CkVhZQ0C4k8aVPbvuy60uFz+wThIe5puAjP0fDWpdl719fVwdnbGhg0bYGlpCS8vL/zxxx9Ys2ZNs8krOjoay5cvbzI/NTVV0K/I3LgPAFbIzs7GHxf5joa0Rnu8Xh+kgYut/gGVVfXAbQXgKAGs9TxqK60T4bsCSxxOOwb3rh0aboeqra01qBxvycvJyQmWlpYoLS3VmV9aWgpXV1e96/Tu3RvW1tY6l4hDhw5FSUkJlEolxGJxk3UiIyMhk8m46erqari7uyMwMBAODg7ttDf8KKysQY1Co3eZoqQKuJAL50HD4eHardltdJFYmvRfaaG4dKsaMRey8XLAOPypj/7zUqVSQS6XIyAggLvn+/A2vivIxrPPPtvsNkyB9uqoNbwlL7FYDC8vL6SlpWH69OkAGmpWaWlpCA8P17vOuHHjsG3bNtTX18PCouFPz++//47evXvrTVwAIJFIIJFImsy3trbWewKYimsVhl1mLNmT22qZ9IiJZnOfxFhp72FZWVm1el42d+62ZRvGzNDYeb1slMlkCAkJwZgxY+Dt7Y34+HjU1NRwTx/nzp0LNzc3REdHAwDCwsLw5Zdf4v3338e7776Ly5cvY+XKlXjvvff43A1eaG/sxs/0xCDnptcINXUKHMjIwosTfdHFtmnyBoCCsvv4YOe5Fm8SE2KseE1eM2fORHl5OZYtW4aSkhJ4enri0KFD3E38oqIiroYFAO7u7jh8+DAWLVqEESNGwM3NDe+//z6WLl3K1y7wbpBzVwxza3pZqFKpUNILGO3Rw6T/ChPSHN5v2IeHhzd7mZiRkdFknq+vL7Kzszs4KkKIseP99SBCCHkUvNe8yKNRaB7AwuYPXKvOh4VN03tearUat9S3kHs7t9kGjdeq78PC5g8oNA8ANP9EkhBjRMnLRN2quY4u/b/ARy2/x46vDn3V4vIu/YFbNZ7wgkuL5QgxNpS8TFSfLh6oufYuEmZ6YqCep41qtRonT5zEuGfHNVvzulJ2H+/vPIc+kzw6OlzSitZq0kDrtWlzq0lT8jJREksb1D9wQ3+HwXiqp/6njdesrmGo49BmnzbWP6hC/YNySCxtOjpc0gpDa9JAy7Vpc6pJU/IixAi0VpMGWq9Nm1tNmpIXIUagtZo00Hpt2txq0tRUghBikih5EUJMEiUvQohJouRFCDFJlLwIISaJkhchxCRR8iKEmCRKXoQQk0TJixBikih5EUJMEiUvQohJoncbTVSdqmHIs4t/VOldXlOnwC/lgOv1Oy0OwEGIqaLkZaKu/P/E8+HulkZPtsJ3BWda3VYXCZ0GxPQYxVm7bt06rFmzBiUlJRg5ciS++OILeHt76y27ZcsWbmg0LYlEggcPHnRGqEYj8E8NA/MOdO4KW2vLJsvzi6vw910XsPaV4Rjcu6VBZ61ozEZiknhPXjt37oRMJkNiYiJ8fHwQHx+PoKAg5Ofnw9nZWe86Dg4OyM/P56ZFIlFnhWs0HLuI8Vfvfs0uV6sbxmIc2KuL3qHRCDF1vN+wj42Nxfz58xEaGoqnnnoKiYmJsLOzQ1JSUrPriEQiuLq6cj/acR4JIeaD15qXUqlETk4OIiMjuXkWFhbw9/dHVlZWs+vdv38fHh4eqK+vx+jRo7Fy5Ur86U9/0ltWoVBAoVBw09XV1QAaOnZTqVTttCfGR1vzUqvVgt5PoTDkeGnnN7dcKMfc0Nh5TV4VFRXQaDRNak4uLi7Iy8vTu87gwYORlJSEESNGoKqqCjExMRg7diwuXbqEvn37NikfHR2N5cuXN5mfmpoKOzu79tkRI3TjPgBYITs7G39c5Dsa0hrt8Tpx4gSu6+8FmiOXyx97G8astrbWoHK83/NqK19fX/j6+nLTY8eOxdChQ/H1119jxYoVTcpHRkZCJpNx09XV1XB3d0dgYCAcHBw6JWY+nC+6DVz4Bc888wxG9nPkOxzSiku3qhFzIRvPPvss/tRH/3mpUqkgl8sREBCgtxtoQ7ZhCrRXR63hNXk5OTnB0tISpaWlOvNLS0vh6upq0Dasra0xatQoFBQU6F0ukUggkTRt52Rtbd3sqDpCoB2gwcrKStD7KRRtOV7NnbtCOeaGxs7rDXuxWAwvLy+kpaVx8+rr65GWlqZTu2qJRqPBhQsX0Lt3744KkxBihHi/bJTJZAgJCcGYMWPg7e2N+Ph41NTUcG255s6dCzc3N0RHRwMAPvnkEzzzzDMYNGgQ7t69izVr1uD69et46623+NwNQkgn4z15zZw5E+Xl5Vi2bBlKSkrg6emJQ4cOcTfxi4qKYGHxvwrinTt3MH/+fJSUlKBHjx7w8vLCqVOn8NRTT/G1C4QQHvCevAAgPDwc4eHhepdlZGToTMfFxSEuLq4ToiKEGDPeG6kSQsijoORFCDFJlLwIISbJKO55EWLuWuufDWi9jzZz65+NkhchRsCw/tkAQ/poM5f+2cxjLwkxcq31zwYY1kebOfXPRsmLECPQWv9sAPXR9jC6YU8IMUmUvAghJqnNyevq1asdEQchhLRJm5PXoEGDMGnSJHz//fdmN+gFIcR4tDl5nT17FiNGjIBMJoOrqyv+9re/4fTp0x0RGyGENKvNycvT0xMJCQm4desWkpKSUFxcjGeffRbDhg1DbGwsysvLOyJOQgjR8cg37K2srDBjxgz8+OOPWLVqFQoKChAREQF3d3fMnTsXxcXF7RknIYToeOTk9csvv2DBggXo3bs3YmNjERERgStXrkAul+PWrVuYNm1ae8ZJCCE62txINTY2Fps3b0Z+fj6Cg4Px7bffIjg4mOswsH///tiyZQukUml7x0oIIZw2J6/169dj3rx5eOONN5rtN97Z2RmbNm167OAIIaQ5bU5ecrkc/fr10+maGQAYY7hx4wb69esHsViMkJCQdguSEEIe1uZ7XgMHDkRFRUWT+bdv30b//v3bJShCCGlNm5MXY0zv/Pv378PGxuaRgli3bh2kUilsbGzg4+NjcLuxHTt2QCQSYfr06Y/0uYQQ02XwZaN21GmRSIRly5bBzs6OW6bRaPDzzz/D09OzzQHs3LkTMpkMiYmJ8PHxQXx8PIKCgpCfnw9nZ+dm1yssLERERATGjx/f5s8UotraWuTl5QEAlEol1m36FmWZF7C2cjgWvjkXYrGYKztkyBCd40eIKTI4ef36668AGmpeFy5c0PkyiMVijBw5EhEREW0OIDY2FvPnz+fGaUxMTMTBgweRlJSEDz/8UO86Go0Gr732GpYvX47MzEzcvXu3zZ8rNHl5efDy8moy//v8E/j+m/U683JycjB69OjOCo2QDmFw8kpPTwcAhIaGIiEhAQ4ODo/94UqlEjk5OYiMjOTmWVhYwN/fH1lZWc2u98knn8DZ2RlvvvkmMjMzW/wMhUIBhULBTVdXVwMAVCoVVCrVY+6B8Rg4cCDmzJmD77//Hj169IB/QCDu1qrQ3c4aR+SpuHPnDubMmYN3330XAwcOFNS+mwttf15qtVrQx8/QfWvz08bNmze3OZjmVFRUQKPRcAPMarm4uHCXQA87ceIENm3ahHPnzhn0GdHR0Vi+fHmT+ampqYK6dFIqldi2bRvs7OxgbW2NH3/YyS1zdnaGnZ0dtm/fjpdeeonefjBRN+4DgBWys7Pxx0W+o+k4tbW1BpUzKHnNmDEDW7ZsgYODA2bMmNFi2d27dxv0wY/i3r17eP3117Fx40Y4OTkZtE5kZCR3vw5oqHm5u7sjMDCwXWqPxuLzzz9HfX096urqMGnSJERERKCkpASurq6IiYlBSkoKGGMoKirCe++9x3e45BGcL7oNXPgFzzzzDEb2c+Q7nA6jvTpqjUHJq1u3bhCJRNz/24uTkxMsLS1RWlqqM7+0tBSurq5Nyl+5cgWFhYWYOnUqN6++vh5Aw7uW+fn5GDhwoM46EokEEknTkVasra1hbW3dHrthFLT9rAUEBGD//v3QaDRISUnBuHHjMGHCBDz33HOQy+W4evWqoPbbnFhZWXH/CvkYGrpvBiWvxpeK7XnZKBaL4eXlhbS0NK65Q319PdLS0hAeHt6k/JAhQ3Dhgu7oKh9//DHu3buHhIQEuLu7t1tspkb7x2X06NGwsLCARqPhlllYWGDUqFGQy+VcOUJMHe/dQMtkMmzcuBFbt25Fbm4uwsLCUFNTwz19nDt3LndD38bGBsOGDdP56d69O+zt7TFs2DCdJ6DmxsfHBwCwadMm7saullqt5v7oaMsRYuoMqnmNGjXK4L/YZ8+ebVMAM2fORHl5OZYtW4aSkhJ4enri0KFD3E38oqKiJq8ikaa0tc7y8nK4ublh9uzZqK2tRUZGBrZt28b1s2bOtVMiLCLWXJP5RvQ9rWtOVFTUYwXU0aqrq9GtWzdUVVUJ6oa9RqPBoEGD8ODBA5SUlDRZ7urqCltbW1y+fBmWlvrHBSTG7dz1Skxfn429Yc/A06Mn3+F0GEO/owbVvIw9IRHA0tISr776KtasWQMLCwvuQQbQcM+rpKQEixcvpsRFBIOuxwRCo9Fgy5YtANDk6ar2ndOtW7fq3MgnxJQZlLwcHR25niR69OgBR0fHZn8IPzIyMlBeXo5nn30WVVVVkMvlkMlkkMvluHv3Lp599lmUlZUhIyOD71AJaRcGXTbGxcXB3t6e+z89bjc+2qS0fPlyWFtbw8/PDzU1NfDz84O1tTWioqIQEBCAjIwMTJkyhd9gCWkHBiWvxh0LvvHGGx0VC2knGo0Gx44dw/Hjx9GlSxdMmjSJ75AIaXdtfrfR0tISxcXFTbqrqayshLOzM91T4cnEiRPx6aefYuHChairq8P169cBNPTa4eHhAVtbW64cIULQ5uTVXMsKhUJh1o1E+TZx4kR069YNeXl5cHZ2xvr16yGRSKBQKBAVFYXr16+jW7dulLyIYBicvD7//HMADa+hfPPNN+jatSu3TKPR4Pjx4xgyZEj7R0gMpv3jce/ePYSFhXHztb1n6HvHkxBTZXDyiouLA9BQ80pMTNRpLyQWiyGVSpGYmNj+ERKDZGZmory8HNHR0fj6669RWFjILXNxccH8+fPx0UcfITMzk2pfRBAMTl7Xrl0DAEyaNAm7d+9Gjx49Oiwo0nbaPrrCw8OxePFipKen46effsLzzz+PSZMmoba2Fh999BH15UUEo82NVNPT0ylxGSHtGJoXL+rvpU47v7mxNgkxNW2+YT9v3rwWlyclJT1yMOTRjR8/HlKpFO+++y4qKiq4y8bY2FhIpVI4OTmhf//+NGAJEYw2J687d+7oTKtUKly8eBF3797F5MmT2y0w0jaN3218uBHx9evXUVhYSO82EkFpc/Las2dPk3n19fUICwtr0osp6TwajQZbt25tsczWrVsRHR1NCYwIQru8mG1hYQGZTMY9kSSdLyMjA2VlZQCA4OBgJCQkIDw8HAkJCQgODgYAereRCEq79Spx5cqVJj14ks5z9OhRAICvry/279+PsLAw+Pv7IywsDPv378czzzyjU44QU9fmy8bGI/EADe2+iouLcfDgQZ13IEnnKioqAgDMnj1bbx/2s2bNQnZ2NleOEFPX5uSlHTlby8LCAr169cLatWtbfRJJOk6/fv0AANu2bcOCBQt0ltXX12P79u065QgxdW1OXtqRs4lxmTx5MlauXImsrCxMmzYNS5YsQV1dHbKzs7F69WpkZ2dz5QgRAqPoSXXdunWQSqWwsbGBj48PTp8+3WzZ3bt3Y8yYMejevTu6dOkCT09PfPfdd50YrXGaOHEi19PHkSNHMGHCBMyaNQsTJkxAWloagIaRs+nVICIUvCevnTt3QiaTISoqCmfPnsXIkSMRFBTEPTl7mKOjI/7xj38gKysL//3vfxEaGorQ0FAcPny4kyM3LpaWlli/fj1EIpHeziJFIhHWr19PzSSIcDCeeXt7s4ULF3LTGo2G9enTh0VHRxu8jVGjRrGPP/7YoLJVVVUMAKuqqmpzrKYgOTmZSaVSBoD76d+/P0tOTuY7NPKYfi2sYB5LD7BfCyv4DqVDGfod5bXmpVQqkZOTA39/f26ehYUF/P39kZWV1er6jDGkpaUhPz8fEyZM6MhQTcaMGTOQn5+PmJgYBAcHIyYmBnl5eZgxYwbfoRHSrtp8w745N2/exCeffIINGzYYvE5FRQU0Gg03wKyWi4sL8vLyml2vqqoKbm5uUCgUsLS0xFdffYWAgAC9ZRUKBRQKBTddXV0NoOG1JpVKZXCspmLPnj2IiIjAjRs3AAApKSlISEhATEwM/vznP/McHXkc2naUarVakOeulqH71m7Jq7KyEps2bWpT8npU9vb2OHfuHO7fv4+0tDTIZDIMGDBA783o6OhovYPmpqamcp30CUVWVhZWrVrVZP6NGzcwc+ZMLF26FL6+vjxERtrDjfsAYIXs7Gz8ob/zEEGora01qJxBI2Yb4vz58xg9enSb+rBXKpWws7PDrl27MH36dG5+SEgI7t69i3379hm0nbfeegs3btzQe9NeX83L3d0dFRUVghsx29HREXV1dc2WsbW1xe3bt+mmvYk6X3Qbr2z8Bbvmj8HIfsIdZrC6uhpOTk7tM2J2RxGLxfDy8kJaWhqXvOrr65GWlobw8HCDt1NfX6+ToBqTSCR6uz+2traGtbX1I8VtjI4cOcIlruDgYAQFBeHy5ct44okncPjwYaSkpKCurg7p6el4/vnneY6WPAorKyvuXyGduw8zdN94TV5Aw+tGISEhGDNmDLy9vREfH4+amhqEhoYCAObOnQs3NzdER0cDaLgMHDNmDAYOHAiFQoGUlBR89913WL9+PZ+7wbu1a9cCaLhfeOnSJaSkpHDLPDw84OzsjLKyMqxdu5aSFxEEg5NXa0+r7t69+0gBzJw5E+Xl5Vi2bBlKSkrg6emJQ4cOcTfxi4qKYGHxv4eiNTU1WLBgAW7evAlbW1sMGTIE33//PWbOnPlIny8U2hv0paWlePHFF/H3v/8dv//+O5588kmkpqbiwIEDOuUIMXUGJ69u3bq1unzu3LmPFER4eHizl4kPd+Hy6aef4tNPP32kzxEyd3d3/P777xCLxbhw4QKXrICGmpe1tTVUKhXc3d15jJKQ9mNw8tq8eXNHxkEeU0BAANLS0qBUKrkBZ7UaTzfXpIQQU8P760GkffTp06ddyxFi7AyueRna3Q0NwMGP0tLSdi1HiLEzOHlt2bIFHh4eGDVqFNqpaRhpR+fOnQMA9OzZE3fv3tVpb2dpaYnu3bujsrKSK0eMW21tbZO3TPKL70JRUoDci7aor+zOzR8yZIjgGlwbwuDkFRYWhu3bt+PatWsIDQ3FnDlz4Ogo3IZypqampgZAw5sOzs7OmDBhAm7fvg1HR0ccP36c66VDW44Yt7y8PHh5eeldNvuhcVZycnIwevToTojKuLSphb1CocDu3buRlJSEU6dO4YUXXsCbb76JwMBAvd2wGKPq6mp069at1da7piYmJgaLFy+GWCyGWq1GfX09t8zCwgJWVlZQKpVYs2YNIiIieIyUGEJfzet+nQIH07PwwiRfdLX9X8NrodW8DP2OtqmRqkQiwaxZszBr1ixcv34dW7ZswYIFC6BWq3Hp0iV07dr1sQMnj2bkyJEAGl65cnZ2xvjx43Hnzh306NEDmZmZXM1LW44YNzs7uya1KZVKhTsVZfD1HiPoFvaGeuQW9hYWFhCJRGCMtel9RtIxGnfeWFZWhuTk5FbLEWLK2tRUQqFQYPv27QgICMCTTz6JCxcu4Msvv0RRURHVunhWXl7eruUIMXYG17wWLFiAHTt2wN3dHfPmzcP27dvh5OTUkbGRNujZsyeAhp4jlEplk6eNYrEYdXV1XDlCTJ3BySsxMRH9+vXDgAEDcOzYMRw7dkxvud27d7dbcMRwlZWVAIC6ujqIxWK8+uqrsLW1RV1dHXbv3s31OKEtR0xLXV0dZDIZsrOzcejQIcTGxsLW1pbvsHhlcPKaO3euyTxRNEc9evQA0HAvUq1WY8eOHdwyS0tLWFhYoL6+nitHTMf06dN1+rY7d+4cEhMTMW3aNOzdu5e/wHjWpkaqxHidOXMGAHSaSGg1voQ8c+YMjWxuQh5OXI3t27cP06dPN9sERu82CoS+pPU45Qj/6urquMT18CWidnrfvn0t9p4rZJS8BKJxW2Nra2tMmjQJfn5+mDRpkk6bIHq1y3TIZDLu/1OmTEFmZia2b9+OzMxMTJkyRW85c8J7T6qkfTRuqqLRaJCens5NN+7MkZq0mA7trQAvLy/s27cPGo0GlZWV8PHxwb59+/D000/j7NmzXDlzQzUvgWj8wvXDl4aNp+nFbNOhrSU/8cQTOn+AgIY/SIMGDdIpZ24oeQmEoe+2CekdOKHTDsa8a9cu1NXV4dixYzh+/DiOHTvGNYFpXM7c0GWjQPTu3btdyxH+BQYGYvXq1VCr1Tp/dGJjY5uUM0dU8xIIQ7snom6MTMfEiRMNGjtC32DL5sAokte6desglUphY2MDHx8fnD59utmyGzduxPjx49GjRw/06NED/v7+LZY3F9RUQpjEYjEAwMbGRme+tqmEvjFJzQXvyWvnzp2QyWSIiorC2bNnMXLkSAQFBTXb+0FGRgZmzZqF9PR0ZGVlwd3dHYGBgfjjjz86OXLjcunSpXYtR/iXmZmJ8vJyvPbaa1Cr1TrLVCoVZs+ejbKyMmRmZvIUIb/a1BlhR/Dx8cHTTz+NL7/8EkBDzcDd3R3vvvsuPvzww1bX12g06NGjB7788kuDhl4TameETzzxBAoKCgAAzz//PAYOHMiN23jlyhX89NNPAIBBgwbh8uXLfIZKDLR9+3bMnj0bIpEIL7zwAgIDA3XG4jx48CAYY9i2bRtmzZrFd7jtpkM6I2xvSqUSOTk5iIyM5OZZWFjA398fWVlZBm2jtrYWKpWq2Xs5CoUCCoWCm66urgbQ8JdLpVI9RvTG5d69e9z/MzIyuGSVmpqq0zr73r17gtpvIdOe02PHjsWuXbug0Wggl8sREBCAt99+G5MnT8apU6fg6OgoqGNq6L7wmrwqKiqg0Wi40bG1XFxcmnSB25ylS5eiT58+zT4ujo6OxvLly5vMT01NFVSzge7du3MjA3Xp0kXnlZHG0927d0dKSgovMZK2OX/+PADg9u3bSElJ4dp6yeVy1NfX486dOwCAn3/+GQ8ePOAtzvZWW1trUDmTbirx2WefYceOHcjIyGhyQ1MrMjJS5/WJ6upq7j6ZkC4bc3NzuRpsVVUVXn31VXTp0gU1NTU6L+6+8cYbCA4O5ilK0hbaq4Tc3Fx888038Pf3x9WrVzFgwAAcOXIEubm5AID+/fsL6phq97tVjEcKhYJZWlqyPXv26MyfO3cue+mll1pcd82aNaxbt27szJkzbfrMqqoqBoBVVVW1NVyjplAomEgkYgCa/RGJREyhUPAdKjFQeno6A8Bee+01ZmVlpXMsrays2OzZsxkAlp6ezneo7crQ7yivTxvFYjG8vLyQlpbGzauvr0daWhp8fX2bXW/16tVYsWIFDh06hDFjxnRGqEZPLBa3OipQREQE9+idGL/x48ejV69e+Pe//42goCAkJCQgPDwcCQkJCAoKwrZt27jBVswR75eNMpkMISEhGDNmDLy9vREfH4+amhqEhoYCaOgE0c3NDdHR0QCAVatWYdmyZdi2bRukUilKSkoANLxwbO4vHa9evRpAQwvsxn14WVlZYdGiRdxyYjq0HYCKRCKMGjUKzs7OcHNzg1wu5zkyI9BJNcEWffHFF6xfv35MLBYzb29vlp2dzS3z8/NjISEh3LSHh4feS6KoqCiDPkuol42NKRQKFhMTw4KDg1lMTAxdKpoo7WVjdHR0k/NeKpWylStXmvVlI+81LwAIDw9HeHi43mUZGRk604WFhR0fkIkTi8V47733MGjQIAQHB9MYfyaquLgYAFBSUoKbN2/qLLtx4wb3dFlbztwYRfIihDSlfYk+ISEBLi4uWL58OSQSCRQKBaKiopCQkKBTztzw/noQIUQ/Hx8fAA016aKiIsybNw89evTAvHnzUFRUxD180ZYzN5S8CDFSX3/9NYCGFuevvPIKsrOzUVdXh+zsbLzyyitcS3RtOXNDyYsQI3XlyhUADT2pXLhwARMmTMCsWbMwYcIEXLx4ERs2bNApZ24oeRFipAYOHAigoZvngoICyOVyyGQyyOVyXL58meveSFvO3PDeq0RnE2qvEg9TqVRISUmhp40mTKlUokuXLujZsydu3rwJxhh3TEUiEfr27YvKykrU1NQIqvGxod9RqnkRYqTEYjEWLVqE0tJS9O3bF9988w1u376Nb775Bn379kVpaSkWLVokqMTVFtRUghAjpn0rIi4uDgsWLODmW1lZYfHixWb91gTVvAgxcqtXr0ZNTQ1iYmIQHByMmJgY1NTUmHXiAqjmJUgajYYbJqtLly6YNGkSLC0t+Q6LPAZ6a6IpqnkJzO7duzFw4EAEBAQgNjYWAQEBGDhwIDfGHyFCQclLQHbv3o2XX36Z62lDq6SkBC+//DIlMBOmVCrx+eefY8OGDfj888+hVCr5Dol31FRCIDQaDXr37o3y8vJmyzg7O+PWrVt0CWlilixZgri4OJ0RhITczRE1lTAzGRkZXOISiUSYM2cO4uLiMGfOHK5PqLKysia9dBDjtmTJEqxZswY9e/ZEYmIiNm/ejMTERPTs2RNr1qzBkiVL+A6RN5S8BOLw4cMAGhJXbW0tkpKS0L9/fyQlJaG2tpZLYNpyxPgplUrExcXBxcUFN2/e1Hkx++bNm3BxcUFcXJzZXkJS8hKIffv2AQCmTJnSZDASGxsbTJo0SaccMX5fffUV1Go1Pv30U1hZ6TYMsLKywieffAK1Wo2vvvqKpwj5RclLILQ9DJSXl3PvvGnV19ejsrJSpxwxftoXrl988UW9y7Xz6cVsYtKGDx8OoGGsv2nTpul0nzJt2jRuDEBtOWL8tC9cHzhwQO9y7Xx6MdtMCPVp4/3792Fvbw8AXG+bWo2n7927Z/YDlZgKejHbyJ82rlu3DlKpFDY2NvDx8cHp06ebLXvp0iW8/PLLkEqlEIlEiI+P77xAjVzXrl3x9NNPA4BO4mo8/fTTT1PiMiH0YnYrOnYckJbt2LGDicVilpSUxC5dusTmz5/PunfvzkpLS/WWP336NIuIiGDbt29nrq6uLC4urs2fKeTRg5KTk1scdDY5OZnvEMkjWLx4sd5BZxcvXsx3aB3C0O8or8nL29ubLVy4kJvWaDSsT58+LDo6utV1PTw8KHk1olarWa9evRgAZmNjo3Oia6ednZ2ZWq3mO1TyCGpra9k777zDPD092TvvvMNqa2v5DqnDGP3QZ0qlEjk5OYiMjOTmWVhYwN/fH1lZWe32OQqFQucyqrq6GkDDUzchPXk7evQo10h18uTJWLx4MUpKSuDq6oo1a9YgJSUFZWVlOHLkCCZPnsxztKQt9uzZg6VLl3LD/p07dw6HDh3CqlWr8Oc//5nf4DqAod9L3pJXRUUFNBoNXFxcdOa7uLggLy+v3T4nOjoay5cvbzI/NTUVdnZ27fY5fPvuu+8AAIMHD8Zbb72Fqqoq2NraoqqqCm+99RYKCgrw+++/Y+PGjXjw4AHP0RJDZWVlYfXq1RgzZgzCwsLQr18/FBUVYdeuXfjrX/+KJUuWwNfXl+8w21Vtba1B5QTfJU5kZCRkMhk3XV1dDXd3dwQGBgrqaeOPP/4IAAgLC8OLL74IlUoFuVyOgIAAWFtbo7CwEDKZDDY2NggODuY5WmIIjUaDDz74AMHBwUhOToZGo4FcLkd4eDjef/99vPzyy/jhhx/wr3/9S1Dvq2qvjlrDW/JycnKCpaUlN+qvVmlpKVxdXdvtcyQSCSQSSZP51tbWguoTycPDAwCwY8cOLFiwAKdOneL68/Lz88MPP/zAlRPSfgvZyZMnUVhYiO3bt0MikXCXU9pz9x//+AfGjh2L7OxsTJw4kd9g25Gh5ydvTSXEYjG8vLyQlpbGzauvr0daWprgqsGdQXsfKzs7Gw4ODjr9eTk4OCA7O1unHDF+xcXFAIBhw4bpXa6dry1nbnht5yWTybBx40Zs3boVubm5CAsLQ01NDUJDQwEAc+fO1bmhr1Qqce7cOZw7dw5KpRJ//PEHzp07h4KCAr52wWhMnDiRuwx++EVd7bSDg4Og/kILXe/evQEAFy9e1LtcO19bzux00tPPZn3xxResX79+TCwWM29vb5adnc0t8/PzYyEhIdz0tWvX9LZf8vPzM/jzzKGphK2trc7vRztNTSVMi1qtZlKplE2dOpVpNBqmVCrZ3r17mVKpZBqNhk2dOpX1799fcMfUJNp58UGoySs9PZ0BYNHR0czDw0MneUmlUrZy5UoGgKWnp/MdKmmD5ORkJhKJ2NSpU9nx48fZ9u3b2fHjx9nUqVOZSCQSZMNjSl7NEGry2rZtGwPA7t27x9RqNZPL5UwmkzG5XM7UajWrrq5mANi2bdv4DpW0UXJyMpNKpTp/kPr37y/IxMWYCTRSJe2r8f2RZ555Bn5+fqipqYGfnx8sLS3p/ogJmzFjBqZNm4b09HT89NNPeP7552lEKBjBi9mkfYwfPx5SqRQrV67U259XdHQ0+vfvj/Hjx/MUIXkclpaW8PPzw4QJE7g/SOaOkpdAWFpaYu3atThw4ACmT5+u05/X9OnTceDAAcTExNBJTwSDLhsFZMaMGdi1axdkMhkmTJjAzZdKpdi1axdmzJjBY3SEtC+qeZkBZl79TRIzQclLQHbv3o1XXnkFZWVlOvPLysrwyiuv0KCzRFAoeQmERqNBWFgYGGOYMmUKMjMzsX37dmRmZmLKlClgjCEsLAwajYbvUAlpF5S8BCIjIwNlZWV49tlnsW/fPvj4+MDW1hY+Pj7Yt28fxo0bR4POEkGh5CUQ2qS0fPlyWFjoHlYLCwv861//0ilHiKmj5EUIMUmUvARC21tEVFSU3kaq2t5kqVcJIhSUvARi4sSJ6NWrF06cOKF30NkTJ07A2dmZkpeJ0mg0OHbsGI4fP45jx47RgxeA/y5xOptQX8xm7H9Dnz3cJY6dnR0NfWbC9L2YLZVKBXs8Df2OUs1LQGbMmIHk5GQ4OzvrzHd2dkZycjK1sDdB2rZ7w4cP12n+Mnz4cLNvuydizLyaXxs6lLgp02g01AOBAGg0GgwaNAjDhw/H3r17odFokJKSguDgYFhaWmL69Om4ePEiLl++LKjja+h3lN5tFCBtDwSNu8QhpiczM5MbgEOtVuOLL77A0aNHUVBQgHfffReRkZEYO3YsMjMzzfJeJiUvQoyUdmCNHTt2YPz48VCr1QCAlJQUfPjhh1i4cKFOOXNDyYsQI6XtODIhIQEuLi5Yvnw5JBIJFAoFoqKikJCQoFPO7HTK44NWfPnll8zDw4NJJBLm7e3Nfv755xbL//DDD2zw4MFMIpGwYcOGsYMHDxr8WUJ+2qg1depUnSdTU6dO5Tsk8ghqa2sZACYWi9nkyZN1junkyZOZWCxmAFhtbS3fobYrk+nDfseOHUwsFrOkpCR26dIlNn/+fNa9e3dWWlqqt/zJkyeZpaUlW716Nfvtt9/Yxx9/zKytrdmFCxcM+jyhJ6/GJ/jDP8S0xMXFtXg8tT9xcXF8h9quTKapRGxsLObPn4/Q0FA89dRTSExMhJ2dHZKSkvSWT0hIwHPPPYfFixdj6NChWLFiBUaPHo0vv/yykyM3PiKR6LGWE+Ny5cqVdi0nNLwmL6VSiZycHPj7+3PzLCws4O/vj6ysLL3rZGVl6ZQHgKCgoGbLm4uXXnqJ+/+iRYugVCqxd+9eKJVKLFq0SG85YtwOHTrE/f/tt9+GXC6HTCaDXC7H22+/rbecOeH1hn1FRQU0Gg1cXFx05ru4uCAvL0/vOiUlJXrLl5SU6C2vUCigUCi46erqagCASqWCSqV6nPCNyn/+8x/u/6tWreL2TaVSYdWqVYiLi+PKCWm/hazxSPDx8fFgjKGmpgZjx47FuHHjsGHDBq6ckI6pofsi+KeN0dHR3EvJjaWmpsLOzo6HiDpeSkoK93+5XN7icmIa+vTpg1mzZuHpp59GREQEtm/frrNcSMe0trbWoHK8Ji8nJydYWlqitLRUZ35paSlcXV31ruPq6tqm8pGRkZDJZNx0dXU13N3dERgYKNgW9sHBwVCpVJDL5QgICIC1tXWT5cS03L9/H+vXr8f69esBAFZWul9dIR1T7dVRa3hNXmKxGF5eXkhLS8P06dMBNHTfkpaWhvDwcL3r+Pr6Ii0tDR988AE3Ty6Xw9fXV295iUQCiUTSZL61tXWTL7Upmzp1KnfpuHTpUqxatQpAw34uXbpUp5yQ9lvI/P39ceTIEQBAaGgoBg8ejKNHj2Ly5MnIz8/Hxo0buXJCOqYG70vnPPxs3o4dO5hEImFbtmxhv/32G3v77bdZ9+7dWUlJCWOMsddff519+OGHXPmTJ08yKysrFhMTw3Jzc1lUVBQ1lfj/YMBjdWJazPGYGvod5f2e18yZM1FeXo5ly5ahpKQEnp6eOHToEHdTvqioSKdb47Fjx2Lbtm34+OOP8dFHH+GJJ57A3r17MWzYML52wWgwxlpsDsHM6x18QaBj2jzqVUKAXnrpJZ2nj1OnTsX+/ft5jIg8roCAAO4SEmi4VNT3MEYIDP2OUvISKJVKxXWfIqT7IebMXI6pod9R3lvYE0LIo6DkRQgxSZS8CCEmifenjZ1Ne4vP0IZwpkqlUqG2thbV1dWCvj9iTszlmGq/m63djje75HXv3j0AgLu7O8+REEJacu/ePXTr1q3Z5Wb3tLG+vh63bt2Cvb29oLuI0b4GdePGDUE/VTUn5nJMGWO4d+8e+vTpo9PG82FmV/OysLBA3759+Q6j0zg4OAj6RDdH5nBMW6pxadENe0KISaLkRQgxSZS8BEoikSAqKkpvjxrENNEx1WV2N+wJIcJANS9CiEmi5EUIMUmUvAghJomSF08KCwshEolw7ty5TtnOyZMnMXz4cFhbW3NdbhNiyih5GYmMjAyIRCLcvXu3Q7Yvk8ng6emJa9euYcuWLR3yGaSpiRMn6oy3wDdji+dxUPJ6BEqlku8Q2uzKlSuYPHky+vbti+7duz/SNjpzvxljUKvVnfZ5xswUz7dO0UF96AuKn58fW7hwIXv//fdZz5492cSJE9mFCxfYc889x7p06cKcnZ3ZnDlzWHl5ObfOTz/9xMaNG8e6devGHB0d2QsvvMAKCgq45deuXWMA2K+//sr9v/FPSEhIm7azfft25uvryyQSCfvTn/7EMjIydJY3/tm8eTNjjLGMjAz29NNPM7FYzFxdXdnSpUuZSqVqcb/T09MZAHbo0CHm6enJbGxs2KRJk1hpaSlLSUlhQ4YMYfb29mzWrFmspqaG25ZGo2ErV65kUqmU2djYsBEjRrAff/yRW67dbkpKChs9ejSztrZm6enp7XkYO11ISEiT331BQQGbN28e93t48sknWXx8fJP1pk2bxj799FPWu3dvJpVKGWMNg8+MHDmSSSQS5uXlxfbs2cOdQ1otnZf64rl27Vpn/TraHSUvA/j5+bGuXbuyxYsXs7y8PJadnc169erFIiMjWW5uLjt79iwLCAhgkyZN4tbZtWsXS05OZpcvX2a//vormzp1Khs+fDjTaDSMMd3kpVarWXJyMgPA8vPzWXFxMbt7926bttO3b1+2a9cu9ttvv7G33nqL2dvbs4qKCqZWq1lxcTFzcHBg8fHxrLi4mNXW1rKbN28yOzs7tmDBApabm8v27NnDnJycWFRUVLP7nZeXxyWZZ555hp04cYKdPXuWDRo0iPn5+bHAwEB29uxZdvz4cdazZ0/22Wefcdv69NNP2ZAhQ9ihQ4fYlStX2ObNm5lEIuGSrHa7I0aMYKmpqaygoIBVVlZ29KHtUHfv3mW+vr5s/vz5rLi4mBUXF7MHDx6wZcuWsTNnzrCrV6+y77//ntnZ2bGdO3dy64WEhLCuXbuy119/nV28eJFdvHiRVVVVMUdHRzZnzhx26dIllpKSwp588kmd5HXnzp0Wz0t98ajVaj5+Ne2CkpcB/Pz82KhRo7jpFStWsMDAQJ0yN27c4JKPPuXl5QwAN0Rb4+TF2P++vHfu3Gkxlua20zhRqFQq1rdvX7Zq1SpuXrdu3bgaF2OMffTRR2zw4MGsvr6em7du3TrWtWtXLjE+vN+N4zxy5Ag3Lzo6mgFgV65c4eb97W9/Y0FBQYwxxh48eMDs7OzYqVOndLb15ptvslmzZulsd+/evS3uv6nx8/Nj77//fotlFi5cyF5++WVuOiQkhLm4uDCFQsHNW79+PevZsyerq6vj5m3cuFHnHDLkvDQkHlNB97wM5OXlxf3//PnzSE9PR9euXbmfIUOGAGi4twQAly9fxqxZszBgwAA4ODhAKpUCaBjKrS0M3U7jQXetrKwwZswY5ObmNrvd3Nxc+Pr66nQLNG7cONy/fx83b97Uu9+NjRgxgvu/i4sL7OzsMGDAAJ15ZWVlAICCggLU1tYiICBA53f27bffcr8vrTFjxjQbs1CsW7cOXl5e6NWrF7p27YoNGzY0OZ7Dhw+HWCzmpvPz8zFixAjY2Nhw87y9vXXWMeS8FBKz6xLnUXXp0oX7//379zF16lRuVOrGevfuDaBhuDEPDw9s3LgRffr0QX19PYYNG9bmm6/ttZ1H1Xi/G2vck6dIJGrSs6dIJEJ9fT2Aht8XABw8eBBubm465R5+T6+5zxOKHTt2ICIiAmvXroWvry/s7e2xZs0a/PzzzzrlHuX3YMh5KSSUvB7B6NGjkZycDKlUCiurpr/CyspKbjj28ePHAwBOnDjR4ja1f2U1Gs0jbSc7OxsTJkwAAKjVauTk5CA8PLzZzxs6dCiSk5N1BjU9efIk7O3t272/s6eeegoSiQRFRUXw8/Nr120bO7FYrHNMT548ibFjx2LBggXcPENqRYMHD8b3338PhULBJfwzZ87olGntvNQXjymjy8ZHsHDhQty+fRuzZs3CmTNncOXKFRw+fBihoaHQaDTo0aMHevbsiQ0bNqCgoABHjx6FTCZrcZseHh4QiUQ4cOAAysvLcf/+/TZtZ926ddizZw/y8vKwcOFC3LlzB/PmzWv28xYsWIAbN27g3XffRV5eHvbt24eoqCjIZLIWe698FPb29oiIiMCiRYuwdetWXLlyBWfPnsUXX3yBrVu3tutnGRupVIqff/4ZhYWFqKiowBNPPIFffvkFhw8fxu+//45//vOfTZKQPrNnz0Z9fT3efvtt5Obm4vDhw4iJiQEA7o9Pa+elvni0tWNTRMnrEfTp0wcnT56ERqNBYGAghg8fjg8++ADdu3eHhYUFLCwssGPHDuTk5GDYsGFYtGgR1qxZ0+I23dzcsHz5cnz44YdwcXFBeHh4m7bz2Wef4bPPPsPIkSNx4sQJ7N+/H05OTi1+XkpKCk6fPo2RI0finXfewZtvvomPP/74sX43zVmxYgX++c9/Ijo6GkOHDsVzzz2HgwcPon///h3yecYiIiIClpaWeOqpp9CrVy8EBQVhxowZmDlzJnx8fFBZWalTC2uOg4MD/vOf/+DcuXPw9PTEP/7xDyxbtgwAuPtgrZ2X+uJp6z1YY0Jd4hBiov79738jNDQUVVVVsLW15TucTkf3vAgxEd9++y0GDBgANzc3nD9/HkuXLsVf/vIXs0xcACUvQkxGSUkJli1bhpKSEvTu3Ruvvvoq/u///o/vsHhDl42EEJNEN+wJISaJkhchxCRR8iKEmCRKXoQQk0TJixBikih5kQ71xhtvQCQScS9vu7i4ICAgAElJSW16NWXLli2P3APs43jjjTeoz38jRcmLdLjnnnsOxcXFKCwsxE8//YRJkybh/fffx4svvkhdPZNHx2tvYkTwtF0aPywtLY0BYBs3bmSMMbZ27Vo2bNgwZmdnx/r27cvCwsLYvXv3GGP/66iw8Y+2x9dvv/2WeXl5sa5duzIXFxc2a9YsVlpayn3O7du32ezZs5mTkxOzsbFhgwYNYklJSdzyoqIi9uqrr7Ju3bqxHj16sJdeeonrGjkqKqrJ55p619RCQjUvwovJkydj5MiR2L17NwDAwsICn3/+OS5duoStW7fi6NGjWLJkCQBg7NixiI+Ph4ODA4qLi1FcXIyIiAgAgEqlwooVK3D+/Hns3bsXhYWFeOONN7jP+ec//4nffvsNP/30E3Jzc7F+/XruhXWVSoWgoCDY29sjMzMTJ0+eRNeuXfHcc89BqVQiIiICf/nLX7iaY3FxMcaOHdu5vyjSPL6zJxG25mpejDE2c+ZMNnToUL3LfvzxR9azZ09uevPmzaxbt26tft6ZM2cYAK7WNnXqVBYaGqq37HfffdekK2yFQsFsbW3Z4cOHW42f8ItqXoQ3rFFHiEeOHMGUKVPg5uYGe3t7vP7666isrERtbW2L28jJycHUqVPRr18/2Nvbc50dart6CQsLw44dO+Dp6YklS5bg1KlT3Lrnz59HQUEB7O3tuW6THR0d8eDBA0F2myw0lLwIb3Jzc9G/f38UFhbixRdfxIgRI5CcnIycnBysW7cOQMtjFtbU1CAoKAgODg7497//jTNnzmDPnj066z3//PO4fv06Fi1ahFu3bmHKlCncJef9+/fh5eWFc+fO6fz8/vvvmD17dgfvPXlc1KsE4cXRo0dx4cIFLFq0CDk5Oaivr8fatWu5TvN++OEHnfL6ui/Oy8tDZWUlPvvsM7i7uwMAfvnllyaf1atXL4SEhCAkJATjx4/H4sWLERMTg9GjR2Pnzp1wdnaGg4OD3jiF1G2y0FDNi3Q4hUKBkpIS/PHHHzh79ixWrlyJadOm4cUXX8TcuXMxaNAgqFQqfPHFF7h69Sq+++47JCYm6mxDKpXi/v37SEtLQ0VFBWpra9GvXz+IxWJuvf3792PFihU66y1btgz79u1DQUEBLl26hAMHDmDo0KEAgNdeew1OTk6YNm0aMjMzce3aNWRkZOC9997jRlCSSqX473//i/z8fFRUVEClUnXOL420ju+bbkTYGo/SbGVlxXr16sX8/f1ZUlISNz4kY4zFxsay3r17M1tbWxYUFMS+/fbbJuNYvvPOO6xnz546TSW2bdvGpFIpk0gkzNfXl+3fv7/JWIZDhw5ltra2zNHRkU2bNo1dvXqV22ZxcTGbO3cuc3JyYhKJhA0YMIDNnz+fVVVVMcYYKysrYwEBAaxr167UVMLIUH9ehBCTRJeNhBCTRMmLEGKSKHkRQkwSJS9CiEmi5EUIMUmUvAghJomSFyHEJFHyIoSYJEpehBCTRMmLEGKSKHkRQkwSJS9CiEn6f/FFE2+AyEbjAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T15:26:43.595106Z", + "iopub.status.busy": "2024-03-26T15:26:43.594747Z", + "iopub.status.idle": "2024-03-26T15:26:43.888868Z", + "shell.execute_reply": "2024-03-26T15:26:43.887759Z" + }, + "papermill": { + "duration": 0.31398, + "end_time": "2024-03-26T15:26:43.891384", + "exception": false, + "start_time": "2024-03-26T15:26:43.577404", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEoCAYAAAD8JfzQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABXC0lEQVR4nO2dd3xT1fvH3zdJk+7B6oCWvfcG2RtEnN8fCjjAgYOhIrJUQFFBRUVx4AYZ4kBEEVBAtoBsyobSsqGFQvfIuL8/bpMmbUqTtmnS9rxfr7xIbs69eUiaT855zjMkWZZlBAKBwI2o3G2AQCAQCCESCARuRwiRQCBwO0KIBAKB2xFCJBAI3I4QIoFA4HaEEAkEArcjhEggELgdIUQCgcDtCCESCARuR+NuA8zMmTOHqVOn8vzzzzNv3jyHzjGZTFy+fJmAgAAkSXKtgQKBwGlkWSYlJYWIiAhUqoLnPR4hRHv27OGLL76gRYsWTp13+fJlIiMjXWSVQCAoKS5cuECNGjUKfN7tQpSamsqIESP46quvePPNN506NyAgAFD+k4GBga4wTyAQFIPk5GQiIyMt39WCcLsQjRkzhsGDB9O3b99ChSgrK4usrCzL45SUFAACAwOFEAkEHkxhrhO3CtHy5cvZv38/e/bscWj87Nmzef31111slUAgKG3ctmt24cIFnn/+eZYuXYq3t7dD50ydOpWkpCTL7cKFCy62UiAQlAaSuwqj/fbbb9x3332o1WrLMaPRiCRJqFQqsrKybJ6zR3JyMkFBQSQlJYmlmUDggTj6HXXb0qxPnz5ER0fbHBs1ahSNGjVi8uTJhYpQaWI0GtHr9e42QyAoUdRqNRqNxiNCX9wmRAEBATRr1szmmJ+fH5UrV8533J2kpqZy8eJFREVdQXnE19eX8PBwtFqtW+1w+66ZJ2M0Grl48SK+vr5UrVrVI345BIKSQJZlsrOzSUhIIDY2lvr169824NDVeJQQbd682d0m2KDX65FlmapVq+Lj4+NucwSCkkXthclHT+r1K2RnZzu8aeQKRK6ZA4iZkKA8Ep+SRYbeRGKantlrjpOS6T4/qBAigaACkpZlICkjV3g2HL/GnR9vY9+5m26xp8IIkXA2CwQKsixz+VYGAEHeXlQN0FEtwJsLiRkM/WIn8zeexmgq3e9LhRCipAw9oxfvY8Oxa+42pcwSFxeHJEkcPHiwVK6zY8cOmjdvjpeXF/fee2+xXrOisnnzZiRJ4tatWzbHb6brydAbUUsSlQN06DQqvnqsLXe3jMBoknl//SmGfbmLSzliVRpUCCFauCOO9ceu8eJPB4m7nuZuc8oFBf2RlxQTJkygVatWxMbGsnDhQpe8RkUlLcsAQGV/LV5qRQL8dV589FArPhjaEj+tmv/iEhk0byt/Hr5SKjZVCCF6tmdd2tYMISXTwDNL9pGRbXS3SaVKdna2u01wmpiYGHr37k2NGjUIDg4u0jVK8/8tyzIGg6HEr+uK/4Mpx02hUdt+/SVJ4v42NVjzfDdaRgaTnGlgzLL9TPrlkEW8XEWFECKtRsVnI9pQxV/HiaspTFsZXSSfkSzLpGcb3HJzxt6ePXsyduxYXnjhBapUqcKAAQM4cuQIgwYNwt/fn9DQUB555BGuX79uOWfdunV07dqV4OBgKleuzF133UVMTIzd68fFxdGrVy8AQkJCkCSJkSNHOnWdEydOcMcdd+Dt7U2zZs3YsmWL5dqSJHHjxg0ef/xxJEmyzIi2bNlChw4d0Ol0hIeHM2XKFJsvv73/t3nm9tdff9G6dWt8fHzo3bs38fHxrF27lsaNGxMYGMjw4cNJT0+3XMtkMjF79mxq166Nj48PLVu25JdffrE8b77u2rVradu2LTqdju3btxf62bz55ptUq1aNgIAAnnzySaZMmUKrVq0sz48cOZJ7772Xt956i4iICBo2bAjA4sWLadeuHQEBAYSFhTF8+HDi4+Ntrr1mzRoaNGiAj48PvXr1Ii4uzq4N5j+lgjaDa1b245dnOjO2Vz0kCX7ae5G75m/n8MVbhf7/iopHxRG5ktBAbz4Z3poRX+9m5YFLtI4K5tHOtZy6RobeSJPpf7nGwEI49sYAfLWOf1yLFi3i2WefZceOHdy6dYvevXvz5JNP8uGHH5KRkcHkyZMZOnQo//zzDwBpaWlMmDCBFi1akJqayvTp07nvvvs4ePBgvkC3yMhIVqxYwQMPPMDJkycJDAy0xFk5ep2XX36ZefPm0aRJEz744AOGDBlCbGwskZGRXLlyhYYNG/LGG2/w4IMPEhQUxKVLl7jzzjsZOXIk33//PSdOnOCpp57C29ubmTNn2v1/A1y5oiwtZs6cySeffIKvry9Dhw5l6NCh6HQ6li1bRmpqKvfddx/z589n8uTJgFLpYcmSJSxYsID69euzdetWHn74YapWrUqPHj0srzdlyhTmzp1LnTp1CAkJue1nsnTpUt566y0+++wzunTpwvLly3n//fepXbu2zbiNGzcSGBjI+vXrLcf0ej2zZs2iYcOGxMfHM2HCBEaOHMmaNWsAJYn8/vvvZ8yYMYwePZq9e/fy0ksv2bXD/JOmQrJ6ZIuXWsXEAQ3pWr8KL/54kNjradz/2b9MHNCQ0d3qoFKVbEiL25JeS4KiJL1+ve0sb/55HI1K4senO9G2ZqUCx2ZmZhIbG0vt2rXx9vYmPdtQJoSoZ8+eJCcns3//fkD5Fd62bRt//ZVr+8WLF4mMjOTkyZM0aNAg3zWuX79O1apViY6OplmzZsTFxVG7dm0OHDhAq1at2Lx5M7169eLmzZu3XToVdJ05c+ZYvvQGg4HatWszbtw4Jk2aBEBwcDDz5s2zzLReeeUVVqxYwfHjxy1xXZ999hmTJ08mKSkJlUqV7/8NWOzcsGEDffr0AXLLEsfExFCnTh0AnnnmGeLi4li3bh1ZWVlUqlSJDRs20LlzZ8u1nnzySdLT01m2bJnlur/99hv33HOPQ59Lp06daNeuHZ988onlWNeuXUlNTbU470eOHMm6des4f/78bdMu9u7dS/v27UlJScHf359p06axatUqjh49ahkzZcoU3nnnHfacPE+TWuGWv5+YhFTSsgxEVfLFW2Wy+Ru3x630bKb+Gs3aI1cB6FKvMu//XyvCggoPgPT4pFd38UTX2hw4f4s/o6/w3NL9/DGuK9UCHIso9fFSc+yNAS62sODXdoa2bdta7h86dIhNmzbh7++fb1xMTAwNGjTg9OnTTJ8+nd27d3P9+nVMJhMA58+fdyr3z9HrWH/BNRoN7dq14/jx4wVe9/jx43Tu3NkmuLRLly6WXMCoqKh8/29rrMsQh4aG4uvraxEh87H//vsPgDNnzpCenk6/fv1srpGdnU3r1q1tjrVr165Am/Ny8uRJnnvuOZtjHTp0sMxKzTRv3jyfCO3bt4+ZM2dy6NAhbt68afO+NmnShOPHj9OxY0ebc8zvcabeRHKGwSJEuUszx2Y1wb5aPhvRhp/2XmDm78fYceYGgz7aytePtbvtD7kzVDghkiSJd/7XgpPXUjgTn8rYZQdoWSOIED8tz/WsV+i5ziyP3Imfn5/lfmpqKkOGDOGdd97JNy48PByAIUOGULNmTb766isiIiIwmUw0a9bMaWdpSV2nqFj/v63x8vKy3Jckyeax+Zj5y52amgrAn3/+SfXq1W3G6XQ6h16vOOS9ZlpaGgMGDGDAgAEsXbqUqlWrcv78eQYMGODw+2odF2ReBDmzupIkiQfbR9GuViVeWbaVsylQq3LJ/d/LxreqhPHXaVjwcFvu/XQH/8Um8l9sIgDDO0QR7OveLGRX0KZNG1asWEGtWrXQaPJ/5Ddu3ODkyZN89dVXdOvWDaBQx6v5F9tozN2BdOY6u3btonv37oCyNNu3bx9jx44t8PUaN27MihUrkGXZ8ku+Y8cOAgICbluUvSg0adIEnU7H+fPnbfxBxaVhw4bs2bOHRx991HLMkeqkJ06c4MaNG8yZM8fSLGLv3r02Yxo3bszvv/9uc2znzp2W+8YckQUwa5KE836eulX8WBb4GdmaZLzTqoN/U6evYY8KsWtmj3rV/Jn7fy3RWP0sxCSUzxijMWPGkJiYyLBhw9izZw8xMTH89ddfjBo1CqPRSEhICJUrV+bLL7/kzJkz/PPPP0yYMOG216xZsyaSJLF69WoSEhJITU116jqffvopK1eu5MSJE4wZM4abN2/y+OOPF/h6zz33HBcuXGDcuHGcOHGCVatWMWPGDCZMmFDiWeMBAQFMnDiRF198kUWLFhETE8P+/fuZP38+ixYtKvJ1x40bxzfffMOiRYs4ffo0b775JocPHy50iRQVFYVWq2X+/PmcPXuW33//nVmzZtmMeeaZZzh9+jQvv/wyJ0+eZNmyZTa2GqxnRDkO6iKlUJ7+G9W5bXgnngDd7QviO0OFFSKAgc3C2PNKXzrWVta5ZxNS3WyRa4iIiGDHjh0YjUb69+9P8+bNeeGFFwgODkalUqFSqVi+fDn79u2jWbNmvPjii7z33nu3vWb16tV5/fXXmTJlCqGhoYwdO9ap68yZM4c5c+bQsmVLtm/fzu+//06VKlVu+3pr1qzhv//+o2XLljzzzDM88cQTvPrqq8V6bwpi1qxZvPbaa8yePZvGjRszcOBA/vzzz3w7XM4wYsQIpk6dysSJE2nTpg2xsbGMHDmy0Kz3qlWrsnDhQn7++WeaNGnCnDlzmDt3rs2YqKgoVqxYwW+//UbLli1ZsGABr87MFSvbpZnyr9MbX0YDrJ+u3O/4NARHOXmBgqlwu2b2eO23IyzedY5netRlyqBGluN5d80EgpKmX79+hIWFsXjx4hK/dnxKJleTMgHQqlU0Cle+I8cuJ2EwyTQIDQCj3vG/8X0L4Y/nwScExh8En+BCbRC7Zk5Qt6ridCuvMyKBZ5Cens6CBQsYMGAAarWaH374gQ0bNtjEC5UkWfpcv5D1jMhkFdDo8CwkKxU2va3c7z7JIRFyhgq9NDNTp6qyrR0jhEhQDJo2bYq/v7/d29KlS5EkiTVr1tC9e3fatm3LH3/8wYoVK+jbt69L7MkyWAmRLFtSO8zi45SzeucnkHoNQmpB+ydLzsgcxIwIqFtNEaLzienojSZLIqBA4Axr1qwpsMlCaGgoPj4+bNiwoVRskWWZLL1tTqXRJCOpbLfvTfZOzkvKNdjxsXK/zwzQlPzOshAiIDzQG28vFZl6ExcS0y0zJIHAGWrWrOluEywYTDJGWUYiJ0ZKljGaZNRWW2UOVx7d/Dbo06B6O2h6n0vsFT/9gEolUaeKIj5ny+kWvqBiYZ4NaTUqNGpFcIwmGZOVV8ghHYo/Afu/V+73f7OIe/6FI4Qohzo5DmvhJxKUB8z+IZ1GjSYnzspokrHeI3dIUjbMANkEje6Cmp0LH19EhBDlULeqmBEJyg+ZZiHyUqHOCRgymGQr/5BU+NIsdhucWgeSGvrOdKW5QojMmB3WYkYkKA+Yl2Y6jdoiRNYzokJXWCYT/J0TLNpuFFSp7yJLFSqGEBmyYdcCOLu5wCF1quTEEolSsoJyQO7STGVJY1J8RAqFbt0fWQFXDoI2AHpMcZ2hOVQMIdoxD9ZNhnVTwWS/TKzZR5SYls3NtLJXWrUkqVWrFvPmzbM8liSJ3377zW32CJzDaJLRG3OFKHdGZHIs816fCRvfUO53fR78q7rSXMUel7+CJ9D+SfAOgvhjcHCZ3SG+Wg0ROYWezl4XyzNrrly5wqBBg9xthsBBsgzKj61GpUKjzusjUsbc1j/035eQdB4CIqDTGFebC1QUIfKtBN1fVu5veguy7S+/LH6ieLE8syYsLCxfHZ7SxlXF6csj5tQOnZfy9bb1ERWSeZ+eCNtyEmp7vwJaX5faaqZiCBFAh9FKtnDKFdj5md0h5p2zU9dS7F9DlhURc8fNydzklJQURowYgZ+fH+Hh4Xz44Yf07NmTF154wanrgO3SzFzc/tdff6VXr174+vrSsmVLm9o3oNQh6tatGz4+PkRGRjJ+/HjS0nIFvrBi8EUtTi/InRF5a/ILkakwZ/XWuZCZBNWaQsthrjbVQsWJrNbolPD0FU8oPqO2j4F/NZshzasHAfD9znP0blSNNjXyRFjr0+HtiFIyOA/TLoPW8Yp4EyZMYMeOHfz++++EhoYyffp09u/fb9Mxoji88sorzJ07l/r16/PKK68wbNgwzpw5g0ajISYmhoEDB/Lmm2/y7bffkpCQwNixYxk7dizfffcdUHgxeDPOFKcXKFjHEAFoJKsZUU5Ao8qeszoxVlmWAfR/A1TOlScuDhVnRgTQ9H6IaA3ZqbB5Tr6n72kVwYCmoWQbTTz5/V6OXU5yg5HFJyUlhUWLFjF37lz69OlDs2bN+O6772yqKRaXiRMnMnjwYBo0aMDrr7/OuXPnOHPmDKB0wBgxYgQvvPAC9evX54477uDjjz/m+++/JzNTKUvx+OOPM2jQIOrUqUOnTp34+OOPWbt2raVMq5k33niDfv36UbduXSpVKpn6yOWdzAKWZobCtu83vg4mPdTpBfVck4hbEBVnRgSgUilh6gsHK7VVOj4DVXM7WGjUKj4e1ponFu5l+5nrTP01mvcGhuee7+WrzEzcgZfja/WzZ8+i1+vp0KGD5VhQUJClR1ZJYF2M3lz3Oj4+nkaNGnHo0CEOHz7M0qVLLWNkWcZkUjpGNG7cuNBi8GacKU4vUJonZueZEZmFyJxvBnac1VePwNGVgAT9bas/lgYVS4gAanWFBoPg1FrYMBOG2e6i6TRqvnikLQ9/s5v4mylcT82mlsGINyg/I04sj8ozeYvRAzbF559++mnGjx+f77yoqCinisG7ojh9eSbbYEJGRiVJeOXkmKmt9urNJWPzbd/vmKf822o4hDUvBUttqVhLMzP9XlfC1k/+Cef+zfe0n07DwpEdqF3FD6NJ5uLNDPQGhwomeAR16tTBy8vLpjB7UlISp06dKpXXb9OmDceOHaNevXr5blqt1qYYfLdu3WjUqFG+rqWComEdyGj+gZAkySJG5vgim4BGfYYSvKjxgV6vlKq9ZiqmEFVtCG1yOin8/ardHakgXy/e+V8LNCoJvdFE7PU0DMayIUYBAQE89thjvPzyy2zatImjR4/yxBNPoFKpHC/9UAwmT57Mv//+y9ixYzl48CCnT59m1apVli4djhSDFxQN846ZLk8fPHN0tcGYZ/teNkHGLeV+5+cgyLZ9UmlRMYUIoOdU8PKDS/vg6K92h1T201ElQItGpSLTYCT2RppNWxZP5oMPPqBz587cdddd9O3bly5dutC4ceNSqb3dokULtmzZwqlTp+jWrRutW7dm+vTpREQoO46OFIMXFA1LDJHG9qutzsnAN8+ILEuzjFuKg9o7BLq8UEpW5qdiF8/f/I5S9Cm4Jozdo2zxW2Eunh9RI4pLKUYMJhN+Og21K/uVeO9vV5OWlkb16tV5//33eeKJJ9xtjsBFnIlPJT3bQM1KvgRZ9eiLvZ5GSqYeL7UKvdFEFX8dEYFaMi8dJfbiFWqrr+Hd4dHbXLloOPodrbgzIoA7xoJ/GNw6B3u+LnCYzktNrSq+qCWJtCwD5xPTLfV/PZUDBw7www8/WHpyjRgxAsDhPu2Csod1edi8SzPLFr7ZRySh1KCWDaDyclnlRUep2EKk9YNe05T7W96FjJsFDvXVaqhZxQ+VJJGcqediYgaePpmcO3cuLVu2pG/fvqSlpbFt2zaOHz9eYIF3f39RIrcsY10eVptnaWb2EZn/YjWyAVITlAc+QaC2bcFd2lS87fu8tH4Ydn0OCcdh2/tKnFEB+Os0RFXy5dyNdG5lZKO+BRHBPqXiAHaW1q1bs2/fvnzHMzIyOHjwYOkbJHA5ueVh1ajy/E2q87gS/LMTAJOyU+ZeDQKEEClh7P3egGX/B7u/gPZPQUjBRdADfbyIrOTD+cR0bqRlo1ZJhAX5lKLBxcPHx4d69eq52wyBC8g02HdUg60QeZONtyEna8A/FJKulYp9t6NiL83M1O8HtbuDMRv+yb+NnHcJFuyrpXqwIj7xKVnEp2SWipkCwe3IMtimdlijsRKicClRiSLyDkb28owfUSFEoHju+uUIUPTPcGk/AGq14vDLG+0LUNlfR1hO/aKrSZncSM0qHVsFggKwLg+bF/OMyJ8MAqQMZCQIjCA9PR2wjZR3B2JpZiaiFbR4EA7/COunw2N/oNFo8PX1JSEhAS8vL1QqW90O9IIsnVLV8eL1bIx6bwJ9Sr75nEDgCBkZmcgmExg1ZGbaxrsZsg1gyKaSdJ1Mk0ymJhBjUirx8fEEBwdbfnTdhVuF6PPPP+fzzz8nLi4OUFr2Tp8+3X3VAHu/Ckd/g7htcOovpIYDCQ8PJzY2lnPnzhV4Wnp6NqlZRhIuQ2V/Ld5e7v1QBRUPkyxz+ZbiItCke+dzVhuMJlKTk7gipWBCRZaPBh9dNsHBwYSFhbnDZBvcKkQ1atRgzpw51K9fH1mWWbRoEffccw8HDhygadOmpW9QcBR0egZ2fATrX4N6fdFqtdSvX9/u8syMySQze+1x/jkRj1atYvYDzWkVKWrnCEqP41eSmblyPyF+Wn555o58z6ekpBK0djTVpCS+0A+m2ZBxtGwQ7vaZkBm3CtGQIUNsHr/11lt8/vnn7Nq1yz1CBNB1AuxfDNdPwYHvod3jqFSqQlMjpt/bisuL97HxRDxPLT3Mtkm9CPYVyzRB6XA28TqXUoxEVvWx+7eq/e9TVGlHuChX4bOsTnyp1XmMCIEHOauNRiPLly8nLS2Nzp3td5TMysoiOTnZ5lbi+ARDj8nK/U2zIauAsrF58FKr+HREG+pW9SMl08C6I1dL3jaBoADM/fjqVbMTlJp2HdX2DwGYqx9KFlq7W/zuxO3WREdH4+/vj06n45lnnmHlypU2hbGsmT17NkFBQZZbZGSka4xq9zhUqgNp8fDvfIdP8/ZSc3+bGgCsPnzFNbYJBHY4E68Ikbnuug1b3oHsFE6p6rLKpCzb8kZeuxu3W9OwYUMOHjzI7t27efbZZ3nsscc4duyY3bFTp04lKSnJcrtw4YJrjNJolfrWoAhRsuOiMqSFkmH+b8x1ElLElr6gdChwRnT9DOz9FoDFAU8g53zltWq3f/VtcLs1Wq2WevXq0bZtW2bPnk3Lli356KOP7I7V6XQEBgba3FxGk3ugRgelYP7mtx0+LaqyLy0jgzHJsPaImBUJXI/eaOLcDSUeKN+MaONMMBmgfn8uBre3HPa2E/ToTjzLGpRyo1lZHjCTkKTcvLMDSyD+uMOnDmmh1HD+45Cb6lsLKhTnbqRjMMn4atWEB1k5qs/vguN/gKSCfm/YbJ5oPchRDW4WoqlTp7J161bi4uKIjo5m6tSpbN682VKywu1EdYTGQ5QqduunO3za4Bwh2hN3kytJGa6yTiAAbP1DlgRsWVaqj4KS2F2tMUE+udHTwkdkRXx8PI8++igNGzakT58+7Nmzh7/++ot+/fq50yxb+r4OKg2c/hvObnHolPAgHzrUUlrf/Cmc1gIXY9c/dOw3uLhH6f6SU4c6xGpGJHbNrPjmm2+Ii4sjKyuL+Ph4NmzY4FkiBFC5rrKLBsovjIOlYoe0FMszQekQY5kR5XQ8MWTDhteV+3eMgwAlcjrYV8yIyjY9JoMuEK4eVpJiHWBgs3BUEhy6mMT5HEeiQOAK8s2I9n4DN2PBrxrckdvSSQhRWcevCnR9Qbn/zyzQF172o2qAjjvqVgHgj8NiViRwDbIsE5OQBuTsmGXcUuKGQKk+qstdrpmd1SrJtiyIJyCEyFE6PQeB1SHpAuxe4NApYnkmcDXXkrNIzTKgVknUrOwH2z9QSh5XaQitH7EZG5zjrNZqSqetlDMUS4hSU1Ndn3LhKXj5KNn5ANs+gPTEQk8Z0DQMjUrixNUUzsQ7lioiEDiDecesZiVftKkXYVfOj2S/N0Btm0paI8QHL7VEhAdWFHVaiGJjYxk8eDB+fn4EBQUREhJCSEgIwcHBhISU84zzFg9CaHPISlKK7RdCsK+W7g2qAvDHIbF7Jih5zP6hOlX94Z83wZgFtbpBgwH5xlb217FqTFcWP9mxtM0sFKez7x9++GFkWebbb78lNDTU46Z4LkWlhv5vwOL7lPZDHUcrOWm3YUjLcP45Ec8fhy/zQt/6Fev9Ergc84yoi99FpagfQP9ZVq1cbWkS4cJshGLgtBAdOnSIffv20bBhQ1fY4/nU7Q11+0DMRmWLdOii2w7v2zgUnUbF2YQ0jl1JpmlEUCkZKqgIKDMimTuvfKYcaP5/ENHarTYVBaeXZu3bt3ddsmlZod8bgKQEjV3Yc9uhAd5e9GpYDRDLM0HJcyY+lZ6qg4Te2A1qLfR+zd0mFQmnhejrr7/mnXfeYdGiRezbt4/Dhw/b3CoEYc2gVU4ayt+vKuH0t2FISyUjf/Xhyx7flFFQdkjO1HMjJZ2pmh+UAx2fvm0rLE/G6aVZQkICMTExjBo1ynJMkiRkWUaSJIxGY4ka6LH0fgWOrIALu+DEaiUnraChjarhq1Vz8WYGBy/conVUrlP/3I00jlxK5s7mYcJ/JHCKmPhU/qfeSkPVRfAOhm4vudukIuP0jOjxxx+ndevW7Ny5k7NnzxIbG2vzb4UhMAI6j1Hur58BRn2BQ320avo2DgXyL896vLeZMcv2s/6Y+5vcCcoWcVcSmKDJifTvMQl8yu6utdNCdO7cOd555x06duxIrVq1qFmzps2tQtHlefCtAokxsG/hbYeal2d/Rl/GZMq/PNt37qYrLBSUY6oc/pJQ6RaJ2gho/6S7zSkWTgtR7969OXTokCtsKXt4B0LPKcr9zbMhM6nAod0bVCHAW8O15Cz2xCnBkNb+Ik/LhhZ4OCnXaH95MQCHGz4PGp2bDSoeTvuIhgwZwosvvkh0dDTNmzfP1yHy7rvvLjHjygRtRyopHzfOwPZ50HeG3WE6jZqBTcP4ed9FVh++Qsc6lUnONFie97QkRIGHs3k23nImB0110TR/wN3WFBunheiZZ54B4I033sj3XIVyVptReynb+cuHw67PoP0TEFTD7tA+javx876LlmWYdZvqbINj5UUEAhJOIu//Hgl4Sz+Cj0PtFMwvYzj9M2wymQq8VTgRMtPwToi6AwyZ8M9bBQ5rVl0JZjwdn0KWwcjyPbnxWClZhoJOEwhsWT8DSTbyt7Etx7yaEhZ4+557ZQGnhEiv16PRaDhy5Iir7CmbWNe3PvQDXI22O6x6sA9BPl7ojTLv/32KL7fm7jKmZAohEjhA3HY4tRaTpGaOYRh1q/mXi7APp4TIy8uLqKioijvzuR012kLT+wEZ/rYf3SpJEk1zcn2sRQggVQiRoDBMJksd6qPh93FWjqCevT5mZRCnl2avvPIK06ZNIzGx8DIYFY4+00HlBWc3wZkNdoeYl2d5SckqOA5JIADg6K9w+QBo/fnJT4nsr2uvs2sZxGln9SeffMKZM2eIiIigZs2a+Pn52Ty/f//+EjOuzFGpNnQYDbs+hb+nQ51eSsa+FU0LyH4WSzPBbTFkwcacOtRdXuBQtBbIsN/ZtQzitBDde++9LjCjHNF9IhxcAvFHFX9R64dtni4o+14szQS35b8v4dZ5CAhH7vwcMRu3AVCvml8hJ5YNnBaiGTPsx8kIcvCtBN0mwvrXlEJVTe8Hra/l6dpV/PDxUpOht/WzJQshEhREeiJsfU+53+sVLqVJpGUb0ZjLw5YDihxFt2/fPpYsWcKSJUs4cOBASdpU9ukwGoKiIOWKskyzQq2SaBwekO+UVOEjEhTE1rlK1H61ptBqOD/lhH00qx6El4f1sC8qTv8v4uPj6d27N+3bt2f8+PGMHz+etm3b0qdPHxISElxhY9nDy1txXANs/whSbd+XRzvXopKf1uZYpt6E3iiCGgV5SIxVlmUA/d4gJdvEwn/jABjd/fbVQcsSTgvRuHHjSElJ4ejRoyQmJpKYmMiRI0dITk5m/PjxhV+gotDsAQhvBdkpsGWOzVP3tq7O/tf68ea9zfhgaEvLceEnEuRj4xtg0isbH/X6sHjXOZIzDdSt6sfApmHutq7EcFqI1q1bx2effUbjxo0tx5o0acKnn37K2rVrS9S4Mo1KlRvkuPc7uH4635CHO9Xk/jY18PFSdtbEzpnAhov7lC17JOg/iwy9iW+2xQLwXM96qDysN1lxKFKKR95EV1CCHU0OtmOuMNTuBg0GgmyEDTMLHObvrewZJGcKP5EgB1m2BC/SchiENWf5nvPcSMumRogPd7eKcK99JUyRyoA8//zzXL6c2zTw0qVLvPjii/Tp06dEjSsX9H0dJJVSxfHcv3aHBOQIUarINxOYObkGzv8LGm/o/SrZBpMlGv+ZHnXLjZPajNP/m08++YTk5GRq1apF3bp1qVu3LrVr1yY5OZn58+e7wsayTbVG0OZR5f7fr9mtbx3grcwwxdKsfONwvXKjXqn6CUqH4aDq/Lr/IleSMqkWoON/be1XdyjLOB1HFBkZyf79+9mwYQMnTpwAoHHjxvTt27fEjSs39JwGh3+GS3vh6Epodr/N035axUeUni2EqLwSfTGJ4V/volOdynz8UGt8tOqCB+9fBDdOg29l6PoCBqOJz7fEAMpOmbfXbc4tozgtRKAkb/br149+/fqVtD3lk4BQ6DJeqeK48XVoNNimop5vzh9lRrZIJi6vHLxwk5RMA+uPXWP5nvOM6lLb/sCsFNics8vaYwp4B/HnwUucu5FOiK8XwztGlZ7RpUiRhGjjxo1s3LiR+Pj4fA7qb7/9tkQMK3d0Hgt7v4WbcbDnG+j8nOUp8y9cuhCicot1mfJzN9ILHrjjI0hLgEp1od0oTCaZTzedAeDxLrXx1RbpK+vxOO0jev311+nfvz8bN27k+vXr3Lx50+YmKACdP/Saptzf+i5k3LI8ZZkR6YUQlVdm/H7Ucv9KUob9QcmX4d9PlPt9Z4Lai/XHr3HqWioBOg2P3lHL5Xa6C6fldcGCBSxcuJBHHnnEFfaUb1o9DLs+h4QTsO19pUc5WH7lhI+oYnAhsQAh2vQWGDIgshM0HoIs586GHulckyCf/GEz5QWnZ0TZ2dnccccdrrCl/KPW5LSrRim4f/McgMVxKZZm5ZPjV5JtHp9PTM+/g3btKBxYqtzvPwskiW2nr3P4YhLeXiqe6FqAT6mc4LQQPfnkkyxbtswVtlQM6veHWt3AmA3/KDMic2R1plialTsS07IZ9NE2m2OpWQYS07JtB66fDsjQ5B6I7ADAJzmzoWEdoqjsX7bbBRWG00uzzMxMvvzySzZs2ECLFi3yRVl/8MEHJWZcucRc3/rLHhD9M3R6Dl9tMCBmROWR7Weu2z1+LjE9V1xicip6qrygjxI/tCcukf9iE/FSS+UqubUgnBaiw4cP06pVK4B8RfTLQxHvUiGiFbR4EA7/COun49NIcVAKISp/pBSQtnPuRhptokKUOtTrc2qct38CKtcF4JN/lNnQ/9rWIDzIp1RsdSdOC9GmTZtcYUfFo/ercPQ3iNtGndDtQIiIIyqHpBWQtmPZwj/8o9L1RRcI3ScBSvDjllMJqCQlnaMiUL4SVsoSwVHQSWlW2fzYB6gxiu37ckhqlv3P9PyNdNBnKFU8AbpNAL/KAJadsrtbRpSbCoyFIYTInXSdAD6V8E+JYah6s1ialUMKmhHF3UhTQjmSL0JgDeio/ChFX0xi3dGrADzXq16p2eluhBC5E59g6KFMxydoflHC+wXlioKEKPnGFdj+ofKgz2vg5UNCShZPL94LwOAW4TQIzV9SuLwihMjdtHuCrMCaVJWSuD9zpbutEZQweUu7mGuZjcj6CbKSIawFNB9KlsHIs0v2cTkpkzpV/Hj73uZusNZ9OC1EW7duxWDIr/IGg4GtW7eWiFEVCo2WW52nAjDCuApSrrrZIEFJkndG5KfV0NL3Bg+rcxpw9p+FLEm89tsR9p67SYC3hq8ea0eQb/mNoraH00LUq1cvu11ek5KS6NWrl1PXmj17Nu3btycgIIBq1apx7733cvLkSWdNKvOYGt/DflM9fKUs2PS2u80RlCBpeZzVRllmsmY5XpKR+NDuUKcnC/+N46e9F1FJMH9Y63LTNNEZnBYiWZbtxgvduHEjX9fXwtiyZQtjxoxh165drF+/Hr1eT//+/UlLS3PWrDKNr9aLt/RKC2H5wGKIP+5miwQlRd6lWQvTCe7I3oFRltgUNZZtpxOYtfoYANPubEzPhtXcYabbcTiO6P77lWJekiQxcuRIdLrckHOj0cjhw4edzkFbt26dzeOFCxdSrVo19u3bR/fu3fONz8rKIisry/I4OTk535iyiI9WzT65IeuM7Rmo3qNU5xvxk82Y5Ew97647wUPto2hW3X63WIHnkWaTyCwzSbUEgJ+NPTiUFc78X6MxyfBAmxrlPp/sdjg8IwoKCiIoKAhZlgkICLA8DgoKIiwsjNGjR7NkyZJiGZOUlARApUqV7D4/e/Zsm9eNjIws1ut5ClqNCo1K4h3DQ8gqDZz+C2Jt/W2rD11hya7zvLIy2k1WCoqCtY9okOo/2qhOY1D78IHh/4i7ns6lW0om/qSBDSt0ZoLDM6LvvvsOgFq1ajFx4kSnl2GFYTKZeOGFF+jSpQvNmjWzO2bq1KlMmDDB8jg5ObnciJGPVk1sZjjJTR8mKHqh0sHhqc1KWyJyUwUOXUzi/I10oir7FnwxgcdgXpp5YWCyZjkA8c2eIn53CKb4FEsJcz9d+Sx45ihO+4hmzJhR4iIEMGbMGI4cOcLy5csLHKPT6QgMDLS5lRfMGfiXWz4P2gC4cgiO/GJ5PlOfWwlzdfTlfOcLPA+D0WT53EaoN1BLdY0EOQiv7i8AcD01NwPfpxzWoXYGp4Xo2rVrPPLII0RERKDRaFCr1Ta3ojB27FhWr17Npk2bqFGj/HUocARzlcZUTTB0fUE5uPEN0GcCttUbVx+6UsrWCYqCeccskDTGa34F4EPD/6hSqZLl8wZlaa4uR80Si4LT88GRI0dy/vx5XnvtNcLDw4u1rpVlmXHjxrFy5Uo2b95M7doV11nnY6nSaFRayOz5BpIuwH9fQJfnbWoVHbuSzNmEVOpUwG1eT+HQhVscvHCLRzvXLPA7sOlkPAAv+62hkjGV06bq/GjsyduSRFQlX05cVSLpfW/X0aOC4LQQbd++nW3btllKgRSHMWPGsGzZMlatWkVAQABXryrBfEFBQfj4lP/SB9bkdvIwgNZXyc5f9RxsfR9aP0KWwTYeZfXhK4zvU98dpgqAez7dAUDVAB13Ng+3O2bxrnNEcJ1hpj8BmGN4CCPK51yzcq4QVfRlGRRhaRYZGel4o7hC+Pzzz0lKSqJnz56Eh4dbbj/++GOJXL8sYf5jtCzBWj4Eoc0gKwm2vmcpEdIwJ/9o9WHhJ/IEYq/bj3k7fiWZfedu8rLXz2jkbHaZGrPR1MbyfC2rrPrb9jirIDgtRPPmzWPKlCnExcUV+8VlWbZ7GzlyZLGvXdbIV7dapc6tb/3fV/inXwDg7lYReKklTl1L5dQ1kSTrbnQa+1+hH/dcoKkUxz3q7QA5Aau5SzjrXU8xIyqCED344INs3ryZunXrEhAQQKVKlWxugqJht8livT5QtzeY9Nx9/SsAqgXo6F6/KgCrD4lZkbspSIj2xSUyVbMUFTI0+x/Rsm2515qVcmdEwkdUBB/RvHnzXGCGwLegTh79ZkHMJjqkb6W11AcfbWvuahnOxhPxrD58hRf7NajQgXDuRqfJLyIGo4nQhG10VR9FVmmR+kyHvbZllWtazYjKYwtpZ3FaiB577DFX2FHh8fFSPop8VRrDmkGrEXBwCdO8lpKkfogeDaqi1ag4ez2NY1eSaRohUj5Ki6R0PQHeuV+bnWdvMLS9bVBtXEIyE6WcTjcdRkNITcBWiCKCffBSS+iNspgRUcR6RDExMbz66qsMGzaM+Hhli3Lt2rUcPXq0kDMFBeGjVT4Ku3Wre79CJlraq05R/dpGAry96NUwZ3l2WMQUlRZbTyXQ8o2/eXrJPsuxlQcu5RuXuvt7GqkukCL5I/WYaPdaapVEjRBlVlRe20g7g9NCtGXLFpo3b87u3bv59ddfSU1NBeDQoUPMmDGjxA2sKNy222tgBD953QNA7YPvglHPXS0iAGX3rKR2MQW3592/TgCw/ti1ggdlp1HvyEcAbA0bCT4hBQ41L8/E0qwIQjRlyhTefPNN1q9fj1artRzv3bs3u3btKlHjKhLmnZOC6lYv5G6uy4F4J8fBvoX0aVwNHy81FxIzOHwxqRQtrbgYjA4I/s5P8ddf57ypKjeb3d6NYd7Ct17qVVScFqLo6Gjuu+++fMerVavG9ev2m8kJCsfurpkV1/U65hkeUB5snoOvKZ3ejZXaNSKmqHQwmAoRotR42KHMht4zPEiDiCqWp+ztJzzauSYPtKnBg+3LR+J2cXBaiIKDg7lyJb9f4sCBA1SvXr1EjKqImOOICmoplGkwsdzYC31IXUi/DjvmMaSFEtH75+ErmAr7kgiKjcFouv2AzbMhO5WDpjr8YepMw7Dc4vcaO7lkdar68/7QlhWyImNenBaihx56iMmTJ3P16lUkScJkMrFjxw4mTpzIo48+6gobKwS3W5qZTDLZBhMGNGR0z+kKuvNTeobr8dOquZyUyYELN0vT3AqJ/nZLs4RTsG8RAG/rR1A92Jcgn9y60yoRYnFbnBait99+m0aNGhEZGUlqaipNmjShe/fu3HHHHbz66quusLFCYHZW21uaZVrlmakb3wVRncGQife2OfRrEgrAHyIj3+UYC5h1pmYZYMMMkI2cr9qT/+TGNA63bQVU0bPrC8NpIdJqtXz11VfExMSwevVqlixZwokTJ1i8eHGRy4AIrFI89Pl3zaxrEXlrNdA/pzvowWU8FKU4qtdEXynwiyIoGQwm+0uzYzvXwsk1IKn5OfhJABqF2dbKEkJ0e4rsro+KiiIqKqokbanQ5Dqr8/+xm0uAaNU5dWtqtIOm98HRlXQ4M48A72eJT8liT1winepULlW7KxL2lmYSJsJ3v6U8aPsYW89VAm7RSMyInMIhIZowYQKzZs3Cz8/PplSrPT744IMSMayiYcm+txNHZHZg67ysJrB9ZsDx1ajObmJczcG8fTKC1YcvCyFyIfac1XepdhGZcRxZ64+p+xROvrcfgMbhtjMie85qQS4OCdGBAwfQ6/WW+wUhcp6KjiXXTG/M17LJPCOyydKuVBs6PAW7PmN40tfM4VXWRl/l9bubiV9fF6HPs/TVomeSRilZ87VpCBFxSmlYnUZlU+YD4LW7mvD88oM81a3iFv+7HQ4J0aZNm+zeF5QcZh+RLEOWwWQTbWv2EeWLwO3+MhxYiv+tE9yv3sYvaT24mZ5NFX8dgpIn74zoUfXfRKoSiCeED1L7Ef630hy0YVhAvh+De1pVp2u9KlTy0yLIT5FyzQQlj3W+Ud4tfLszIgDfStD9JQAman7Gmyz0hcW6CIqM9YQoiFTGaVYCsKf2c2TgzdmcImn1qwXYO53K/jqxaigAh2ZE5uaKjvDrr78W2ZiKjFolodWoyDaY8gU1moXI28vO70aHp+G/rwlLOs/j6rVkGwaUhrnlmoU7YrmSlMnUOxsXOGas5jeCpHQu6+pwMuwuOB5rea5qgJiROotDMyLrpoaBgYFs3LiRvXv3Wp7ft28fGzduJChIlKMoDgU5rM1LM5295Egvb+ijBDk+q/mDz/7cRVK64s/bGXOD55cf4EZqVv7zBAUy849jfLH1LCev2q+AWUOK51H13wCsCX2WsGDbyOhKfl72ThPcBodmRObmigCTJ09m6NChLFiwwBI3ZDQaee6558pVnzF34KtVk5Shz7c0yyhoaWam2f+4vv4DqqQcp8mpz7nvczUbJ/Rg2FdKErJKkvjwwVauNL3cYL20tY4bMje4BJik+RGdZGCbsRkXK99B/zzNLkN8hR/IWZz2EX377bdMnDjRJnhRrVYzYcIEvv322xI1rqKRr251DrddmgGoVFS5/10Ahqv/Qb5+hixD7pfofGK6C6wtn1gvi62F//U/jgHQQorhbvVOTLLEbMNwxvWpT80qtjtkwiHtPE4LkcFg4MSJE/mOnzhxAlMBkacCx/AtIPE1V4huE7leuzvGev3xkoxM1iy3ibIW7lHHsU6x8VLnfj1+2XcRkHnFaykAK01dqVyvHZX9dYQHeqO1ql0dIoTIaZyOrB41ahRPPPEEMTExdOjQAYDdu3czZ84cRo0aVeIGViRyfUQO7prlwdhnJpxez0D1HhLObLccFxs1jlNQGRaAvqr9dFSdIFP2Yq5+KPVyjqtUEpEhPsQkKLtmlYUQOY3TQjR37lzCwsJ4//33LeVAwsPDefnll3nppZdK3MCKhE23VysKjCPKg6paY3409mS4ZhOpf0wBXgUkJDEnchh7ZVhkWUaNkSmaHwD41jiIK1S2CBFAWJC3RYjEjMh5nF6aqVQqJk2axKVLl7h16xa3bt3i0qVLTJo0SSS9FhPfAnbN7KZ42EGtkvjQ8D/SZB21M48zWLXbNYaWI0wmmYs3c31o9sqwxKdk8ZB6E/VUl7khB/C54W7ANpMg0Dt3pyxAJyouOkuxAhoDAwPFTlkJUlBLIUeXZpIkcUMK4UvDXQBM0ixHi144iW7DpBWH6frOphwfUO57DUqUO0Ds5au8oPkFgI8N95OCsktmLkwHtkIkghadp0hC9MsvvzB06FA6depEmzZtbG6ColNQlUZHl2agzIq+Mg7mOsHUVMXzsHpDyRtajjAL0LwNpwDbHwEZRYmC9y+gqpRMrCmUZcY+lucfaFPDcr9TXdFctDg4LUQff/wxo0aNIjQ0lAMHDtChQwcqV67M2bNnGTRokCtsrDAU5qz2LqCrqDUqSSIdbz5lKADjNCvxM6WWsKXlD3MFResfgUs3M0i4HEfdM0oc3TuGYehz3KpqlYTKKp/snpbVmTGkCb+P7VKKVpcfnBaizz77jC+//JL58+ej1WqZNGkS69evZ/z48SQliW4SxaHQpZkDjfjM5SZ+NvbklKk6IVIq96UuL2FLyx/m1ZS1f27417vZ+NnzeJky2WtqQJX2D1iey1uETqWSGNWlNi1qBJeGueUOp4Xo/Pnz3HHHHQD4+PiQkqKEwT/yyCP88MMPJWtdBaPAXTODA3FEOZh/pTOMSsAdwMC0VXDrPKBEDi/bfZ7zN0SQozXmuY31bLSBdIH/U28B4G39cCr5iRwyV+G0EIWFhZGYmAgoVRrNvcxiY2NFo79iYp4RZebxEZm/HPb6rOfFXH7CaJLZZGrFv8YmisN64ywAvtgSw7SV0fT9cEtJml7mkSxLs9yg3KmaZaglmTXGDuyXGxAsUjdchtNC1Lt3b37//XdACW588cUX6devHw8++KDdfmcCx8nt5GE/6dWZpZmCxNs5syKif4LLB9h0MgGAbIOIgrcmd0akvPd3qI7QS30IvazmXcODAISIZFaX4XTAw5dffmlJ5RgzZgyVK1fm33//5e677+bpp58ucQMrEgXmmhmcc1Zbc0Suw3bf3nRN/4fk36ey79xYxH6+Hcw+Ir0RCRPTNMsAWGLsS5ysbNOLZFbX4dSMyGAw8Oabb3L16lXLsYceeoiPP/6YcePG2bSgFjhPgblm2Y77iOyVif05cCSodQRe3Ukv1cFi21keMb9r6dlG7lXtoJkqjmTZh48NubP8EF8tLw9oCMD4PvXdYGX5xSkh0mg0vPvuuxgM+Qu8C4pPwTMix5dm9oQoQR0KHZXZ6lTNMtQUnE9VUTHPJA1Z6Uz0+gmAzw33cJPcgN1Kflqe61mXbZN68WJfIUQlidM+oj59+rBli3B0uoKCmizmxhEVTYgMRhm6vcRN2Z8GqkuWnSBBLpKkpHuEHl9IdekGl+TKfGscaDMm2NcLSZKIrOQroqdLGKd9RIMGDWLKlClER0fTtm1b/Pxsa7HcfffdJWZcRcMS0GiTZiBbHntrC//dUNv5guhNJvAJZr7hPqZ7LWaC5hd+N95RQlaXDyQkVu86wpModajf1/8fWdi6GvxFDpnLcPqdfe655wD7/cskScJoFNP+opIb0Ji79M02miw5T0X1EZmrDi429uMx9V/UVMXzlPpP4IF8Y8sjeqOJ0d/vpVVkCM8XsKQyyTK6f98jUMrgqKkmK01d840RsyDX4fTSzGQyFXgTIlQ8fCxxRCZMOZG7Nu2mi7g00xtk9EYTejS8a3gIgNGa1ZByDaDcx39tOHaNTScT+DAnn8we+oTT9E5ZDcDbhuHIosFNqSLebQ/C18oZbd6yN/uH1CoJL3Xhv8h5t+9BmRGYr/OnqSMHTPXwk7Jg89tkG0wMnLeNMcv2l8R/wSNJu02xMzOTND/iJRnZbGzJDlNzQInJEh05SgeHl2YZGRls3LiRu+5SSkxMnTqVrKzc7hBqtZpZs2bh7e1d8lZWEKxnPOnZRny1GpuEV0eWBho7YpWebbTyO0m8pR/OL7o3YP/3HAp7kJPXUjh5LYVDF/5hwcNtaVa9fHVjMRUy42sjneJO9X8YZasAUKBLvSrsPHvD1eYJcGJGtGjRIr744gvL408++YR///2XAwcOcODAAZYsWcLnn3/uEiMrCiqVZCmQb945y3CkXrX1NeyI1dXkTK4mZVoe75UbsVFuD7KJWvvftRy/eDODseVwZmSySlC9lpzJppPxyLLMmugrWNeh/tnYg1NypGWsv7dGRKCXEg4L0dKlSxk9erTNsWXLlrFp0yY2bdrEe++9x08//VTiBlY0fPMkvjpTiwjs+4gAvt95zubxAu0jIKmpemUTnVTHLMevp2Y7bbOnY50o3//DrYz6bg8/7b3Ac0v3M1C1h7aq06TLOj4w/J/NeQE6DcM6RAHwf21rIHAdDgvRmTNnaN68ueWxt7c3KlXu6R06dODYsWP2ThU4Qd58s0JbCeWhICE6fiXZ5nG8NgraKc0OpmmWIqEIXlp2+QtWNVotzZIylP5kk1dE44WByTl1qL8y3kk8ITbn+ek0zLy7CYuf6MCse5uVnsEVEIeF6NatWzY+oYSEBGrVqmV5bDKZbJ53hK1btzJkyBAiIiKQJInffvvNqfPLI3nTPJxdmtmLIwI4etlWiGQZ6DGFFNmHFqpYhqh2Wo6nZpUvMSpoV3C4eiO1VddIkAMt5XWHd4yyPO+v06DTqOlWv6rD77+gaDgsRDVq1ODIkSMFPn/48GFq1HBu+pqWlkbLli359NNPnTqvPGMpF5uzNMtysF61mYJmRHkxyTL4V2WBYQgAk7x+RIeyLGs24y++2BLjlN2eTN4iZgABpPO8ZgUA8wz/Iw0fwLZ6gQhgLD0cFqI777yT6dOnk5mZme+5jIwMXn/9dQYPHuzUiw8aNIg333xTlA+xIndpVrI+oryYHbjfGAdxRa5EDek6j6n/sjw/e23+JpplFTs6xLOa36kkpXLGFMFyYy/L8cS0XB+Zv7cQotLCYSGaNm0aiYmJNGzYkPfee49Vq1axatUq3n33XRo2bMjNmzeZNm2aK20lKyuL5ORkm1t5wzfPjCijmD6ijx5qxagutfKNM/tNMtHxfo6TdqxmFcGkFMluTybv0iycGzyuXgvAR9LDNKmeW/g+22Di3lYRVA/2YUjLiFK1syLjsOSHhoby77//8uyzzzJlyhTLhytJEv369eOzzz4jNDTUZYYCzJ49m9dff92lr+FuLImvetuAxqJu3zcIDWBA0zC+2xFnc9wkgyEn9eNXYzeeUK+lseo84zS/McvwCKAsaRydYXkyeeOIJnr9hLekZ7epEWOeHUvDsEC+2R7L55tjmDigIfWr+SPL2BTHF7gWpyKra9euzbp160hISGDXrl3s2rWLhIQE1q1bR506dVxlo4WpU6eSlJRkuV24cMHlr1naeBdzaabJ8+XxUqvw9lLn+3U3mWTSspTXMKGyBPI9ov6bKElJ/Yi9nkZypp7FO+NISHFuI8KTMFqFAjWR4rhPpbTjfks/gjpVA5AkiSe71WHvq31pEKo8FiJUuhRpEVypUiVL3/vSRKfTodOV75D73KWZsnNV3KWZOS3krfuaUcVfy7HLyeyOTSQly0BCaq6/b5upBVuNzemujmaS5kfG6sfjp1MzdUU0f0ZfYenu86x7oXux/3/uIHdGJDNVswyVJPO7sTOH5bporapeiqRW9yFyzTyMvC2FnN01y/tLrlErH3GgtxczhjRl+ehORFbyIdtgYsX+SzZj3zaMwCRL3KXeRSvpDBISfx1VqnGeuFp2fUdmx3wP1WG6qY+QJWssdagFnoFbhSg1NZWDBw9y8OBBQOkEcvDgQc6fP+9Os9yKpUpjEX1E+Zdmto8lSeL+1kqYxff/xtk8d0KOYoWxGwDTvJZiMpkoD5MEkwwqTEzNqUP9vbE/F+VqbrZKYI1bhWjv3r20bt2a1q1bAzBhwgRat27N9OnT3WmWW7G0FCqhXDMvVf6P+P421QH7WenvG/6PDFlLB9VJEvevRG8s+yVCjLLMA+qtNFJdIEn25RPDve42SZAHtwpRz549kWU5323hwoXuNMutFD+OyPaxvWz8mpX9uLN5mN3zr1KZb4xK63DfrbPQUHairJftPs/rfxzNt12vNqTzkuZnAOYb7iMJf7QaFVMGNXKHmQI7CB+Rh2Hp9ppvaeaos9p2nFdeZcrh44da8+nwNnafW2AYwnU5kDrSFR5Sb7Icv5KU4ZAN7mLaymi+2xHH7thEm+NtLi0jTLrJBVNVvjf2p141f469PoBnetR1k6WCvAgh8jAK3DVzoDoj5J8RFSREGrWKwS3CWT66U77nUvHlI8P9ALygWYE/Sntqc8Kop5OaaTWLS42nw+XFALxneJBsvDCZZIsTX+AZiE/Dw/DJk/Sa5USXV7BNelVJhad8dKpTmS8eaZvv+A/G3sSYwqkiJfO0Rimhmre7iKfy7l8n+H5nnPJg8xx0pnQOmerwh0kR3QAf0bHV0xBC5GHk9RE5H0eUO87RX/0BTcOICLKtrGmwqm/9pHoNoSQWWunQUzh1LZXpq45CwinYtxCAt/UjkFER4K3h7ftESQ9PQwiRh5E318zZ7Xtr7fFyIjp44eP5A1T/MrVjj6kBPlI2L2l+5tFv/uOnvWUnml3eMANkI+uNbdgtN6ZTnUpEzxxA04jyVQq3PCCEyMPIG9Bo6XtfhIBGL43jH2+D0AA7RyXe1o8A4H/qrUTqY5n0y2GHr+lOOkjHkU6uwSCrmGMYBkC3+lXdbJWgIIQQeRg+ebq9ZmTnbN876Ky2DmjU2IkhcpYDcn1WGzuikmRLQKCnI2FiWk4d6h+NvYiRlbgpR7qgCNyDECIPwzdn5pNtNGEwmnJTPIrgrHb2i/fbmC52S1+8a3iIbFlND/VhuqqinbpmaWEdO3SXahetVGdJk3XMM+Q2kbTuESfwLIQQeRjWgpOhN1otzRz7qGyWZk5uUbeKDGb+sNaseNa2HfV5OZQlxn4ATNMsA5Pn7Z6ZdUiLnkmaHwElHiqBYMuYD9YX3GBR4F6EEHkYOo3Kkt+VkmmwpFgUaWlWxKVI25oh+Y59bLiPZNmXJqpz/PjNexYnurvQG02sOnjJ0ibJXOjtEfXfRKoSuCYH87XxTptzKvtp811H4BkIIfIwJEmyLM9upueWLXV0aWY9I9IWI2hv88SeNo9vEcCnhnsA6HbxC5ZsO17ka5cE32yP5fnlB+n34RZAKfURSCrjNL8BOTlzKCEJY3vVA2DN893cYqugcIQQeSBmh/XNtNxIZp2DO2DWPqKizogAalXxo08j2wz1hcYBXJSrECElUvvM90W+dknwz/F4QJk1AuiNslLqVkrjhCmSX4w9AHjtriZMHNCQuDmDCQ0UXYg9FSFEHoh5Cz8xZ0akc7DdNNhGUhd31+ybke1tHmehZa5+KACdryzGkHyNGauOsO7I1WK9TlEw5gmufP/Hvy3F/+cYhmHK+dNuHGYvLEHgaQgh8kDM0dU3czpKOLosA1shKs7SrCBWme4g2lQLXzmdPz95gUU7z/HMkn1sPhlf4q91O6xbBD2+cA+tTs9HJxnYbmzKZlNLy3ONwgNL1S5B0RBC5IGYhedGjhA56qiGPDOiEoibeah9pM1jGRVvG5Qgxzuz/qK2dAWAkd/tKfZrOYO1ECWc3MU96n8xyRKzDSOA3P93JeGgLhMIIfJAzEsz84zI0a17yCtExf945zzQIt+xnaambDS2xksyMlmzvNivURRyhUjmlZzgxZWmLhyVa1nG2GujJPBMhBB5IHl9RM60O7Z2VmtLKJJ468u98h2bbRiGUZYYqN5DO6n0mzGaE3D7qPbTSXWcLNmL93P8V2am39Wk1O0SFA0hRB6It1feGZHjQqQq4RQPgKjKvux5pa/NsTNyDX409gRgmtcyoHQz802yjBojUzU/APCtcSCXqQJAm6hg9r/WT3TlKEMIIfJALDOiIizNNEVMei2MqgE6Ph7W2ubYh4b/kSbraKM6w52q3ZxNSC2x17sdeqOJU9dSeVC9mXqqyyTK/nyWE+MEML5PfeEbKmMIIfJAzN1ezQGNjrYSAlsfkTNlQBzh7jx5aAmE8JVxMACTND/y7KKdJfp6eTEYTWw9lcC0X6PxI4MXNb8A8LHhflLwtYzr0UBk2Zc1hBB5ID4WZ7US0OjU0qyEAhoLYu7/tbR5/KXhLuLlYGqprtEt6fcSfz1rPt54mke//Y+f911ktOZPqkpJxJpCWWpUlo2HZ/bn9FuDxJKsDCKEyAOxzsAH54RIU4ykV0f4X9saHJrR3/I4HW8+zMlwH6P6FVP6zRJ/TTPfbI8FoBo3eUr9J6BUBtCj4cW+DQj09nLJ/1ngesSn5oHkDWAsqrPaVV/KIB8vYmfnJpT+ZOzJaVN1QqRU9i+dToyLfEXmPmwvan7BV8pin6k+a00dWPxEB8b3qeeS1xSUDkKIPJD8QuREHJH10qyEfUTWSJLEmvFKEqkRNbNzqiA2v/gDj77/i8tet750kaHqzQC8pVeCF7vVryqWY2UcIUQeiG8eIXLKWa12za6ZPRqH5+Zx/WNqzU5jE3SSnpe8fibL4JoyIVM1y1BLMmuN7dkvN3DJawhKHyFEHoiPl8bmcVEDGkt61ywvkiRx/I2B5ke8ZRgOwP3q7Twy68sSrVm0M+YGnVVH6a0+iF5W805OhxFB+UAIkQdSrKVZKfiIrPHRqnm2p9Ix9Yhch9+MSnXH502LOX01pcRe55VfD/GKRknlWGrsQ5wczvg+9Tn15qASew2B+xBC5IEUa2lWwrlmjjB5YCP+GNsVgLmGB8mSNXRRHyXx0OoSuX5qloEWN9fTTBVHsuzDxzldaCf0a4DWxctPQekgPkUPJK/w6Ioa0FiKXSua11B6hV2Uq7LQOACAsP9mc+bqrWJdNy3LQNsZfzDR6ycAFhjuJpFAXh3cuFjXFXgWQog8kLwzoqIGNJZ2TE2jnCJknxru4ZbsR0PVRY7++VmxrvndjlhGqddRQ7rOZbkS3xgHMb53PZ7sVqckTBZ4CEKIPBBziocZZ5ZmJVE8v6h8n9MtNhl/5hvuA6DT+S/4+0BMka+Zfiue5zSrAHhfP5QstEzo37D4xgo8CiFEHkhe4Smys7qEsu8dpVqgN6ffUpzHi439OG+qSqh0iyO/vI3JVLTs/Kr7PyJQyuCoqSYrTV15pFPNkjRZ4CEIIfJAihVZbb0005R+kJ+XWsW0OxuRjRfv5myxP635gwV//uv0taIP7+dh9QYA3jYMx4SKWfc2K1F7BZ6BECIPRKtR2SyxnFqaqUu+HpGzjO6ubOevNnXioKkuflIWwbvnYjA63mnVaJI5//NkvCQjm40t2WFqzs6pvV1lssDNCCHyUKxnRc4szdzprLbmv2l9ACknDQMeVG/i29/WOXz+218uYrD6P4yyZEkfCQ/ycYWpAg9ACJGHYj0L0hWxeH5pbt/npVqgNz881Yk9ciP+MrZDLcnUOfgu+88Xnp2fmqln0OVPAfjF2IOTchTLR3dytckCNyKEyEOxnhE5005I44aAxoLoXLcy4/vU5x3DQxhkFX3VB3h3wdeFnrd33SLaqU6RIWv5wPA/lj7ZkU51KpeCxQJ3IYTIQ7HOGSt6GRD3Z6SP7VWP7p3vYJmxDwDTNEvBVLCv6KfdMUTtfw+Ar4x3ogmuTpd6VUrFVoH7EELkoVhXtfB2Io1B7SE+IjNajYoxverxkeF+UmQfWqhi2fvnVwWOP/L7R9RRXSVBDuQLwxC+eKRtKVorcBfu/0sV2EWSirbEsm057f4ZESiF928QxALDEADC9r4L+sx847JTb/K85lcAPjI8QGRYNZpVDypVWwXuQQiRh1JUCSnt7HtHiZ19J98YB3FFrkQN6Ton/3g/35hfPp5IZSmFGFM4txoPY+3z3dxgqcAdeM5fqsAGVRErDnra0syMJEksGt2DDwz/AyDs0CckXLtieV6feJ77s5RUjjmGYcwf0UFUXaxAeM5fqsCGon4HbSo0eoCz2pqOdSpzrfZ9HDdFEiSls2r+i5bnVn04Bm9Jz25TI8Y+M16IUAVDCFE5w1NnRGYWPt6Z2TmVHB9V/03c6aPEHdnN/aptALytH07LqBB3mihwAx7xl/rpp59Sq1YtvL296dixI//995+7TXI7RV2aWWd1lHb2vSOoVBJzJr3IVmNztJKR6O8ncP7Hl1BJMn8YO/HjG2PdbaLADbhdiH788UcmTJjAjBkz2L9/Py1btmTAgAHEx8e72zS30jg8sEjnWeeXuSvXrDAign0402oSJlliiHoX3dXRZMtqDjV43qmYKUH5we1/qR988AFPPfUUo0aNokmTJixYsABfX1++/fbbfGOzsrJITk62uZVXxvepR6c6lVj6ZEenzlOrJO5sHkaXepWp4u+5/d8ff+BufjXl7op9b+zPq4/ceZszBOUZtwpRdnY2+/bto2/fvpZjKpWKvn37snNn/j7qs2fPJigoyHKLjIwsTXNLlZqV/Vg+unORooo/G9GWpU928niH71z9/5Es+5AgB/J/L37kbnMEbsStQnT9+nWMRiOhoaE2x0NDQ7l69Wq+8VOnTiUpKclyu3DhQmmZKnABG18fzqouK9GM2UlQ5dDCTxCUWzSFD/EcdDodOp3O3WYISgg/nYZH+nd2txkCD8CtM6IqVaqgVqu5du2azfFr164RFhbmJqsEAkFp41Yh0mq1tG3blo0bN1qOmUwmNm7cSOfO4pdSIKgouH1pNmHCBB577DHatWtHhw4dmDdvHmlpaYwaNcrdpgkEglLC7UL04IMPkpCQwPTp07l69SqtWrVi3bp1+RzYAoGg/CLJsly0Pi8eQHJyMkFBQSQlJREYWLQAQIFA4Doc/Y66PaBRIBAIhBAJBAK343YfUXEwryrLc6qHQFCWMX83C/MAlWkhSklJASjXqR4CQXkgJSWFoKCCy/6WaWe1yWTi8uXLBAQElFheVXJyMpGRkVy4cEE4wEsB8X6XLqX9fsuyTEpKChEREahuUw2iTM+IVCoVNWrUcMm1AwMDxRejFBHvd+lSmu/37WZCZoSzWiAQuB0hRAKBwO0IIcqDTqdjxowZIsu/lBDvd+niqe93mXZWCwSC8oGYEQkEArcjhEggELgdIUQCgcDtCCESCARup8ILUWJiIiNGjCAwMJDg4GCeeOIJUlNTbzt+3LhxNGzYEB8fH6Kiohg/fjxJSUmlaHXZwtkGmj///DONGjXC29ub5s2bs2bNmlKytHzgzPv91Vdf0a1bN0JCQggJCaFv377uaXAqV3AGDhwot2zZUt61a5e8bds2uV69evKwYcMKHB8dHS3ff//98u+//y6fOXNG3rhxo1y/fn35gQceKEWryw7Lly+XtVqt/O2338pHjx6Vn3rqKTk4OFi+du2a3fE7duyQ1Wq1/O6778rHjh2TX331VdnLy0uOjo4uZcvLJs6+38OHD5c//fRT+cCBA/Lx48flkSNHykFBQfLFixdL1e4KLUTHjh2TAXnPnj2WY2vXrpUlSZIvXbrk8HV++uknWavVynq93hVmlmk6dOggjxkzxvLYaDTKERER8uzZs+2OHzp0qDx48GCbYx07dpSffvppl9pZXnD2/c6LwWCQAwIC5EWLFrnKRLtU6KXZzp07CQ4Opl27dpZjffv2RaVSsXv3boevY64+p9GU6dS9EsfZBpqgfCbW4wEGDBhQ4HhBLkV5v/OSnp6OXq+nUqVKrjLTLhVaiK5evUq1atVsjmk0GipVqmS3waM9rl+/zqxZsxg9erQrTCzTONtAE5TPxJnxglyK8n7nZfLkyUREROT7MXA15VKIpkyZgiRJt72dOHGi2K+TnJzM4MGDadKkCTNnziy+4QKBG5kzZw7Lly9n5cqVeHt7l+prl8u1xEsvvcTIkSNvO6ZOnTqEhYURHx9vc9xgMJCYmFhog8eUlBQGDhxIQEAAK1euxMvLq7hmlzuK0kAzLCxMNNwsIsVpWDp37lzmzJnDhg0baNGihSvNtE+peqQ8DLOzeu/evZZjf/31V6HO6qSkJLlTp05yjx495LS0tNIwtczSoUMHeezYsZbHRqNRrl69+m2d1XfddZfNsc6dOwtntYM4+37Lsiy/8847cmBgoLxz587SMNEuFVqIZFnZvm/durW8e/duefv27XL9+vVttu8vXrwoN2zYUN69e7csy4oIdezYUW7evLl85swZ+cqVK5abwWBw13/DY1m+fLms0+nkhQsXyseOHZNHjx4tBwcHy1evXpVlWZYfeeQRecqUKZbxO3bskDUajTx37lz5+PHj8owZM8T2vRM4+37PmTNH1mq18i+//GLzt5ySklKqdld4Ibpx44Y8bNgw2d/fXw4MDJRHjRpl8yHExsbKgLxp0yZZlmV506ZNMmD3Fhsb657/hIczf/58OSoqStZqtXKHDh3kXbt2WZ7r0aOH/Nhjj9mM/+mnn+QGDRrIWq1Wbtq0qfznn3+WssVlG2fe75o1a9r9W54xY0ap2izKgAgEArdTLnfNBAJB2UIIkUAgcDtCiAQCgdsRQiQQCNyOECKBQOB2hBAJBAK3I4RIIBC4HSFEAoHA7QghEggEbkcIkaDEGTlypN3SKwMHDnS3aQIPpVyWARG4n4EDB/Ldd9/ZHCuozbFer89XRiU7OxutVuv06xb1PIF7ETMigUvQ6XSEhYXZ3EJCQgCQJInPP/+cu+++Gz8/P9566y1mzpxJq1at+Prrr6ldu7alMNf58+e555578Pf3JzAwkKFDh9rU2ynoPEHZQgiRwC3MnDmT++67j+joaB5//HEAzpw5w4oVK/j11185ePAgJpOJe+65h8TERLZs2cL69es5e/YsDz74oM218p4nKHuIpZnAJaxevRp/f3+bY9OmTWPatGkADB8+nFGjRtk8n52dzffff0/VqlUBWL9+PdHR0cTGxhIZGQnA999/T9OmTdmzZw/t27e3e56g7CGESOASevXqxeeff25zzLozhHXnFDM1a9a0EZPjx48TGRlpESGAJk2aEBwczPHjxy1ClPc8QdlDCJHAJfj5+VGvXr3bPu/IMUdfS1C2ET4igcfSuHFjLly4wIULFyzHjh07xq1bt2jSpIkbLROUNGJGJHAJWVlZ+XppaTQaqlSp4vA1+vbtS/PmzRkxYgTz5s3DYDDw3HPP0aNHD7tLO0HZRcyIBC5h3bp1hIeH29y6du3q1DUkSWLVqlWEhITQvXt3+vbtS506dfjxxx9dZLXAXYia1QKBwO2IGZFAIHA7QogEAoHbEUIkEAjcjhAigUDgdoQQCQQCtyOESCAQuB0hRAKBwO0IIRIIBG5HCJFAIHA7QogEAoHbEUIkEAjczv8DJX3tKXHHP34AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.018485, + "end_time": "2024-03-26T15:26:43.929165", + "exception": false, + "start_time": "2024-03-26T15:26:43.910680", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 671.652428, + "end_time": "2024-03-26T15:26:46.669116", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/realtabformer/4/mlu-eval-load.ipynb", + "output_path": "eval/treatment/realtabformer/4/mlu-eval-load.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/treatment/realtabformer/4", + "path_prefix": "../../../../", + "random_seed": 4, + "single_model": "realtabformer" + }, + "start_time": "2024-03-26T15:15:35.016688", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/realtabformer/mlu-eval.ipynb b/treatment/realtabformer/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..2dc569d57d7230366db4bc8e7cc681f23eef7465 --- /dev/null +++ b/treatment/realtabformer/mlu-eval.ipynb @@ -0,0 +1,2301 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:24.898687Z", + "iopub.status.busy": "2024-03-24T02:37:24.898314Z", + "iopub.status.idle": "2024-03-24T02:37:24.937619Z", + "shell.execute_reply": "2024-03-24T02:37:24.936699Z" + }, + "papermill": { + "duration": 0.054866, + "end_time": "2024-03-24T02:37:24.939882", + "exception": false, + "start_time": "2024-03-24T02:37:24.885016", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:24.965582Z", + "iopub.status.busy": "2024-03-24T02:37:24.965206Z", + "iopub.status.idle": "2024-03-24T02:37:24.972135Z", + "shell.execute_reply": "2024-03-24T02:37:24.971336Z" + }, + "papermill": { + "duration": 0.022194, + "end_time": "2024-03-24T02:37:24.974191", + "exception": false, + "start_time": "2024-03-24T02:37:24.951997", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:24.997846Z", + "iopub.status.busy": "2024-03-24T02:37:24.997602Z", + "iopub.status.idle": "2024-03-24T02:37:25.001601Z", + "shell.execute_reply": "2024-03-24T02:37:25.000755Z" + }, + "papermill": { + "duration": 0.018167, + "end_time": "2024-03-24T02:37:25.003536", + "exception": false, + "start_time": "2024-03-24T02:37:24.985369", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:25.026962Z", + "iopub.status.busy": "2024-03-24T02:37:25.026719Z", + "iopub.status.idle": "2024-03-24T02:37:25.030928Z", + "shell.execute_reply": "2024-03-24T02:37:25.030004Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018379, + "end_time": "2024-03-24T02:37:25.032950", + "exception": false, + "start_time": "2024-03-24T02:37:25.014571", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:25.057558Z", + "iopub.status.busy": "2024-03-24T02:37:25.056902Z", + "iopub.status.idle": "2024-03-24T02:37:25.062619Z", + "shell.execute_reply": "2024-03-24T02:37:25.061856Z" + }, + "papermill": { + "duration": 0.019803, + "end_time": "2024-03-24T02:37:25.064397", + "exception": false, + "start_time": "2024-03-24T02:37:25.044594", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f78f4de4", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:25.091369Z", + "iopub.status.busy": "2024-03-24T02:37:25.091086Z", + "iopub.status.idle": "2024-03-24T02:37:25.095808Z", + "shell.execute_reply": "2024-03-24T02:37:25.094938Z" + }, + "papermill": { + "duration": 0.021072, + "end_time": "2024-03-24T02:37:25.097732", + "exception": false, + "start_time": "2024-03-24T02:37:25.076660", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"realtabformer\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 4\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/realtabformer/4\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011619, + "end_time": "2024-03-24T02:37:25.120767", + "exception": false, + "start_time": "2024-03-24T02:37:25.109148", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:25.145470Z", + "iopub.status.busy": "2024-03-24T02:37:25.144598Z", + "iopub.status.idle": "2024-03-24T02:37:25.154094Z", + "shell.execute_reply": "2024-03-24T02:37:25.153310Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.024061, + "end_time": "2024-03-24T02:37:25.156079", + "exception": false, + "start_time": "2024-03-24T02:37:25.132018", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/realtabformer/4\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:25.179841Z", + "iopub.status.busy": "2024-03-24T02:37:25.179582Z", + "iopub.status.idle": "2024-03-24T02:37:27.299047Z", + "shell.execute_reply": "2024-03-24T02:37:27.298117Z" + }, + "papermill": { + "duration": 2.13365, + "end_time": "2024-03-24T02:37:27.301104", + "exception": false, + "start_time": "2024-03-24T02:37:25.167454", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:27.329498Z", + "iopub.status.busy": "2024-03-24T02:37:27.329064Z", + "iopub.status.idle": "2024-03-24T02:37:27.355367Z", + "shell.execute_reply": "2024-03-24T02:37:27.354504Z" + }, + "papermill": { + "duration": 0.041965, + "end_time": "2024-03-24T02:37:27.357350", + "exception": false, + "start_time": "2024-03-24T02:37:27.315385", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:27.381288Z", + "iopub.status.busy": "2024-03-24T02:37:27.381008Z", + "iopub.status.idle": "2024-03-24T02:37:27.388387Z", + "shell.execute_reply": "2024-03-24T02:37:27.387676Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021574, + "end_time": "2024-03-24T02:37:27.390221", + "exception": false, + "start_time": "2024-03-24T02:37:27.368647", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:27.414095Z", + "iopub.status.busy": "2024-03-24T02:37:27.413803Z", + "iopub.status.idle": "2024-03-24T02:37:27.506670Z", + "shell.execute_reply": "2024-03-24T02:37:27.505823Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.107493, + "end_time": "2024-03-24T02:37:27.509018", + "exception": false, + "start_time": "2024-03-24T02:37:27.401525", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:27.534570Z", + "iopub.status.busy": "2024-03-24T02:37:27.534271Z", + "iopub.status.idle": "2024-03-24T02:37:32.234512Z", + "shell.execute_reply": "2024-03-24T02:37:32.233674Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.715704, + "end_time": "2024-03-24T02:37:32.236975", + "exception": false, + "start_time": "2024-03-24T02:37:27.521271", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-24 02:37:29.781312: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-24 02:37:29.781382: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-24 02:37:29.783146: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:32.263204Z", + "iopub.status.busy": "2024-03-24T02:37:32.262574Z", + "iopub.status.idle": "2024-03-24T02:37:32.278379Z", + "shell.execute_reply": "2024-03-24T02:37:32.277552Z" + }, + "papermill": { + "duration": 0.031553, + "end_time": "2024-03-24T02:37:32.280238", + "exception": false, + "start_time": "2024-03-24T02:37:32.248685", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:32.306724Z", + "iopub.status.busy": "2024-03-24T02:37:32.306402Z", + "iopub.status.idle": "2024-03-24T02:37:55.106494Z", + "shell.execute_reply": "2024-03-24T02:37:55.105392Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 22.816518, + "end_time": "2024-03-24T02:37:55.108910", + "exception": false, + "start_time": "2024-03-24T02:37:32.292392", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-24T02:37:55.140073Z", + "iopub.status.busy": "2024-03-24T02:37:55.139739Z", + "iopub.status.idle": "2024-03-24T02:37:55.146490Z", + "shell.execute_reply": "2024-03-24T02:37:55.145489Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.025962, + "end_time": "2024-03-24T02:37:55.148626", + "exception": false, + "start_time": "2024-03-24T02:37:55.122664", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 95,\n", + " 'realtabformer': (69, 281, Embedding(281, 768), True),\n", + " 'lct_gan': 75,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:55.175195Z", + "iopub.status.busy": "2024-03-24T02:37:55.174888Z", + "iopub.status.idle": "2024-03-24T02:37:55.179680Z", + "shell.execute_reply": "2024-03-24T02:37:55.178806Z" + }, + "papermill": { + "duration": 0.020338, + "end_time": "2024-03-24T02:37:55.181562", + "exception": false, + "start_time": "2024-03-24T02:37:55.161224", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:55.207493Z", + "iopub.status.busy": "2024-03-24T02:37:55.207215Z", + "iopub.status.idle": "2024-03-24T02:37:55.856633Z", + "shell.execute_reply": "2024-03-24T02:37:55.855732Z" + }, + "papermill": { + "duration": 0.665081, + "end_time": "2024-03-24T02:37:55.858927", + "exception": false, + "start_time": "2024-03-24T02:37:55.193846", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/realtabformer/all inf False\n", + "../../../../ml-utility-loss/aug_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_bs_test/realtabformer/all inf False\n", + "../../../../ml-utility-loss/bs_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_synth_test/realtabformer/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:55.889423Z", + "iopub.status.busy": "2024-03-24T02:37:55.889092Z", + "iopub.status.idle": "2024-03-24T02:37:56.209831Z", + "shell.execute_reply": "2024-03-24T02:37:56.208970Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.338602, + "end_time": "2024-03-24T02:37:56.212006", + "exception": false, + "start_time": "2024-03-24T02:37:55.873404", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'loss_balancer_beta': 0.7999999999999999,\n", + " 'loss_balancer_r': 0.95,\n", + " 'tf_pma_low': 8,\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.75,\n", + " 'bias_weight_decay': 0.05,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 2,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.09,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'realtabformer',\n", + " 'g_loss_mul': 0.1,\n", + " 'd_model': 512,\n", + " 'attn_activation': torch.nn.modules.activation.SELU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 8,\n", + " 'ada_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'single_model': True,\n", + " 'models': ['realtabformer'],\n", + " 'max_seconds': 3600,\n", + " 'Body': 'twin_encoder',\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.2, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "# params = {\n", + "# **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "# }\n", + "params = getattr(PARAMS, dataset_name).BEST_DICT[gp][gp_multiply][single_model]\n", + "if isinstance(params, (list, tuple)):\n", + " params = params[param_index]\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:56.241585Z", + "iopub.status.busy": "2024-03-24T02:37:56.241198Z", + "iopub.status.idle": "2024-03-24T02:37:56.353969Z", + "shell.execute_reply": "2024-03-24T02:37:56.353033Z" + }, + "papermill": { + "duration": 0.130192, + "end_time": "2024-03-24T02:37:56.356000", + "exception": false, + "start_time": "2024-03-24T02:37:56.225808", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_train/realtabformer/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/treatment [400, 0]\n", + "Caching in ../../../../treatment/_cache_aug_val/realtabformer/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/treatment [0, 200]\n", + "Caching in ../../../../treatment/_cache_bs_train/realtabformer/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/treatment [100, 0]\n", + "Caching in ../../../../treatment/_cache_bs_val/realtabformer/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/treatment [0, 50]\n", + "Caching in ../../../../treatment/_cache_synth/realtabformer/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/treatment [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-24T02:37:56.384951Z", + "iopub.status.busy": "2024-03-24T02:37:56.384388Z", + "iopub.status.idle": "2024-03-24T02:37:56.912695Z", + "shell.execute_reply": "2024-03-24T02:37:56.911807Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.545055, + "end_time": "2024-03-24T02:37:56.914745", + "exception": false, + "start_time": "2024-03-24T02:37:56.369690", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding True True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['realtabformer'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:56.944944Z", + "iopub.status.busy": "2024-03-24T02:37:56.944148Z", + "iopub.status.idle": "2024-03-24T02:37:56.948771Z", + "shell.execute_reply": "2024-03-24T02:37:56.947957Z" + }, + "papermill": { + "duration": 0.021849, + "end_time": "2024-03-24T02:37:56.950726", + "exception": false, + "start_time": "2024-03-24T02:37:56.928877", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:56.977790Z", + "iopub.status.busy": "2024-03-24T02:37:56.977523Z", + "iopub.status.idle": "2024-03-24T02:37:56.984157Z", + "shell.execute_reply": "2024-03-24T02:37:56.983390Z" + }, + "papermill": { + "duration": 0.022525, + "end_time": "2024-03-24T02:37:56.986121", + "exception": false, + "start_time": "2024-03-24T02:37:56.963596", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "19911750" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:57.013833Z", + "iopub.status.busy": "2024-03-24T02:37:57.013379Z", + "iopub.status.idle": "2024-03-24T02:37:57.144361Z", + "shell.execute_reply": "2024-03-24T02:37:57.143437Z" + }, + "papermill": { + "duration": 0.147505, + "end_time": "2024-03-24T02:37:57.146647", + "exception": false, + "start_time": "2024-03-24T02:37:56.999142", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 52992] --\n", + "├─Adapter: 1-1 [2, 2648, 52992] --\n", + "│ └─Embedding: 2-1 [2, 2648, 69, 768] (215,808)\n", + "│ └─TensorInductionPoint: 2-2 [69, 1] 69\n", + "│ └─Sequential: 2-3 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 1024] 787,456\n", + "│ │ │ └─LeakyHardtanh: 4-2 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─LeakyHardtanh: 4-4 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─LeakyHardtanh: 4-6 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─LeakyHardtanh: 4-8 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─LeakyHardtanh: 4-10 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─LeakyHardtanh: 4-12 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─LeakyHardtanh: 4-14 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-15 [2, 2648, 512] 524,800\n", + "│ │ │ └─LeakyHardsigmoid: 4-16 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 52992] (recursive)\n", + "│ └─Embedding: 2-4 [2, 661, 69, 768] (recursive)\n", + "│ └─TensorInductionPoint: 2-5 [69, 1] (recursive)\n", + "│ └─Sequential: 2-6 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-9 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 1024] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-18 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 1024] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-20 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 1024] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-22 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 1024] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-24 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 661, 1024] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-26 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 661, 1024] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-28 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 661, 1024] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-30 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-32 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 4096] --\n", + "│ └─Encoder: 2-7 [2, 8, 512] --\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-6 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-18 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-30 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 8, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 64, 512] 32,768\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 64, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 64, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 64, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-42 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 64, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 2648, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 262,656\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 8, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 8, 512] 4,096\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 8, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 8, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 8, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 8, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 8, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-54 [2, 8, 512] --\n", + "│ └─Encoder: 2-8 [2, 8, 512] (recursive)\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-60 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-31 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-72 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-39 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-40 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-84 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-42 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-43 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-40 [2, 8, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-45 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-46 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 64, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-96 [2, 64, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 64, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 661, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-48 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-49 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-18 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-51 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-52 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 8, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 8, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-108 [2, 8, 512] --\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-9 [2, 1] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 524,416\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 128] --\n", + "│ │ │ └─Linear: 4-45 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 128] --\n", + "│ │ └─FeedForward: 3-22 [2, 128] --\n", + "│ │ │ └─Linear: 4-47 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-48 [2, 128] --\n", + "│ │ └─FeedForward: 3-23 [2, 128] --\n", + "│ │ │ └─Linear: 4-49 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-50 [2, 128] --\n", + "│ │ └─FeedForward: 3-24 [2, 128] --\n", + "│ │ │ └─Linear: 4-51 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-52 [2, 128] --\n", + "│ │ └─FeedForward: 3-25 [2, 128] --\n", + "│ │ │ └─Linear: 4-53 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-54 [2, 128] --\n", + "│ │ └─FeedForward: 3-26 [2, 1] --\n", + "│ │ │ └─Linear: 4-55 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-56 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 20,127,558\n", + "Trainable params: 19,911,750\n", + "Non-trainable params: 215,808\n", + "Total mult-adds (M): 78.72\n", + "========================================================================================================================\n", + "Input size (MB): 1.83\n", + "Forward/backward pass size (MB): 3938.39\n", + "Params size (MB): 80.51\n", + "Estimated Total Size (MB): 4020.73\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:37:57.178038Z", + "iopub.status.busy": "2024-03-24T02:37:57.177758Z", + "iopub.status.idle": "2024-03-24T03:55:38.411233Z", + "shell.execute_reply": "2024-03-24T03:55:38.410333Z" + }, + "papermill": { + "duration": 4661.251912, + "end_time": "2024-03-24T03:55:38.413820", + "exception": false, + "start_time": "2024-03-24T02:37:57.161908", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding True True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.09495536066487066, 'avg_role_model_std_loss': 8.93220916427853, 'avg_role_model_mean_pred_loss': 0.0419871234360975, 'avg_role_model_g_mag_loss': 1.3698969186252603, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.09889276345333473, 'n_size': 900, 'n_batch': 450, 'duration': 658.6492667198181, 'duration_batch': 1.4636650371551514, 'duration_size': 0.7318325185775757, 'avg_pred_std': 0.12809311661881004}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01607477423283967, 'avg_role_model_std_loss': 5.248249570016049, 'avg_role_model_mean_pred_loss': 0.0017426262642197404, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01607477423283967, 'n_size': 450, 'n_batch': 225, 'duration': 186.72706723213196, 'duration_batch': 0.8298980765872531, 'duration_size': 0.41494903829362656, 'avg_pred_std': 0.07684096147841084}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.011189657653926462, 'avg_role_model_std_loss': 1.6530850822855043, 'avg_role_model_mean_pred_loss': 0.0004210917485972683, 'avg_role_model_g_mag_loss': 1.341028964128718, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.011419157582775041, 'n_size': 900, 'n_batch': 450, 'duration': 666.2451884746552, 'duration_batch': 1.4805448632770115, 'duration_size': 0.7402724316385058, 'avg_pred_std': 0.19066866852136274}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01324691695151491, 'avg_role_model_std_loss': 2.477538332505353, 'avg_role_model_mean_pred_loss': 0.0012280172330747016, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01324691695151491, 'n_size': 450, 'n_batch': 225, 'duration': 199.9650740623474, 'duration_batch': 0.8887336624993218, 'duration_size': 0.4443668312496609, 'avg_pred_std': 0.10047116354194967}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007910739361629894, 'avg_role_model_std_loss': 1.2925524352938216, 'avg_role_model_mean_pred_loss': 0.0002544926988752502, 'avg_role_model_g_mag_loss': 1.0837882141060295, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008088903618937845, 'n_size': 900, 'n_batch': 450, 'duration': 677.3252453804016, 'duration_batch': 1.505167211956448, 'duration_size': 0.752583605978224, 'avg_pred_std': 0.20559132802500749}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.009221325316331483, 'avg_role_model_std_loss': 3.315624680383264, 'avg_role_model_mean_pred_loss': 0.0006904058667853433, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009221325316331483, 'n_size': 450, 'n_batch': 225, 'duration': 199.83344054222107, 'duration_batch': 0.8881486246320937, 'duration_size': 0.4440743123160468, 'avg_pred_std': 0.08771737627721059}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006322342480156698, 'avg_role_model_std_loss': 0.9110217978435494, 'avg_role_model_mean_pred_loss': 0.00014503141735743212, 'avg_role_model_g_mag_loss': 0.9366931309483294, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006472822059344278, 'n_size': 900, 'n_batch': 450, 'duration': 680.0380289554596, 'duration_batch': 1.5111956199010212, 'duration_size': 0.7555978099505106, 'avg_pred_std': 0.18851194770978813}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01230627063448872, 'avg_role_model_std_loss': 2.257654165002171, 'avg_role_model_mean_pred_loss': 0.0008331807685687578, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01230627063448872, 'n_size': 450, 'n_batch': 225, 'duration': 201.17853093147278, 'duration_batch': 0.894126804139879, 'duration_size': 0.4470634020699395, 'avg_pred_std': 0.0902620027468543}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005821315827197143, 'avg_role_model_std_loss': 1.3421934734654035, 'avg_role_model_mean_pred_loss': 9.012431415762501e-05, 'avg_role_model_g_mag_loss': 0.9289716981382824, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.005968375098755764, 'n_size': 900, 'n_batch': 450, 'duration': 680.3976809978485, 'duration_batch': 1.5119948466618855, 'duration_size': 0.7559974233309428, 'avg_pred_std': 0.19308502603797528}\n", + "Time out: 4155.438156604767/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'realtabformer', 'n_size': 1050, 'n_batch': 525, 'role_model_metrics': {'avg_loss': 0.004561402160108541, 'avg_g_mag_loss': 0.15238095238095345, 'avg_g_cos_loss': 0.0, 'pred_duration': 30.614413499832153, 'grad_duration': 5.782175064086914, 'total_duration': 36.39658856391907, 'pred_std': 0.23253320157527924, 'std_loss': 0.0018148290691897273, 'mean_pred_loss': 7.176899089245126e-05, 'pred_rmse': 0.06753815710544586, 'pred_mae': 0.04342292249202728, 'pred_mape': 6809961.5, 'grad_rmse': 1.2579528093338013, 'grad_mae': 0.6942744851112366, 'grad_mape': 12.301761627197266}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.004561402160108541, 'avg_g_mag_loss': 0.15238095238095345, 'avg_g_cos_loss': 0.0, 'avg_pred_duration': 30.614413499832153, 'avg_grad_duration': 5.782175064086914, 'avg_total_duration': 36.39658856391907, 'avg_pred_std': 0.23253320157527924, 'avg_std_loss': 0.0018148290691897273, 'avg_mean_pred_loss': 7.176899089245126e-05}, 'min_metrics': {'avg_loss': 0.004561402160108541, 'avg_g_mag_loss': 0.15238095238095345, 'avg_g_cos_loss': 0.0, 'pred_duration': 30.614413499832153, 'grad_duration': 5.782175064086914, 'total_duration': 36.39658856391907, 'pred_std': 0.23253320157527924, 'std_loss': 0.0018148290691897273, 'mean_pred_loss': 7.176899089245126e-05, 'pred_rmse': 0.06753815710544586, 'pred_mae': 0.04342292249202728, 'pred_mape': 6809961.5, 'grad_rmse': 1.2579528093338013, 'grad_mae': 0.6942744851112366, 'grad_mape': 12.301761627197266}, 'model_metrics': {'realtabformer': {'avg_loss': 0.004561402160108541, 'avg_g_mag_loss': 0.15238095238095345, 'avg_g_cos_loss': 0.0, 'pred_duration': 30.614413499832153, 'grad_duration': 5.782175064086914, 'total_duration': 36.39658856391907, 'pred_std': 0.23253320157527924, 'std_loss': 0.0018148290691897273, 'mean_pred_loss': 7.176899089245126e-05, 'pred_rmse': 0.06753815710544586, 'pred_mae': 0.04342292249202728, 'pred_mape': 6809961.5, 'grad_rmse': 1.2579528093338013, 'grad_mae': 0.6942744851112366, 'grad_mape': 12.301761627197266}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:55:38.448469Z", + "iopub.status.busy": "2024-03-24T03:55:38.448127Z", + "iopub.status.idle": "2024-03-24T03:55:38.452592Z", + "shell.execute_reply": "2024-03-24T03:55:38.451690Z" + }, + "papermill": { + "duration": 0.024677, + "end_time": "2024-03-24T03:55:38.455118", + "exception": false, + "start_time": "2024-03-24T03:55:38.430441", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:55:38.487469Z", + "iopub.status.busy": "2024-03-24T03:55:38.487149Z", + "iopub.status.idle": "2024-03-24T03:55:38.619304Z", + "shell.execute_reply": "2024-03-24T03:55:38.618509Z" + }, + "papermill": { + "duration": 0.151783, + "end_time": "2024-03-24T03:55:38.621919", + "exception": false, + "start_time": "2024-03-24T03:55:38.470136", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:55:38.655553Z", + "iopub.status.busy": "2024-03-24T03:55:38.655167Z", + "iopub.status.idle": "2024-03-24T03:55:38.932457Z", + "shell.execute_reply": "2024-03-24T03:55:38.931600Z" + }, + "papermill": { + "duration": 0.296351, + "end_time": "2024-03-24T03:55:38.934653", + "exception": false, + "start_time": "2024-03-24T03:55:38.638302", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAESCAYAAABU2qhcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyR0lEQVR4nO3de1RTZ7438O/OnVtAuYUgglYRRQWrwmBbHStT2uIF6xyty6XWaWvbFztjaXuUXqTT866hM2PfsbWetrbn6ExnPNp2Kjrq8UZFW8VSuSgIUrUIKgTES4AACSTP+0dIJBoggcDO5fdZay+TnWfv/JKSb599ezbHGGMghBAnIOC7AEIIMaFAIoQ4DQokQojToEAihDgNCiRCiNOgQCKEOA0KJEKI0xDxXYAjGAwG1NbWws/PDxzH8V0OIeQejDE0NzdDqVRCIOi5H+QWgVRbW4uIiAi+yyCE9OHq1asYMWJEj6+7RSD5+fkBMH5YuVzOczWEkHs1NTUhIiLC/FvtiVsEkmkzTS6XUyAR4sT62qVCO7UJIU6jX4G0ZcsWREVFQSaTITExEQUFBT22PX/+PBYtWoSoqChwHIdNmzYNeJ2EEPdkdyDt2rULGRkZyMrKQlFREeLi4pCSkoKGhgar7VtbWzF69Gi89957UCgUDlknIcQ9cfYOP5KYmIjp06fjo48+AmA85B4REYGXX34Z69ev73XZqKgorF27FmvXrnXYOgHjDjN/f3+o1Wrah9QPer0eHR0dfJdBXJhYLIZQKOzxdVt/o3bt1NbpdCgsLERmZqZ5nkAgQHJyMvLz8+1Z1YDWqdVqodVqzc+bmpr69d6ejjEGlUqFO3fu8F0KcQMBAQFQKBQDOhfQrkBqbGyEXq9HaGioxfzQ0FBcuHChXwX0Z53Z2dn4/e9/36/3I3eZwigkJATe3t50UinpF8YYWltbzbtYwsLC+r0ulzzsn5mZiYyMDPNz0zkOtrjSqMHZa3ewID58sMpzCXq93hxGgYGBfJdDXJyXlxcAoKGhASEhIb1uvvXGrkAKCgqCUChEfX29xfz6+voed1gPxjqlUimkUqnd71V9U4NfbsyDWMhhVnQwArwl/arZHZj2GXl7e/NcCXEXpr+ljo6OfgeSXUfZJBIJpk6ditzcXPM8g8GA3NxcJCUl9auAwVhnTyIDfTA+TI4OPcP+0jqHrttV0WYacRRH/C3Zfdg/IyMDn332Gf7617+ioqICL730EjQaDVatWgUAWLFihcUOap1Oh5KSEpSUlECn0+H69esoKSnBpUuXbF6nI6XFKwEAe4prHb5uQsjA2L0PacmSJbhx4wY2bNgAlUqF+Ph4HDx40LxTuqamxuJq3traWkyZMsX8fOPGjdi4cSNmzZqFvLw8m9bpSPPjlXjv4AUUXLmF63faEB7g5fD3IIT0j93nITkje89DWvJpPn6ouoV1j8fgpV8+MAQVOp/29nZUVVVh1KhRkMlkfJfjUjiOw+7du5GWlsZ3KQ7zzjvvICcnByUlJf1eR29/U7b+Rj3yWra0KcYjbHtKrvNcCSH9s337dgQEBDhsfa+99prFfly+eGQgPTkxDBKhABdUzbigopMqifvS6XQ2tfP19XWK0z88MpD8vcX45bhgAMCeEtq5bcIYQ6uuc8gne/caHDx4EA8//DACAgIQGBiIuXPn4vLlywCAGTNmYN26dRbtb9y4AbFYjBMnTgAA6urqkJqaCi8vL4waNQo7duxAVFRUjxd+96W0tBSPPvoovLy8EBgYiNWrV6OlpcX8el5eHhISEuDj44OAgAA89NBDqK6uBgCcPXsWs2fPhp+fH+RyOaZOnYozZ870+n55eXlYtWoV1Go1OI4Dx3F45513ABgvz/qP//gPrFixAnK5HKtXrwYArFu3DtHR0fD29sbo0aPx9ttvW1wu9M477yA+Pt78/JlnnkFaWho2btyIsLAwBAYGIj09fdAvMXLJEyMdYUF8OA6X12NvSS1ef2wcBAI6/N3WoceEDYeG/H3L302Bt8T2P0WNRoOMjAxMnjwZLS0t2LBhAxYuXIiSkhIsW7YMf/rTn/Dee++ZD0Pv2rULSqUSjzzyCADjkeDGxkbk5eVBLBYjIyOj3xdyazQapKSkICkpCT/++CMaGhrw3HPPYc2aNdi+fTs6OzuRlpaG559/Hv/zP/8DnU6HgoICc23Lli3DlClT8PHHH0MoFKKkpARisbjX95wxYwY2bdqEDRs2oLKyEoCxh2OyceNGbNiwAVlZWeZ5fn5+2L59O5RKJUpLS/H888/Dz88P//7v/97j+xw7dgxhYWE4duwYLl26hCVLliA+Ph7PP/98v74rW3hsIM0ZHwJfqQjX77ThTPVtJIwazndJxEaLFi2yeP7f//3fCA4ORnl5ORYvXoy1a9fi+++/NwfQjh07sHTpUnAchwsXLuDo0aP48ccfMW3aNADA559/jrFjx/arlh07dqC9vR1/+9vf4OPjAwD46KOPMG/ePPzxj3+EWCyGWq3G3Llz8cADxgMo48ePNy9fU1OD119/HTExMQBgUx0SiQT+/v7gOM7qycOPPvooXn31VYt5b731lvlxVFQUXnvtNezcubPXQBo2bBg++ugjCIVCxMTEIDU1Fbm5uRRIg0EmFuLxiQp8XXgNOSXXKZAAeImFKH83hZf3tcfFixexYcMG/PDDD2hsbITBYABg/HFPnDgRjz32GP7xj3/gkUceQVVVFfLz8/Hpp58CACorKyESifDggw+a1zdmzBgMGzasX7VXVFQgLi7OHEYA8NBDD8FgMKCyshIzZ87EM888g5SUFPzqV79CcnIyFi9ebL7eKyMjA8899xy++OILJCcn49/+7d/MwdVfpqDtbteuXfjwww9x+fJltLS0oLOzs88j0rGxsRZnXIeFhaG0tHRAtfXFI/chmaR1Xc92oLQOuk4Dz9Xwj+M4eEtEQz7Ze4bvvHnzcOvWLXz22Wf44Ycf8MMPPwC4uwN32bJl+Prrr9HR0YEdO3Zg0qRJmDRpksO/L1tt27YN+fn5mDFjBnbt2oXo6GicPn0agHHfzfnz55Gamopvv/0WEyZMwO7duwf0ft3DEQDy8/OxbNkyPPnkk9i3bx+Ki4vx5ptv9rnD+95NR47jzOE/WDw6kJIeCESwnxR3Wjtw4qcbfJdDbHDz5k1UVlbirbfewpw5czB+/Hjcvn3bos2CBQvQ3t6OgwcPYseOHVi2bJn5tXHjxqGzsxPFxcXmeZcuXbpvHbYaP348zp49C41GY5538uRJCAQCjBs3zjxvypQpyMzMxKlTpzBx4kTs2LHD/Fp0dDReeeUVHD58GE899RS2bdvW5/tKJBLo9Xqbajx16hQiIyPx5ptvYtq0aRg7dqx5p7qz8ehAEgo4zJtsvJQkh85JcgnDhg1DYGAgtm7dikuXLuHbb7+1GPkBMPYQ0tLS8Pbbb6OiogJLly41vxYTE4Pk5GSsXr0aBQUFKC4uxurVq+Hl5dWva7GWLVsGmUyGlStXoqysDMeOHcPLL7+M5cuXIzQ0FFVVVcjMzER+fj6qq6tx+PBhXLx4EePHj0dbWxvWrFmDvLw8VFdX4+TJk/jxxx8t9jH1JCoqCi0tLcjNzUVjYyNaW1t7bDt27FjU1NRg586duHz5Mj788MMB98IGi0cHEgCkTTEG0tGKerRoO3muhvRFIBBg586dKCwsxMSJE/HKK6/gz3/+833tli1bhrNnz+KRRx7ByJEjLV7729/+htDQUMycORMLFy40H3Hqzxnr3t7eOHToEG7duoXp06fj17/+NebMmWMe/dTb2xsXLlzAokWLEB0djdWrVyM9PR0vvPAChEIhbt68iRUrViA6OhqLFy/GE088YdNYXzNmzMCLL76IJUuWIDg4GH/60596bDt//ny88sorWLNmDeLj43Hq1Cm8/fbbdn/WoeCRl450xxjDnPeP4+dGDf7f4jg89WDPN7FzJ3TpyF3Xrl1DREQEjh49ijlz5vBdjsuiS0ccgOM4zI83bbbRSZKe4Ntvv8XevXtRVVWFU6dO4emnn0ZUVBRmzpzJd2kez+MDCbh7tO37izdwo1nbR2vi6jo6OvDGG28gNjYWCxcuRHBwsPkkyX/84x/w9fW1OsXGxg5ZjU888USPdfzhD38YsjqGmseeh9RdVJAP4iICcPbqHew7V4tVD43iuyQyiFJSUpCSYv18q/nz5yMxMdHqa32dQe1In3/+Odra2qy+Nny4+54zR4HUZUGcEmev3sGeEgokT+bn59fn/eeHQni4Z475TptsXebGhUHAASVX7+BKo6bvBQghDkeB1CXET4aHxgQBoBEACOELBVI3pp3be0qu2z0kBiFk4CiQunksNhRSkQA/N2pQdp0GbiNkqFEgdeMnEyN5gvHGAnQpCSFDjwLpHqbNtn+drYXeQJtt5H4cxyEnJ4fvMtwSBdI9ZkUHw99LjIZmLfIv3+S7HEKscvQg/4BxaFyO43Dnzh2HrtceFEj3kIgEeHKScfAsuisJIUOLAskK091tD5ap0N5h25gzboExQKcZ+okG+XfYIP9arRavvfYawsPD4ePjg8TERPMNWQGguroa8+bNw7Bhw+Dj44PY2FgcOHAAV65cwezZswEYh3jhOA7PPPNMv76PgaAzta2YHjUcSn8ZatXt+PZCg7nH5PY6WoE/KIf+fd+oBSQ+fbfrQoP89zzI/5o1a1BeXo6dO3dCqVRi9+7dePzxx1FaWoqxY8ciPT0dOp0OJ06cgI+PD8rLy+Hr64uIiAj885//xKJFi1BZWQm5XA4vr6G/qzMFkhUCAYf58eH45Phl7Cm57jmB5CJokH/rg/zX1NRg27ZtqKmpgVJp/B/La6+9hoMHD2Lbtm34wx/+gJqaGixatMg8pO/o0aPNy5uukQsJCXH4/ilbUSD1YEG8Ep8cv4xjF25A3doBf++hu7CSN2JvY2+Fj/e1Aw3yb11paSn0ej2io6Mt5mu1WvNNIH/729/ipZdewuHDh5GcnIxFixZh8uTJ/Xq/wUD7kHowPkyOcaF+0OkN+N+yOr7LGRocZ9x0GuqJBvl3yCD/LS0tEAqFKCwsRElJiXmqqKjABx98AAB47rnn8PPPP2P58uUoLS3FtGnTsHnzZod91oGiQOrFgik03razoUH+jawN8j9lyhTo9Xo0NDRgzJgxFlP3TbuIiAi8+OKL+Oabb/Dqq6/is88+M68TgM03DxgMFEi9mB9nDKQfqm6hTm19bBoytGiQfyNrg/xHR0dj2bJlWLFiBb755htUVVWhoKAA2dnZ2L9/PwBg7dq1OHToEKqqqlBUVIRjx46Z3y8yMhIcx2Hfvn24ceOGxZHCIcPcgFqtZgCYWq12+Lp//fFJFrluH/v0+CWHr5tPbW1trLy8nLW1tfFdit2OHDnCxo8fz6RSKZs8eTLLy8tjANju3bvNbQ4cOMAAsJkzZ963fG1tLXviiSeYVCplkZGRbMeOHSwkJIR98sknNr3/ve917tw5Nnv2bCaTydjw4cPZ888/z5qbmxljjKlUKpaWlsbCwsKYRCJhkZGRbMOGDUyv1zOtVsuefvppFhERwSQSCVMqlWzNmjU2/zd58cUXWWBgIAPAsrKyGGOM6XQ6tmHDBhYVFcXEYjELCwtjCxcuZOfOnWOMMbZmzRr2wAMPMKlUyoKDg9ny5ctZY2OjeZ3vvvsuUygUjOM4tnLlSpvqMOntb8rW36jHD/Lfl7+frsZbOWWYECbHgd894tB184kG+b+LBvl3DBrkfwikTgqDSMChvK4JF+ub+S6HOAAN8u+8KJD6MMxHgl+OCwZAA7e5Cxrk33nReUg2mB8fjqMVDdhz9jpefSy6Xzs/ifOgQf6dV796SFu2bEFUVBRkMhkSExNRUFDQa/uvvvoKMTExkMlkmDRpEg4cOGDxektLC9asWYMRI0bAy8sLEyZMwCeffNKf0gbFr8aHwkcixNVbbSiq6d/hYeIa/Pz87jtkbpoiIyOHrI7w8PAe66BA6mbXrl3IyMhAVlYWioqKEBcXh5SUlB6vBTp16hSWLl2KZ599FsXFxUhLS0NaWhrKysrMbTIyMnDw4EH8/e9/R0VFBdauXYs1a9Zg7969/f9kDuQlESIl1ngeR06xe222ucExDeIkHPK3ZNdxPcZYQkICS09PNz/X6/VMqVSy7Oxsq+0XL17MUlNTLeYlJiayF154wfw8NjaWvfvuuxZtHnzwQfbmm2/aVNNgHvY3yatsYJHr9rEp7x5muk79oL3PUOns7GTl5eUWh3wJGYjGxkZWXl7OOjs773vN1t+oXfuQdDodCgsLkZmZaZ4nEAiQnJyM/Px8q8vk5+ffd+JaSkqKxYh7M2bMwN69e/Gb3/wGSqUSeXl5+Omnn/CXv/zF6jq1Wi202rt3mG1qGvzxrx96IBBBvhI0tujw/cVGzI4JGfT3HExCoRABAQHmnq23tzftGyP9whhDa2srGhoaEBAQAKFQ2O912RVIjY2N0Ov1CA0NtZgfGhqKCxcuWF1GpVJZba9SqczPN2/ejNWrV2PEiBEQiUQQCAT47LPPejwMm52djd///vf2lD5gIqEAcycrsf3UFeSUXHf5QAJgvpygv0NvENJdQECAxSUq/eEUR9k2b96M06dPY+/evYiMjMSJEyeQnp4OpVKJ5OTk+9pnZmZa9LqampoQEREx6HUuiDcG0uHz9dBoO+EjdYqvr984jkNYWBhCQkLQ0dHBdznEhYnF4gH1jEzs+kUFBQVBKBSivr7eYn59fX2PyahQKHpt39bWhjfeeAO7d+9GamoqAGDy5MkoKSnBxo0brQaSVCqFVCq1p3SHiI8IQGSgN6pvtuJoRT0WxLvH7Y6FQqFD/pgIGSi7jrJJJBJMnToVubm55nkGgwG5ublISkqyukxSUpJFewA4cuSIuX1HRwc6OjogEFiWIhQKzePcOAuO47Cg64LbnGIaAYAQh7N3T/rOnTuZVCpl27dvZ+Xl5Wz16tUsICCAqVQqxhhjy5cvZ+vXrze3P3nyJBOJRGzjxo2soqKCZWVlMbFYzEpLS81tZs2axWJjY9mxY8fYzz//zLZt28ZkMhn7z//8T5tqGoqjbCaXGppZ5Lp9bHTmftbY3D7o70eIO7D1N9qvq/03b97MRo4cySQSCUtISGCnT582vzZr1qz7rhL+8ssvWXR0NJNIJCw2Npbt37/f4vW6ujr2zDPPMKVSyWQyGRs3bhx7//33mcFgsKmeoQwkxhib++F3LHLdPvbXU1VD8n6EuDq62n8Qff7dz/i/+yswNXIY/vnSjEF/P0JcHV3tP4jmxSnBcUBh9W1cvdXKdzmEuA0KpH4Ilcsw4wHjoOl0M0lCHIcCqZ9Mh/xzSmrpejBCHIQCqZ8en6iARCTApYYWlNcN/qUrhHgCCqR+ksvEmNN1+QgN3EaIY1AgDYBps21vSS30BtpsI2SgKJAGYHZMMPxkIqia2vFD1U2+yyHE5VEgDYBUJMSTE423RN5Lm22EDBgF0gCZ7m57oLQO2k7+7vhJiDugQBqgX4wKhEIuQ1N7J45duMF3OYS4NAqkARIIOMyPN/aS9p6lkyQJGQgKJAeY3zUkydGKBjS100BnhPQXBZIDxCrlGBPiC12nAQfLVH0vQAixigLJATiOQ1rXZhtd20ZI/1EgOYjpJMlTl2+ioamd52oIcU0USA4SMdwbD44MAGPA3rN0ThIh/UGB5EBpU4y9JLq2jZD+oUByoNRJYRAKOJReV+PyjRa+yyHE5VAgOVCgrxQzxwYBoF4SIf1BgeRgpp3be0qu08BthNiJAsnBfjUhFF5iIapvtqLk6h2+yyHEpVAgOZiPVITHYkMB0GYbIfaiQBoEaV2bbfvO1aJT71x33yXEmVEgDYKHxwZhuI8EjS06nLxMA7cRYisKpEEgFgqQOsk4cNueYrqUhBBbUSANkrSugdsOnVehTUcDtxFiCwqkQfLgyGEYMcwLGp0eRyvq+S6HEJdAgTRIOI7DAhoBgBC7UCANItPRtrzKG7it0fFcDSHOjwJpEI0N9cOEMDk6DQz7S+v4LocQp0eBNMhMm210myRC+kaBNMjmxyvBcUDBlVu4druV73IIcWoUSIMszN8LiaOGA6CB2wjpCwXSEDDt3N5TTIFESG/6FUhbtmxBVFQUZDIZEhMTUVBQ0Gv7r776CjExMZDJZJg0aRIOHDhwX5uKigrMnz8f/v7+8PHxwfTp01FTU9Of8pzOExPDIBEKUFnfjAuqJr7LIcRp2R1Iu3btQkZGBrKyslBUVIS4uDikpKSgoaHBavtTp05h6dKlePbZZ1FcXIy0tDSkpaWhrKzM3Oby5ct4+OGHERMTg7y8PJw7dw5vv/02ZDJZ/z+ZE/H3FuOX44IBADnUSyKkRxyzcxSxxMRETJ8+HR999BEAwGAwICIiAi+//DLWr19/X/slS5ZAo9Fg37595nm/+MUvEB8fj08++QQA8PTTT0MsFuOLL77o14doamqCv78/1Go15HJ5v9Yx2A6U1uH//KMISn8Zvl/3KAQCju+SCBkytv5G7eoh6XQ6FBYWIjk5+e4KBAIkJycjPz/f6jL5+fkW7QEgJSXF3N5gMGD//v2Ijo5GSkoKQkJCkJiYiJycnB7r0Gq1aGpqspic3aMxIfCTilCrbsePV27xXQ4hTsmuQGpsbIRer0doaKjF/NDQUKhU1u/YqlKpem3f0NCAlpYWvPfee3j88cdx+PBhLFy4EE899RSOHz9udZ3Z2dnw9/c3TxEREfZ8DF7IxEKkTFQAAPbQ0TZCrOL9KJvBYBzAbMGCBXjllVcQHx+P9evXY+7cueZNuntlZmZCrVabp6tXrw5lyf1mOtp2oLQOuk4auI2Qe9kVSEFBQRAKhaivt7x6vb6+HgqFwuoyCoWi1/ZBQUEQiUSYMGGCRZvx48f3eJRNKpVCLpdbTK4g6YFAhPhJcae1A8d/usF3OYQ4HbsCSSKRYOrUqcjNzTXPMxgMyM3NRVJSktVlkpKSLNoDwJEjR8ztJRIJpk+fjsrKSos2P/30EyIjI+0pz+kJBRzmxdEIAIT0iNlp586dTCqVsu3bt7Py8nK2evVqFhAQwFQqFWOMseXLl7P169eb2588eZKJRCK2ceNGVlFRwbKysphYLGalpaXmNt988w0Ti8Vs69at7OLFi2zz5s1MKBSy7777zqaa1Go1A8DUarW9H2fInb16m0Wu28fGvXWANbd38F0OIUPC1t+o3YHEGGObN29mI0eOZBKJhCUkJLDTp0+bX5s1axZbuXKlRfsvv/ySRUdHM4lEwmJjY9n+/fvvW+d//dd/sTFjxjCZTMbi4uJYTk6OzfW4UiAZDAY2+8/HWOS6fezrM1f5LoeQIWHrb9Tu85CckSuch9TdB0cv4i9Hf8IjY4PwxbOJfJdDyKAblPOQiGOYhiQ5eakRN5q1PFdDiPOgQOJBVJAP4iICYGDGe7cRQowokHiS1tVLyqGB2wgxo0DiydzJSggFHM5evYOqRg3f5RDiFCiQeBLsJ8VDY4IA0PC2hJhQIPFoQbeTJN3gYCchA0aBxKOUiQrIxAL83KhB6XU13+UQwjsKJB75SkVIHm8cCYEGbiOEAol3phEA/nWuFnoDbbYRz0aBxLOZ0cEI8BbjRrMW+Zdv8l0OIbyiQOKZRCTAk5PCAAA5NAIA8XAUSE7AtNl2sEyF9g49z9UQwh8KJCcwLXIYwgO80KLtxLcXrN+9hRBPQIHkBATdBm7LKabNNuK5KJCcRNoUYyDlVd6AurWD52oI4QcFkpOIUcgRo/CDTm/AgbI6vsshhBcUSE5kfjyNt008GwWSE5nftR/ph6pbqFO38VwNIUOPAsmJjBjmjYSo4WCMRgAgnokCycksmEIDtxHPRYHkZJ6cGAaRgENFXRN+qm/muxxChhQFkpMZ5iPBL8cFA6Cd28TzUCA5oQVdl5LsKamlgduIR6FAckLJ40PhIxHi2u02FFbf5rscQoYMBZIT8pIIkRKrAGDsJRHiKSiQnNSCKcbNtv2ldejQG3iuhpChQYHkpB56IBBBvhLc0ujw3cUbfJdDyJCgQHJSIqEAcyebLiWhzTbiGSiQnNiCrmvbDp+vh0bbyXM1hAw+CiQnFh8RgMhAb7R16HGkvJ7vcggZdBRITozjOPM5STTeNvEEFEhOzrTZ9t3FRtxs0fJcDSGDiwLJyT0Q7ItJ4f7QGxj2l9LAbcS9USC5AFMvicbbJu6OAskFzI9TQsABRTV3UHOzle9yCBk0/QqkLVu2ICoqCjKZDImJiSgoKOi1/VdffYWYmBjIZDJMmjQJBw4c6LHtiy++CI7jsGnTpv6U5pZC5DLMeCAIALD3LPWSiPuyO5B27dqFjIwMZGVloaioCHFxcUhJSUFDg/X7iZ06dQpLly7Fs88+i+LiYqSlpSEtLQ1lZWX3td29ezdOnz4NpVJp/ydxc6bxtnNoBADizpidEhISWHp6uvm5Xq9nSqWSZWdnW22/ePFilpqaajEvMTGRvfDCCxbzrl27xsLDw1lZWRmLjIxkf/nLX2yuSa1WMwBMrVbb/kFcjLpNx8a+eYBFrtvHSq/d4bscQuxi62/Urh6STqdDYWEhkpOTzfMEAgGSk5ORn59vdZn8/HyL9gCQkpJi0d5gMGD58uV4/fXXERsb22cdWq0WTU1NFpO7k8vESB4fAoAGbiPuy65AamxshF6vR2hoqMX80NBQqFQqq8uoVKo+2//xj3+ESCTCb3/7W5vqyM7Ohr+/v3mKiIiw52O4LNNJknvP1kJvoM024n54P8pWWFiIDz74ANu3bwfHcTYtk5mZCbVabZ6uXr06yFU6h1+OC4ZcJkJ9kxY/VN3kuxxCHM6uQAoKCoJQKER9veV1VfX19VAoFFaXUSgUvbb/7rvv0NDQgJEjR0IkEkEkEqG6uhqvvvoqoqKirK5TKpVCLpdbTJ5AKhLiyUlhAIA9xTQCAHE/dgWSRCLB1KlTkZuba55nMBiQm5uLpKQkq8skJSVZtAeAI0eOmNsvX74c586dQ0lJiXlSKpV4/fXXcejQIXs/j9szbbYdKKtDe4ee52oIcSyRvQtkZGRg5cqVmDZtGhISErBp0yZoNBqsWrUKALBixQqEh4cjOzsbAPC73/0Os2bNwvvvv4/U1FTs3LkTZ86cwdatWwEAgYGBCAwMtHgPsVgMhUKBcePGDfTzuZ3EUcOhkMugampHXuUNPD7Res+UEFdk9z6kJUuWYOPGjdiwYQPi4+NRUlKCgwcPmndc19TUoK7u7jVXM2bMwI4dO7B161bExcXh66+/Rk5ODiZOnOi4T+FBBALOfE4SHW0j7oZjzPXPsmtqaoK/vz/UarVH7E86X6tG6offQyIS4MxbyZDLxHyXREivbP2N8n6UjdhvQpgcY0N8oes04GCp9dMtCHFFFEguyDhwW9dmG13bRtwIBZKLMh1tO3X5Juqb2nmuhhDHoEByURHDvTE1chgYA/51ls5JIu6BAsmFpZlHAKDNNuIeKJBc2JOTwiAUcCi73oRLDS18l0PIgFEgubBAXylmju0auI16ScQNUCC5uLQpptsk0cBtxPVRILm4X00IhZdYiJpbrSi+eofvcggZEAokF+ctEeGxWONlO3tL6GgbcW0USG4greucpH3natGpN/BcDSH9R4HkBh4eG4ThPhI0tujw/aVGvsshpN8okNyAWCjA3MnGgdtos424MgokN2G6tu3QeRXadDRwG3FNFEhu4sGRwxAx3AsanR5HKur7XoAQJ0SB5CY4jsOCOOPO7T3FdJIkcU0USG4kbYpxs+34TzdwW6PjuRpC7EeB5EbGhPhhQpgcnQaG/aV1fS9AiJOhQHIzpl4SjbdNXBEFkpuZHxcOjgN+vHIb12638l0OIXahQHIzCn8ZfjHKeFupvTRwG3ExFEhuyDzeNt3dlrgYCiQ39MSkMEiEAlTWN6OironvcgixGQWSG/L3EmN2TDAAGt6WuBYKJDdlGgHgXyW1MBho4DbiGiiQ3NTsmBD4SUWoVbfjxyu3+C6HEJtQILkpmViIxycqABiHtyXEFVAguTHTeNsHSuug66SB24jzo0ByY78YHYgQPynUbR04/tMNvsshpE8USG5MKOAwL45uJklcBwWSmzMdbTtaXo/m9g6eqyGkdxRIbm5iuByjg32g7TTg0HkauI04NwokN2cxcBttthEnR4HkAUzXtp281IiG5naeqyGkZ/0KpC1btiAqKgoymQyJiYkoKCjotf1XX32FmJgYyGQyTJo0CQcOHDC/1tHRgXXr1mHSpEnw8fGBUqnEihUrUFtL5844SlSQD+IjAmBgwL6zNHAbcV52B9KuXbuQkZGBrKwsFBUVIS4uDikpKWhoaLDa/tSpU1i6dCmeffZZFBcXIy0tDWlpaSgrKwMAtLa2oqioCG+//TaKiorwzTffoLKyEvPnzx/YJyMW0uJp4DbiApidEhISWHp6uvm5Xq9nSqWSZWdnW22/ePFilpqaajEvMTGRvfDCCz2+R0FBAQPAqqurbapJrVYzAEytVtvU3hM1NLWz0Zn7WeS6feznGy18l0M8jK2/Ubt6SDqdDoWFhUhOTjbPEwgESE5ORn5+vtVl8vPzLdoDQEpKSo/tAUCtVoPjOAQEBFh9XavVoqmpyWIivQv2k+KhMUEAqJdEnJddgdTY2Ai9Xo/Q0FCL+aGhoVCpVFaXUalUdrVvb2/HunXrsHTpUsjlcqttsrOz4e/vb54iIiLs+Rge6+5mWy0YoxEAiPNxqqNsHR0dWLx4MRhj+Pjjj3tsl5mZCbVabZ6uXr06hFW6rsdiFZCJBahq1ODcNTXf5RByH7sCKSgoCEKhEPX1lifY1dfXQ6FQWF1GoVDY1N4URtXV1Thy5EiPvSMAkEqlkMvlFhPpm69UhOTxxt7qHhoBgDghuwJJIpFg6tSpyM3NNc8zGAzIzc1FUlKS1WWSkpIs2gPAkSNHLNqbwujixYs4evQoAgMD7SmL2ME8cNu5Wuhp4DbiZET2LpCRkYGVK1di2rRpSEhIwKZNm6DRaLBq1SoAwIoVKxAeHo7s7GwAwO9+9zvMmjUL77//PlJTU7Fz506cOXMGW7duBWAMo1//+tcoKirCvn37oNfrzfuXhg8fDolE4qjPSgDMjA5GgLcYN5q1OHW5EY+MDea7JELu6s8hvM2bN7ORI0cyiUTCEhIS2OnTp82vzZo1i61cudKi/Zdffsmio6OZRCJhsbGxbP/+/ebXqqqqGACr07Fjx2yqhw772+eNb86xyHX72KtflvBdCvEQtv5GOcZc/3BLU1MT/P39oVaraX+SDQqqbmHxp/nwlYpw5q1kyMRCvksibs7W36hTHWUbEndqAG0L31XwalrkMIQHeKFF24ncCutn2BPCB7v3Ibk0xoCPpgOd7YDYG/ANAXxCjP+aHwcDvqGW8yU+fFfuUAIBh/nxSnycdxk5JdeROjmM75IIAeBpgaTTAFxXp7CjFbh9xTj1RezTLbS6AqunxxLvwfwEDpMWH46P8y4jr7IB6tYO+HuL+S6JEA8LJKkv8GadcZNN0wC0dE0Wj28ALfV3n3e2AR0a4HaVceqLxLdbUAV39bRC7+l5dc3nMbzGKfwQo/DDBVUzDpTVYWnCSN5qIcTEswLJROprnIaP7r0dY4CupYfgagBausLLNL+z3dhe12JjePl1C6rgrl5YD4/FXo757N0siA/HhYMXkFN8nQKJOAXPDCRbcRwg9TNOgQ/03pYxQNvc1cNq6Aqqbr0t8+MbxgDrbAd0zcCtZuDWz33XIpX33PO697FYZtPHmx+vxB8PXsAPVbdQe6cNygDHhx4h9qBAchSOA2Ry42RTeDXdDafuQWXxuGvSa43ttU3Arct91yL17xZU3XfYWz4P9w1BwqjhKKi6hX+drcULs/qom5BBRoHEB44DZP7GKWhM720ZA9rVVnpe9z7u2ozU6wCt2jjdvNRnKV+I/HBN4ovW7wKBhrFAwEggIBIYFgkERAEBEYBI6pjPTUgfKJCcHccBXgHGKWhs721N4WV1f5eVnpehA9LOZjwgaAY664DzZdYKAOTKuyE1LKpbYEUCfmGAwPNOZyODgwLJnXQPr+Do3tsyBrTfAVoa8Od/fofqmit4aqwQjyragdvVxtMh7lQbT49oum6cak7dvx6h5J5eVVdomR57DTPWRVyfrhVorjP+z625DmiuB1pUQMxcICLBIW9BgeSpOM4YFl7DMH6GH7ZcKUZxvRdmPzMbnClAGAM0jcZgMp2zdaf6bmCprxk3EW9e6nnzUCq3DCqLwBo5KEcPiZ20LV0ho7o/cMzPVcZ9mNb4hFAgEceZExMKH4kQ1++0YfO3lxCrlEPhL4NCLsNwnyBwvsHAiGn3L6jvNPaczIFVfTew7lQb/5C1TYCq1DhZ4xt6f6/KtGkoDwcEdJ1dv2mb7wmVuq7QUXULIJXxaK+txN7G/2Z+YYBf179hkx1WMl1cSwAAr355Fv8sunbffIlIgFC5FGFyL4T6yxDmL0Oo3PLfYD8pxEIr+5F0rcZrB7v3qroHVk//xzURiAD/EVYCK8r4r0+Q520Omk4vsdqLuee5zo5rNsU+gJ/i7uSruBs43QNIKu/Xd27rb5QCiQAAVOp2fHL8Mq7dboOqqQ0qdTsaW3Q2LctxQLCv1NyrUvjLLB93/est6dYhZwxou33/ZqDpsfqqcXOwN2Kf+3tV3fdlSX37+3UMPdOpIKZeS7PKuH/G2vOOVtvXK/HtFjD3Bk63Seo3eJ8NFEjEAXSdBtQ3taO+qR116rv/qpraUa82Pm5obkeH3rY/IblMhDD/rp6WXGbucXUPrgBvsXEflsFg/D/9vb0q06Zhcx2Mw2b1wjuo58DyjwCEQ3D9nunIZ28BY5o622xfr1Te1XPpIWRMPZxBDhpbUSCRIWEwMNzU6CzCSqVug0qtNfe0VOp2aHR6m9YnFQms9rTubiJ6IchXAhHrAO5c7QqsK5b7r25fMR5B7A0nMO6jurdXZdo09A3tfdPE1MOzZdOp047bl0v9uzaV+ujVuNgIFBRIxKk0t3cYw8nU21K3o65bT6u+qR03NbZtIgo4433mFP5eUMilxl7XPfu1FFIdZC1Xe97h3ldIiGTdTmeIMgbYvYGj19r+Bcj8798fc+9zX4XLjBZhLwok4nK0nXo0NGl77WnVN2ttvjlBgLf4vv1YCrkMCrkU4eImhBka4NN6DZzFJmE10HQNYAbbivYa1vMOYPNzhcef3kCBRNyS3sBws0V7t6fVvcfV7Xlbh22biDKxoKuHdbenpfQTIlJ0GyPQgODOOvi21ULAsa6g6bbp5Btq84XMno4CiXgsxhia2jvNm4j39rRMwXW7tcOm9QkFHAK8xPCVieAjEcFXKjI+lnY9lgq7Pe6aL+t6LBHBr6utj1QIqcgzz6uy9TdKJ0YSt8NxHPy9xPD3EmOcouejTO0d+vuPIJqmJuO/Dc3txl6ZRmfzPq7eSIQC+PQUYJLuQSeEr1QMH6nQsl23sJOKBHfPqncTFEjEY8nEQkQG+iAysOcjVp16AxpbdLjTpkNLeydatJ3QaPVo0XagRauHRmucZ5zfebeNzvTY2Ma0CanTG6BrNdjcO+uNUMB1CyuhZWjdE2A+UhH8pHd7an7dw04mgpdY6BThRoFESC9EQoH59IOB6NQboNH1EmDm+Vba3NPedAqF3sCgbuuAum3g4SbgYNwcNW9emgJMeE+YdW2CSu4+Hh3sgzB/x+y0p0AiZAiIhAL4ewng7zXwkzENBobWDr05zCyCq72rd2Z63BVyLdqOrp6dZXuNthMGBhgY0KztRLO20+561j8RgxcdNLgfBRIhLkbQbVNtoBhjaOvQdwuw+0PLeuDp0dJuDDmF3HFHGimQCPFgHMfBWyKCt0SEECe4yoSG+iOEOA0KJEKI06BAIoQ4DQokQojToEAihDgNCiRCiNOgQCKEOA23OA/JNGBBU1Mfg8YTQnhh+m32NbiIWwRSc7PxNi4RERE8V0II6U1zczP8/f17fN0txkMyGAyora2Fn59fn1csNzU1ISIiAlevXqWxk7qh76Vn9N1YZ8/3whhDc3MzlEolBL3cet0tekgCgQAjRoywaxm5XE5/XFbQ99Iz+m6ss/V76a1nZEI7tQkhToMCiRDiNDwukKRSKbKysiCVSvkuxanQ99Iz+m6sG4zvxS12ahNC3IPH9ZAIIc6LAokQ4jQokAghToMCiRDiNCiQCCFOw+MCacuWLYiKioJMJkNiYiIKCgr4Lol3J06cwLx586BUKsFxHHJycvguySlkZ2dj+vTp8PPzQ0hICNLS0lBZWcl3Wbz7+OOPMXnyZPMZ2klJSfjf//1fh6zbowJp165dyMjIQFZWFoqKihAXF4eUlBQ0NDTwXRqvNBoN4uLisGXLFr5LcSrHjx9Heno6Tp8+jSNHjqCjowOPPfYYNBoN36XxasSIEXjvvfdQWFiIM2fO4NFHH8WCBQtw/vz5ga+ceZCEhASWnp5ufq7X65lSqWTZ2dk8VuVcALDdu3fzXYZTamhoYADY8ePH+S7F6QwbNox9/vnnA16Px/SQdDodCgsLkZycbJ4nEAiQnJyM/Px8HisjrkKtVgMAhg8fznMlzkOv12Pnzp3QaDRISkoa8Prc4mp/WzQ2NkKv1yM0NNRifmhoKC5cuMBTVcRVGAwGrF27Fg899BAmTpzIdzm8Ky0tRVJSEtrb2+Hr64vdu3djwoQJA16vxwQSIQORnp6OsrIyfP/993yX4hTGjRuHkpISqNVqfP3111i5ciWOHz8+4FDymEAKCgqCUChEfX29xfz6+nooFAqeqiKuYM2aNdi3bx9OnDhh97hb7koikWDMmDEAgKlTp+LHH3/EBx98gE8//XRA6/WYfUgSiQRTp05Fbm6ueZ7BYEBubq5Dtn2J+2GMYc2aNdi9eze+/fZbjBo1iu+SnJbBYIBWqx3wejymhwQAGRkZWLlyJaZNm4aEhARs2rQJGo0Gq1at4rs0XrW0tODSpUvm51VVVSgpKcHw4cMxcuRIHivjV3p6Onbs2IE9e/bAz88PKpUKgHHkQy8vL56r409mZiaeeOIJjBw5Es3NzdixYwfy8vJw6NChga984Af8XMvmzZvZyJEjmUQiYQkJCez06dN8l8S7Y8eOMQD3TStXruS7NF5Z+04AsG3btvFdGq9+85vfsMjISCaRSFhwcDCbM2cOO3z4sEPWTeMhEUKchsfsQyKEOD8KJEKI06BAIoQ4DQokQojToEAihDgNCiRCiNOgQCKEOA0KJEKI06BAIoQ4DQokQojToEAihDiN/w/clwbY8Wo5zwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:55:38.970480Z", + "iopub.status.busy": "2024-03-24T03:55:38.969567Z", + "iopub.status.idle": "2024-03-24T04:04:06.935963Z", + "shell.execute_reply": "2024-03-24T04:04:06.934992Z" + }, + "papermill": { + "duration": 507.987456, + "end_time": "2024-03-24T04:04:06.938980", + "exception": false, + "start_time": "2024-03-24T03:55:38.951524", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T04:04:06.982694Z", + "iopub.status.busy": "2024-03-24T04:04:06.982272Z", + "iopub.status.idle": "2024-03-24T04:04:07.007608Z", + "shell.execute_reply": "2024-03-24T04:04:07.006486Z" + }, + "papermill": { + "duration": 0.051336, + "end_time": "2024-03-24T04:04:07.010245", + "exception": false, + "start_time": "2024-03-24T04:04:06.958909", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
realtabformer0.00.1371430.0045615.8267610.69427412.3017611.2579530.00007230.6795190.0434236809962.00.0675380.2325330.00181536.50628
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "realtabformer 0.0 0.137143 0.004561 5.826761 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss pred_duration \\\n", + "realtabformer 0.694274 12.301761 1.257953 0.000072 30.679519 \n", + "\n", + " pred_mae pred_mape pred_rmse pred_std std_loss \\\n", + "realtabformer 0.043423 6809962.0 0.067538 0.232533 0.001815 \n", + "\n", + " total_duration \n", + "realtabformer 36.50628 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T04:04:07.051207Z", + "iopub.status.busy": "2024-03-24T04:04:07.050910Z", + "iopub.status.idle": "2024-03-24T04:04:07.512292Z", + "shell.execute_reply": "2024-03-24T04:04:07.511247Z" + }, + "papermill": { + "duration": 0.483789, + "end_time": "2024-03-24T04:04:07.514448", + "exception": false, + "start_time": "2024-03-24T04:04:07.030659", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T04:04:07.552420Z", + "iopub.status.busy": "2024-03-24T04:04:07.552048Z", + "iopub.status.idle": "2024-03-24T04:13:43.762875Z", + "shell.execute_reply": "2024-03-24T04:13:43.762061Z" + }, + "papermill": { + "duration": 576.232872, + "end_time": "2024-03-24T04:13:43.765470", + "exception": false, + "start_time": "2024-03-24T04:04:07.532598", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/realtabformer/all inf False\n", + "Caching in ../../../../treatment/_cache_bs_test/realtabformer/all inf False\n", + "Caching in ../../../../treatment/_cache_synth_test/realtabformer/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T04:13:43.803085Z", + "iopub.status.busy": "2024-03-24T04:13:43.802166Z", + "iopub.status.idle": "2024-03-24T04:13:43.829682Z", + "shell.execute_reply": "2024-03-24T04:13:43.828845Z" + }, + "papermill": { + "duration": 0.048844, + "end_time": "2024-03-24T04:13:43.831904", + "exception": false, + "start_time": "2024-03-24T04:13:43.783060", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T04:13:43.865157Z", + "iopub.status.busy": "2024-03-24T04:13:43.864849Z", + "iopub.status.idle": "2024-03-24T04:13:43.870559Z", + "shell.execute_reply": "2024-03-24T04:13:43.869725Z" + }, + "papermill": { + "duration": 0.024819, + "end_time": "2024-03-24T04:13:43.872628", + "exception": false, + "start_time": "2024-03-24T04:13:43.847809", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'realtabformer': 0.42817036741746484}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T04:13:43.909150Z", + "iopub.status.busy": "2024-03-24T04:13:43.908782Z", + "iopub.status.idle": "2024-03-24T04:13:44.332787Z", + "shell.execute_reply": "2024-03-24T04:13:44.331780Z" + }, + "papermill": { + "duration": 0.445549, + "end_time": "2024-03-24T04:13:44.335048", + "exception": false, + "start_time": "2024-03-24T04:13:43.889499", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABFxElEQVR4nO3deXhU9b348feZfSaZJfu+kbAoyCIK4opLVbRWal1rBbSIvVdrLfXeSp9fsXbjLtVqvZZrF4lUra29Lm2tW6to64KKoNSFnRBCIPvs+5zfH2dmQiCBZDIzZ5J8X8+TJ8mZM+d8ciCffPevJMuyjCAIgoo0agcgCIIgEpEgCKoTiUgQBNWJRCQIgupEIhIEQXUiEQmCoDqRiARBUJ1IRIIgqE4kIkEQVCcSkTCk733ve0iSlNZrNjc3I0kS77///rDO/+///m8mTZqEVqtl9uzZaY1FyB0iEQkj8uMf/5hnn302K/d6+eWX+fd//3fOOOMM1q1bx49//OOs3FfIPp3aAQhjy49//GOuvPJKFi9enPF7vfrqq2g0Gn79619jMBgyfj9BPaJENE54vV61Q0i7jo4OzGZz2pKQLMv4/f60XGs0fD6f2iHkHJGIxqBE280nn3zCl7/8ZQoKCjjzzDMBeOyxx5g7dy5ms5nCwkKuvfZaWltbB7z/73//O1dddRW1tbUYjUZqamr45je/edxfUkmS8Hq9PProo0iShCRJLFu2DICWlhb+9V//lalTp2I2mykqKuKqq65i7969g17L5/Nxyy23UFRUhM1mY8mSJfT29g6417p16/B6vcl7NTc3AxCJRPjBD35AY2MjRqOR+vp6vvOd7xAMBgfco76+ns9//vO89NJLnHLKKZjNZh5++GE2bNiAJEn8/ve/55577qGqqgqr1cqVV16J0+kkGAxyxx13UFpaSn5+PjfeeONR1x7us164cCEzZsxg06ZNnH322VgsFr7zne8c8zlPRKJqNoZdddVVTJ48mR//+MfIssyPfvQjvvvd73L11VezfPlyOjs7efDBBzn77LPZvHkzDocDgKeeegqfz8e//Mu/UFRUxLvvvsuDDz7I/v37eeqpp4a8329+8xuWL1/OvHnzWLFiBQCNjY0AvPfee7z11ltce+21VFdXs3fvXtauXcvChQv55JNPsFgsA65122234XA4+N73vse2bdtYu3YtLS0tySTxm9/8hl/84he8++67/OpXvwLg9NNPB2D58uU8+uijXHnllXzrW99i48aNrFmzhk8//ZRnnnlmwH22bdvGddddxy233MLNN9/M1KlTk6+tWbMGs9nMXXfdxc6dO3nwwQfR6/VoNBp6e3v53ve+xzvvvENzczMNDQ2sXr06+d7hPmuA7u5uFi1axLXXXstXvvIVysrKRvgvPQHIwphz9913y4B83XXXJY/t3btX1mq18o9+9KMB527dulXW6XQDjvt8vqOuuWbNGlmSJLmlpeWo+xwuLy9PXrp06VHvH+yab7/9tgzI69evTx5bt26dDMhz586VQ6FQ8vh//dd/yYD83HPPJY8tXbpUzsvLG3DNLVu2yIC8fPnyAcfvvPNOGZBfffXV5LG6ujoZkF988cUB57722msyIM+YMWNADNddd50sSZK8aNGiAecvWLBArqurS34/kmd9zjnnyID8v//7v0c9H6GfqJqNYV/72teSXz/99NPEYjGuvvpqurq6kh/l5eVMnjyZ1157LXmu2WxOfu31eunq6uL0009HlmU2b96cUiyHXzMcDtPd3U1TUxMOh4MPPvjgqPNXrFiBXq9Pfv8v//Iv6HQ6/vKXvxzzPonXV65cOeD4t771LQCef/75AccbGhq46KKLBr3WkiVLBsQwf/58ZFnmpptuGnDe/PnzaW1tJRKJACN71gBGo5Ebb7zxmD/XRCeqZmNYQ0ND8usdO3YgyzKTJ08e9NzDf+H27dvH6tWr+eMf/zigXQbA6XSmFIvf72fNmjWsW7eOtrY25MMW/hzsmkfGmZ+fT0VFxZBtSgktLS1oNBqampoGHC8vL8fhcNDS0jLg+OHP6Ei1tbUDvrfb7QDU1NQcdTwWi+F0OikqKhrRswaoqqoSvX7HIRLRGHZ4KSQWiyFJEi+88AJarfaoc/Pz8wGIRqN87nOfo6enh29/+9tMmzaNvLw82traWLZsGbFYLKVYvv71r7Nu3TruuOMOFixYgN1uR5Ikrr322pSveSzDHWh5+DM60mDP6VjHE8l1uM96ODEICpGIxonGxkZkWaahoYEpU6YMed7WrVvZvn07jz76KEuWLEkef+WVV4Z1n6ESwB/+8AeWLl3KvffemzwWCATo6+sb9PwdO3Zw7rnnJr/3eDy0t7dzySWXHPP+dXV1xGIxduzYwQknnJA8fujQIfr6+qirqxvWzzEaw33WwvCJNqJx4oorrkCr1XLPPfcMqBaB8pe8u7sb6P9rf/g5sizzwAMPDOs+eXl5gyYXrVZ71H0ffPBBotHooNf5xS9+QTgcTn6/du1aIpEIixYtOub9E4nq/vvvH3D8vvvuA+DSSy893o8wasN91sLwiRLRONHY2MgPf/hDVq1axd69e1m8eDFWq5U9e/bwzDPPsGLFCu68806mTZtGY2Mjd955J21tbdhsNv7v//7vqLaiocydO5e//vWv3HfffVRWVtLQ0MD8+fP5/Oc/z29+8xvsdjsnnngib7/9Nn/9618pKioa9DqhUIjzzz+fq6++mm3btvHzn/+cM888ky984QvHvP+sWbNYunQpv/jFL+jr6+Occ87h3Xff5dFHH2Xx4sUDSlmZMtxnLQyfSETjyF133cWUKVP46U9/yj333AMoDa8XXnhh8hdcr9fzpz/9idtvv501a9ZgMpn44he/yG233casWbOOe4/77ruPFStW8P/+3//D7/ezdOlS5s+fzwMPPIBWq+Xxxx8nEAhwxhln8Ne//nXIHqv/+Z//4fHHH2f16tWEw2Guu+46fvaznw2r7edXv/oVkyZNorm5mWeeeYby8nJWrVrF3XffPYKnNTrDedbC8EnykWVLQRCELBNtRIIgqE4kIkEQVCcSkSAIqhOJSBAE1YlEJAiC6kQiEgRBdWN6HFEsFuPAgQNYrda0L/IuCMLoybKM2+2msrISjWbocs+YTkQHDhw4aqa0IAi5p7W1lerq6iFfH9OJyGq1AsoPabPZVI5GEIQjuVwuampqkr+rQxnTiShRHbPZbCIRCUIOO17TiWisFgRBdSIRCYKgOpGIBEFQ3ZhuIxoOWZaJRCJDLtAlDE2r1aLT6cTQCCHjxnUiCoVCtLe3i501R8FisVBRUSEWfxcyatwmolgsxp49e9BqtVRWVmIwGMRf9hGQZZlQKERnZyd79uxh8uTJxxyQJgijMW4TUSgUIhaLUVNTc9Quo8LwmM1m9Ho9LS0thEIhTCaT2iFlzJ4uL5v39dJYks+sGofa4Uw44zYRJYi/4qMzEZ6fKxDmzx8eIBKTaen2YTfrqS/OUzusCWX8/y8ThOP4534nkZiyYvLcugJKrEaVI5p4xn2JSBCOZ1enB4CLZ5RzQoUYoa8GUSKa4Orr64/aI2wicQfCdHlCaCSJBlEdU40oEQkT2kFnAIBiqwGTXosvFKHdGUCv0VBbJDo5skUkonEgFAqJcT4pMuq0TCrJozhfaRfafsjDa5910FCcJxJRFk3IqlkoEhvyIxKNDfvc8DDOTcXChQu57bbbuO2227Db7RQXF/Pd7343ub1xfX09P/jBD1iyZAk2m40VK1YA8I9//IOzzjoLs9lMTU0Nt99+O16vN3ndjo4OLrvsMsxmMw0NDTz++OMpxTee1BZZuHx2FWc0FQNQblOGKBx0BY7aTlrInAlZInrotZ1DvtZQnMfiOVXJ73/xxi7C0cH/Q1YXmLnqlP6F2R55cw/+0MCpJN/83JSUYnz00Uf56le/yrvvvsv777/PihUrqK2t5eabbwbgJz/5CatXr07ubrpr1y4uvvhifvjDH/LII4/Q2dmZTGbr1q0DYNmyZRw4cIDXXnsNvV7P7bffTkdHR0rxjVdF+UrJ0h+K4g9HsRgm5K9I1uVMieg//uM/kCSJO+64Q+1QckJNTQ0//elPmTp1Ktdffz1f//rX+elPf5p8/bzzzuNb3/oWjY2NNDY2smbNGq6//nruuOMOJk+ezOmnn87PfvYz1q9fTyAQYPv27bzwwgv88pe/5LTTTmPu3Ln8+te/xu/3q/hTqisak/EGIwNKPnqtBqtJST69vrBaoU04OZHu33vvPR5++GFmzpyZlfvdem7TkK9pjpgFsuLsxiHPPXLGyE1nNIwmrAFOO+20AVNSFixYwL333pucvHvKKacMOP/DDz/ko48+GlDdkmU5OdVl+/bt6HQ65s6dm3x92rRpOByOtMU81nS4Azz5bitF+QaWLKhPHi+wGHAHIvR6Q1Q5zOoFOIGonog8Hg/XX389v/zlL/nhD3+YlXsadMMvCGbq3NHKyxvY1ezxeLjlllu4/fbbjzq3traW7du3Zyu0MaPHGwIg74jqV2GegX09Pnp9ITXCmpBUT0S33norl156KRdccMFxE1EwGCQYDCa/d7lcmQ5PNRs3bhzw/TvvvMPkyZPRarWDnn/yySfzySef0NQ0eGlv2rRpRCIRNm3axKmnngrAtm3b6OvrS2vcY0kiERXmDexxdFj0gKiaZZOqbURPPvkkH3zwAWvWrBnW+WvWrMFutyc/xvMOHvv27WPlypVs27aN3/72tzz44IN84xvfGPL8b3/727z11lvcdtttbNmyhR07dvDcc89x2223ATB16lQuvvhibrnlFjZu3MimTZtYvnw5ZvPErXo4/UqisccTT0JDcR6XnFTBGY1FaoQ1IamWiFpbW/nGN77B448/PuxZ3atWrcLpdCY/WltbMxylepYsWYLf72fevHnceuutfOMb30h20w9m5syZvP7662zfvp2zzjqLOXPmsHr1aiorK5PnrFu3jsrKSs455xyuuOIKVqxYQWlpaTZ+nJzk8kcAsJkGJiKHxcDUcitF+WLOWbZIskqDJZ599lm++MUvDqhqRKNRJElCo9EQDAaHrIYkuFwu7HY7TqfzqF08AoEAe/bsoaGhYcwtX7Fw4UJmz56dE1MvxvJzPJ6HX9+FLxTl+tNqKbWOr58tVxzrd/RwqrURnX/++WzdunXAsRtvvJFp06bx7W9/+7hJSBBGIxSJ4YuP+TqyRASwt8tLjy9EU2n+oK8L6aVaIrJarcyYMWPAsby8PIqKio46LgjpFpNl5tQ68IWimPRH/9F7e3c3B50BbCa9SERZoHqvmXC0DRs2qB3CuGfSa1k4dej2MatJx0GnMjtfyLycSkTiF1DIFdZ4KcgdiKgcycSQM1M8BCGbPMEI/lB0yImtiWkeIhFlR06ViAQhW/6+vZPPDro5e0oJc+sKjno936j8aniDIhFlgygRCROSJ55gEgnnSHnx4x6RiLJCJCJhQkqUdCyGwYeJ5Bv6S0RiXaLME1UzYULyxscQDVUiyjfpuGxWRbJkJGSWeMrChHP46pkW4+AlIq1GoqnUms2wJjRRNRMmnES1zKDTYNSJEfy5QJSIhAnHGzp2+1BCa4+PTk+QaoeZUpuYi5ZJE6tEJMsQCWX/YwSNnevXr6eoqGjAuksAixcv5oYbbkj3E5mQzHots2sdTC0/dtXr4wNOXt/Wyb4eX5Yim7gmVokoGoa/35v9+571LdANb7ufq666ittvv50//vGPXHXVVYCy+8bzzz/Pyy+/nMkoJ4yifCPnHmN6R4Lows+eiVUiGgPMZjNf/vKXkztvADz22GPU1taycOFC9QKbgBI7ePiO2JlFSL+JVSLS6pXSiRr3HYGbb76ZU089lba2NqqqqmhubmbZsmUDFtMXUucKhNFpJMx67TGfqRhdnT0TKxFJ0rCrSGqaM2cOs2bNYv369Vx44YV8/PHHPP/882qHNW688vEh9vX4uGh6OSdWDr1YV6Ix2x8WJaJMm1iJaAxZvnw5999/P21tbVxwwQXjen3ubEv0mg01mDEhsU7RkZtmCukn2ohy1Je//GX279/PL3/5S2666Sa1wxlXEo3PeUMMZkw4vEQUi4lpHpkkElGOstvtfOlLXyI/P5/FixerHc64EY7GCIaVUdXHm75h1mu5bFYlV58iSqOZJqpmOaytrY3rr78eo1HsJpEuvqBSzdJpJIyDbYjp6wF/Lzhq0Wj1NJXmZznCiUkkohzU29vLhg0b2LBhAz//+c/VDmdcSTQ8mw2D9Jh1boePnwE5BvmlMOeGMdG5MR6IRJSD5syZQ29vL//5n//J1KlT1Q5nXPHFG6rNR07vCAdg21+UJATg6YCWf7DPcRpd3iC1hRaKxT5nGSMSUQ7au3ev2iGMW/kmHbNrHUf3mB3cCmE/WIqg4WylZHRgM1udk9jeHeK8aaUiEWWQSETChFJqNVE69YgJrLIM7VuUr6vnQslUsBSCr4ei4D6gXIyuzrBx32smVtcbnQnx/Hw94O0CjRZKpysDX0uUKnGBvwUAf1iMrs6kcZuI9HplWoXPJ2ZOj0bi+SWe51jnCUbwBiMDxwX17FI+O2pBHy8tFU0GwOptATmGPxTLcqQTy7itmmm1WhwOBx0dHQBYLBYxV2sEZFnG5/PR0dGBw+EYN1uA/+3TQ+zu9HLBCWWcVG1XDnbHE1FhY/+JtkrQGTHgIi/UjS+Ul/1gJ5Bxm4gAysvLAZLJSBg5h8ORfI7jQWK6htkQrwzEYuBqU74uqOs/UZLAXo3e/RnWwCH84aosRzqxjOtEJEkSFRUVlJaWEg6LrYNHSq/Xj5uSUEL/OKL4f31vh7JOlc4AeSUDT7ZVodNuwxo8RJtorM6ocZ2IErRa7bj7hRJSk+j9MscntOKMl4ZsVUop6HC2Skw6LaeVBAnPrsxilBPPhEhEggAQiR62e0diQKNrv/LZNkjVK78MrUaiGD/kjdt+nZwgnq4wYQTiSUgjHTbPzH1Q+WwbpMRjsIAxvq61tzMLEU5cIhEJE0b/9A6N0oMaCSkTXAHyywZ/U34pPb4Qn+3cidMn2hkzRSQiYcIw6pTdO06oiK/K6O1URlUbLGAcYpZ9XgkH+vx8tnMXnZ5A9oKdYEQbkTBh2M36gbt3eOPDOoYqDQHkl6LTSFjCPQTCYlBjpogSkTBxeeLtPkd22x/OUoxOo8Ecdoq1qzNIJCJhwvCFjpje4TmkfD5WichcgE4roYsFCPm9mQ9yghKJSJgw3t7VzS/e2M3GPT1K21CiJyz/GJst6gxIJqXnLOLpzkKUE5NIRMKEcfjqjIQ8EAkqgxjNhcd8n8aivC77ezIe40QlEpEwYQwYVe2LJxWTA7TH7rPR5BUBIHt7MxnehCZ6zYQJIxAvEVkMWvDFq1mWouO+r6C4HHO5FW2p6DXLFJGIhAkjMfPepNdCd7xEZCk47vss9hIsFgPI7kyGN6GpWjVbu3YtM2fOxGazYbPZWLBgAS+88IKaIQnjlCzLA9uIRlAiwhxPVr54I7eQdqomourqav7jP/6DTZs28f7773Peeedx+eWX8/HHH6sZljAOBcKxZA4x67WQaHgeRiKSTXY63UHae1yEg6ILPxNUrZpddtllA77/0Y9+xNq1a3nnnXeYPn36UecHg0GCwWDye5fLlfEYhfFjTq2DcFRGK0cg4FQOHqfHDEDS6tnWK6ON+Mhz9aA3iU0X0y1nes2i0ShPPvkkXq+XBQsWDHrOmjVrsNvtyY+aGrEVsDA8ZoOWhVNL+dyJZcpEV1kGnREMw1sCNhafhR8SPWcZoXoi2rp1K/n5+RiNRr72ta/xzDPPcOKJJw567qpVq3A6ncmP1tbWLEcrjAu+w6plw1zHXDYp61tHRCLKCNV7zaZOncqWLVtwOp384Q9/YOnSpbz++uuDJiOj0Sj2gRdSEghHicRkzHot2pE0VCcYlUQU9jkzEJ2geiIyGAw0NTUBMHfuXN577z0eeOABHn74YZUjE8aTD1v7eGtXN9MrbVyoiZdqzMfvuk/QmJVEFPX1ZSA6QfWq2ZFisdiABmlBSAd/cjCjDgJ9ykGzY9jv18TPjYlElBGqlohWrVrFokWLqK2txe1288QTT7BhwwZeeuklNcMSxqFA+LBthPx9ykGTY9jv1+U5iAFyQFTNMkHVRNTR0cGSJUtob2/Hbrczc+ZMXnrpJT73uc+pGZYwDiXnmWlRJrzCiEpEjTWVaFusmPQaCAf6d4QV0kLVRPTrX/9azdsLE0iiapYne5Sue60e9JZhv7/YbgW7HcJ+ZQySSERplXNtRIKQCYl5ZpboYaWhkW5BHu/CR1TP0k71XjNByDRZlvu3mo7GJ66OoH0IlDamLr8BgztAaVCM6E83kYiEcU+W4aRqO4FwFGOkRTk4gvYhUNqY3msPUe31URoUs/DTTSQiYdzTaCQWJnbv2Pqm8tk0/DFEACa9hpA2n0hMJhZwiTaNNBPPU5hYUhhDBGDSaQlqlXlpYb9oI0o3kYiEcS8YieIJRohGYymNIQKlVEViEX2faCNKN1E1E8a9nR0eXv74EJPscHk0rPSWJXrARkATn4Ef9buUhqeR9roJQxIlImHcS4yqthFf1MyQf9wF8wejMytbVUciYYiI7afTSSQiYdxLjKrOl0c+ovpwRqOBiMZEJBoD0XOWVqJqJox7iTFE+bF4Ihph+1DCqfWFGPsqsUV6lER0rI0ZhRERiUgY95KL5sdGVyKqLrBAUQl0O0EMakwrUTUTxr3+UdXx5JFiiQgAY3y9alE1SyuRiIRxL1EiMkbiySPFEpHTH2avW0OvLwRBT5qiEyDFRLR79+50xyEIGTO1zMoJ5RZMsXivWYolonann3+0hmh3BkSJKM1SSkRNTU2ce+65PPbYYwQCohtTyG2nNxVzcaMFk1ajdNsPc+eOI5l0WkLaPMLRmGgjSrOUEtEHH3zAzJkzWblyJeXl5dxyyy28++676Y5NENLHH1+n2uRIeSCi2aAlpMsjGpNFiSjNUkpEs2fP5oEHHuDAgQM88sgjtLe3c+aZZzJjxgzuu+8+Ojs70x2nIKQkHI0p0zsSa02PYMH8IyklIosyjigShEgoPUEKo2us1ul0XHHFFTz11FP853/+Jzt37uTOO++kpqYmuQSsIKipvS/AL9/YzWtbtikHRtFjZjJoiEoGQuiJynL/krPCqI0qEb3//vv867/+KxUVFdx3333ceeed7Nq1i1deeYUDBw5w+eWXpytOQUiJLxwB4kvEQso9ZgAGrQaNRkM4USoS7URpk9KAxvvuu49169axbds2LrnkEtavX88ll1yCRqPktYaGBpqbm6mvr09nrIIwYv2jqkfXYwYgSZKyLpEuj0jMi1F04adNSolo7dq13HTTTSxbtoyKiopBzyktLRWL4wuq84eiIMtYYm5AP6oSEcD5J5SRb6zG6N0hqmZplFIieuWVV6itrU2WgBJkWaa1tZXa2loMBgNLly5NS5CCkCp/OIouFsRAGNCntPzH4ZpK88FdDP5dYlBjGqXURtTY2EhXV9dRx3t6emhoaBh1UIKQLr5QFGPEhV6rUaZnaPWjv6hBWZdItBGlT0qJSJblQY97PB5MJrHfk5A7/OEopogLvVYa3RyzuA53gJ0uCU8wIqpmaTSiqtnKlSsBpdFu9erVWCz9G9RFo1E2btzI7Nmz0xqgIIzGpOI8ou4YppB21O1DAJ+1u9m2x8eFgSD5dpGI0mVEiWjz5s2AUiLaunUrBoMh+ZrBYGDWrFnceeed6Y1QEEbhlPpCCGigXZeWEpHZoCWszSMci48jEkvGpsWIEtFrr70GwI033sgDDzyAzWbLSFCCkFYp7twxmAGjq2NRCPtSnrsm9Eup12zdunXpjkMQ0i4ak/GFIuT5epXG0DSUiEx6DbKkJSDF20KDHpGI0mDYieiKK66gubkZm83GFVdcccxzn3766VEHJgij1esL8dhbuznjQAun1hWkp0Sk1wLgl8xANN5gXTbq6050w05EdrsdKV4XtttHNxZDELLBH4piiHjQa6T48h/5o75mIhH5MAMeMQs/TYadiA6vjomqmTAWKF337v6u+zQ0Kpv0yogXLxZk3EgiEaVFSuOI/H4/Pp8v+X1LSwv3338/L7/8ctoCE4TR8oWUMUQ6rWZUy38czmLQcdH0ck6dWgsyYixRmqSUiC6//HLWr18PQF9fH/PmzePee+/l8ssvZ+3atWkNUBBS5Q9FMUbdaRvMCKDVSJxYaaOitFhpqhDTPNIi5RUazzrrLAD+8Ic/UF5eTktLC+vXr+dnP/tZWgMUhFT5wxFlVLVGk5aG6gES0zxComqWDiklIp/Ph9Wq/EO8/PLLXHHFFWg0Gk477TRaWlrSGqAgpMofimEKu9ClsUQEsL/Xx7Y+WdnKWrQRpUXKi+c/++yztLa28tJLL3HhhRcC0NHRIQY5Cjmj2mGixhLCYtCltUS0cXcPL+/0xeeb+ZSBjcKopJSIVq9ezZ133kl9fT3z589nwYIFgFI6mjNnTloDFIRUzSo3MqXIgN08+uU/DmfSa4loTITkeC+caLAetZRGVl955ZWceeaZtLe3M2vWrOTx888/ny9+8YtpC04QRiUxtSNdy3/EmfQakCRCiUGNQU9aE91ElPKa1eXl5cyZM2fA4mjz5s1j2rRpw77GmjVrOPXUU7FarZSWlrJ48WK2bduWakiCkCTLMl5nFzFZTmv7EIA5PqgxoIlP7RDtRKOWUiLyer1897vf5fTTT6epqYlJkyYN+Biu119/nVtvvZV33nmHV155hXA4zIUXXojX600lLEFICoRjPL/xUzbu6SGa5tKKMTHNQ2NWDoiq2ailVDVbvnw5r7/+OjfccAMVFRXJqR8j9eKLLw74vrm5mdLSUjZt2sTZZ5+d0jUFAcAXimCMuNBpJLRpGsyYkBhdrUzzQJSI0iClRPTCCy/w/PPPc8YZZ6Q1GKfTCUBhYeGgrweDQYLBYPJ7l0ss1SkMzh+OYoq6la77NCci84D5ZogSURqkVDUrKCgYMlmkKhaLcccdd3DGGWcwY8aMQc9Zs2YNdrs9+VFTU5PWGITxw3/4WtVpHsxYajNx8YxyTmqsVg6IEtGopZSIfvCDH7B69eoB881G69Zbb+Wf//wnTz755JDnrFq1CqfTmfxobW1N2/2F8cUbCGGMeJVElObG6nyjjhMqbJSXFCsHxDSPUUupanbvvfeya9cuysrKqK+vR68f2DX6wQcfjOh6t912G3/+85954403qK6uHvI8o9GI0WhMJWRhggm5uwEZnd6QuYXLxDSPtEkpES1evDgtN5dlma9//es888wzbNiwQWxFJKRN2NuDBpAshRlZU3p3p4dAIMrUmIw2EoJIEHTij2SqUkpEd999d1pufuutt/LEE0/w3HPPYbVaOXjwIKAsvGY2m9NyD2FiqjD4iOUbySsozcj1X/z4IMFwjDqNjrzEoEaRiFKW8oDGvr4+fvWrX7Fq1Sp6enoApUrW1tY27GusXbsWp9PJwoULqaioSH787ne/SzUsQQBgkiVEU2k+ZaXlGbl+oucsrI1vqSWqZ6OSUonoo48+4oILLsBut7N3715uvvlmCgsLefrpp9m3b19yraLjGWqjRkEYNX+v8tmc3t7dBGXJ2DAhrQUibtFgPUoplYhWrlzJsmXL2LFjx4CdXS+55BLeeOONtAUnCKmQZZmAq1OZ3mHJTCJKTvOQEiUikYhGI6VE9N5773HLLbccdbyqqirZziMIagkGg2zesY+Ne3qIGDIzGTUxujqQqJqJsUSjklIiMhqNg45q3r59OyUlJaMOShBGw+fsUr7QGdGZRr9zx2CMydHVIhGlQ0qJ6Atf+ALf//73CYfDAEiSxL59+/j2t7/Nl770pbQGKAgjFXQriUg2Z6brHvqrZn4xzSMtUkpE9957Lx6Ph5KSEvx+P+eccw5NTU1YrVZ+9KMfpTtGQRiRsLtb+SLNc8wO11iSz8UzyplcG++VE43Vo5JSr5ndbueVV17hzTff5MMPP8Tj8XDyySdzwQUXpDs+QRixsFdJRJq8zDRUA5RYjZRYjRCI9/yGPCDLGSuBjXcjTkSxWIzm5maefvpp9u7diyRJNDQ0UF5ejizLKS8JIgjpEvUo49q0eUWZv1li99hYFMK+zE0nGedGVDWTZZkvfOELLF++nLa2Nk466SSmT59OS0sLy5YtE8vECjlB9iklIn1+5hJRKBJjV6eHbR0+MCQarEX1LFUjKhE1Nzfzxhtv8Le//Y1zzz13wGuvvvoqixcvZv369SxZsiStQQrCsEWCFOhCaK1G9MVlGbtNIBLlj1sOoNVITLHlI4V88QbrzN1zPBtRiei3v/0t3/nOd45KQgDnnXced911F48//njaghOEEfN2UWo10VRdTl1Z5kpEiV6zaEwmrItXz4Jiob5UjSgRffTRR1x88cVDvr5o0SI+/PDDUQclCCmLV8uwFGf0NnqtBoNO+fUJaUTVbLRGlIh6enooKxu66FlWVkZvb++ogxKEVMneLgLhKFFT5rruE/p38xBjiUZrRIkoGo2i0w3drKTVaolEIqMOShBSFXAeYnNrH099GiAWy+ykaoshPqhREiWi0RpRY7UsyyxbtmzIVRIPX9heENQQdnUqX+QVo9FkdiiJOZ6IfMlEJNqIUjWiRLR06dLjniN6zATVRMOEfUrTgDY/s21E0F818xJfgUJUzVI2okS0bt26TMUhCKPn6yYUjhLRmDDnWTN+u+lVdmoKLZSZotABhHzKwEaNNuP3Hm9SmuIhCDnJ20UoGsOnd5BvSt9e90OpcpipcpiVqR0arZKEQh5I886yE0HKS8UKQs7xdRGOxAjoHVhNWfwbK0n9Uz3EciApESUiYfzwdROMxPCZC8gzZv6/diAcpa3PjyzLNBnzIeAUPWcpEolorIqEoOMTkGNQeiLoTcd/z3jn7aIwz0BVRQ1FeZnfUcPlD/PHLQfIM2ppKoyXiESDdUpEIhqLQl7Y/Hj/KOLWjTD7ejDZ1I1LTZEQ+HupsJupmH0iZGEjzkT3vT8UQzbkI4GomqVItBGNRdteUJKQIQ+MVvD3wad/UhpNJypvp/LzG/LAmJnlYY+U6L6PyTIhbXz5D5GIUiIS0VjTtw+6doCkgVnXwuwvg0anHO/epXZ06vEcIhqT8RmKiERjWbmlTqvBmFhEXyN28xgNkYjGmtZ3lc8VMyG/VNkup3qucmzf2+rFpTZvJ65AmBf2xnjyvdas3daSWERfis83E43VKRGJaCwJuvtLPdWn9h+vPlUpITn3g6dTndjU5jlEOBrDqy/Katd9cppHchF9UTVLhUhEY8mhj5VeMnsV5B02hcFoheKm+Dlb1YlNTbIMng6l615fSH4Wuu4TzAblXslpHpEQRMScy5ESiWgs6dqufC6bcfRrpScqnzu3T7xGa38vRMP4oxJ+vQObOfOjqhNmVztYdFI5taUFoDMoB0X1bMREIhorQl5wHVC+LmpKHnb6wvz+vVZcebVKo7W/FzwdKgWpkvjP69LYQdJgy8L0joTaIgvTym04LAYwxOe3ierZiIlENFZ071RKOtayAeOFOtwB2vr8PPtRJxFHffzcHerEqBavkoh6JAcANrNKw+MSwwZEiWjERCIaK+KN1HJRE3/95BBOn7LLbqnNRL5RR7cnxMfBeLtRzx61olSH+yAxWaZHUlZlzGaJKBCOsqvTw84Oj9JWB2IsUQpEIhoLZBn6WgBoiZWxtc3JU5taicVk7GY9C6eWALCxx0Y4GlOqcBOlwVSWwd2OLENT42ROrLQlV07Mhh5viD9uOcDr2zv7J76KsUQjJqZ4jAXeTggHQKvjvR4jEOKECltyBcKm0nzKbCYOuWB/0EyDNgi9LVAyRd24syHghJAPrVbLKdNPAG12/0vnxXvNfMGImOYxCqJENBb0KQP0fOZK9veFkCSYWd2/5o0kSZxc5wBgW7AIWZahd68KgarA3a58zi/NehICyDMqpa9ITCakFaOrUyUS0VgQr5btiSh7udcUWLAe0Q4yudSKSa/lkLYMVyACzn1ZD1MV8Z5Er7EUpy9MNMML5h9Jp9VgSiwZm1y7WpSIRkokolwny+BUSkSf+R0ATKs4ehlUrUZiVrWdhsapyn5b3i6lOjfeuQ8CsKXPxCNv7mHzvuxvZ5UoFXnlxOhq78QbyzVKoo0o1/l6IOQjEJNojTqQNNBYMvjs8tObioFi8JYo73O1QVFjduPNplgsWTXrQCktZnMwY0KeQUc3Idwxo7JaYyyqJKMsrQIwHogSUa5zx6sehhJMRgOVdnOyKjAke7Xy2Zm9yZ+q8HVBNAxaPQcjyi+9w6JCIoqXiHwRub/nLODMehxjmSgR5TqX8he/qKKOFZMm4Q9Hj3l6NCbTSTEGb5BC5/5sRKieeKIN5VUQ6FGqQg6zIethzKiyU1+cR5nVBL0OpY0o0KfMCRSGRSSiXOdqUz5bK9FopOOuxbyny8sruyVO6/ZRYD2AFI2o0puUFfFE6zIo26BbTbrkfvTZVF1g6f/GZAdaRYlohFStmr3xxhtcdtllVFZWIkkSzz77rJrh5J5oBLydSne8rXJYb6kttBAxOHDHjPgDQfAczHCQKoonoh69kogKLNkvDR3F5FA++/vUjGLMUTUReb1eZs2axUMPPaRmGLnLcwhiUfb7JH71XjebWnqO+xaDTkNNUR5uYxl9vnDyl3Xc8fdBwAWShi6pCICCvOy3DwEEI8o0j88Ouvr3NBMlohFRtcy+aNEiFi1apGYIuS0+RqaDItzBY7cNHa62yMJ2YzlOfxuV4zURJX4uaxm1pQ7QGqhwmFUJxR+K8sctBzDoNEydbVdGVwf6VIllrBpTjQfBYJBgsH8OlcvlUjGaLHAfQEamLeoAoMI+vF+0ukILm4xluJxhon2taGVZ6VYeTxI9gvYaqgssA9tpsswSn+YRisQI6W0YQSmtxWKgER3TwzGmntKaNWuw2+3Jj5qaGrVDyixXO4FwjC6pGJ1GotQ6vC1yCvMMSNZyImhxu13KGkXjTV985Lhd/f8DBl3/Ivoe2aRsPy3HIDjO/1Cm0ZhKRKtWrcLpdCY/WlvH8TiZkA/8vbgDYTzGEspsJnTa4f1zSZJETbEVr6EEtz8y/tqJAk5lwKakwZ9Xxd4uL55gRNWQElNu3MEYGOPrRYl2omEbU1Uzo9GIMQsb5+WE+IjhHtlKVGOkwjGynVxPqStAI5+Eo3OTkogqZmYiSnUk1luyVbDfHePPH7VTZjPx5fm1qoVkM+nocgdxByJgdiilUJGIhm1MlYgmlHhDdbusLPY13PahhKJ8IwXlk5CQ+scijReJlQUK6unyhAAozle36z6xc4g7ED6s56xPvYDGGFVLRB6Ph507dya/37NnD1u2bKGwsJDaWvX+uuUEdzsyMrbSWsq0JipHWCICwBYf2evtgrAf9Or0KqXV4UucFDTQ1aJ0XhQPs/0sUxJVM1cgAlaHclCMJRo2VRPR+++/z7nnnpv8fuXKlQAsXbqU5uZmlaLKAfFVByUk5k4/kbkpThU4FNDQ7dSRF3VS5zowPibAeg4pSVVnAFslXR6l0bo4T91E1FCch9WkoyjP2J+AxmMnQYaomogWLlyojBoWBoqvOoikgfyylC/jD0XZEXBQE+yktq8VaTwkou54CdpRRygm4fQra3cXW9WtmhXnGynOjydDSVkJAH+P8kdlvA2dyADRRpSL4g3VLl0B4VH8E1UVmPGZywlGYni7WtIVnbq64juUFE/mkCugbGxi0iXH8uQEs9KuRziglN6E4xKJKBe524nJMm+06/j5a7uSO3aMlF6rIa+kDgDXoX3KOjljWcClLIQmSVDUxCGXsvBbuT2F9rMM2NPlZUtrH4GYpn/LJ//xp+UIY6z7fsJwteMLRenTlWDQaUa1T1dlRRX+bUZcXh+Vng6wVaQx0CxL7NdmqwJDHpPLDJgN2qxuMX0sf/v0EO5AhDKbkQpzoZI4fT3960MJQxIlolwTi4HnIO5AGK+hmEqHCWkUbQz1Jfl4DKU4/WEivWN8HevDqmUAdrOe6ZV26oryVAyqX2I/NXcgApbD2omE4xKJKNf4eyASwhWS8OkLRjx+6EhFeQYi1ipiMvS0j+GNF0M+ZYskgKLJ6sYyhETJ1ekPgzmeiHwiEQ2HSES5xqVMdD0YU/Zxrxhl+4ckSRRUNJBn0KJxt43dRd07P1Pmb1nLIK+I1h4fm1p66fGG1I4syR5fHbLPFxYlohESiSjXuA8SjMTo1hShkaS0NMSePvskZtYUUqwLjd2JmB2fKJ9LpwPw2UE3b2zv5J9tuTONIrFedp8v1N9z5usdu8k/i0QiyjXuA7gDETyGEkptRvTDnOh6LBq9QdmAEMbmBNiAU9lkUpKg9AQAWnt8ANQUqrf8x5ESicjpDysrNWp0EIuIgY3DIBJRLolFwdOB1ahj1vQTmFXtSN+17TVEYzKejr3pu2a2HIqXhuw1YLLh9Idx+sNoJCm1qS8Zkli43x2IEJaBPGXlSLxd6gU1RuRGv6eg8HRALIrJksespvq0jsjdFy2gvaUHXd825p50adqum3GyDO1blK/LZwCwu1PZ0rnCbsKoO87WSllk0mtYdFI5drMerSRBXgm4D4G3A0qmqB1eThMlolySmCVvq0r7tAB7eQMxGULOdnw+b1qvnVG9e5S5WzojlCjVst2dSvyNpbnRbZ8gSRLTym1U2M1oNBLkxavD3k51AxsDRCLKJc5WvKEIu0OOlEdTD8XuKMSQVwCyzL4929N67Yw6sFn5XDYDdAYC4Sj7e5VpE0PteJsz8oqVzx6RiI5HJKJcIcvgbKPbE+K1dgPv7OlO+y2sVcr4m659n6X92hkRcEJXfJJr5RwAujxBdFqJ4nwDjlzYPugIvd4QW1r72H7I3d9B4O9VtoYShiQSUa4IuiDoxhWM4jGUUJWBHSkq6k4EINy5S/WlVYel9T1l7FBBHeSXAMpmhjefNYlFJ+XmVJWDrgCvfdbBltY+ZftpvUn5GXyiwfpYRCLKFc79xGSZjpidmEafkURkrWjCajZgCvexbW+Od+OHfNAer5bVnjbgJYNO07/kRo4piq8U2e0JIYPSYA1KR4QwJJGIcoWzDU8wQp+hDItBmxyTklZ6EwXlymz8A3s+ze21oNo2KdUZaxkUNADQ7QnmdsxAocWARpIIhKN4Q1HIL1de8BxSN7AcJxJRrnC2KgMZjWVUOsyjmuh6LKW1J1BbaOGCMk/G7jFqIS+0blS+rl0AkoQ7EObxjft44t19BMK5u5yJTqtJ/hHp9gT7VzuIr0EuDE4kolwQDoC3E3cgjMtQTlVB5taW1pdOpsphxuxugWh6e+bSZu+bSmzWciiZBsCbO7uJxmQMWg0mfe6MHRpMonrW5QmBNZ6IPIdEg/UxiESUC/r2IcsxOqP5hHV5GWkfSrKWK7tMRMPQs4dINJa5e6XC293fZd94LkgSbX1+Pm1X5sidNblExeCGpyi+fna3J6jMOdOblFHzXtFONBSRiHJBXwsSEvNPPpnLZlVQksmGWEmCkil4QxHe2vgmz245kDvtLrIM2/6i9DIVNUFBPYFwlBe2Kkvnzqiy58xqjMdSfHiJSJLAWqm84GpXMarcJhJRLohvj2MobqCp1KqMys2kkmnKFISuHezvdrGjw5PZ+w3XgQ+USblaPUy5kHA0xh8/VCYBOyx6zp5SrHaEw1JTaOGqU6q54uT47iu2eCJyi3aioYhEpLagW5kUKUnKeJlssFVhshZQY9NS6NvLa591KBsDqsnTATtfVb6etBBMdl7f1klbrx+DTsOlMytyal7ZsZj0WqoLLP1tWYlE5BxnG12mkZj0qrbeFiKxGFv6TIT3+ZjfYMp8iUiSoPwkqvxOmnp3sjHUxJ8/aueLc6qG3RAsy/KAXrfN+3rZ0eFBI0noNBJ5Rh12sx6HRU+BxUBhngHtUD9XOAD/fFpZMqNwElTNBWD+pEIOOP2cf0IZpdbcr5INyV6tbA2V2IY6sROskCQSkdq6d+Dyh9kbq8Nz0MWCxqLs3Ld8JpqWtzjZ6uIz3Bx0wlOb9rNoRvmggwVlWabXF2Z/r4/9vX4O9Pn5yml1ycTlDkRo6x1665yvnFZHSXw31tYeH05/GLtZj0aOYPzkKaI97fTGzOy2ncIl8QRnNen5yvy6zCfmDOh0B/mk3YVZr2VeQ6HSSeA6oCx3WzFT7fByjkhEaopGoGc3Pb4wvZY6JhdncTa52QGFkzB17+KKgjZ+55xOlzvIu3t6uCQ+faLDHWDnIQ+dniAHnQF8oYHjd9r6/MmJp9PKrZTbTcgyhKMx3IFIfN2gEL2+MAWHDdD87KCbf7Y50cTCTOl6BUdgP1GNgY9LL8LXE+UUdyBZAhqLSQjAHQjzQUsvhXkGJREV1CmJqE8kosGIRKQm5z7kcJCOgA6PozT7s8lrT4PuXTicn3H9nDN5dU+AU+oLki93e0Js3NO/5rJOI1HhMFPlMFNdYB6wnnapzUSpbXjVp+J8A422GMV7/4Il2olsNNLRsJgTS+upK8xTffvodEhsetDjDeENRshz1EHL20qJSOz+ehSRiNTUtQNXIEKnsQazUZfZ8UODcdSCowb6Wsk/8A5fmHXJgJeL8g3MrLZTkGeg3Gai1GpEN9qla2WZOcZ25oRfgtIIGKpgxpVgrxrddXOM2aCl1GakwxWktdfHtJJq0OqUzgnPIaWqJiSJXjO1xGLQuY0eb4gecx0NxXnqVEMazlE+t3941HrWpVYT559Qxsm1BVQ6zKNPQs422PIEfPKc0kBtLYc5N4y7JJRQU6Csp72v26cMSSicpLzQNYbWg8oSkYjU0ruHWNDDIb+E01TNlDKrOnE4avrbLD79s5Ig0ikShINbYdOj8MF66NunLCpffyacvKR/251xqDa+sP++Hp8yaLQ4vlxs5zYVo8pNomqmlkP/JBqT0ZVPx2YxUafmbhSTzlXaLvy98PEzcNKVyl/wVPn7lEGa3TuhZ7cyvQFAo4Wy6UoSmgBd2FUFZnQaCXcgQqcnSGlRk9KN7+1SprLkZamHdAwQiUgNIR90bUev1XDKvLM5Ob9C3d4hgwVmXAGbf6MkkC1PwAmXDa+0IstKAnO1KaWd3hZlrMzhLEVQdiJUzAZjji/vmkZ6rYb64jwOuQJ4AhFKrflQ2ADdu+Dgh9B4ntoh5gyRiNTQvqV/rR1rBZpc6EGxlsOs6+Cj3yvdzO/9Spn5XtSoJBKtQRlwGPYpJR5/j7JDhbtdqX4dTtIoy18UNCjXyM/9iaqZ8rkTyzDqNP2DPytmxxPRVqV9TjM2RotnmkhE2RaLQtsmujxBtHWzKMiFJJRgr4ZTboLtL0LPHjj0sfJxPBqdsj6zowYcdcr+Y7rcW09aDUeNVC9qBEOesuZS5zalpCiIRJR1B7YQ9rnY1hvjvZ1mrisOUDbM8TdZYXbArGuVHq6OT8G1X+lyjgSVdiOdSdnF1Fyg7FJhq1SWQxV/2Y8pGpM56AooQzQq58Def0DLm8rOtbn0x0glIhFlUyQELf9gf6+flvy5lNjzKLXm6OA9e9W47VbPtkA4ym/ebsEXirLs9Hrs1afA/neVRuvOz5LbaE9kovs+m3ZvwOtx0eI30pE/jbMnl+Tucq1C2pj0WoryDcRkmTd3dYHeDNXzlBd3/u3oNrYJSCSibOnZTWz/++zs8LDbcQZN5XZq1OyyF7LqzMnFSBJsO+hWBjjWnqZUg4Nu2PWq2uGpTiSibPB0In/8LDs7POwxTCFsr+O8aaVqRyVkUanVxMxqZezUSx8fxB0GplystA8d2AL731c1PrXlRCJ66KGHqK+vx2QyMX/+fN599121Q0qfvlbY8hh9Ljd7I4W0Fp3OJSdVYDGI5rmJ5qzJJRTlG/AEIzz9QRtOU3X/FJsdryiTYnNl2d4sUz0R/e53v2PlypXcfffdfPDBB8yaNYuLLrqIjo4xvtB40I28/WXY8jiEAxSU12E79RouOqlaVMkmKL1Ww+Wzq7CadPR4Q0p7Ue1pUBNvL9q9AT787YRcyVGSVV45ff78+Zx66qn8z//8DwCxWIyamhq+/vWvc9dddx3zvS6XC7vdjtPpxGazZSPco8kyRMPIIS9BdzfBvnZ8HbsIduyhz+tnSpkVS+1smHyRGFsjAOAKhHljeyfnTyvDbFCGPez/5z/Q7XkNs1bGqNOgt5eiKZ6MZC0HS7Ey9khvHnNd/cP9HVW1fhAKhdi0aROrVq1KHtNoNFxwwQW8/fbbR50fDAYJBvt7GFwu17Dv1dbn59VPDyEDpb0f4HDvRJZjSMggQ12RRVlfR5Zx+kN8ckCZpiAhA3K8yKzk7JoCM1V2EyDj8fv5dH8v0ZjMkRndbSzH5Tibs0+YN+w4hfHPZtLz+ZmVA469H27ggM5ElXMzJb4dSK19SGxHo1GW3j25tkAZsa7VsaPTT28ghixpQdJwZGo6paEoOVp/Z4ebbu+R65H3v2NufQG6+PSi3Z1e2n0Su6svHzTuL51clbEmBVUTUVdXF9FolLKysgHHy8rK+Oyzz446f82aNdxzzz0p3SsciSnbuwBWtxN7fC/yZPIw+5W/OIAUCEPANfD1w4VkCMdfiUaJxGRikpaQNp+wuQhsVRhKp1BXWU19saiGCcdXU2hGppxe2wW0eU/D6mvBFmjHEu7BEo3/wZVjEAkRDXqJ+ofe7EDy9i+8JnvdxOL/7wflCYNGaaGRPR6iPg1d7sGHE8QyWHcaUy2mq1atYuXKlcnvXS4XNTU1w3pvmc3El06uRpJA478AKXSaMoZHkpAkCatJDwYdSBLGaIxGXyT5GsQ/S0piyjPqwaDMTjeiZUpEh8FgxGLUD71AvCAcw9y6QubWKZOMZVkmFD2RSFQmHI0Riclg0Snz/KJhSt0+7OEwxCLIsUE2yLQbUUo9MkXeEJZQjCH+pKKxm5IFpEJfCFMY6m3Vg55r0mWuSVnVRFRcXIxWq+XQoUMDjh86dIjy8qNXsDMajRiNqY1ENhu01BYlSifHLqUYgdJhLpOjB8RiDkI6SZKEUafFeORvp1FZs6pwBGs4FRYc/5yEgkIYwelppWqvmcFgYO7cufztb39LHovFYvztb39jwYIFKkYmCEI2qV41W7lyJUuXLuWUU05h3rx53H///Xi9Xm688Ua1QxMEIUtUT0TXXHMNnZ2drF69moMHDzJ79mxefPHFoxqwBUEYv1QfRzQaOTGOSBCEIQ33d1T1kdWCIAgiEQmCoDqRiARBUJ3qjdWjkWjeGslUD0EQsifxu3m8pugxnYjcbjfAsEdXC4KgDrfbjd0+9F52Y7rXLBaLceDAAaxW64RccjUxxaW1tVX0Gg5CPJ/jy/QzkmUZt9tNZWUlGs3QLUFjukSk0Wiorh58XsxEYrPZxC/aMYjnc3yZfEbHKgkliMZqQRBUJxKRIAiqE4loDDMajdx9990pr0gw3onnc3y58ozGdGO1IAjjgygRCYKgOpGIBEFQnUhEgiCoTiQiQRBUJxJRjhvpLrhPPfUU06ZNw2QycdJJJ/GXv/wlS5GqYyTPp7m5GSm+IULiw2QyZTHa7HrjjTe47LLLqKysRJIknn322eO+Z8OGDZx88skYjUaamppobm7OeJwgElFOG+kuuG+99RbXXXcdX/3qV9m8eTOLFy9m8eLF/POf/8xy5NmRyi7BNpuN9vb25EdLS0sWI84ur9fLrFmzeOihh4Z1/p49e7j00ks599xz2bJlC3fccQfLly/npZdeynCkgCzkrHnz5sm33npr8vtoNCpXVlbKa9asGfT8q6++Wr700ksHHJs/f758yy23ZDROtYz0+axbt0622+1Zii63APIzzzxzzHP+/d//XZ4+ffqAY9dcc4180UUXZTAyhSgR5ajELrgXXHBB8tixdsEFePvttwecD3DRRRcNef5YlsrzAfB4PNTV1VFTU8Pll1/Oxx9/nI1wxwQ1//+IRJSjjrUL7sGDBwd9z8GDB0d0/liWyvOZOnUqjzzyCM899xyPPfYYsViM008/nf3792cj5Jw31P8fl8uF3+/P6L3H9Ox7QRiJBQsWDNgv7/TTT+eEE07g4Ycf5gc/+IGKkQmiRJSjRroLLkB5efmIzh/LUnk+R9Lr9cyZM4edO3dmIsQxZ6j/PzabDbPZnNF7i0SUo1LZBXfBggUDzgd45ZVXxuWuuenYJTgajbJ161YqKioyFeaYour/n4w3hwspe/LJJ2Wj0Sg3NzfLn3zyibxixQrZ4XDIBw8elGVZlm+44Qb5rrvuSp7/5ptvyjqdTv7JT34if/rpp/Ldd98t6/V6eevWrWr9CBk10udzzz33yC+99JK8a9cuedOmTfK1114rm0wm+eOPP1brR8got9stb968Wd68ebMMyPfdd5+8efNmuaWlRZZlWb7rrrvkG264IXn+7t27ZYvFIv/bv/2b/Omnn8oPPfSQrNVq5RdffDHjsYpElOMefPBBuba2VjYYDPK8efPkd955J/naOeecIy9dunTA+b///e/lKVOmyAaDQZ4+fbr8/PPPZzni7BrJ87njjjuS55aVlcmXXHKJ/MEHH6gQdXa89tprMnDUR+KZLF26VD7nnHOOes/s2bNlg8EgT5o0SV63bl1WYhXLgAiCoDrRRiQIgupEIhIEQXUiEQmCoDqRiARBUJ1IRIIgqE4kIkEQVCcSkSAIqhOJSBAE1YlEJIwpzc3NOByO5Pff+973mD17dvL7ZcuWsXjx4qzHJYyOSETCoJYtW4YkSXzta1876rVbb70VSZJYtmzZgPPTnQDq6+u5//77Bxy75ppr2L59+5DveeCBBwass7xw4ULuuOOOtMYlpJ9IRMKQampqePLJJwcsihUIBHjiiSeora1VJSaz2UxpaemQr9vt9gElJmFsEIlIGNLJJ59MTU0NTz/9dPLY008/TW1tLXPmzBnVtQcrqSxevDhZylq4cCEtLS1885vfTO64AUdXzY50eMls2bJlvP766zzwwAPJa+zZs4empiZ+8pOfDHjfli1bkCRJrE2kEpGIhGO66aabWLduXfL7Rx55hBtvvDHj93366aeprq7m+9//fnLHjZF64IEHWLBgATfffHPyGrW1tUf9TADr1q3j7LPPpqmpKV0/gjACIhEJx/SVr3yFf/zjH7S0tNDS0sKbb77JV77ylYzft7CwEK1Wi9Vqpby8PKVVJu12OwaDAYvFkryGVqtl2bJlbNu2LbkHWjgc5oknnuCmm25K948hDJNYs1o4ppKSEi699FKam5uRZZlLL72U4uJitcMalcrKSi699FIeeeQR5s2bx5/+9CeCwSBXXXWV2qFNWKJEJBzXTTfdRHNzM48++mjaSg0ajYYjl8IKh8NpufZwLF++PNkQv27dOq655hosFkvW7i8MJBKRcFwXX3wxoVCIcDjMRRddlJZrlpSUDGj3iUajR+1IazAYiEajo7rPUNe45JJLyMvLY+3atbz44ouiWqYyUTUTjkur1fLpp58mvx6K0+lky5YtA44VFRVRU1Nz1LnnnXceK1eu5Pnnn6exsZH77ruPvr6+AefU19fzxhtvcO2112I0GlOqEtbX17Nx40b27t1Lfn4+hYWFaDSaZFvRqlWrmDx58rjcYGAsESUiYVhsNhs2m+2Y52zYsIE5c+YM+LjnnnsGPfemm25i6dKlLFmyhHPOOYdJkyZx7rnnDjjn+9//Pnv37qWxsZGSkpKU4r7zzjvRarWceOKJlJSUsG/fvuRrX/3qVwmFQlnpBRSOTaxZLUxYf//73zn//PNpbW09aodTIbtEIhImnGAwSGdnJ0uXLqW8vJzHH39c7ZAmPFE1Eyac3/72t9TV1dHX18d//dd/qR2OgCgRCYKQA0SJSBAE1YlEJAiC6kQiEgRBdSIRCYKgOpGIBEFQnUhEgiCoTiQiQRBUJxKRIAiq+/8Or/FWKg8oXgAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T04:13:44.373995Z", + "iopub.status.busy": "2024-03-24T04:13:44.373668Z", + "iopub.status.idle": "2024-03-24T04:13:44.789705Z", + "shell.execute_reply": "2024-03-24T04:13:44.788752Z" + }, + "papermill": { + "duration": 0.438372, + "end_time": "2024-03-24T04:13:44.791847", + "exception": false, + "start_time": "2024-03-24T04:13:44.353475", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABDDklEQVR4nO29eXxb5ZX//77aJVuS9y1eErKQfYVAKBACFApMCsPQ0nagCYF2OoW+vjQ/CqTz/cLAMA3TFgozZRhKh2SYQplCgS4UAkMJKSmEkIUGspHNcWIn3m3t231+f1xJiWM7sWVJV7ae9+vll6SruxzJ0kfnOc8551GEEAKJRCLREYPeBkgkEokUIolEojtSiCQSie5IIZJIJLojhUgikeiOFCKJRKI7UogkEonuSCGSSCS6Y9LbgJGgqirNzc04nU4URdHbHIlEcgpCCDweDzU1NRgMg/s9o1qImpubqaur09sMiURyBpqamqitrR30+VEtRE6nE9BepMvl0tkaiURyKr29vdTV1SW/q4MxqoUoMRxzuVxSiCSSHOZMoRMZrJZIJLojhUgikeiOFCKJRKI7UogkEonuSCGSSCS6I4VIIpHojhQiiQQ42O7j5a1H+LipW29T8pJRnUckkaSD3mCE33/cTFQVNHb4cdvNjC8r0NusvEJ6RJK855MjPURVbQ2JBQ3FlDutOluUf0iPSJL37G/zAvCFmVVMq5YZ+nogPSJJXuMJRmj3hjEoChPkcEw3pEckyWuO9QQBKHNasJmN+MNRWnqCmA0G6ksdOluXP0ghkuQ1VpORs8oLKCvU4kJ7j3t5Z3crE8oKpBBlESlEkrymvtTRR3CqXDYAjvUGEULIhntZQsaIJJKTKC20ABAIxwhEYjpbkz/kjBA9/PDDKIrCnXfeqbcpkjwhpgp8oShCiOQ2s9GA06YNFLr8Eb1MyztyQog2b97MU089xezZs/U2RZJHtHqC/GzDAf77g8Y+24sdmlfU5QvrYVZeorsQeb1e/vZv/5ann36a4uJivc2R5BGdcaEpsPQNlZYUxIXIL4UoW+guRLfffjvXXHMNl19++Rn3DYVC9Pb29vmTSFIlIUQJ4UlQ5DADcmiWTXSdNXvhhRfYunUrmzdvHtL+q1ev5oEHHsiwVZJ8oSegCY07LjwJJpQV4LCYKCu0DHSYJAPo5hE1NTXxf/7P/+G5557DZrMN6ZhVq1bR09OT/GtqasqwlZKxTG8gCoDL1leIihwWzq5yUlooa86yhW4e0ZYtW2htbWX+/PnJbbFYjA0bNvDTn/6UUCiE0Wjsc4zVasVqlR8OSXrwBDWPyGWX6XR6o9t/4LLLLmPHjh19tt1yyy1MnTqVe+65p58ISSTpJBxV8Ye1PKFTPSKAQ+0+Ov1hJlUUDvi8JL3oJkROp5OZM2f22VZQUEBpaWm/7RJJulGFYF59Ef5wDJu5/4/e+wc6ONYTxGUzSyHKAtInleQlNrORS86uGPR5p83EsZ4TwzdJZskpIVq/fr3eJkgkADjjXpAnGNXZkvxA9zwiiUQPvKEogXCsT3nHySTKPKQQZYec8ogkkmzxp71t7D7m4eIp5Sxo6J/RX2jVvhq+kBSibCA9Ikle4o0LTEJwTqUgvt0rhSgrSCGS5CUJT8dhGThNpNBywiMabPgmSR9yaCbJS3zxHKLBPKJCm4mlc6qTnpEks8h3WZJ3hKMq4agKgMM6sEdkNChMqnBm06y8Rg7NJHlHYlhmMRmwmmQGfy4gPSJJ3uELnz4+lKCp00+bN0RtkZ0K19AKsyWpIYVIknfYzUbm1hdhNZ1+QPBpcw+7WjxcNLlMClGGkUIkyTtKC60sOU15RwI5hZ89ZIxIIhkER3wKP1GlL8kcUogkeUdvMII/fOb8IJldnT3k0EySd7z16XEOd/q5ckYV02tcg+6XCGbL9c0yj/SIJHlHYtZssGTGBIk+RQE5NMs4UogkeUci+FwwSDJjgpM9IlWVZR6ZRA7NJHlFJKYSimhZ1Wcq37CbjSydU3PGfCPJyJFCJMkr/CFtmGUyKAPnEfk7IdAFRfUYjGYmVRRm2cL8RAqRJK9IBJ7tFiOKovR9sm0vfPoKCBUKK2DezWCSa5tlAxkjkuQV/nig2n7qcCsShD1/0EQIwNsKje9xuMPP1sNdtHtDWbY0v5BCJMkrCm0m5tYXMaXylMr6YzsgEgBHKcz4a21b8zZ2NLby7p42jnYFsm9sHiGHZpK8osJpo+LsU+rGhICW7dr92gVQfjY4SsDfSWnoMFAls6szjPSIJBJ/J/jawWCEihmgKJoYAcWBRgACEZldnUmkEEnyCm8oii8U7ZsX1Llfuy2qB3PcWyqdDIDT1whCJRBWs2xpfiGFSJJXvL3rOD/bcIBPm3tPbOyIC1HJxBPbXDVgsmIhSkG4IxnklmQGKUSSvCJRrmG3xD/6qgq9R7X7xQ0ndlQUcNdiNhpwho/LerMMI4VIklecyCOKz9P4WiEW0fKFCsr77uwah8mo4Awdl/VmGUbOmknyisTslz1e0EpP3BtyjdO8oJNx1WAzGTm/PERkbk0Wrcw/pBBJ8oZo7KTVOxIJjb1HtFvXuP4HFFZiNCiUEYACOXjIJPLdleQNwbgIGZST6sw8x7Rb1wAej8UB1njio68tCxbmL1KIJHnDifIOg1ZnFg1rBa4AhZUDH1RYQac/zO59++jxR7Jkaf4hhUiSN1hN2uod06rjXRl9bVpWtcUB1kGq7AvKae4OsHvfftq8wewZm2fIGJEkb3DbzX1X7/C1areDeUMAhRWYDAqOSCfBiExqzBTSI5LkL9543OfUafuTcZRhMhiwR3pkLlEGkUIkyRv84VPKO7zHtdvTeUT2YkxGBZMaJBzwZd7IPEUKkSRveH9/Bz/bcIBNBzu12FBiJqzwNIstmiwoNm3mLOrtyIKV+YkUIknecHJ3RsJeiIa0JEZ7yWmPMzi050WgM+M25itSiCR5Q5+san9cVGxFYDz9nI2hoBQA4evKpHl5jZw1k+QNwbhH5LAYwR8fZjlKz3hccVkV9ionxgo5a5YpdPWInnzySWbPno3L5cLlcrFo0SJef/11PU2SjGEShau2kz0iR/EZj3O4yylxWHALTybNy2t0FaLa2loefvhhtmzZwkcffcSll17Ktddey6effqqnWZIxiBCib4xoGB4R9rhY+eNBbkna0XVotnTp0j6P//mf/5knn3ySDz74gBkzZuhklWQsEoyoSQ2xm42QCDwPQYiEzU27J0RUDVAW8mG2ybXO0k3OxIhisRgvvvgiPp+PRYsWDbhPKBQiFDqxrEtvb++A+0kkAzGvvohITGAUUQj2aBvPMGMGoBjN7OkSGKN+Cno7pRBlAN2FaMeOHSxatIhgMEhhYSGvvPIK06dPH3Df1atX88ADD2TZQslYwG4xckmivMPbqg2xTFawFAzpeNXqxBj1EfZ1AfWZMzRP0X36/uyzz2b79u1s2rSJv//7v2fZsmXs3LlzwH1XrVpFT09P8q+pqSnL1krGBP6ThmWnNkMbBGFzAxCVU/gZQXePyGKxMGnSJAAWLFjA5s2befzxx3nqqaf67Wu1WrFardk2UTIGCEZiRFWB3WzEOJxAdQKrJkQRf08GrJPoLkSnoqpqnziQRJIOPm7q5s/7O5hR4+IKQ9yrsZ956j6Bwa4JUczfnQHrJLoK0apVq7jqqquor6/H4/Hw/PPPs379etatW6enWZIxSCCZzGgCb7e20V405OMN8X1VKUQZQVcham1t5etf/zotLS243W5mz57NunXr+PznP6+nWZIxSDBy0jJCgW5to61oyMebCopQARGUQ7NMoKsQ/ed//qeel5fkEck6MyNawSsMyyOaWFeDsdGJzWyASPDEirCStJBzMSKJJBMkhmYFwqtN3RvNYHYM+fgytxPcbogEtBwkKURpRffpe4kkGyTqzByxk7yhIU7dJ4lP4SOHZ2lHekSSMY8Q4sRS07F44eow4kOgxZjaAxYsniAVIZnRn26kEEnGPELArFo3wUgMa7RR2ziM+BBoMabNLWFqfX4qQrIKP91IIZKMeQwG5UR5x46N2q1t6DlEADazgbCxkKgqUIO9MqaRZuT7Kckvgt3a7TA9IpvJSMio1aVFAjJGlG6kEEnGPKFoDG8oSiymppRDBJpXRaKJvl/GiNKNHJpJxjz7Wr28+elxznLDtbGINluWmAEbBgarJkSxQK8WeBrurJtkUKRHJBnzJLKqXcTXJbMUnrFh/kCY7NpS1dFoBKJy+el0IoVIMuZJZFUXiuFnVJ+M1WoharARjakgZ87SihyaScY8iRyiQjUuRMOMDyU4d3wJ1u4aXNFOTYhOtzCjZFhIIZKMeZJN89WReUS1xQ4oLYeOHpBJjWlFDs0kY54TWdVx8UjRIwLAGu9XLYdmaSUvPKJYLEYkEtHbDIlOqNEwdkMMRY0SNDjAWAjBoQWbLRYLBoP2e90TiNDlMeD2hykOeTNpct6RkhAdOHCAs846K922pB0hBMeOHaO7u1tvUyQ6Mr0whlogaFfn0AHQ5oP2g0M61mAwMGHCBCwWCy09AT5qCjM/EKRYekRpJSUhmjRpEosXL+bWW2/lhhtuwGbLzZYICRGqqKjA4XCgyLyP/CUW1dYyUxRwlA0pB0hVVZqbm2lpaaG+vh6byUjYWEAkpsoYUZpJSYi2bt3KmjVrWLlyJXfccQc33ngjt956KwsXLky3fSkTi8WSIlRaOowm6ZKxSTQIMbOWP2S3D/mw8vJympubiUaj2C1GwqYCYqqQMaI0k1Kweu7cuTz++OM0NzfzzDPP0NLSwoUXXsjMmTN59NFHaWtrS7edwyYRE3I4ht78SjL2UIUgpgqEqgWsUYzDOt5isQDaD5vmETm0PKJoCKLhdJubt4xo1sxkMnH99dfz4osv8i//8i/s27ePu+66i7q6umQvar2Rw7H8JhJTafeG8PjjwWnD8AYBJ39+bBYDMcVCGDMxIU60nJWMmBEJ0UcffcS3v/1tqqurefTRR7nrrrvYv38/b731Fs3NzVx77bXpslMiSQk1vt69ATV+Z3ge0clYjAYMBgORhFck40RpIyUhevTRR5k1axYXXHABzc3NPPvsszQ2NvLQQw8xYcIELrroItauXcvWrVvTba8kRQ4dOoSiKGzfvj0r59m4cSOzZs3CbDZz3XXXjeiaI0HElcgg4kI0zKHZySiKovUlMhUQVQXIKfy0kVKw+sknn2TFihUsX76c6urqAfepqKiQq3TkMOvXr2fJkiV0dXVRVFSU9vOvXLmSuXPn8vrrr1NYWJj28w8VVQgQYCAGKCPyiAAum1ZJobUWq+8zOTRLIykJ0VtvvUV9fX0y0SuBEIKmpibq6+uxWCwsW7YsLUbmO+FwOBk0HS3s37+fb33rW9TW1qZ8jnS8bs0hUjEgOJ0QCSGIxWKYTKf/SkyqKARPGQT2S48ojaQ0NJs4cSLt7e39tnd2djJhwoQRG5XvXHLJJdxxxx3ceeedlJWVceWVV/LJJ59w1VVXUVhYSGVlJTfffHOf/8Ebb7zBhRdeSFFREaWlpfzVX/0V+/fvH/D8hw4dYsmSJQAUFxejKArLly8f1nl2797NBRdcgM1mY+bMmbz77rvJcyuKQkdHBytWrEBRFNauXQvAu+++y8KFC7FarVRXV3PvvfcSjUZP+7rXr1+PoiisW7eOefPmYbfbufTSS2ltbeX1119n2rRpuFwuvva1r+H3+5PnUlWV1atXM2vaZMZXlXLORZ/npd++Bor2kU+c9/XXX2fBggVYrVbee++9of2DLFpfIhkjSiMiBRRFEcePH++3/dChQ8LhcKRyypTo6ekRgOjp6en3XCAQEDt37hSBQKDfc6FIbNC/SDQ25H3DQ9g3FRYvXiwKCwvF9773PbF7927xwQcfiPLycrFq1Sqxa9cusXXrVvH5z39eLFmyJHnMSy+9JH7961+Lzz77TGzbtk0sXbpUzJo1S8Rimg0HDx4UgNi2bZuIRqPi17/+tQDEnj17REtLi+ju7h7WeWpra8VLL70kdu7cKW677TbhdDpFe3u7iEajoqWlRbhcLvHYY4+JlpYW4ff7xZEjR4TD4RDf/va3xa5du8Qrr7wiysrKxP333z/o6969e7d45513BCDOP/988d5774mtW7eKSZMmicWLF4srrrhCbN26VWzYsEGUlpaKhx9+OHmuhx56SEydOlW8+OrvxOat28XPn3hUWK1WsX79eiGESJ539uzZ4s033xT79u0THR0d/f4Xp36OjvcGxGefbBae1x8UYst/pfT/zSdO9x09mWENzVauXAloQbv77ruvT45OLBZj06ZNzJ07Nz0KmUGeeGffoM9NKCvgunnjko9/tmE/kZgYcN/aYjtfOqcu+fiZjQeTBZYJvvv5KSnZOHnyZH74wx8C8NBDDzFv3jx+8IMfnLjWM89QV1fH3r17mTJlCn/zN3/T5/hnnnmG8vJydu7cycyZM/s8ZzQaKSkpAbRY3skxoqGe54477kju++STT/LGG2/wn//5n9x9991UVVWhKAput5uqqioA/v3f/526ujp++tOfoigKU6dOpbm5mXvuuYf77rsvOcw/+XUDyRSQhx56iM997nMA3HrrraxatYr9+/cnS41uuOEG3nnnHe655x5CoRA/+MEP+N///V9mzz8XEfIy76wbef/DbTz11FMsXrw4ef4HH3xwWEuc727xsOegnyuCIQrdcmiWLoYlRNu2bQO08fSOHTv6jN8tFgtz5szhrrvuSq+FecqCBQuS9z/++GPeeeedAYO++/fvZ8qUKXz22Wfcd999bNq0ifb2dlRVmyU6fPhwPyE6HUM9z6JFi5L3TSYT55xzDrt27Rr0vLt27WLRokV98nI+97nP4fV6OXLkCPX19f1e98nMnj07eb+yshKHw9Gn3rGyspIPP/wQgH379uH3+08RGEE4HGHevHl9znvOOecMavNA2C1GIsYCImo8j0i2jE0LwxKid955B4BbbrmFxx9/HJfLlRGjMs3tSyYN+pzhlM/UNy+eOOi+p37+VnwuffGxgoKC5H2v18vSpUv5l3/5l377JWYtly5dSkNDA08//TQ1NTWoqsrMmTMJh4eX/Zuu86TKya/7ZMxmc/K+oih9Hie2JUTT69U8lddee41x48ZBoAuiEbA5sRYWDel6g9Enu1qNQcQPluGdQ9KflGbN1qxZk247sorFNPQYfab2HQ7z58/n17/+NePHjx9wVqejo4M9e/bw9NNPc9FFFwGcMfB6culCKuf54IMPuPjiiwGIRqNs2bKFO+64Y9DrTZs2jV//+tcIIZJe0caNG3E6nSOaWRuI6dOnY7VaaWxs5MKLLsLgb0dRY+AoBdPIZuFsZgNCMRJU4oXeIa8UojQwZCG6/vrrWbt2LS6Xi+uvv/60+7788ssjNkxygttvv52nn36ar371q9x9992UlJSwb98+XnjhBX7+859TXFxMaWkpP/vZz6iurubw4cPce++9pz1nQ0MDiqLw+9//nquvvhq73T6s8zzxxBNMnjyZadOm8ZOf/ISuri5WrFgx6PW+/e1v89hjj/Gd73yHO+64gz179nD//fezcuXKfmkgI8XpdHLXXXexcuVKun0hlpwzjUjAx8btu3G5i0aUVmIza9P/AcUOxOK5RJXpMTyPGfInwO12J3/J3G73af8k6aWmpoaNGzcSi8W44oormDVrFnfeeSdFRUUYDFrZwQsvvMCWLVuYOXMm3/3ud/nRj3502nOOGzeOBx54gHvvvZfKykruuOOOYZ3n4Ycf5uGHH2bOnDm89957/Pa3v6WsrOy01/vDH/7Ahx9+yJw5c/jWt77Frbfeyv/9v/93RO/NYPzTP/0Tq77/D/zbT37Egs9dyhf+5m957Q+vjzi9JCFEfuIV/LIKPy0oQoiBp4RGAb29vbjdbnp6evrFq4LBIAcPHmTChAk52y9JklmCkRhen4+CWC92mxUKyod/jlM+R55ghJ//6SATO//E0orjKOMvggkXZcD6scHpvqMnk5JPHAgE+iSPNTY28thjj/Hmm2+mcjqJJCOoQqCImDapMIIas5NxWExcOaOKc8+uB4Es80gTKQnRtddey7PPPgtAd3c3Cxcu5JFHHuHaa6/lySefTKuBEkmqqKpW7KqgDLv9x2AYDQrTa1xUV5RpoQpZ5pEWUhKirVu3JmdVXnrpJaqqqmhsbOTZZ5/lX//1X9NqoESSKn08ohEWu/YjUeYRljGidJCSEPn9fpxO7R/x5ptvcv3112MwGDj//PNpbGxMq4ESSaqoQmAQqXVmPB1Huvzs6RbaUtYyWJ0WUhKiSZMm8eqrr9LU1MS6deu44oorAGhtbR21SY6SsYfFaMBiEBgMI2//cTKbDnTy5j4/3lAUwn4tsVEyIlISovvuu4+77rqL8ePHc9555yXT/d98881+KfQSiV44zAo2k4JJSa8Q2cxGogYbYRFPrZcB6xGTUgTvhhtu4MILL6SlpYU5c+Ykt1922WX89V//ddqMk0hGRGJYZjAk23+kA5vZAIpCOJHUGPKCTebPjYSU/ztVVVXMmzevT1bswoULmTp16pDPsXr1as4991ycTicVFRVcd9117NmzJ1WTJJIkQghisSgCQEnvgsb2eFJj0BAv7ZBxohGT0n/I5/Px8MMP8/bbb9Pa2posNkxw4MCBIZ3n3Xff5fbbb+fcc88lGo3y/e9/nyuuuIKdO3cOuxhRIjkZIaDHG8QSi1JQaCed9fHWRJmHwQ70yqFZGkhJiG677Tbeffddbr75Zqqrq1NesueNN97o83jt2rVUVFSwZcuWZEGlRJIKqhAY0KbulTRP3dvM2ihAlnmkj5SE6PXXX+e1115LNqpKFz09PQDJpl2nEgqFCIVCyce9vWOzVecll1zC3Llzeeyxx/Q2Bcg9e4ZCMocojcmMCeyn1ptJj2jEpBQjKi4uHlQsUkVVVe68804+97nPDdrIa/Xq1X2Ka+vq6gbcT0LW+gflKqqIZ1WnsbwjQYXLxhdmVjFrYrx9ifSIRk4qfWj/+7//W9xwww3C5/OlcviAfOtb3xINDQ2iqalp0H2CwaDo6elJ/jU1NaXUszqXWbZsmUCrYkr+7du3T6xYsUKMHz9e2Gw2MWXKFPHYY4/1O+7aa68VDz30kKiurhbjx48XQgixceNGMWfOHGG1WsWCBQvEK6+8kuxdnWDHjh3iC1/4gigoKBAVFRXipptuEm1tbYPac/DgwWy9HSnjC0aEp/WwCHQ0CRGLpnye036OOg8K8ccfCPHBU6kbOsbJSM/qBI888gj79++nsrKS8ePH9+uWN9yFFe+44w5+//vfs2HDhtM2ybJarVit1lRM1hACYpHUj08Vo3nI7UQff/xx9u7dy8yZM3nwwQcBzQOtra3lxRdfpLS0lD//+c9885vfpLq6mi9/+cvJY99++21cLhdvvfUWoA1dly5dytVXX83zzz9PY2Mjd955Z5/rdXd3c+mll3Lbbbfxk5/8hEAgwD333MOXv/xl/vjHPw5oT3n58KvYs41Q46uDKOmduu+DLPNIGykJUbpW7hRC8J3vfIdXXnmF9evXZ34polgE/vRIZq8xEBf9f0PuDOh2u7FYLDgcjmTjeYAHHnggeX/ChAm8//77/OpXv+ojRAUFBfz85z9Pdl/8j//4DxRF4emnn8ZmszF9+nSOHj3KN77xjeQxP/3pT8/YmH8ge3IdEdOESDEYM9JT+kCbl2AwxtmqwBgNQzQEphH8SOY5KQnR/fffn5aL33777Tz//PP85je/wel0cuzYMUD7Mtrt9rRcY6zwxBNP8Mwzz3D48GECgQDhcLjfiimzZs3qs6DBnj17mD17dp9+TAsXLuxzzFAa849GzAYVYTRgMJrPvHMKvPHpMUIRlQaDiYJEUqMUopRJeTqhu7ubl156if379/O9732PkpIStm7dSmVlpdawfAgkWoZccsklfbavWbMmueBfWjGaNe8k24zwy/DCCy9w11138cgjj7Bo0SKcTic/+tGP2LRpU5/9Usm9Gkpj/tGI1SDAZABzZoTIbjYSiqhEjA4QHm14VlCakWvlAykJ0V/+8hcuv/xy3G43hw4d4hvf+AYlJSW8/PLLHD58ONmr6EyIbDeHVJQRN0/PBhaLpU9T+40bN3LBBRfw7W9/O7ltsFVcT+bss8/mF7/4BaFQKBlb27x5c599ztSYfyB7RgXJGFGa23/E0VrGRggbHRD1yL5EIySlKN7KlStZvnw5n332WR+3/+qrr2bDhg1pMy5fGT9+PJs2beLQoUO0t7czefJkPvroI9atW8fevXv5f//v//UTlIH42te+hqqqfPOb32TXrl2sW7eOH//4xwDJJNTbb7+dzs5OvvrVr7J582b279/PunXruOWWW5Lic6o9p2bS5xpCCNREeUeac4gSJMs8lPgiozKXaESkJESbN2/m7/7u7/ptHzduXDLOI0mdu+66C6PRyPTp0ykvL+fKK6/k+uuv58Ybb+S8886jo6Ojj3c0GC6Xi9/97nds376duXPn8g//8A/cd999AMkfkDM15h/InsOHD2fuxacBIQT+YBhfKIpId0O0OIns6qAxLkQyl2hEpPRzYbVaB8xq3rt376iY2s11pkyZwvvvv99n25o1a/qtJ7d69erk/bVr1w54rgsuuICPP/44+fi5557DbDYnV1YFbZnn0y0BNZA9uYwaOzF1r2Ro6t6azK6WQpQOUvovffGLX+TBBx8kEtFychRF4fDhw9xzzz391k6X6Muzzz7Le++9x8GDB3n11VeTOUJjeVYyKUQZmrqHE0OzgCzzSAspCdEjjzyC1+ulvLycQCDA4sWLmTRpEk6nk3/+539Ot42SEXDs2DFuuukmpk2bxne/+12+9KUv8bOf/UxvszJKMpkxQ/EhgInlhXxhZhWT6+O5VTJYPSJS+k+53W7eeustNm7cyMcff4zX62X+/Plcfvnl6bZPMkLuvvtu7r77br3NyCriZI8oQ5Q7rZQ7rRCMz/yGvVrmfoY8sLHOsIVIVVXWrl3Lyy+/zKFDh1AUhQkTJlBVVdVnXXOJRDfURFZ15jyiJJZ4Iqgag4gfLLKPVioMa2gmhOCLX/wit912G0ePHmXWrFnMmDGDxsZGli9fnpNtYrOeqyTRn3gze8U4ciEa7PMTjqrsb/Oyp9UPlkTAWg7PUmVY/6m1a9eyYcMG3n77bZYsWdLnuT/+8Y9cd911PPvss3z9619Pq5GpkCjE9fv9YzowKzkFoWJUVDAaUEwjz6pOtFMxGvsO84LRGL/d3ozRoDDFVYgS9scD1pUjvmY+Miwh+uUvf8n3v//9fiIEcOmll3Lvvffy3HPP5YQQGY1GioqKaG1tBcDhcMhhYz4QDUM0BooBEYsSTMSLUkBVVdra2nA4HP2yzhOzZjFVEDEVYqEVQmOzUV82GJYQ/eUvf+GHP/zhoM9fddVVObXSa6JaPCFGkjwgFoZIQJsxs4w8t8dgMFBfX9/vR8xsNGAxGQhHVcIGBxaQQ7MRMCwh6uzspLJycNezsrKSrq6uERuVLhRFobq6moqKimTOk2RsIxo/IHxsG6aamRgnDNzpczhYLJY+K9WcjN1sJBxVCRrsFILMJRoBwxKiWCw2aGEkaMOhaDR1VzhTGI3GfmN8ydgk4G1hZ+NRGj0T+PJUq7bKa4ZwWIz0BCIEFBmsHinDEiIhBMuXLx+0S+LJje0lEj2I9LZpdwrKMipCAHZLvMwjKUQyRpQqwxKiZcuWnXGfXAhUS/KUWISIXwsNGAvLMn65RMDaR7wDhRyapcywhOjUokuJJKfwdxCOxIgabNgLnBm/3IxxbupKHFTaYtAKhP1aDlMGM7rHKllIPZVIsoSvnXBMxW8uotCWmc6MJzOuyM64IrtW2mEwaiIU9oLNnfFrjzUytLyBRKID/nYiUZWguQinLYu/sYpyotRDtgNJCekRScYO/g5CURW/vZgCa+Y/2sFIjKPdAYQQTLIWQrBHzpyliBSi0Uo0DK07QahQMR3MtjMfM9bxtVNSYGFcdR2lBZlfUaM3EOG325spsBqZVBL3iGTAOiWkEI1Gwj7Y9hz4O7THTZtg7t+CzaWvXXoSDUOgi2q3neq502EkC3EOkcT0fSCsIiyFKCCHZikiY0SjkT2vayJkKQCrEwLdsOt3WtA0X/G1aa/fUgDW/mu0ZYLE9L0qBGFjvP2HFKKUkEI02ug+DO2facsoz/kKzP2aVlfVfRg6zrzE0JjFe5yYKvBbSonGsrPKiMlowJpoom+Qq3mMBClEo42mD7Xb6tlQWAGOEqhdoG07PHoa3KcdXxu9wQivH1J5YXNT1i7rSDTRV+KtZmSwOiWkEI0mQp4TXk/tuSe2156reUg9R8Dbpo9teuM9TiSm4jOXZnXqPlnmkWyiL4dmqSCFaDRx/FNtlsw9DgpOKmGwOqFsUnyfHfrYpidCgLdVm7o3l1CYhan7BHaLdq1kmUc0DFFZczlcpBCNJtr3areVA7S3qJiu3bbtzb+gdaALYhECMYWAuQiXPfNZ1Qnm1hZx1awq6iuKTyxnLodnw0YK0Wgh7IPeZu1+6aTk5h5/hF9tbqK3oF4LWge6wJtnjeDir7fX4AbFgCsL5R0J6ksdTK1yUeSwgCVe3yaHZ8NGCtFooWOf5uk4K/vkC7V6ghztDvDqX9qIFo2P7/uZPjbqhU8Tok6lCACXXaf0uETagPSIho0UotFCPEgtSifxvzuP0+PXOk5WuGwUWk10eMN8GorHjToP6mWlPniOoQpBp1IMkFWPKBiJsb/Ny75WrxarA5lLlAJSiEYDQkB3IwCNaiU7jvbw4pYmVFXgtpu55OxyADZ1uojEVG0Ily8BUyHA04IQMGniZKbXuHBYsteGo9MX5rfbm3l3b9uJwleZSzRsZInHaMDXBpEgGE1s7rQCYaZVu5IdCCdVFFLpsnG8F46E7EwwhqCrEcqn6Gt3Ngj2QNiP0WjknBnTIA1rmQ2HgvismT8UlWUeI0B6RKOBbi1Bz2+v4Uh3GEWB2bUnet4oisL8hiIA9oRKtUUBuw7pYKgOeFq028KKrIsQQIFV876iqiBslNnVqSKFaDQQH5YdjJYAUFfswHlKHGRyhROb2chxYyW9wSj0HM66mboQn0n0WSvo8UeIqdlNXTAZDdgSLWOTvaulRzRcpBDlOkJAj+YR7Q4UATC1un8bVKNBYU6tmwkTz8ZiMoCvXRvOjXU8xwDY3m3jmY0H2XY4+8tZJbwin0hkV/vyL5drhMgYUa7j74Swn6Cq0BQrQjHAxPKBq8svmFQGlIGvXDuu9yiUTsyuvdlEVZNDs1Y0bzGbyYwJCiwmOgjjUa1at0Y1polRlroAjAWkR5TreOJDD0s5NquFGrc9ORQYFHetdtuTveJPXfC3QywCRjPHotqXvsihgxDFPSJ/VJyYOQv2ZN2O0Yz0iHKdXu0Xv7S6gW+edRaBSOy0u8dUQRtlWHwhSnqOZMNC/YgLbbigmmCnNhQqsluybsbMcW7GlxVQ6bRBV5EWIwp2azWBkiGhq0e0YcMGli5dSk1NDYqi8Oqrr+ppTm7Se1S7ddZgMChn7MV8sN3HKwcUGjv8iN5miOXeyrtpIy60vRZtGXSnzaTFx7JMbbFW5lFcYDmxgof0iIaFrkLk8/mYM2cOTzzxhJ5m5C6xKPjatOl4V82QDqkvcRC1FOFRrQSCIfAey7CROhIXok6zJkTFjux7Q/2wFWm3gW49rRh16Do0u+qqq7jqqqv0NCG38R4HNcYRv8K6zR3Ma1BZ0FBy2kMsJgN1pQV4Wivp9rfi6DlyImY0lgh0Q7AXFAPtSingpbgg+/EhgFA0xpGuAJGYylTpEaWEjBHlMvEcmVZK8YROHxs6mfpSB3utVfQEjlIzVuNEidflrKS+ogiMFqqL7LqYEgjH+O32ZiwmA2fPdWvZ1cFuXWwZrYwqIQqFQoRCJ2qoent7dbQmC3iaEQiOxooAqHYP7YvWUOJgi7WS3p4Ise4mjEJo08pjicSMoLuO2mIHtcUO3UxxxMs8wlGVsNmFFTRvTVXBICemh8KoepdWr16N2+1O/tXV1eltUmbpbSEYUWlXyjAZFCqcQ1sip6TAguKsIooRj6dX61E01uiOZ4679f8MWEwnmuh7hU1bflqoEBrjP5RpZFQJ0apVq+jp6Un+NTWN4TyZsB8CXXiCEbzWcipdNkzGof27FEWhrsyJz1KOJxA9MYwZKwR7tIRNxUCgYByH2n14Q/rODiZKbjwhFazxflEyTjRkRtXQzGq1Ys3Cwnk5QTxjuFM4iRmsVBcNbyXXcxqKMYhZFLVt0YSoenYmrNSHRL8lVzVHPCq//0sLlS4bXzuvXjeTXDYT7Z4QnmAU7EWaFyqFaMjoKkRer5d9+/YlHx88eJDt27dTUlJCfb1+H6qcIB6obhFas6+hxocSlBZaoeosaNt6IhdprJDoLFA8nnZvGICyQn2n7hMrh3iCkZNyibr1M2iUoasQffTRRyxZsiT5eOXKlQAsW7aMtWvX6mRVjuBpQSBwVdRTabRRM0yPCABXPLPX1w6RAJj1mVVKKye3OCmeQHujNnlRNsT4WaZIDM16g1FwFmkbZS7RkNFViC655BItWU/Sl3jXQQWFBTOmsyDFUoHjQQMdPSYKYj009DaPjQJY73FNVE0WcNXQ7tWC1mUF+grRhLICnDYTpQXWEwI0FicJMsSoClbnDfGugygGKKxM+TSBcIzPgkV0eMOI7jES2O+ID+WLGgirCj0BrXd3mVPfoVlZoZWpVS7KnVZt9V2AQKdsBzJEpBDlIvFAda+pmMgI/kXjiu347VWEoiq+9sZ0Wacv7fEVSsomc7w3qC1sYjMlc3lyArsW1yMS1Lw3yRmRQpSLeFpQhWBDi4l/f2d/csWO4WI2GigobwCg9/hhrU/OaCbYqzVCUxQoncTxXq3xW5U7hfhZBjjY7mN7UzdB1XBiyadAp75GjRJy6GdEkqS3BX84RrepHIvJMKJ1umqqxxHYY6XX56fG2wqu6jQammUS67W5xoGlgMmVFuwWY1aXmD4db+86jicYpdJlpdpeogmnv3Ns1vqlGekR5RqqCt5jeIIRfJYyaopsKCMozxhfXojXUkFPIEK0a5T3sT5pWAbgtpuZUeOmobRAR6NOkFhPzROM9o0TSc6IFKJcI9AJ0TC9YQW/uXjY+UOnUlpgIeochyqgs2UUL7wY9mtLJAGUTtbXlkFIeK49gQjY40Lkl0I0FKQQ5Rq9WqHrMVVbx716hPEPRVEorp5AgcWIwXN09M7itO3W6reclVBQSlOnny2NXXT6wnpblsQd7w7Z7Y9Ij2iYSCHKNTzHCEVVOgylGBQlLYHYC+bOYnZdCWWm8OgtxGzdqd1WzABg9zEPG/a28cnR3CmjSPTL7vaHT8yc+btGr/hnESlEuYanGU8witdSToXLinmIha6nw2C2aAsQwugsgA32aItMKgpUTAOgqdMPQF2Jfu0/TiUhRD2BiNap0WACNSoTG4eAFKJcQo2BtxWn1cScGdOYU1uUvnO764ipAm/rofSdM1scj3tD7jqwuegJROgJRDAoSmqlLxki0bjfE4wSEUBBqfaEr10/o0YJuTHvKdHwtoIaw+YoYM6k8WltZnY4VkxLYyem7j0smHVN2s6bcYSAlu3a/aqZABxo05Z0rnbbsJrOsLRSFrGZDVw1qwq33YxRUaCgHDzHwdcK5VP0Ni+nkR5RLpGokneNS3tHRXfVBFQB4Z4W/H5fWs+dUboOarVbJiuUa8OyA22a/RMrcmPaPoGiKEytclHttmMwKFAQHw772vQ1bBQghSiX6GnCF45yIFyUcjb1YLiLSrAUFIMQHD64N63nzijN27TbyplgshCMaI3qYfAVb3OGgjLt1iuF6ExIIcoVhICeo3R4w7zTYuGDgx1pv4RznJZ/0354d9rPnRGCPdAeL3KtmQdAuzeEyahQVmihKBeWDzqFLl+Y7U3d7D3uOTFBEOga2+vLpQEpRLlCqBdCHnpDMbyWcsZlYEWK6obpAETa9uveWnVING3WcoeKG6CwHNAWM/zGRWdx1azcLFU51hvknd2tbG/q1pafNtu01+CXAevTIYUoV+g5gioEraob1WDOiBA5qyfhtFuwRbrZcyjHp/HDfmiJD8vqz+/zlMVkoKwwN1sGl8Y7RXZ4wwjQAtagTURIBkUKUa7QcxRvKEq3pRKHxZjMSUkrZhvFVVo1fvPBXbndlO7oFm0446yE4gkAdHhDuW0zUOKwYFAUgpEYvnAMCqu0J7zH9TUsx5FClCv0NGmJjNZKaorsIyp0PR0V9dOoL3FweaU3Y9cYMWEfNG3S7tcvAkXBE4zw3KbDPP/hYYKR3G1nYjIakj8iHd7QiW4H8R7kkoGRQpQLRILga8MTjNBrqWJcceZ6S5srJjOuyI7d0wix9M7MpY1DGzXbnFVQPhWAjfs6iKkCi9GAzZw7uUMDkRietXvD4IwLkfe4DFifBilEuUD3YYRQaYsVEjEVZCQ+lMRZpa0yEYtA50GiMTVz10oFX8eJKfuJS0BRONodYFeLViN30eRyHY0bGqXx/tkd3pBWc2a2aVnzPhknGgwpRLlAdyMKCufNn8/SOdWUZzIQqyhQPgVfOMqfN23k1e3NuRN3EQL2/EGbZSqdBMXjCUZivL5Da507c5w7Z7oxno6ykz0iRQFnjfZEb4uOVuU2UohygfjyOJayCUyqcGpZuZmkfKpWgtD+GUc6evms1ZvZ6w2V5q1aUa7RDFOuIBJT+e3HWhFwkcPMxVPK9LZwSNSVOPjSObVcPz+++oorLkQeGScaDClEehPyaEWRiqLly2QD1zhszmLqXEZK/Id4Z3ertjCgnnhbYd8ftftnXQI2N+/uaeNoVwCLycA1s6tzqq7sdNjMRmqLHSdiWQkh6hljC12mEVn0qjddjURVle3dNiKH/Zw3wZZ5j0hRoGoW4wI9TOrax6bwJH7/lxb+et64IQeChRB9Zt22He7is1YvBkXBZFAosJpw280UOcwUOyyUFFgwDva6IkH45GWtZUbJWTBuAQDnnVVCc0+Ay6ZVUuHM/SHZoLhrtaWhEstQJ1aClSSRQqQ3HZ/RG4hwSG3Ae6yXRRNLs3PdqtkYGv/MfGcvu/FwrAde3HKEq2ZWDZgsKISgyx/hSJefI10BmrsD3HR+Q1K4PMEoR7sGXzrnpvMbtDW/0HoJ9QQiuO1mDCKKdeeLxDpb6FLtHHCdw9VxgXPazNx0XkPmhTkDtHlC7GzpxW42snBCiTZJ0Nustbutnq23eTmHFCI9iUWh8wCd/ghdjgYml2WxmtxeBCVnYevYz/XFR/mfnhm0e0J8eLCTq+PlE62eIPuOe2nzhjjWE8Qf7pu/c7Q7kCw8nVrlpMptQwiIxFQ8wWi8b1CYLn+E4pMSNHcf8/DJ0R4MaoQp7W9RFDxCzGDh04or8XfGOMcTTHpAo1GEADzBCFsbuygpsGhCVNygCVG3FKKBkEKkJz2HEZEQrUET3qKK7FeT158PHfsp6tnN3867kD8eDHLO+OLk0x3eMJsOnui5bDIoVBfZGVdkp7bY3qefdoXLRoVraMOnskILE10qZYf+gCPWhrBaaZ1wHdMrxtNQUqD78tHpILHoQacvjC8UpaCoARrf1zwiIdLe5mW0I4VIT9o/ozcYpc1ah91qymz+0EAU1UNRHXQ3Udj8AV+cc3Wfp0sLLcyudVNcYKHKZaPCacU00ta1QjDP2sK8yDqoiIJlHMy8AdzjRnbeHMNuMVLhstLaG6Kpy8/U8lowmrTJCe9xbagmSSJnzfRCVaFtD52+MJ32BiaUFegzDJmwWLtt+bhfP+sKp43LplUyv76YmiL7yEWo5yhsfx52/kYLUDurYN7NY06EEtQVa/20D3f4tZSEkrO0J9pHUT+oLCGFSC+6DqKGvBwPKPTYaplS6dTHjqK6EzGLXb/XBCKdRENwbAds+S/Y+ix0H9aayo+/EOZ//cSyO2OQ+nhj/8Odfi1ptCzeLrZtj45W5SZyaKYXxz8hpgpMVTNwOWw06LkaxVlLtNhFoAs+fQVm3aD9gqdKoFtL0uzYB50HtPIGAIMRKmdoIpQHU9jjiu2YDAqeYJQ2b4iK0knaNL6vXStlKcjSDOkoQAqRHoT90L4Xs9HAOQsvZn5htb6zQxYHzLwetv23JiDbn4dpS4fmrQihCVjvUc3b6WrUcmVOxlEKldOhei5Yc7y9axoxGw2MLyvgeG8QbzBKhbMQSiZAx3449jFMvFRvE3MGKUR60LL9RK8dZzWGXJhBcVbBnK/CX36lTTNv/rlW+V46URMSo0VLOIz4NY8n0KmtUOFp0YZfJ6MYtPYXxRO0cxTmfqFqpvj89EqsJsOJ5M/quXEh2qHF5wyjI1s800ghyjZqDI5uod0bwtgwh+JcEKEE7lo4ZwXsfQM6D8LxT7W/M2Ewaf2Zi+qgqEFbf8yUe/2k9aBfpnrpRLAUaD2X2vZonqJEClHWad5OxN/Lni6VzfvsfLUsSOUQ82+ygr0I5nxFm+Fq3QW9R7Qp52hIixuZbNoqpvZibZUKV43WDlX+sp+WmCo41hvUUjRq5sGh96Bxo7ZybS79GOmEFKJsEg1D43sc6QrQWLiAcncBFc4cTd5zjxuz0+rZJhiJ8d/vN+IPx1h+wXjctefAkQ+1oHXb7uQy2vmMnL7PJgfW4/P20hiw0lo4lYsnl+duu1ZJ2rCZjZQWWlCFYOP+djDboXah9uS+t/vH2PIQKUTZovMA6pGP2Nfq5UDR55hU5aZOzyl7SVa5cHIZigJ7jnm0BMf687VhcMgD+/+ot3m6I4UoG3jbEJ++yr5WLwctU4i4G7h0aoXeVkmySIXTxuxaLXdq3afH8ESAKV/Q4kPN2+HIR7rapzc5IURPPPEE48ePx2azcd555/Hhhx/qbVL66G6C7b+gu9fDoWgJTaUXcPWsahwWGZ7LNy6aXE5poQVvKMrLW4/SY6s9UWLz2VtaUWyutO3NMroL0f/8z/+wcuVK7r//frZu3cqcOXO48soraW0d5Y3GQx7E3jdh+3MQCVJc1YDr3Bu5clatHJLlKWajgWvnjsNpM9HpC2vxovrzoS4eLzqwHj7+ZV52clSEzp3TzzvvPM4991x++tOfAqCqKnV1dXznO9/h3nvvPe2xvb29uN1uenp6cLlc2TC3P0JALIII+wh5Ogh1t+Bv3U+o9SDdvgBTKp046ufC5Ctlbo0EgN5ghA1727hsaiV2i5b2cOST9zAdfAe7UWA1GTC7KzCUTUZxVoGjTMs9MttH3VT/UL+juo4PwuEwW7ZsYdWqVcltBoOByy+/nPfff7/f/qFQiFDoxAxDb2/vkK91tDvAH3cdRwAVXVsp8uxDCBUFAQIaSh1afx0h6AmE2dmslSkoCEDEXWZNs+uK7Yxz2wCBNxBg15EuYqrgVEX3WKvoLbqYi6ctHLKdkrGPy2bmr2bX9Nn2UWQCzSYb43q2Ue7/DKWpG4W9GAxa69359cVaxrrRxGdtAbqCKkIxgmLgVGk6Z0JpMlt/X6uHDt+p/chPHLFgfDGmeHnRgTYfLX6FA7XXDmj338wfl7GQgq5C1N7eTiwWo7Kyss/2yspKdu/e3W//1atX88ADD6R0rUhU1ZZ3AZyeHtzxtciT4mEPaL84gBKMQLC37/MnExYQiT8TixFVBapiJGwsJGIvBdc4LBVTaKipZXyZHIZJzkxdiR1BFV2uyznqOx+nvxFXsAVHpBNHLP6DK1SIhomFfMQCgy92oPhONF4TPg9q/HM/IN4IGLQIjfB6ifkNtHsGTidQMzh2GlUR01WrVrFy5crk497eXurq6oZ0bKXLxt/Mr0VRwBC4HCV8vpbDoygoioLTZgaLCRQFa0xloj+afA7it4omTAVWM1i06nQrRqZETVgsVhxW8+AN4iWS07CgoYQFDVqRsRCCcGw60ZggElOJqgIcJq3OLxahwuPHHYmAGkWoAyyQ6baieT2CUl8YR1hlkJ9UDG5b0kEq8YexRWC8q3bAfW2mzIWUdRWisrIyjEYjx48f77P9+PHjVFX172BntVqxWlPLRLZbjNSXJryT03spVqBiiG1yzIBs5iBJJ4qiYDUZsZ767bRqPatKhtHDqaT4zPskKC6BYeyeVnSdNbNYLCxYsIC33347uU1VVd5++20WLVqko2USiSSb6D40W7lyJcuWLeOcc85h4cKFPPbYY/h8Pm655Ra9TZNIJFlCdyG68cYbaWtr47777uPYsWPMnTuXN954o18AWyKRjF10zyMaCTmRRySRSAZlqN9R3TOrJRKJRAqRRCLRHSlEEolEd3QPVo+ERHhrOKUeEokkeyS+m2cKRY9qIfJ4PABDzq6WSCT64PF4cLsHX8tuVM+aqapKc3MzTqczL1uuJkpcmpqa5KzhAMj358xk+j0SQuDxeKipqcFgGDwSNKo9IoPBQG3twHUx+YTL5ZJftNMg358zk8n36HSeUAIZrJZIJLojhUgikeiOFKJRjNVq5f7770+5I8FYR74/ZyZX3qNRHayWSCRjA+kRSSQS3ZFCJJFIdEcKkUQi0R0pRBKJRHekEOU4w10F98UXX2Tq1KnYbDZmzZrFH/7whyxZqg/DeX/Wrl2LEl8QIfFns9myaG122bBhA0uXLqWmpgZFUXj11VfPeMz69euZP38+VquVSZMmsXbt2ozbCVKIcprhroL75z//ma9+9avceuutbNu2jeuuu47rrruOTz75JMuWZ4dUVgl2uVy0tLQk/xobG7NocXbx+XzMmTOHJ554Ykj7Hzx4kGuuuYYlS5awfft27rzzTm677TbWrVuXYUsBIclZFi5cKG6//fbk41gsJmpqasTq1asH3P/LX/6yuOaaa/psO++888Tf/d3fZdROvRju+7NmzRrhdruzZF1uAYhXXnnltPvcfffdYsaMGX223XjjjeLKK6/MoGUa0iPKURKr4F5++eXJbadbBRfg/fff77M/wJVXXjno/qOZVN4fAK/XS0NDA3V1dVx77bV8+umn2TB3VKDn50cKUY5yulVwjx07NuAxx44dG9b+o5lU3p+zzz6bZ555ht/85jf84he/QFVVLrjgAo4cOZINk3OewT4/vb29BAKBjF57VFffSyTDYdGiRX3Wy7vggguYNm0aTz31FP/0T/+ko2US6RHlKMNdBRegqqpqWPuPZlJ5f07FbDYzb9489u3blwkTRx2DfX5cLhd2uz2j15ZClKOksgruokWL+uwP8NZbb43JVXPTsUpwLBZjx44dVFdXZ8rMUYWun5+Mh8MlKfPCCy8Iq9Uq1q5dK3bu3Cm++c1viqKiInHs2DEhhBA333yzuPfee5P7b9y4UZhMJvHjH/9Y7Nq1S9x///3CbDaLHTt26PUSMspw358HHnhArFu3Tuzfv19s2bJFfOUrXxE2m018+umner2EjOLxeMS2bdvEtm3bBCAeffRRsW3bNtHY2CiEEOLee+8VN998c3L/AwcOCIfDIb73ve+JXbt2iSeeeEIYjUbxxhtvZNxWKUQ5zr/927+J+vp6YbFYxMKFC8UHH3yQfG7x4sVi2bJlffb/1a9+JaZMmSIsFouYMWOGeO2117JscXYZzvtz5513JvetrKwUV199tdi6dasOVmeHd955RwD9/hLvybJly8TixYv7HTN37lxhsVjEWWedJdasWZMVW2UbEIlEojsyRiSRSHRHCpFEItEdKUQSiUR3pBBJJBLdkUIkkUh0RwqRRCLRHSlEEolEd6QQSUYVa9eupaioKPn4H//xH5k7d27y8fLly7nuuuuybpdkZEghkgzI8uXLURSFb33rW/2eu/3221EUheXLl/fZP90CMH78eB577LE+22688Ub27t076DGPP/54n/aml1xyCXfeeWda7ZKkHylEkkGpq6vjhRde6NOLJhgM8vzzz1NfX6+LTXa7nYqKikGfd7vdfTwmyehACpFkUObPn09dXR0vv/xyctvLL79MfX098+bNG9G5B/JUrrvuuqSXdckll9DY2Mh3v/vdZKN76D80O5WTPbPly5fz7rvv8vjjjyfPcfDgQSZNmsSPf/zjPsdt374dRVFkSxCdkEIkOS0rVqxgzZo1ycfPPPMMt9xyS8av+/LLL1NbW8uDDz6YbHQ/XB5//HEWLVrEN77xjeQ56uvr+70mgDVr1nDxxRczadKkdL0EyTCQQiQ5LTfddBPvvfcejY2NNDY2snHjRm666aaMX7ekpASj0YjT6aSqqiql5m5utxuLxYLD4Uiew2g0snz5cvbs2ZNceigSifD888+zYsWKdL8MyRCRrWIlp6W8vJxrrrmGtWvXIoTgmmuuoaysTG+zRkRNTQ3XXHMNzzzzDAsXLuR3v/sdoVCIL33pS3qblrdIj0hyRlasWMHatWv5r//6r7R5DQaDgVM70EQikbSceyjcdtttyUD8mjVruPHGG3E4HFm7vqQvUogkZ+QLX/gC4XCYSCTClVdemZZzlpeX94n7xGKxfgtBWiwWYrHYiK4z2DmuvvpqCgoKePLJJ3njjTfksExn5NBMckaMRiO7du1K3h+Mnp4etm/f3mdbaWkpdXV1/fa99NJLWblyJa+99hoTJ07k0Ucfpbu7u88+48ePZ8OGDXzlK1/BarWmNCQcP348mzZt4tChQxQWFlJSUoLBYEjGilatWsXkyZPHZF/v0YT0iCRDwuVy4XK5TrvP+vXrmTdvXp+/Bx54YMB9V6xYwbJly/j617/O4sWLOeuss1iyZEmffR588EEOHTrExIkTKS8vT8nuu+66C6PRyPTp0ykvL+fw4cPJ52699VbC4XBWZgElp0e2ipXkLX/605+47LLLaGpq6rewoCS7SCGS5B2hUIi2tjaWLVtGVVUVzz33nN4m5T1yaCbJO375y1/S0NBAd3c3P/zhD/U2R4L0iCQSSQ4gPSKJRKI7UogkEonuSCGSSCS6I4VIIpHojhQiiUSiO1KIJBKJ7kghkkgkuiOFSCKR6I4UIolEojv/PyFgdTUvON14AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T04:13:44.828903Z", + "iopub.status.busy": "2024-03-24T04:13:44.828567Z", + "iopub.status.idle": "2024-03-24T04:13:44.975036Z", + "shell.execute_reply": "2024-03-24T04:13:44.974020Z" + }, + "papermill": { + "duration": 0.169217, + "end_time": "2024-03-24T04:13:44.978934", + "exception": false, + "start_time": "2024-03-24T04:13:44.809717", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAkOUlEQVR4nO3de1xUdf4/8BfDZRC5eEEQER0vJZgCiqvhZc1CcEtXH+6WP6xAdK1EypyljLZgXdvQTQkrvvLQQrMyXVPLkryE8i1vqaAummAgpMmA4A2BHAb4fP/wx1nHmYEBgeEMr+fjwUPmzOeceZ/5DC/P+czM59gIIQSIiGRGYekCiIhaguFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyZKdpQtob/X19SguLoaLiwtsbGwsXQ4R3UMIgVu3bqFPnz5QKEwfX3W68CouLoaPj4+lyyCiJly6dAl9+/Y1eX+nCy8XFxcAd54YV1dXC1fTdnQ6Hfbu3YvQ0FDY29tbuhxqBZ2lTysqKuDj4yP9rZrS6cKr4VTR1dXV6sPLyckJrq6uVv1C70w6W582NazDAXsikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSx1uo9KWKvq6mrk5uZKtyt/0+JwTgG6u5+AcxelXltfX184OTm1d4lErYrhZSVyc3MRFBRksPxfRtpmZWVh5MiRbV8UURtieFkJX19fZGVlSbfzNDeg3pqDpCeHY4hXN4O2RHLH8LISTk5OekdTil+uQvnDb/AbFoDA/j0tWBlR2+CAPRHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJXw8i6oDunSUEMD1TSGedJYThRdQBmZolBDCcKaSzzhLC8CLqgO6dJQQwPVNIZ50lhOFF1AHdO0sIwJlC7sUBeyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsWD6+UlBSoVCo4OjpizJgxOHbsWKPtb9y4gYULF8LLywtKpRIPPvgg0tPT26laIuooLDqT6pYtW6BWq5GamooxY8YgOTkZYWFhyMvLg4eHh0H7mpoaTJ48GR4eHvjiiy/g7e2NX375Bd26dWv/4onIoiwaXklJSZg/fz6ioqIAAKmpqdi1axfS0tLw2muvGbRPS0vDtWvXcPjwYdjb2wMAVCpVe5ZM1GYKy6tQpa01eX9BWZX0r52d8T/drko7DHDv2ib1dTQWC6+amhpkZWUhLi5OWqZQKBASEoIjR44YXWfnzp0IDg7GwoUL8dVXX6FXr16YPXs2lixZAltbW6PraLVaaLVa6XZFRQUAQKfTQafTteIedSy1tbXSv9a8n9ai6GoVJicfMqvtX7/IafT+fS+Pg6qnfAPM3NerxcKrvLwcdXV18PT01Fvu6elpcL26BhcuXMD+/fvx9NNPIz09Hfn5+YiOjoZOp0NCQoLRdRITE7F06VKD5Xv37rXqa91dqgQAOxw9ehSXz1i6GmpKQ389O7gOnl2E0Ta6euCaFuihBOyNjFaX/maDT/JtsSfjf+Hj3Kbltqnq6mqz2snq6kH19fXw8PDA2rVrYWtri6CgIFy+fBnvvPOOyfCKi4uDWq2WbldUVMDHxwehoaFwdXVtr9Lb3emL14CcE3j44YcR0K+HpcuhJpwtrsDKnKP40+RxeKiP8delTqfDvn37MHnyZGnY5N5tfJJ/FOPHjze5DTloODtqisXCy93dHba2tigtLdVbXlpait69extdx8vLC/b29nqniH5+figpKUFNTQ0cHBwM1lEqlVAqlQbL7e3tjb4ArEXDmIidnZ1V76e1aE5/mXrtWkufm1u7xcLLwcEBQUFByMjIwIwZMwDcObLKyMhATEyM0XXGjRuHTZs2ob6+HgrFnePm8+fPw8vLy2hwWbvGBnjNGdwFOtcAL1kXi542qtVqREZGYtSoURg9ejSSk5NRVVUlvfsYEREBb29vJCYmAgAWLFiADz74AIsWLcKLL76In3/+GW+//TZeeuklS+6GRRSWV2HSyswm2zU1uAsAB2IfYYCR7Fg0vGbNmoWysjLEx8ejpKQEgYGB2L17tzSIf/HiRekICwB8fHywZ88eLF68GP7+/vD29saiRYuwZMkSS+2CxTQccSXPCsRgD8PR2arftPgm8wimPhKMrl0MT5sBIP9KJV7ecqrRt+eJOiqLD9jHxMSYPE3MzMw0WBYcHIyjR4+2cVXyMdjDGcO83QyW63Q6lPQCRvbvLuvxDyJTLP71ICKilmB4EZEsWfy0kVpGW3cbCsfLKKzIg8LRcMyrtrYWxbXFOHftnMl3GwsrKqFwvAxt3W0AhqeeRB0Zw0umiqt+QdcB7+P1xifhwP/s/p9G7+86ACiuCkQQPBttR9TRMLxkqk/X/qgqfBGrZwVikJF3G2tra3Ho4CGMGz/O5JFXwZVKLNpyCn0m9W/rcqkJTR1JA00fTXe2I2mGl0wpbR1Rf9sbA1yHYGhP4+82FtoVwq+Hn8l3G+tv30T97TIobR3bulxqgrlH0kDjR9Od6Uia4UXUATR1JA00fTTd2Y6kGV5EHUBTR9JA00fTne1Imh+VICJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJUrPD68KFC21RBxFRszQ7vAYPHoxJkybh008/xe3bt9uiJiKiJjU7vLKzs+Hv7w+1Wo3evXvj+eefx7FjZlyjnIioFTU7vAIDA7F69WoUFxcjLS0NGo0G48ePx7Bhw5CUlISysrK2qJOISE+LB+zt7Owwc+ZMbN26FStWrEB+fj5iY2Ph4+ODiIgIaDSa1qyTiEhPi8PrxIkTiI6OhpeXF5KSkhAbG4uCggLs27cPxcXFmD59emvWSUSkx665KyQlJWH9+vXIy8vD448/jo0bN+Lxxx+HQnEnBwcMGIANGzZApVK1dq1ERJJmh9eaNWswd+5czJkzB15eXkbbeHh44KOPPrrv4oiITGl2eO3btw/9+vWTjrQaCCFw6dIl9OvXDw4ODoiMjGy1IomI7tXsMa9BgwahvLzcYPm1a9cwYMCAVimKiKgpzQ4vIYTR5ZWVlXB0dLzvgoiIzGH2aaNarQYA2NjYID4+Hk5OTtJ9dXV1+PHHHxEYGNjqBRIRGWN2eJ08eRLAnSOvnJwcODg4SPc5ODggICAAsbGxrV8hEZERZofXgQMHAABRUVFYvXo1XF1d26woIqKmNPvdxvXr17dFHUREzWJWeM2cORMbNmyAq6srZs6c2Wjb7du3t0phRESNMevdRjc3N9jY2Ei/N/bTEikpKVCpVHB0dMSYMWPMnqVi8+bNsLGxwYwZM1r0uEQkX2Yded19qtjap41btmyBWq1GamoqxowZg+TkZISFhSEvLw8eHh4m1ysqKkJsbCwmTJjQqvUQkTxYfBropKQkzJ8/H1FRURg6dChSU1Ph5OSEtLQ0k+vU1dXh6aefxtKlSzFw4MB2rJaIOgqzjrxGjBghnTY2JTs72+wHr6mpQVZWFuLi4qRlCoUCISEhOHLkiMn1/vGPf8DDwwPz5s3DDz/8YPbjEZH1MCu82mpMqby8HHV1dfD09NRb7unpidzcXKPrHDx4EB999BFOnTpl1mNotVpotVrpdkVFBQBAp9NBp9O1rPAOoLa2VvrX2H40LGtsH5vaBrUfc/qiqT61lv40t3azwishIeG+imktt27dwrPPPot169bB3d3drHUSExOxdOlSg+V79+7V+5aA3FyqBAA7HDx4EL84m263b9+++94Gtb3m9IWpPrWW/qyurjarXbM/59Wa3N3dYWtri9LSUr3lpaWl6N27t0H7goICFBUVYdq0adKy+vp6AHdmds3Ly8OgQYP01omLi5O+2gTcOfLy8fFBaGiorD9oe7a4AitzjmL8+PF4qI/hfuh0Ouzbtw+TJ0+Gvb19i7ZB7cecvmiqT62lPxvOjppiVnj16NED58+fh7u7O7p3797o+Ne1a9fMqxB3vlYUFBSEjIwM6dS0vr4eGRkZiImJMWjv6+uLnJwcvWVvvPEGbt26hdWrV8PHx8dgHaVSCaVSabDc3t7e5B+1HNjZ2Un/NrYfje2nudugttecvjDVp9bSn+bWblZ4vfvuu3BxcZF+N3fw3hxqtRqRkZEYNWoURo8ejeTkZFRVVSEqKgoAEBERAW9vbyQmJsLR0RHDhg3TW79bt24AYLCciKybWeF198SCc+bMadUCZs2ahbKyMsTHx6OkpASBgYHYvXu3NIh/8eJFg4kPiYiaPeZla2sLjUZj8AHSq1evwsPDA3V1dc0uIiYmxuhpIgBkZmY2uu6GDRua/XhEJH+tNhmhVqvVmyaHiKgtmX3k9d577wG4Mxnhhx9+CGfn/74XW1dXh++//x6+vr6tXyERkRFmh9e7774L4M6RV2pqKmxtbaX7HBwcoFKpkJqa2voVEhEZYXZ4FRYWAgAmTZqE7du3o3v37m1WFBFRU5o9YN8woyoRkSU1O7zmzp3b6P2NzQZBRNRamh1e169f17ut0+lw5swZ3LhxA48++mirFUZE1Jhmh9eOHTsMltXX12PBggUG3yskImorrfLRdYVCAbVaLb0jSUTU1lrtezcFBQXSfEJERG2t2aeNd08vA9z53JdGo8GuXbv0vgNJRNSWmh1eDVfObqBQKNCrVy+sWrWqyXciiYhaCz/nRUSyxLlmiEiWGF5EJEsMLyKSJYYXEclSq4XXr7/+iueee661NkdE1KhWC6+rV6/io48+aq3NERE1iqeNRCRLDC8ikiWGFxHJktmfsJ85c2aj99+4ceN+ayEiMpvZ4eXm5tbk/REREfddEBGROcwOr/Xr17dlHUREzcIxLyKSJbOPvMyd7oYX4CCi9mB2eG3YsAH9+/fHiBEjIIRoy5qIiJpkdngtWLAAn3/+OQoLCxEVFYVnnnkGPXr0aMvaiIhMMnvMKyUlBRqNBq+++iq+/vpr+Pj44KmnnsKePXt4JEZE7a5ZA/ZKpRLh4eHYt28ffvrpJzz00EOIjo6GSqVCZWVlW9VIRGSgxe82KhQK2NjYQAiBurq61qyJiKhJzZrDXqvVYvv27UhLS8PBgwcxdepUfPDBB5gyZQoUCn7qoj39prvzH8aZyzeN3l/1mxYnyoDev1xH1y5Ko23yr/BouaNoqj+Bpvu0s/Wn2eEVHR2NzZs3w8fHB3PnzsXnn38Od3f3tqyNGlHw/1+or23PaaSVHT7JP97ktroqm30dFmpl5vUnYE6fdpb+tBFmjrYrFAr069cPI0aMgI2Njcl227dvb7Xi2kJFRQXc3Nxw8+ZNuLq6WrqcFrtWVYO9Z0swyMMZXextDe7P09zEX7/Iwao/D8cQL9Nf7eqqtMMA965tWSqZoan+BMzrU2voT3P/Rs2O6IiIiEZDi9pXj64O+H+j+5m8v+Hq5YN6dcUw78a/l0qW11R/AuzTezXrQ6pERB0FR9mJSJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSx0ivFJSUqBSqeDo6IgxY8bg2LFjJtuuW7cOEyZMQPfu3dG9e3eEhIQ02p6IrJPFw2vLli1Qq9VISEhAdnY2AgICEBYWhitXrhhtn5mZifDwcBw4cABHjhyBj48PQkNDcfny5XaunIgsyeLhlZSUhPnz5yMqKgpDhw5FamoqnJycTF5C7bPPPkN0dDQCAwPh6+uLDz/8EPX19cjIyGjnyonIkiw6a1lNTQ2ysrIQFxcnLVMoFAgJCcGRI0fM2kZ1dTV0Op3JKxlptVpotVrpdkVFBQBAp9NBp9PdR/UdW8P0KbW1tVa9n51JZ+lTc/fNouFVXl6Ouro6eHp66i339PREbm6uWdtYsmQJ+vTpg5CQEKP3JyYmYunSpQbL9+7dCycnp+YXLROXKgHADkePHsXlM5auhlpDZ+nT6upqs9rJer7Y5cuXY/PmzcjMzISjo6PRNnFxcVCr1dLtiooKaZxMzjOpNuX0xWtAzgk8/PDDCOjH62tag87Spw1nR02xaHi5u7vD1tYWpaWlestLS0vRu3fvRtdduXIlli9fju+++w7+/v4m2ymVSiiVhhcrsLe3h729fcsKlwE7OzvpX2vez86ks/Spuftm0QF7BwcHBAUF6Q22Nwy+BwcHm1zvX//6F5YtW4bdu3dj1KhR7VEqEXUwFj9tVKvViIyMxKhRozB69GgkJyejqqoKUVFRAO7Mne/t7Y3ExEQAwIoVKxAfH49NmzZBpVKhpKQEAODs7AxnZ2eL7QcRtS+Lh9esWbNQVlaG+Ph4lJSUIDAwELt375YG8S9evKh3Tcg1a9agpqYGf/7zn/W2k5CQgL///e/tWToRWZDFwwsAYmJiEBMTY/S+zMxMvdtFRUVtXxARdXgW/5AqEVFLMLyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkqUOEV0pKClQqFRwdHTFmzBgcO3as0fZbt26Fr68vHB0dMXz4cKSnp7dTpUTUUVg8vLZs2QK1Wo2EhARkZ2cjICAAYWFhuHLlitH2hw8fRnh4OObNm4eTJ09ixowZmDFjBs6cOdPOlRORJVk8vJKSkjB//nxERUVh6NChSE1NhZOTE9LS0oy2X716NaZMmYJXXnkFfn5+WLZsGUaOHIkPPvignSsnIkuyaHjV1NQgKysLISEh0jKFQoGQkBAcOXLE6DpHjhzRaw8AYWFhJtsTkXWys+SDl5eXo66uDp6ennrLPT09kZuba3SdkpISo+1LSkqMttdqtdBqtdLtiooKAIBOp4NOp7uf8juU6upq5OXlSbfPa25CW5KPM6ccUFPqptd2yJAhcHJyau8SqRnu7U/AdJ9aW3+a+3dp0fBqD4mJiVi6dKnB8r1791pVhxcUFOCvf/2rwfJnPzZsu2rVKgwaNKgdqqKWMtWfgGGfWlt/VldXm9XOouHl7u4OW1tblJaW6i0vLS1F7969ja7Tu3fvZrWPi4uDWq2WbldUVMDHxwehoaFwdXW9zz3oOKqrqzF+/HjpduVvWuz54TjCJvwOzl2Uem2t7X9qa3RvfwKm+9Ta+rPh7KgpFg0vBwcHBAUFISMjAzNmzAAA1NfXIyMjAzExMUbXCQ4ORkZGBl5++WVp2b59+xAcHGy0vVKphFKpNFhub28Pe3v7+96HjsLNzQ2jR4+Wbut0Oty6cQ0Txj5sVfvZWdzbn0Dn6VNz983ip41qtRqRkZEYNWoURo8ejeTkZFRVVSEqKgoAEBERAW9vbyQmJgIAFi1ahIkTJ2LVqlV44oknsHnzZpw4cQJr16615G4QUTuzeHjNmjULZWVliI+PR0lJCQIDA7F7925pUP7ixYtQKP77pujYsWOxadMmvPHGG3j99dfxwAMP4Msvv8SwYcMstQtEZAE2Qghh6SLaU0VFBdzc3HDz5k2rGvO6l06nQ3p6Oh5//HGrPsXoTDpLn5r7N2rxD6kSEbUEw4uIZInhRUSyxPAiIlmy+LuN7a3h/QlzPwgnVzqdDtXV1aioqLDqwd3OpLP0acPfZlPvJXa68Lp16xYAwMfHx8KVEFFjbt26BTc3N5P3d7qPStTX16O4uBguLi6wsbGxdDltpuFrUJcuXbLqj4R0Jp2lT4UQuHXrFvr06aP3Gc97dbojL4VCgb59+1q6jHbj6upq1S/0zqgz9GljR1wNOGBPRLLE8CIiWWJ4WSmlUomEhASjM2qQPLFP9XW6AXsisg488iIiWWJ4EZEsMbyISJYYXhZSVFQEGxsbnDp1ql22c+jQIQwfPhz29vbSlNtEcsbw6iAyMzNhY2ODGzdutMn21Wo1AgMDUVhYiA0bNrTJY5ChRx55RO96C5bW0eq5HwyvFqipqbF0Cc1WUFCARx99FH379kW3bt1atI323G8hBGpra9vt8ToyOb7e2oWgJk2cOFEsXLhQLFq0SPTs2VM88sgjIicnR0yZMkV07dpVeHh4iGeeeUaUlZVJ63z77bdi3Lhxws3NTfTo0UM88cQTIj8/X7q/sLBQABAnT56Ufr/7JzIyslnb+fzzz0VwcLBQKpXioYceEpmZmXr33/2zfv16IYQQmZmZ4ne/+51wcHAQvXv3FkuWLBE6na7R/T5w4IAAIHbv3i0CAwOFo6OjmDRpkigtLRXp6enC19dXuLi4iPDwcFFVVSVtq66uTrz99ttCpVIJR0dH4e/vL7Zu3Srd37Dd9PR0MXLkSGFvby8OHDjQmt3Y7iIjIw2e+/z8fDF37lzpeXjwwQdFcnKywXrTp08Xb731lvDy8hIqlUoIIcShQ4dEQECAUCqVIigoSOzYsUN6DTVo7HVprJ7CwsL2ejpaHcPLDBMnThTOzs7ilVdeEbm5ueLo0aOiV69eIi4uTpw7d05kZ2eLyZMni0mTJknrfPHFF2Lbtm3i559/FidPnhTTpk0Tw4cPF3V1dUII/fCqra0V27ZtEwBEXl6e0Gg04saNG83aTt++fcUXX3whfvrpJ/GXv/xFuLi4iPLyclFbWys0Go1wdXUVycnJQqPRiOrqavHrr78KJycnER0dLc6dOyd27Ngh3N3dRUJCgsn9zs3NlULm4YcfFgcPHhTZ2dli8ODBYuLEiSI0NFRkZ2eL77//XvTs2VMsX75c2tZbb70lfH19xe7du0VBQYFYv369UCqVUsg2bNff31/s3btX5Ofni6tXr7Z117apGzduiODgYDF//nyh0WiERqMRt2/fFvHx8eL48ePiwoUL4tNPPxVOTk5iy5Yt0nqRkZHC2dlZPPvss+LMmTPizJkz4ubNm6JHjx7imWeeEWfPnhXp6eniwQcf1Auv69evN/q6NFZPbW2tJZ6aVsHwMsPEiRPFiBEjpNvLli0ToaGhem0uXbokhY8xZWVlAoDIyckRQuiHlxD//eO9fv16o7WY2s7dQaHT6UTfvn3FihUrpGVubm7SEZcQQrz++utiyJAhor6+XlqWkpIinJ2dpWC8d7/vrvO7776TliUmJgoAoqCgQFr2/PPPi7CwMCGEELdv3xZOTk7i8OHDetuaN2+eCA8P19vul19+2ej+y83EiRPFokWLGm2zcOFC8ac//Um6HRkZKTw9PYVWq5WWrVmzRvTs2VP89ttv0rJ169bpvYbMeV2aU49ccMzLTEFBQdLvp0+fxoEDB+Ds7Cz9+Pr6ArgztgQAP//8M8LDwzFw4EC4urpCpVIBuHMpt+Ywdzt3X3TXzs4Oo0aNwrlz50xu99y5cwgODtabFmjcuHGorKzEr7/+anS/7+bv7y/97unpCScnJwwcOFBv2ZUrVwAA+fn5qK6uxuTJk/Wes40bN0rPV4NRo0aZrNlapKSkICgoCL169YKzszPWrl1r0J/Dhw+Hg4ODdDsvLw/+/v5wdHSUlt17UVpzXpfWpNNNidNSXbt2lX6vrKzEtGnTsGLFCoN2Xl5eAIBp06ahf//+WLduHfr06YP6+noMGzas2YOvrbWdlrp7v+9290yeNjY2BjN72tjYoL6+HsCd5wsAdu3aBW9vb712935Pz9TjWYvNmzcjNjYWq1atQnBwMFxcXPDOO+/gxx9/1GvXkufBnNelNWF4tcDIkSOxbds2qFQq2NkZPoVXr15FXl4e1q1bhwkTJgAADh482Og2G/6Xraura9F2jh49it///vcAgNraWmRlZSEmJsbk4/n5+WHbtm0QQkhHX4cOHYKLi0urz3c2dOhQKJVKXLx4ERMnTmzVbXd0Dg4Oen166NAhjB07FtHR0dIyc46KhgwZgk8//RRarVYK/OPHj+u1aep1aaweOeNpYwssXLgQ165dQ3h4OI4fP46CggLs2bMHUVFRqKurQ/fu3dGzZ0+sXbsW+fn52L9/P9RqdaPb7N+/P2xsbPDNN9+grKwMlZWVzdpOSkoKduzYgdzcXCxcuBDXr1/H3LlzTT5edHQ0Ll26hBdffBG5ubn46quvkJCQALVa3ejslS3h4uKC2NhYLF68GB9//DEKCgqQnZ2N999/Hx9//HGrPlZHo1Kp8OOPP6KoqAjl5eV44IEHcOLECezZswfnz5/Hm2++aRBCxsyePRv19fV47rnncO7cOezZswcrV64EAOk/n6Zel8bqaTg6liOGVwv06dMHhw4dQl1dHUJDQzF8+HC8/PLL6NatGxQKBRQKBTZv3oysrCwMGzYMixcvxjvvvNPoNr29vbF06VK89tpr8PT0RExMTLO2s3z5cixfvhwBAQE4ePAgdu7cCXd390YfLz09HceOHUNAQABeeOEFzJs3D2+88cZ9PTemLFu2DG+++SYSExPh5+eHKVOmYNeuXRgwYECbPF5HERsbC1tbWwwdOhS9evVCWFgYZs6ciVmzZmHMmDG4evWq3lGYKa6urvj6669x6tQpBAYG4m9/+xvi4+MBQBoHa+p1aaye5o7BdiScEodIpj777DNERUXh5s2b6NKli6XLaXcc8yKSiY0bN2LgwIHw9vbG6dOnsWTJEjz11FOdMrgAhheRbJSUlCA+Ph4lJSXw8vLCk08+iX/+85+WLstieNpIRLLEAXsikiWGFxHJEsOLiGSJ4UVEssTwIiJZYnhRm5ozZw5sbGykL297enpi8uTJSEtLa9ZXUzZs2NDiGWDvx5w5czjnfwfF8KI2N2XKFGg0GhQVFeHbb7/FpEmTsGjRIkydOpVTPVPLWXQ2MbJ6DVMa3ysjI0MAEOvWrRNCCLFq1SoxbNgw4eTkJPr27SsWLFggbt26JYT470SFd/80zPi6ceNGERQUJJydnYWnp6cIDw8XpaWl0uNcu3ZNzJ49W7i7uwtHR0cxePBgkZaWJt1/8eJF8eSTTwo3NzfRvXt38cc//lGaGjkhIcHgceU+NbU14ZEXWcSjjz6KgIAAbN++HQCgUCjw3nvv4ezZs/j444+xf/9+vPrqqwCAsWPHIjk5Ga6urtBoNNBoNIiNjQUA6HQ6LFu2DKdPn8aXX36JoqIizJkzR3qcN998Ez/99BO+/fZbnDt3DmvWrJG+sK7T6RAWFgYXFxf88MMPOHToEJydnTFlyhTU1NQgNjYWTz31lHTkqNFoMHbs2PZ9osg0S6cnWTdTR15CCDFr1izh5+dn9L6tW7eKnj17SrfXr18v3Nzcmny848ePCwDSUdu0adNEVFSU0baffPKJwVTYWq1WdOnSRezZs6fJ+smyeORFFiPumgjxu+++w2OPPQZvb2+4uLjg2WefxdWrV1FdXd3oNrKysjBt2jT069cPLi4u0mSHDVO9LFiwAJs3b0ZgYCBeffVVHD58WFr39OnTyM/Ph4uLizRtco8ePXD79m2rnDbZ2jC8yGLOnTuHAQMGoKioCFOnToW/vz+2bduGrKwspKSkAGj8moVVVVUICwuDq6srPvvsMxw/fhw7duzQW+8Pf/gDfvnlFyxevBjFxcV47LHHpFPOyspKBAUF4dSpU3o/58+fx+zZs9t47+l+cVYJsoj9+/cjJycHixcvRlZWFurr67Fq1Spp0rx///vfeu2NTV+cm5uLq1evYvny5fDx8QEAnDhxwuCxevXqhcjISERGRmLChAl45ZVXsHLlSowcORJbtmyBh4cHXF1djdZpTdMmWxseeVGb02q1KCkpweXLl5GdnY23334b06dPx9SpUxEREYHBgwdDp9Ph/fffx4ULF/DJJ58gNTVVbxsqlQqVlZXIyMhAeXk5qqur0a9fPzg4OEjr7dy5E8uWLdNbLz4+Hl999RXy8/Nx9uxZfPPNN/Dz8wMAPP3003B3d8f06dPxww8/oLCwEJmZmXjppZekKyipVCr85z//QV5eHsrLy6HT6drnSaOmWXrQjazb3VdptrOzE7169RIhISEiLS1Nuj6kEEIkJSUJLy8v0aVLFxEWFiY2btxocB3LF154QfTs2VPvoxKbNm0SKpVKKJVKERwcLHbu3GlwLUM/Pz/RpUsX0aNHDzF9+nRx4cIFaZsajUZEREQId3d3oVQqxcCBA8X8+fPFzZs3hRBCXLlyRUyePFk4OzvzoxIdDOfzIiJZ4mkjEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIln6PyehK9C+T55EAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T04:13:45.035931Z", + "iopub.status.busy": "2024-03-24T04:13:45.035257Z", + "iopub.status.idle": "2024-03-24T04:13:45.328167Z", + "shell.execute_reply": "2024-03-24T04:13:45.327175Z" + }, + "papermill": { + "duration": 0.31389, + "end_time": "2024-03-24T04:13:45.330416", + "exception": false, + "start_time": "2024-03-24T04:13:45.016526", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAEmCAYAAAA3JMYtAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABXI0lEQVR4nO2dd3iTVfvHP09m92C0tNCy9x6CgqggAspQxI2+giAioAIqQxEUUFCBFwEFQVF8lR8ORFARFQQUBGQLsiwUKKsFSvfIen5/pEmTNm2TkjRJez7XlYunzzrnIck397nPfe5bkmVZRiAQCPwEhbc7IBAIBK4gREsgEPgVQrQEAoFfIURLIBD4FUK0BAKBXyFESyAQ+BVCtAQCgV8hREsgEPgVKm93wNOYTCYuXrxIaGgokiR5uzsCgcAGWZbJzMwkNjYWhcI5G6rSi9bFixeJi4vzdjcEAkEpJCUlUadOHafOrfSiFRoaCpj/U8LCwrzcG4FAYEtGRgZxcXHW76kzVHrRsgwJw8LChGgJBD6KK64b4YgXCAR+hRAtgUDgVwjREggEfkWl92k5gyzLGAwGjEajt7siELgNpVKJSqWqdKE+VV60dDodly5dIicnx9tdEQjcTlBQEDExMWg0Gm93xW1UadEymUwkJiaiVCqJjY1Fo9FUul8lQdVElmV0Oh1XrlwhMTGRxo0bOx286Rby0iEg3CO3rtKipdPpMJlMxMXFERQU5O3uCARuJTAwELVazdmzZ9HpdAQEBFRMw7lp8GF3aHQX9J4FGvd+t4QjHir2F0ggqEAq/LMty/DjBEg7B6c2g8ng9ibEt1UgELiPQ6vhyBqQlDD4Ywhwf0C3EC2BQOAeUk/DhpfM23dMgTqdPNKMEC2BlTNnziBJEgcPHqyQ++zYsYPWrVujVqu57777bqjNqsrWrVuRJIm0tDTvdsSohzUjQJcF8V2h+wSPNSVES1Ainv5CTJgwgXbt2pGYmMinn37qkTYEFcTWOXBhn3nG8P5loFB6rCkhWpUEnU7n7S64zKlTp+jZsyd16tQhIiKiXPeoyOe2BCG7G3987+w4sx3+mGfe7r8AIjybCkqIlg2yLJOjM3jl5Wqh7zvuuIOxY8cybtw4atSoQZ8+fThy5Ah33303ISEhREdH88QTT3D16lXrNRs3buTWW28lIiKC6tWr079/f06dOuXw/mfOnKFHjx4AREZGIkkSQ4cOdek+x48fp2vXrgQEBNCqVSu2bdtmvbckSVy7do2nnnoKSZKslta2bdvo3LkzWq2WmJgYJk+ebCcUjp7bYhH+/PPPtG/fnsDAQHr27ElKSgo//fQTzZs3JywsjMcee8wuiNhkMjF79mzq169PYGAgbdu25ZtvvrEet9z3p59+omPHjmi1WrZv317mezNr1iyioqIIDQ1lxIgRTJ48mXbt2lmPDx06lPvuu48333yT2NhYmjZtCsD//vc/OnXqRGhoKLVq1eKxxx4jJSXF7t4bNmygSZMmBAYG0qNHD86cOVNmfzxK7nX4diQgQ7vHodX9Hm+ySsdpFSVXb6TFtJ+90vbRGX0I0rj2dqxcuZJnn32WHTt2kJaWRs+ePRkxYgT//e9/yc3NZdKkSTz00EP89ttvAGRnZzNhwgTatGlDVlYW06ZNY9CgQRw8eLDY1HhcXBxr1qxh8ODBnDhxgrCwMAIDA126z8svv8yCBQto0aIF8+fPZ8CAASQmJhIXF8elS5do2rQpM2bM4OGHHyY8PJwLFy5wzz33MHToUD777DOOHz/O008/TUBAAK+//rrD5wa4dOkSAK+//jqLFy8mKCiIhx56iIceegitVsuqVavIyspi0KBBLFq0iEmTJgEwe/ZsPv/8c5YuXUrjxo35/fffefzxx6lZsya33367tb3Jkyczd+5cGjRoQGRkZKnvyRdffMGbb77JBx98QLdu3Vi9ejXz5s2jfv36dudt3ryZsLAwfv31V+s+vV7PzJkzadq0KSkpKUyYMIGhQ4eyYcMGwJwT7v7772fMmDGMHDmSvXv38uKLL5b9QfEUsgzfvwAZF6BaQ7j77QppVoiWH9O4cWPeeecdwPzr3r59e9566y3r8RUrVhAXF8fJkydp0qQJgwcPtrt+xYoV1KxZk6NHj9KqVSu7Y0qlkmrVqgEQFRVlN3xz9j5jx461nrtkyRI2btzIxx9/zMSJE6lVqxaSJBEeHk6tWrUA+OCDD4iLi2Px4sVIkkSzZs24ePEikyZNYtq0aVZBtH1uKBStWbNm0a1bNwCGDx/OlClTOHXqFA0aNADggQceYMuWLUyaNIn8/HzeeustNm3axC233AJAgwYN2L59Ox9++KGdaM2YMYO77rqr7DcEWLRoEcOHD2fYsGEATJs2jV9++YWsrCy784KDg/noo4/sltc89dRT1u0GDRqwcOFCbrrpJrKysggJCWHJkiU0bNiQefPMQ7GmTZty+PBh3n67YsSiGAc+h6PrQKGCwctBG1IhzQrRsiFQreTojD5ea9tVOnbsaN0+dOgQW7ZsISSk+Afn1KlTNGnShH///Zdp06axe/durl69islkAuDcuXPFRKs0nL2PRQwAVCoVnTp14tixYyXe99ixY9xyyy12S6m6detGVlYW58+fJz4+vthz29KmTRvrdnR0NEFBQVbBsuz766+/AEhISCAnJ6eYGOl0Otq3b2+3r1Mn56fuT5w4wejRo+32de7c2WrtWmjdunWx9YD79u3j9ddf59ChQ1y/ft3u/7VFixYcO3aMLl262F1j+39coVxNgJ/MFis9p0Jtx++JJxCiZYMkSS4P0bxJcHCwdTsrK4sBAwY4/NWNiYkBYMCAAdStW5fly5cTGxuLyWSiVatWLjuC3XWf8mL73Lao1WrrtiRJdn9b9lmEwGL5/Pjjj9SuXdvuPK1W61R7N0LRe2ZnZ9OnTx/69OnDF198Qc2aNTl37hx9+vTxPUe9QQdrhoM+G+p1h64vVGjz/vMNFZRKhw4dWLNmDfXq1UOlKv62Xrt2jRMnTrB8+XK6d+8OUKZT2WIJ2KbsceU+u3bt4rbbbgPAYDCwb98+xo4dW2J7zZs3Z82aNciybLW2duzYQWhoqNNFD5ylRYsWaLVazp07ZzcUvFGaNm3Knj17+M9//mPdt2fPnjKvO378ONeuXWPOnDnWQix79+61O6d58+asX7/ebt+uXbvc0GsX2fImXDoIgZEw6EOo4KVCYvawkjBmzBhSU1N59NFH2bNnD6dOneLnn39m2LBhGI1GIiMjqV69OsuWLSMhIYHffvuNCRNKDwCsW7cukiTxww8/cOXKFbKysly6z/vvv8/atWs5fvw4Y8aM4fr163Z+m6KMHj2apKQknnvuOY4fP866deuYPn06EyZMcPsautDQUF566SXGjx/PypUrOXXqFPv372fRokWsXLmy3Pd97rnn+Pjjj1m5ciX//vsvs2bN4u+//y4ze0h8fDwajYZFixZx+vRp1q9fz8yZM+3OGTVqFP/++y8vv/wyJ06cYNWqVRUf33Z6G+x4z7w9cBGE1y79fA/gVdH6/fffGTBgALGxsUiSxHfffWd3XJZlpk2bRkxMDIGBgfTq1Yt///3XO531cWJjY9mxYwdGo5HevXvTunVrxo0bR0REBAqFAoVCwerVq9m3bx+tWrVi/PjxvPvuu6Xes3bt2rzxxhtMnjyZ6Ohoxo4d69J95syZw5w5c2jbti3bt29n/fr11KhRo9T2NmzYwF9//UXbtm0ZNWoUw4cPZ+rUqTf0f1MSM2fO5LXXXmP27Nk0b96cvn378uOPPxab6XOFIUOGMGXKFF566SU6dOhAYmIiQ4cOLTPDQs2aNfn000/5+uuvadGiBXPmzGHu3Ll258THx7NmzRq+++472rZty9KlS+0mXjxOTiqsfQaQoeNQaD6g4tq2RfYiGzZskF999VX522+/lQF57dq1dsfnzJkjh4eHy99995186NAheeDAgXL9+vXl3Nxcp9tIT0+XATk9Pb3YsdzcXPno0aMu3U8gcJVevXrJjz/+uFfadttn3GSS5f97TJanh8nywo6ynJ/llv6V9v0sCa/6tO6++27uvvtuh8dkWWbBggVMnTqVe++9F4DPPvuM6OhovvvuOx555JGK7KpA4BQ5OTksXbqUPn36oFQq+b//+z82bdpkF4/ll+z7FI7/AAo1PPAxaNw/OeEsPuvTSkxM5PLly/Tq1cu6Lzw8nC5durBz584Sr8vPzycjI8PuJRC4i5YtWxISEuLw9cUXXyBJEhs2bOC2226jY8eOfP/996xZs8buc+x3XDkBG6eYt3tNh5i2Xu2Oz84eXr58GTDH1tgSHR1tPeaI2bNn88Ybb3i0b4Kqy4YNG9Dr9Q6PRUdHExgYyKZNmyq4Vx7EkG8ObzDkQoMecPMYb/fId0WrvEyZMsVuNstSdlsgcAd169b1dhcqls0z4PJhCKoOg5ZWeHiDI7zfgxKwLO1ITk6225+cnGw95gitVktYWJjdSyAQlIOEzbBzsXl74GIILfl7V5H4rGjVr1+fWrVqsXnzZuu+jIwMdu/e7b2lCwJBVSH7Knz3rHn7phHQ7B7v9scGrw4Ps7KySEhIsP6dmJjIwYMHqVatGvHx8YwbN45Zs2bRuHFj6tevz2uvvUZsbKzIcikQeBJZhnVjICsZajYzV9TxIbwqWnv37rXmbAKsvqgnn3ySTz/9lIkTJ5Kdnc3IkSNJS0vj1ltvZePGjRVXCkkgqIrs+QhObgSl1lycQh3o7R7ZIcmyi9nn/IyMjAzCw8NJT08v5t/Ky8sjMTGR+vXrCyEUVEpc/oynHINld4AhD/rOgZuf9Wj/Svt+loTP+rQEnqNevXosWLDA+rejJVSCKog+D74ZbhasRndBl1He7pFDKl3Ig8B1Ll26VGZGTkEVYNN0SPkHgmvCfR9AGYu8vYUQLUGpISQVhSzLGI1Gh2l1BBXAyV9g91Lz9r0fQEiUd/tTCmJ4aIssgy7bOy8XXYuZmZkMGTKE4OBgYmJi+O9//8sdd9zBuHHjXH5s2+GhpejEt99+S48ePQgKCqJt27bFlk5t376d7t27ExgYSFxcHM8//zzZ2dnW42UVaShv0QiBB8hKgXUF2Va7jIImvb3bnzIQP2u26HPgrVjvtP3KRZcWoU6YMIEdO3awfv16oqOjmTZtGvv377er+nIjvPrqq8ydO5fGjRvz6quv8uijj5KQkIBKpeLUqVP07duXWbNmsWLFCq5cucLYsWMZO3Ysn3zyCVB2kQYLrhSNEHgAk8kcj5V9BaJaQi/fXwInRMsPyczMZOXKlaxatYo777wTgE8++YTYWPcJ7ksvvUS/fv0AeOONN2jZsiUJCQk0a9aM2bNnM2TIEKtV17hxYxYuXMjtt9/OkiVLCAgIKLNIgwVXikYIPMBfH0LCJlAFmLM3qH1/Fl2Ili3qILPF4622neT06dPo9Xo6d+5s3RceHm6tn+cObItEWHLMp6Sk0KxZMw4dOsTff//NF198YT1HlmVMJhOJiYk0b968zCINFlwpGiFwM5ePwK/TzNu9Z0FUc+/2x0mEaNkiSV7NE+RLFC0SAdgVhXjmmWd4/vnni10XHx/vUpEGTxSNEDiBPtecvcGogyZ3m5fq+AlCtPyQBg0aoFar2bNnj7WsVnp6OidPnrQWkvAkHTp04OjRozRq1Mjh8cOHD5dZpKGyYInNLisHvM/xy1S4chxCouHexT4b3uAIIVp+SGhoKE8++SQvv/wy1apVIyoqiunTp6NQKCrkyzNp0iRuvvlmxo4dy4gRIwgODubo0aP8+uuvLF682K5Iw6hRozhy5EixIg2VAYPRxL8pWRhNMhqVAq1KUfivUoFGpUStlHxP0I5vMC/VAXO6meCS8/b7IkK0/JT58+czatQo+vfvT1hYGBMnTiQpKalCliO1adOGbdu28eqrr9K9e3dkWaZhw4Y8/PDDQGGRhldeeYWFCxfSoUMH5s6dy8CBAz3et4okO9+A3mgeMufpjeTpjcXOkSQJrdIsZrbCpikQtgoXtIxL5sXQALeMhYY9K7Z9NyDWHlaStYfZ2dnUrl2befPmMXz4cG93p0pwJTOPS+l5hAaoqR6sQWc0oTOYyDeY/9UZTMiU/PWSkNCoJDQqpVnQbMRNo1KgcIOg2X3GNRr4fBCc3gq1WsOIzaDSlnkPT1KetYfC0vJTDhw4wPHjx+ncuTPp6enMmDEDwFoEROB5dAazlRWoVhIWqC52XJZl9MZCEbOKWYG4mWSZfINMfsF9iqIpZqEprRaaUlEOQdv1vlmwVIEweIXXBau8CNHyY+bOncuJEyfQaDR07NiRP/74g2PHjpVY4QgKy8ELbhyL2GhUjheWSJLFilIWO2YWNLlAwIx21ll+gaDpjGaBI7/4vdVKhVXUbIecWpUCpaOUyCnHYFNB4Gjf2VCzSbmf29sI0fJT2rdvz759+4rtz83N5eDBgxXfoSqIzli6aJWGWdAk87Va+6+hLMsYTHKhiBltBc2I0WS24PRGE9m64vdWKWyGmCY9Ofl68re9QoBJj9ysP1LHoeV5XJ9BiFYlIzAwsMRQBIH7kGUZvcHsr9Io3buEV5Ik1EoJtVJBsIMRnMFocug/yzeaMBhNGEwmDDoTOTqQDTrk3Otos5K4LEfywLFBRCzeTt3qwdStFkS96sHUrR5EvRrBRIVqfW+m0wFCtCiMtREInEVvNDvZLQJTkaiUClRKBUGa4seMVgvNSL7RhDEzizTyMMkwQT+a86ZAzl/I4MiF4vVAA9QK6lYrFLG61YOsf8dGBJbPj+YBqrRoWaK+c3JyCAz0rZSyAt/G4oT3SthCKSgVEoEaJYEaJRh0XEu5CkYd2nYPseKOFzl/PYczV3M4cy2bs9dyOJuaw9lr2Zy/nkue3sSJ5ExOJGcWu69aKRFXLYhezaOZcnczrz5zlRYtpVJJRESENWVKUFCQT30ABb5LZrYO2aBDqVSRl5fn7e4UQzaZyElJJOVaKhGpB1HeMxmlSkmjqFAaRYUWO19vNHHheq5VxM5cLfj3WjZJqbnojCZOX8lm2ZXT3NG0Jl0bei8gtUqLFhQmwLPN9SQQlEV6rp7MPAMhWhX6tOLhDl4nLwOyU4hI2kytXmNA5WAsaYNaqaBejWDq1QgGatodM5pkLmfkMf+Xk6zZf57FvyUI0fImkiQRExNDVFRUieXOBYKivPH9P/x+8grP3tGIB1rW8XZ37En+BzY8jTonBWW/d6Fm4xu6nVIhUTsikAm9m7D+0AX+PHWNfWdT6Vi3mps67BpVXrQsKJVKlMri8TQCgSMOX87hQqaR6MhQ31pNkZ8F60ZA5llocR+0f9xtt64dEcjgDnVYvSeJRb8l8OmwzmVf5AFEumWBwEVkWebstRwA6lZ3Pg9ahfDTJEg9DWF1YMACt2dvePaOhigVEltPXOHw+XS33ttZhGgJBC5i8WcBxEX6kGgd+RYOfg5IcP8yCHR/+uq61YO5t605Q+7iLf+6/f7OIERLIHCRc6lmKysqVGsOLfAF0pLg+3Hm7e4vQr1uHmtqdI+GSBL8/E8yJy4XD4/wNEK0BAIXsQwN46v5iJVlMsK3IyE/HWp3gjsme7S5RlGh3NPKnIL7/S0JHm3LEUK0BAIXsVha8b7iz/pjPpz7EzQhMHg5KD0fgjGmh3mp2A9/X+T0lYpdhC9ESyBwkXO+ZGkl7YGts83b98yFag0qpNkWsWH0ah6FSYYPtp6qkDYt+LRoGY1GXnvtNerXr09gYCANGzZk5syZYq2gwKtYLS1vi1Zehrk4hWyEVg9A20cqtHmLtbX2wAWSCv5PKgKfFq23336bJUuWsHjxYo4dO8bbb7/NO++8w6JFi7zdNUEVxiJaXg932PAypJ2F8HjoP7/Ci1O0j4+ke+MaGE0yS7dVnLXl06L1559/cu+999KvXz/q1avHAw88QO/evfnrr7+83TVBFUVnMHEpPReAOG9aWn9/DX+vBklh9mMFhHulG8/1NEfbf733PJfTK2YNpk+LVteuXdm8eTMnT54E4NChQ2zfvr3UzJz5+flkZGTYvQQCd3EhLReTbE6xXDPES+mKr5+BHyeYt2+bCPE3e6cfQOf61ehcvxo6o4kPf68Ya+uGRCsrK8ujAjF58mQeeeQRmjVrhlqtpn379owbN44hQ4aUeM3s2bMJDw+3vix19wQCd3D2WjZg9md5JSOI0VAQ3pABcV3gtpcrvg9FeK6n2bf1f3+d40qmg9zQbsZl0UpMTKRfv34EBwcTHh5OZGQkkZGRREREEBnp3gjcr776ii+++IJVq1axf/9+Vq5cydy5c1m5cmWJ10yZMoX09HTrKykpqVxty7JMrq54SShB1SbJ2+EOv78LSbtBGwb3Lwel95cP39qoBm3jIsjTm/h4e6LH23P5iR9//HFkWWbFihVER0d79Nfm5ZdftlpbAK1bt+bs2bPMnj2bJ5980uE1Wq0WrfbGzfap3x3hm33n+fH5Wx3mHxJUTbw6c3huF/z+jnm7/38hsm7F98EBkiTxXI9GjPhsL//beYZRtzcgwlFaVTfhsmgdOnSIffv20bRpU0/0x46cnBwURSqLKJVKTCbHJZfchd5oYt3Bi+QbTJxMzhKiJbDitWj43DRY8zTIJmjzCLR+oGLbL4M7m0fRPCaMY5cy+GTHGcbf5blqPy4PD2+66aZyD7lcZcCAAbz55pv8+OOPnDlzhrVr1zJ//nwGDRrk0Xb/Pp9GVr55QaylgrBAAF6Khpdls+M9/RxE1oN73q24tp1EkiSrb+uTHYlk5nkuN53LltZHH33EqFGjuHDhAq1atbLmWbfQpk0bt3Vu0aJFvPbaa4wePZqUlBRiY2N55plnmDZtmtvacMT2f69Ztw1GEcgqMCPLsneGh4dWw5E1IClh8McQ4Fwl5oqmb8taNIoKISEli892nrUGn7obl0XrypUrnDp1imHDhln3SZKELJsrkxiN7nNeh4aGsmDBAhYsWOC2ezrD9oQr1m2Dh4eiAv/hWraOHJ0RSYI6kRVUCCX1NGx4ybzdYwrU6VQx7ZYDhUJiTI+GjP/yEB9vT2RYt3oEadw/UeDy8PCpp56iffv27Ny5k9OnT5OYmGj3r7+TlW/gwLk06996YWkJCrBYWTFhAWgdVI12O0Y9rBkBuiyo2w1uneD5Nm+QAW1iia8WRGq2jlW7z3mkDZdl8OzZs6xfv77SFgTdffoaBlOhUBmET0tQgGWhdIVFwm+dAxf2maPdB30ICh/J3VUKKqWC0Xc0ZPK3h1n2+2kev7kuAWr39ttlS6tnz54cOnTIrZ3wJbYnXLX721bABFWbCl1zeGY7/DHPvD3gPYjwnyDp+zvUoXXtcIZ1q++R+7tsaQ0YMIDx48dz+PBhWrduXcwRP3DgQLd1zhvsKBCt0AAVmXkGIVoCKxUW7pB73Rz1jmwuTNHSs7Pl7kajUrB+bDePxXC6LFqjRo0CYMaMGcWOudsRX9EkZ+RxMjkLSYJuDWuw8Z/LYnhYCTlyIR2TLNOmToRL1xVGwwd7oFcFyDJ8/wJkXIBqDaHv255ry4N4MujcZdHydGCnN7FYWa1rh1Mj1BzRKxzxlYs8vZH+i7YDcHxmX5f8LRUS7nDgczi6DhQqc/YGbYjn2vJTXPJp6fV6VCoVR44c8VR/vMr2f82idWujGqgKIvFFyEPl4kJarnU7x4W1pXl6I5czzKlX6npKtK4mwE8Tzds9p0Ltjp5px89xSbTUajXx8fF+PQQsCVmWrU74WxvVQK00m7ciuLRycSmtMOeTK0P/89fNVlaoVkVEkAdysBt05iyk+hyo1x26vuD+NioJLs8evvrqq7zyyiukpqZ6oj9eIyEli5TMfLQqBR3qRqJSmv9rxPCwcnEhrTAtsM4F0TprE+7gEX/Nljfh0kFzrcJBH4LCp1PdeRWXfVqLFy8mISGB2NhY6tatS3CwvVNy//79butcRfJHwdCwc/1qBKiVqBQFlpYYHlYqLlwvHB7qDM6/tx4Ndzi9DXa8Z94euAjCa7u/jUqEy6J13333eaAb3md3onm94a2NagBYfVrC0qpcnLcRres5zi/q9Vi4Q04qrH0GkKHjUGg+wL33r4S4LFrTp0/3RD+8zsJH23PgXJr1l1Rl9WkJS6sycd7GEX8l0/mc5h5J/ifLsP45yLwE1RtDn7fcd+9KTLlXM+7bt49jx44B0LJlS9q3b++2TnkDrUrJzQ2qW/+2OuJFcGmlwnZ4mJzhfGpgj4Q77PsUjv8ACjU88DFoPBj/VYlwWbRSUlJ45JFH2Lp1KxEREQCkpaXRo0cPVq9eTc2aNd3dR69QODwUllZlwWA0WcMWAFKctLRMJg+kpLlyAjZOMW/3mg4xbd1z3yqAy1MUzz33HJmZmfzzzz+kpqaSmprKkSNHyMjI4Pnnn/dEH72CCHmofCRn5mO0sZxTnLS0rmTlk28woVRIxEa4ISWNId8c3mDIhQY94OYxN37PKoTLltbGjRvZtGkTzZs3t+5r0aIF77//Pr1793Zr57yJJeRBzB5WHmyHhgApTlaOsTjhYyMCUCvdEIqweQZcPgxB1WHQUhHe4CIu/2+ZTKZii6TBHHhamZb4WEIexOyh/yLL9u+dJUDUgrOiZQ13qOYGn1PCZti52Lx97/sQWuvG71nFKFdqmhdeeIGLFy9a9124cIHx48dz5513urVz3sTyi2oUjni/5EpmPl3n/Ma7Px+37rNYWi1jwwrOcc6nZRGtG86jlX0VvnvWvH3TCGhactFhQcm4LFqLFy8mIyODevXq0bBhQxo2bEj9+vXJyMhg0aJFnuijV7CEPAhHvH9yMCmNS+l5/Ha8MHW2Zd1hh3hzfc5r2TqnQlrOFRRovaHAUlmGdWMgKxlqNoPes8p/ryqOyz6tuLg49u/fz6ZNmzh+3Pwr1rx5c3r16uX2znmTwgXTwtLyR7LyzYGjJpv3zyJarWuHo1RIGE0yV7N01AoPKPVebpk53PMRnNwISq25OIW6gnLMV0LKFaclSRJ33XUXd911l7v74zOoRXCpX5OVZy4BZ7Txa1mGh3WqBVIjRENyRj4pmXmeF63ko/DLVPP2XTOgVqvy3UcAlFO0Nm/ezObNm0lJSSnmfF+xYoVbOuZtxIJp/yazoG6lraV1JcvseI8KDSAqNMAsWmWEPWTnG7iapQPKGQ2vzysIb8iDRndBl2dcv4fADpdF64033mDGjBl06tSJmJgYj2Yo9CZqsWDar3FkaVkWSAeoFUSFaoGyZxCTCmYcI4LUhAWUIyXNpumQchSCa8J9H0Al/b5UJC6L1tKlS/n000954oknPNEfn0GpEMGl/oylQrjt7K9lUkWjVBAVZh4SJmeUPoNoidEqV+K/k7/A7qXm7Xs/gJAo1+8hKIbLs4c6nY6uXbt6oi8+hXV4KCwtv8RiaVmGh0aTjEW/1EoXLK3yhjtkpcC60ebtLqOgSeUJvPY2LovWiBEjWLVqlSf64lOIZTz+jdXSKhge2oauaFQKosLMolVWrFa5UtKYTOZ4rOwrENUSer3hStcFZeDy8DAvL49ly5axadMm2rRpUyw6fv78+W7rnDcR+bT8G4toJWfks/dMKo2jQ63HzJaWeXhYlqVVruR/f30ICZtAFWDO3qAufXZS4BouW1p///037dq1Q6FQcOTIEQ4cOGB9HTx40O0dvHDhAo8//jjVq1cnMDCQ1q1bs3fvXre3U5TC1DRieOiPWEQL4IGlO+0sLbVSKhweljF76PLw8PIR+HWaebv3LIhqXvr5Apdx2dLasmWLJ/rhkOvXr9OtWzd69OjBTz/9RM2aNfn333+JjIz0eNvWBdPC0vJLLD4tCxbRUislJEmyDg+vZuVjMskoFMVn9Ywm2Tp7WNeZWof6XHN4g1EHTe42L9URuJ1yJwGsCN5++23i4uL45JNPrPvq1/dMqe2iFC6YFpaWv5CUmsPJ5Ex6NouyxmlZsIQ7aAp+jGqEaJEk84qH1BwdNUK0xe53OSMPvVFGrZSoFebEEO+XqXDlOIREw72LRXiDh/DpnBjr16+nU6dOPPjgg0RFRdG+fXuWL19eIW2rlWIZj7/R/Z0tDF+5ly0nUkq2tFTm91WtVFAtyFyQt6Qh4tmCNYdxkUHWEJgSOb7BvFQHzOlmgmuU9zEEZeDTonX69GmWLFlC48aN+fnnn3n22Wd5/vnnWblyZYnX5Ofnk5GRYfcqD5YF00aTXCzFicA3SM/VM+mbv9lzxr6c3Y6Ea+Tq7Wtz6gzm99A2H1ZNa9iD4xlEp/1ZGZfMi6EBbhkLDXs6/QwC1/Fp0TKZTHTo0IG33nqL9u3bM3LkSJ5++mmWLl1a4jWzZ88mPDzc+oqLiytX22qbxGzC2vJNXv76EF/uTeLRZbvs9mfkFq+yYxtYasESYFrSDKJTaw5NJvhuFOSmQq3WcOc0l55B4Doui9bvv/+OwWAott9gMPD777+7pVMWYmJiaNGihd2+5s2bc+7cuRKvmTJlCunp6dZXUlJSudq2WFognPHeZEfCVZ79fJ9Da+iXo8lA8R+VdAeiZSnMqlHZiFaoJVarpOGhE+EOu96H01tBFQiDV4CquG9M4F5cdsT36NGDS5cuERVlvyQhPT2dHj16YDQaS7jSdbp168aJEyfs9p08eZK6deuWeI1Wq0WrvfEPjq1o6U0mAlHe8D0FrjPko90AKBQS7z/WwbrfdsgeGmD/MXYkWnpD4eyhheiCGcSSlvKUOTy8eBA2FQSO9p0NNZuU8iQCd+GypSXLssNF0teuXStWbfpGGT9+PLt27eKtt94iISGBVatWsWzZMsaM8XwhALvhobC0vE5yur2w5NtUh25YM8TuWGmWlq1PyxpgWpIjvrTAUl02rBkBJj00628utCqoEJy2tO6//37AnEtr6NChdtaM0Wjk77//dvuaxJtuuom1a9cyZcoUZsyYQf369VmwYAFDhgxxazuOUCgkJMmccFLk1PI+tpYvFIoQ2PupoCSflvmHx9Hw0NHQMz1XT1pBBeq4SAeitXEKXPsXQmPNpexFeEOF4bRohYeHA2ZLKzQ0lMDAwsyLGo2Gm2++maefftrtHezfvz/9+/d3+32dQa1QoDOa0AtHvNdRFalYo7OxtIqGIzi0tAwOLK2wkhdNW4aGNUI0BGuLfE2Orof9KwHJHN4QVM35BxHcME6LliXAs169erz00ktuHwr6IiqlhM4oLC1foKgw2Q4Pi5KtK+5XdTh7aLP+sKjbo8SZw/QL5lL2AN1egAa3O/8QArfgsiN++vTpnuiHTyLKiPkO6qLDQxvRMjkRR6czFnfEW+K0dAYTGbkGwoMKF/87FC2TEdY+A3lpENMOerzq6mMI3IDLjvjk5GSeeOIJYmNjUalUKJVKu1dlQi0KtvoMRS0tW9Fy5ifFuozHxqcVoFYSVjDzWNSvZU1JY7vm8M+FcOYPUAebi1OoNK48gsBNuGxpDR06lHPnzvHaa69V6nTLUOj8FbOH3kdVxNmebygcAjqzYiG7YC1i0QrRUWEBZORlkZKZb5e+JqmopXVhH/xWUPbr7rehRiOXn0HgHlwWre3bt/PHH3/Qrl07D3THtyjMqSUsLW+jKs3ScuI3xZKqpuhMY1SoloSUrOKWVqpNrcP8rILwBgO0uA/aP16OJxC4i3LVPawqa/EKc2pVjef1ZUqbPXTGp5VZsIDadngIOMyrpTeauJhmFrH4akHw04uQehrC6sCABSK8wcu47NNasGABkydP5syZMx7ojm9RWEZMWFrepqille+iT8siWo6Gh2Af9nApLQ+jSUarUlDz7AY4+Dkgwf3LINDzudwEpeOypfXwww+Tk5NDw4YNCQoKKpZuOTU1tYQr/Q+VqMjjMyiVJYuWM4ZwZp45dquYaIUWX8pjGRp2jMhG8cME887uL0K9bi73W+B+XBatBQsWeKAbvonlA24Uw0OvYOuGUBeztIy2J5Z5L6ulpbK/jyNL61xqDgpMvK5fAPnpULsT3DHZ1e4LPITLovXkk096oh8+iWX2UAwPvYNtfJyyVJ9W2feyOOK1JVhatpkezl3LYbRyHU3yD4MmBAYvB2U5CrUKPEK58mmdOnWKqVOn8uijj5KSkgLATz/9xD///OPWznkby6Jp4Yj3DroixShKOuacI7704WGKzfBQeXEv41RrzH/0mwfVGrjWcYFHcVm0tm3bRuvWrdm9ezfffvstWVlZABw6dKjSRcsLS8u76EtZX5ivdy3koXB46NgRn60zmmO58jL4z8WZqCQTl+P7Q5uHy9t9gYdwWbQmT57MrFmz+PXXX9FoCiOCe/bsya5du0q50v9QCke8V7G1popGGbhsaZUQXBqiVRGkMa/kSMnMR97wErVMySSZapJz1zsivMEHcVm0Dh8+zKBBg4rtj4qK4urVq27plK8glvFUHCmZeYz/8iD7zhbOPpfmt9KVsmAaoGgdCkfLeCxYhojGg18i/f0lRllinH40sbVqufIIggrCZdGKiIjg0qVLxfYfOHCA2rVru6VTvoJYMF1xTF17hLUHLjB4yU7rPn0p1pTt7KEjSyukaDqZAjTK4pZTVGgAdaQU6u56DYCFhvu5ENqWAHXlWktbWXBZtB555BEmTZrE5cuXkSQJk8nEjh07eOmll/jPf/7jiT56DaulJXxaHseSVcEW2yFgUV0qaxlPaIDj2b6iw0OA6FAV76nfR23I4lq1Diw23ld6MQuBV3FZtN566y2aNWtGXFwcWVlZtGjRgttuu42uXbsydepUT/TRa6jEMp4KQ+tg2KY3FP6/m4q8B2Ut4ynR0nLQzuCsVXRU/EueMpgfG7+BESXxpRWzEHgVl+O0NBoNy5cv57XXXuPIkSNkZWXRvn17Gjdu7In+eZXCBdOeEy1ZlsnKN5RoGVR29EYTJll2aAHpjLZDQPtjZS3jCQlw/NEu1s7Zndx26VMAvo6ewJHscCBTWFo+jMuiZSE+Pp74+Hh39sXnsC6Y9uDwcOp3R/hi9zm+HHkzXRpU91g7vogsyzR+9ScAGkWFFDuus7W05JItLUfDw5J9WjailZsG345EgYk1xlvZqLgVozO1DgVexSnRmjBhAjNnziQ4OJgJEyaUeu78+fPd0jFfwBqn5cHh4Re7zTUc/7vpJKtH3uKxdnwRW59VQkqWdfufi+kcu5RpzSwKxXNm5dv5uxwMD8uytGQZfpwA6efIDYln+tWhxGTkW/NuieGh7+KUaB04cAC9Xm/dLonKlhDQMjysCEe8ROX6v7sR+i3cDsCQLoWWfLHhob70ZTyhZfm0Dq2GI2tAUnKl92KyVmVxMS2XHL15SCosLd/FKdHasmWLw+3Kjiv5tHJ0BpIz8qlfo3wFPyqZ3jtFWUL99/l063ax4aGtpeXAq1XS8FCtlMy5sTa8ZN7RYwphjW4BfrUWxAjWKKkeLFIp+yrlWntYaTHYl5JSWUMeyhat51YdoMfcrRy7lFGupquiaJVF6cGlNk56B4ZwicNDyWjOQqrLgrrd4NYJhAeq7WYV46oFVbpRQ2XCKUvLUqjVGb799ttyd8arnNgIP4yDR1ZBbXP5dUs6lLIi4mVZZu/Z6wDsPZNK85gwl5sXw8Pi6ErxW9nNHroQ8hB74D1zvveAcBj0ISiUSEDNEC0X0nKBEipKC3wGpyyt8PBw6yssLIzNmzezd+9e6/F9+/axefNma0FXv+TQKsi8ZP4Vzjc7hQszl5ZuaaVm66wFQk8mZ5V6rqCQcwXJ9koiX19y1HtZ1XgciVYX6RiR+xeZ/xjwHkTEWY9ZCreC8Gf5Ok5ZWpZCrQCTJk3ioYceYunSpdaSYUajkdGjRxMW5rqF4TP0XwDn90LqKdg4Ce5932bBdOmWVuLVwi/fieTMcjVfFUcjH/2RWOpx+0XRRY6VFVxaZHgYRhb/1byPhGwuTNHSfv1sVKgQLX/BZZ/WihUreOmll+xqHCqVSiZMmMCKFSvc2rkKJaiaOQc4Ehz4HP5Z67Qj/vSVQtH6NzmzyhT+KC9Gk8zQT/5i9Z6kUs+znyEsbXhY/Fp7S0tmtvojYqVUDJENoO/bxc63VJuGIrUOBT6Hy6JlMBg4fvx4sf3Hjx/H5O/ZEOrdCt0L4tC+f4FwXTJQdj6tU1cLh4TXc/RczdK53HRVcvz+fvIKW09cKfO8fCfXHjoMebCxtB5UbqOf8i/0spK8/h+Ctnggq7C0/AeXRWvYsGEMHz6c+fPns337drZv3868efMYMWIEw4YN80QfrcyZMwdJkhg3bpznGrljCtTuCHnp3PHPVBSYypw9tLW0wGxtuUrVkSzItfFVlUZpQ0Cd3Q+JI0e8eVlUfekSb6hWAjDP8CCKuA4O24ouSAaokKB2RKBT/RN4B5eX8cydO5datWoxb948a4qamJgYXn75ZV588UW3d9DCnj17+PDDD2nTpo3H2gDMucDvXw4f3kb09X08q1zPCdPIUi+x+LRCtSoy8w2cTM6ka6Manu2nH1OeBejFg0tLXpcIZp+WGgPvqRcTJOWzw9iSD439edHBGkcodMTHhAc6XFQt8B1cfncUCgUTJ07kwoULpKWlkZaWxoULF5g4caKdn8udZGVlMWTIEJYvX05kZAXUnaveEO55F4Dxqm+Iyzla4qkGo4mz18yi1bN5FAAnU1yfQaxCo0OM5XAjlBpcWkLIw4uqr2mjSOS6HMIE/bPIKIrVT7Rwc4Pq9G4RzXM9Rbl7X+eGflLCwsIqZMZwzJgx9OvXj169epV5bn5+PhkZGXavctH2Uc7XvgeVZGLU1dmQ5/g+56/nojeaC3ve3qQmIIaHZVGe9NW2wmQyyXZhKA4trQs7GKn8AYDJ+qdJphoalaJE32GAWsmy/3Tikc6VOwlAZaBcWR6++eYbvvrqK86dO4dOZ+903r9/v1s6ZmH16tXs37+fPXv2OHX+7NmzeeONN268YUniSPvpcP4v6hgvwU8TYdDSYqedLnDC168RTLNaZgE/mZyFLMsuOderkiO+PHUkbY0zXZGJEVmWuZ5d+DmMIBPluhdBklll6MnPppuAIhkeBH6Ly+/iwoULGTZsGNHR0Rw4cIDOnTtTvXp1Tp8+zd133+3WziUlJfHCCy/wxRdfEBAQUPYFwJQpU0hPT7e+kpJKn1YvDSkwgnG60RhRwKH/g8PfFDvH4oRvUDOYBjWDUUiQnqu3q6PnVFvl7qX/UT6fVuE1+YaiogVj/8/yYykzL+AjyLzEKVMMMw2PW88rWoZM4J+4LFoffPABy5YtY9GiRWg0GiZOnMivv/7K888/T3p6etk3cIF9+/aRkpJChw4dUKlUqFQqtm3bxsKFC1GpVBiNxWehtFqtddh6o8PXlrFh7JWb8b7hPvOOH8bD9bN255wucMI3qBFCgFpJvYIYH1cj46uQoVU+S8vmkqJFLbJ0Bv48dQ2AR5W/cSd7QKHmef1z5FL4Yycc7JUDl9/Fc+fO0bVrVwACAwPJzDT7b5544gn+7//+z62du/POOzl8+DAHDx60vjp16sSQIUM4ePCgxxz/FupEBtGsVijvGQZxLbId5GfAt0+D0WA95/SVwuEhQONocwxQeSPjqwLlsbRkO0vLWOSY+dVEcZHp6v+Zd/aazpD7BtilqHGUHVXgf7j8LtaqVYvUVHOZp/j4eGutw8TERLdHgoeGhtKqVSu7V3BwMNWrV6dVq1Zubask7mwehREl70VMBG0YJO2GP+Zaj9sODwGaRIcC5XHGVx1T60ZnDx2VD9Og538RywhABw16wM1jeKxLPIem97YOC4VPq3Lg8rvYs2dP1q9fD5gDTcePH89dd93Fww8/7LAeor/Ts1k0AGtPqzh78wzzzm1vw7ldZOUbSCnwXTWoabawGheI1kkXRauyDg/3nEnlqU/3WMNC4MbjtIo64gEWR31PdM5JCKpunjApSOCoUEhoVWaLXFhalQOXZw+XLVtmXa4zZswYqlevzp9//snAgQN55pln3N7BomzdutXjbdjSLi6C6sEarmXruP3nKBaob+U+5XZY8zRn790AQI0QDeGB5gjsJgXDw39dnEGspJrFg0vNdQxTMvP44bnuABjLEfJg54gvWJNYKyyAqDAtXTlE76sFkyT3vg+h9kVWtSoFWfnCp1VZcOldNBgMzJo1i8uXL1v3PfLIIyxcuJDnnnsOjabyZXtUKiQeK0j7q5Bgqn4omYG1If0c4b9NAmS7bKX1awSjVEhk5hu4nJHndDuV1dKycOF6rnW7fD6twm2LpRWoUbJuaFMm5b1nPnDTCGhafAbbUnRVzB5WDlwSLZVKxTvvvIPBYCj75ErEi72bcnxmX8b1akIWQSyrMQUkJXXO/8ggxXYa1ChcgKtVKalXkETOlRnEyp4E0Famyjd7WNynpVVKSOvHImUlQ81m0HuWw2stNRXF8LBy4PK7eOedd7Jt2zZP9MWnCVArualeNQC+So5Bvn0SADPUn9IuJNXu3PI746sGNx6nZZ49vM/wE5zcCEotDP4Y1I4XOmsLLC0xPKwcuOzTuvvuu5k8eTKHDx+mY8eOBAfb5x4aOHCg2zrna7SPj0ClkEjOyOel5F4MU62jleEf7j45De7abF5sjdkZ/9ORyy454yv78NCW8s0eFm7rDCaaSEkMy/7YvOOuGVCr5Nlki6UlZg8rBy6L1ujRowHH9Q0lSXIY8FlZCFArUSokDCaZNQcus5Nn2KidTETqIfOMYs+pADS1ziA6Hh7Kssz3f1+iXZ0I676qJFo3Gqelz89loXoxWnTQ6C7oUvoEkBgeVi5cfhdNJlOJr8osWBY6169m3b5IDV41jjD/8ftcOLMDsJ1BNGcxPXDuut1QcfWeJJ7/vwM89OFO675K79Oy0akbjYhvenguzRRJpCsi4L4PylR8qyNeDA8rBeJddJF3H2hLj6Y1rX8fibgT2g0BZPh2JORep16NYNRKiWydkX8uZjDogz+567+/W62F/+00LwWym12sJJqVkJLJ7A3HSM0uOXvrDfm0Tv5CkzNfALAyaiKERJV5rRgeVi6cHh7m5uayefNm+vfvD5gXJufnFy4KViqVzJw50+mFzf5KrfAAPhnWmYSULMau2s/9HWpDl7fh3E5zEdDvx6F+8FPq1wjmZHIW+wpKi4G5ak/1EC1Hy1kb0R/os+APjCaZ01ezWf6fTtb9tsO78sVpAVkpsM7snlhh6MulqNucujbA6oivJL8MVRynRWvlypX8+OOPVtFavHgxLVu2JDDQPGNz/PhxYmNjGT9+vGd66mM0igph4zibL83gj+Dj3nD0Ozj4BY2jW3AyOcu6NhHgUnoeYQVBqEWpLF8ny9DvUFJaieeUx9KSTEb47lnIvsIZVX3eznuEmfERTl0rfFqVC6ffxS+++IKRI+3TDq9atYotW7awZcsW3n33Xb766iu3d9BvqN0Rerxq3t4wkc6hZgvr+OVCX9al9Dy7CtS2pdcrWz6tlFJS8zgq+VUW9+R+DwmbkFUBjMkbTT4aOsQ7l8VWqxaiVZlw+l1MSEigdevW1r8DAgJQKAov79y5M0ePlpyWuErQ7QWo1x302dx7ehpqDEVEK5cD59Ksf9su/K1cklUcW5ly1dJqJp1jWI65PN2Fm17lH0NtIoLUNKjhXKkvS3mwGiHaMs4U+ANODw/T0tLsfFhXrtiXgDKZTHbHqyQKpbnU+pKuRFw/wgTV17yd+6j18KX0PEwm2yBJG9Gq7KplgytxWlp0LFQvQo0BmtzNL0H9gWN0iI9EUUK+96IM61aP+GpB9GoRXc4eC3wJpy2tOnXqcOTIkRKP//3339SpU8ctnfJrwmvDQHPp9WeUP3CL4h/roUtpueTZVJGxzVZQ2TUrM8/AhK8OIsuySzniX1V9QRPFBVKlSLh3MfsKfGUd6zpf4CQ0QM197WsXKeAq8FecFq177rmHadOmkZdXfBFwbm4ub7zxBv369XNr5/yWFgOhw5MoJJn/qj8gAvMQ8VJ6Hnl616PBKwvf7r/A7sRUp+O0ein28R/VrwC8EzgOgmuwv2A2tr2TTnhB5cNp0XrllVdITU2ladOmvPvuu6xbt45169bxzjvv0LRpU65fv84rr7ziyb76F31nk6yJo5Z0nTnqjwDZLFoGxwG4lc0RXxL5BpNTPq0orvOO+kMAlhn6sVfVnotpuVxKz0OpkGhrs5pAULVwWrSio6P5888/ad68OZMnT2bQoEEMGjSIKVOm0KJFC7Zv3050tPAZWNEEs7XlbHSykr7KPTyi3MLl9DxydY5Fa+2BC9w5b2vF9tEDlKW9CqnsiHgJE/PUS6gmZfGPqS5zDQ9hkmX2nzNbWc1jQgkWQ70qi0vvfP369dm4cSOpqakkJCQA0KhRI6pVq1bGlVWTiEY38e5fD/OqehXTVZ+xR9eUS+mhJZ5/6kp2iccqC0pJwlCGI3648ie6K4+QK2t4Xj8WHWpkGWugbkcnQx0ElZNyBa5Uq1aNzp0707lzZyFYpdAkOpSPjPfwh7EVgZKO99Tvc+FqWqnXWGYXz1/P4ZMdiWTn+1fusrJCsBQKqVRLq6WUyETVagC+qjGa+3v3BMyxXRZ/VgcXnPCCyoeItvMg8dWCkFHwov5ZUuUQWinOMMq4qtRrjAXf+n4Lt/PG90d59+cTFdHVCsOSJcMRgeSxUL0YjWRko/Emuj30El0bVgcgp2AdJ+B0UKmgciJEy4MoC+KIUohkot6cPmWk6kduVRwu8RqLFZKeqwdgZ0E9v8qCshRL6zXV/2iouMQluRqT9SNAklAUOMmuZOZjMMlEhWqpE+k42Z+gaiBEq4LYZOrI/wy9AJivXkLTUMf544taIZUt26ZSkhzGafVR/MVjqi2YZIkJ+mdJIxRJwipaFjrWjawyM60Cx1Sub4QP8sNzt6JRKehcrxpvGoZw0lSbKCmNGdJS7Be3mCmaAcFSjMHgoGyWP+LI0qrFNd5WLwfgQ2N/dppaWo8V1ScxNBQI0fIwrWqHc3LW3Qy5OZ48tDyvfw4dKrro/uJx5aZi5xedWVMrFZy9lk2L6T8z8wffW9u59sB5Dp9Pd/p8RZHZQwUm5quXECFlc8jUgPmGB63HJIpbWh3qRtxolwV+jhCtCqJDfCShASqOy/H8EmPOCTVV9TmNpfN25zkaHi76LQGdwcTH2xMrrL/O8GfCVcZ/eYgBi7e7dJ2tpfWM8ge6Ko+SLWt5QT8GfZEoHJs1+VQP1tBGBJVWeYRoVRBx1YL4Y2IPlj7ekZ5DX+Nw4E0ESHoWqheZc50XUEy0lAq7Rda+xIlyVBuSka3P2EY6xQTV1wC8bniSM3KM3bmSjSMeYEDbWJFeRiBEqyKJCNLQt1UtgrQavoiZzFU5jOaKJCYVxCVBcZ+WRqUoV9I8X0WWzZZWEHm8p16MWjLyg7ELXxtvd3i+bSKHO5uXnVpZUPkRouUl9AE1eakgDOIp1UbuUBwEivu0NCqFNXbL1yhPt85ey+FSeh6vq1ZSX5HMBbk6r+iH4yjPhYT9mswgjbL8nRVUGnxatGbPns1NN91EaGgoUVFR3HfffZw4UTmCLbVqBVtN7fnE0AeAd9VLqUF6sZk1tQ8PD8vTqzGr9tNPsYuHVNswyRLjdaPJIKTE822Hh0Wd8oKqiU+L1rZt2xgzZgy7du3i119/Ra/X07t3b7Kz/X+NniVv+RzDoxw3xVFTyuBd9VIMRpOdcGlUimJCtmbfed74/h+7YhH+QixXma3+CID3jffyl9zc7rglxAMoiNMqPCZESwA+LlobN25k6NChtGzZkrZt2/Lpp59y7tw59u3b5+2u3TBalXmok495UXC+rKaH8hDhh1dYy76D2RF/rUg5rhe/PsQnO87w2/GUCu1zaVxIyy3zHAUm/qv5gDAphwOmRrxnuL/YOS/3aWrdDtGq7IRK6WSmUkHlxqdFqyjp6eZ4oMqwSDtAXfhff1KO4x35CQBq/TUb/YXCZT4/Hr5kV4bMltJqC1YEtpZetzm/lZh2x8Jo5Tq6KI6TJQfwgn4MBgdJRupVD+aH527lf8M7Uz1EaxdcKiwtAfiRaJlMJsaNG0e3bt1o1apViefl5+eTkZFh9/JFLJaWha8VfdhkbI/CmE/Q9yOtYRBXSqlqY4vOYOKvxFT0Xoycv5pVcl/bS/8yTrUGgNf0wzgnO869JkkSrWqH072xuSCunU/Lbz6tAk/iNx+DMWPGcOTIEVavXl3qebNnzyY8PNz6iouLq6Aeuoa2yJpCjUrJRP0zZGtqoE49ySuqL8q8h61H65W1h3now50+GTUfQg7vqRejkkysM3ZlrenWEs8tOgK0Gx4KS0uAn4jW2LFj+eGHH9iyZUuZxTOmTJlCenq69ZWUlFRBvXQNS9VjCyqFglTCGJU1AoAnVb9yp8J53903+8yR9Z/tPOu+TrpISfMCb6g/JV5xhSRTTabqn6K0Mh5FdcnOES98WgJ8XLRkWWbs2LGsXbuW3377jfr165d5jVarJSwszO7lixS1tFQFs2Z/mNqw3HAPAO+ol1ETx/4sX8CZycuBih0MVm7HKEuM048mk6BSz7ek5LEgiZAHQRF8WrTGjBnD559/zqpVqwgNDeXy5ctcvnyZ3NyyZ6p8Ha26yPDQZnnKu4aH+cdUl+pSJvPUS5Eo9FPZZTL1csSDXKQDRf9e+eNWZqnNRVYXGu5nn9wUR9zTupZ1OznD3i9ma1yJ4aEAfFy0lixZQnp6OnfccQcxMTHW15dffuntrt0wAUUc8Sqb+CQdap7XjyVX1nCb8jBPKX+yHluzv3CBdVGR8DTJGXkMWLSdL/ecM7dfpHnbcDIlRlrtfokwKZc9piYsNt5X4n2n9S9MRdO6drjdMVvrSmiWAHxctGRZdvgaOnSot7t2wxS1tIouBD4l12amwRwGMUm1mpbSGcA+Q0Jajv1QytPM+ek4hy+kM2lNyZlXLTynWktHxb9kyIGM14/BSMlLcCKC1OyY3JNPh91kTa9sQQwJBUXxadGqzBQNeVA5yF6wytiTn42d0EhG3lMvJoB8jl0qDOH48PfTHu8nwOAlf1Jv8o/FglyLYonb6iQd5znlWgCm6odzXq5Z6nValYLaEYHc0TSqWFZSSXxCBUUQHwkvUSzkQenIopCYpH+ay3IkjRQXeU31OV/tLRwepmbryNOXHtDpDizBrb+fvGK3v+jgVAbCyGaB5gOUkswaY3fWm7qWef/S0ieL4aGgKEK0vETRkIeS8kSlEcoE/bOYZIkhqs30UeyxO24pYOoNis0eyjKz1CuoI13lrCmKafqhDq/r3rgG2yf1oF71IO5sVnq6GaFTgqKIMr1eoqilVVpyuz9NrVhm7M8o1ffMUS/nYH5DkvG9pUwhx79hoHInBlnBC/qxZFO8as47D7ThoU7mgN/NL95RLJi0KLa6KApaCEBYWl6juCO+9C/kPMOD/G2qT6SUxXz1ksIwCB9J9BAvJVPzj1cB+K/hAQ7KjYqd06t5FA92LAwOViqkMoVIZaNqRYVeUDURnwIvUTTkwZGl1adl4fo8PSpe0I8lR9bSTfkPI5U/Oryvu42R0nJ5WUIuVBhYqF6MQp/NblMzlhgHOjz/gyEdXbaWAtRKpvZrzsS+TakRonXpWkHlRIiWlyhqadnOHoYFqNj60h10rm8//Z8oxzDd8CQAL6m+orVUfPbQ3QOokrKm5huMVp/WC6pvaac4RZ4ylPG60ZhK+FiVV1BHdG/A6DuKW26CqokQLS+hKWJZqW2GQZ+P6EK9GsHEhAcUu+5r4+38aOyMuiAM4pOt/9gdd7ffp6Rq0N/uv4DeaKKLdIwxynUA/NxwChepUezcxY+158MnOoqiFAK3IBzxXkKlVKBSSBhMMpJkn+DOMs0fX83ROj2JKfoRtFOcooHiMr3OzAdut7nWvf0sSbSmfHuYMLLYqH0fhSTzpeEOJv1dz+G5/dvEurdTgiqN+OnzIpawB4Uk2S3jsWy3jHW82DuDECboRmOSJR5WbSVlV2G6HsnNA8SSi2rIzFZ/RKyUymlTLd4w/Met7QoEJSFEy4tYZsMk7C0ty4yZJEnEOhgiAuyWm/NBgcM7+JcXieVqwTXu69+Jy5l8/IfjArEPKrfRT/kXellpniDAcT/nPdjWfR0SCBCi5VUsoqWQJFQ2aTmVNtsbXuhe4vULDIM5aGpIsCmL/2o+QIHJbaJ16koWfRb8znub/y12rL50iTdUKwFzKMZhuUGxc1rXDmfV010Y3LH0/GcCgasI0fIi1qh4yT4eyXY7IkhDu7gIh9cbUPG8fixZcgBdFMd5VrnebcPDO+dtc7hfjYH31IsJkvLZYWzJh8b+Ds97oGMdujYs7pQXCG4UIVpeRGM7PLTxaRWtOrN2dFfeGtTa4T3OydHW5TLjVd/QVipuGbmTF1Vf00aRyHU5hAn6Z5FtPkK2Ymvyw/JmAv9AiJYX0do64hUli5YkSTxyUxw1QjQO7/OtqTvrjbegkky8Ky2C/EyP9Ler4ggjlT8AMFn/dLGlREfe6GPdLmnWUSC4UYRoeZEAi6Ul2WfldFTfT6GQ2Dv1rmLxXWYkpuqf4rxcgzgpGXnDSwAYjCZGfraX97ck3HBfI8hkvnoJCklmlaEnP5tuKt5HyX5YKxB4AiFaXsTW0rINClWVEmz13ZhuDvdnEMwLujEYZQnp0Go4/A2bjqXwy9Fk3v35xA32VOZt9XJqSddJMMUy0/B4sTPmPdgWjUrBvAfbMrhDHe5rJ2KzBJ5BBJd6EduQB1tKq6Rc2uzgPrkpi4yDGKf6FuP341igmw1uyAbxmPI3+ij3oisIb8i1CW94qlt9nr6tPjHh5owOgzvWETOGAo8iLC0vorUZHtqiKqUqqe25nesXF6RFhkHsNTVBqctkpmkhSlxPEmi7SLqhdIHXVP8D4G3DI/wj17Meu6NpTaYNaGEVLIGgIhCi5UUsIQ+SJNmXfy/lXbH1Gy14uB2v9W9hd9yIknH60WTIgdykOMlY5XeAuSiFM/xv11kavLIBAA16FqoXEyjp+N3YmhXGu+3OfXtwG6fuKRC4EyFaXqQwuNR+f6mWls22UiEx/Nb6nH7rHrtzzstRBUVR4XnVt3SUTtDlrc2l9uX0lSwmr/mb1747Yt33supLWirOck0O5UX9KLvwhv2v3UV0mOMoeIHAkwjR8iKW4haSJNkFhZa26NnWIrNsKxQSmybcZnfeelM31hhvRSnJvKd5n1BySu3LI8t2sXpPYTXu7oq/eVpltrgm6kdyhUjrsd9f7kG1YDE7KPAOQrS8SIDasaXlbHoZ2zCJRlGhJM62t7im64dy1hRFHekqs9QrSEkvuchtSmZhkdRqZDBPvRSAzwx3sdnU0Xps84u3E1+99CrRAoEnEaLlRQrLiDm/9MauIKqDIFRbsghinH4MBlnBvco/0R9YjSPScmxLg8m8o/6QKCmNk6bavGkYAsDCR9tzcNpdNKwZ4nRfBQJPIETLi2hLsLRKw3Z1jMKJCw/IjVlgGAxA2JYp7N63t9g5mXkG6/YTyl/ppTxAvqzmef1z5KOhY91IBraNFQGjAp9AiJYXKSnkoTRs1/QpS7mwp01prg+M97Lb1IxQKRfNumfAaF+Z2pK/q4mUxKuqLwCYbXiU43I8AJ8P7+J8BwUCDyNEy4sE2EXEO3eNnaVVRpHTM3P60bZOOCYUjNeNJl0Oor0ige8XvmB3rlIhoUXHQvViAiQ9W4xt+dTYh9a1wzkxqy+BmpJL2gsEFY0QLS9iGxHvrLFla2mVFs9lWQq0buytBKqVXKQGr+hHAHBP2irkM9ut5yolicmq/6OZIokrchgv60cBEt+N6WbjdxMIfAO/EK3333+fevXqERAQQJcuXfjrr7+83SW3YBvyUB5KGx7aOumPzexL27gIfjTdzFeG21FKMhc/+Q87jpgXUl/cs55hqp8BeFk/iquEk/Dm3aUuJxIIvIXPi9aXX37JhAkTmD59Ovv376dt27b06dOHlJQUb3fthrGEPJTbp1WKqBR10q8b042X+zTldcOTnDbVorZ0jetfjuFC0hlqbZ0AwApDX7aa2pE4+x67kmYCgS/h85/M+fPn8/TTTzNs2DBatGjB0qVLCQoKYsWKFd7u2g1TaGk5f41tyENpFpqjgtVjejRi4X9uZZx+DHpZSX/lLmp/3JaaUgbHTPG8bXiEf9+8W5SfF/g0Pi1aOp2Offv20atXL+s+hUJBr1692Llzp8Nr8vPzycjIsHv5KhYHt1qhICTAuYQbjvNpOTivhBLy3ZvU4G+5IfMMD1r3GWWJ5/VjOT77PlGbUODz+PQn9OrVqxiNRqKjo+32R0dHc/nyZYfXzJ49m/DwcOsrLi6uIrpaLtrWCefBjnUY06MRj3aO57YmNZk+oEWp1zSPCWVwhzqMvqOhw+OT+jajdkQgE+5q6vC4xbr70NifP4ytAFhV43l+nf2MsLAEfoEky76bzPvixYvUrl2bP//8k1tuucW6f+LEiWzbto3du3cXuyY/P5/8/MIlKRkZGcTFxZGenk5YmOM6glWN9Bw9+UYjB09f5nrSCR7u36fsiwQCD5CRkUF4eLhL30+fTgJYo0YNlEolycnJdvuTk5OpVauWw2u0Wi1arbYiuue3hAepATW929aDtvW83BuBwDV8enio0Wjo2LEjmzcXplUxmUxs3rzZzvISCARVB5+2tAAmTJjAk08+SadOnejcuTMLFiwgOzubYcOGebtrAoHAC/i8aD388MNcuXKFadOmcfnyZdq1a8fGjRuLOecFAkHVwKcd8e6gPI4+gUBQMZTn++nTPi2BQCAoihAtgUDgVwjREggEfoXPO+JvFIvLzpeX8wgEVRXL99IV13qlF63MzEwAn17OIxBUdTIzMwkPD3fq3Eo/e2gymbh48SKhoaFuW1tnWRqUlJTk1zOSleU5QDyLL+LMc8iyTGZmJrGxsShKy2ppQ6W3tBQKBXXq1PHIvcPCwvz6Q2WhsjwHiGfxRcp6DmctLAvCES8QCPwKIVoCgcCvEKJVDrRaLdOnT/f7bBKV5TlAPIsv4qnnqPSOeIFAULkQlpZAIPArhGgJBAK/QoiWQCDwK4RoCQQCv0KIlhOkpqYyZMgQwsLCiIiIYPjw4WRlZZV6/nPPPUfTpk0JDAwkPj6e559/nvT09ArstRlXq3N//fXXNGvWjICAAFq3bs2GDRsqqKdl48qzLF++nO7duxMZGUlkZCS9evXyqcrk5a2avnr1aiRJ4r777vNsB53E1edIS0tjzJgxxMTEoNVqadKkieufMVlQJn379pXbtm0r79q1S/7jjz/kRo0ayY8++miJ5x8+fFi+//775fXr18sJCQny5s2b5caNG8uDBw+uwF7L8urVq2WNRiOvWLFC/ueff+Snn35ajoiIkJOTkx2ev2PHDlmpVMrvvPOOfPToUXnq1KmyWq2WDx8+XKH9doSrz/LYY4/J77//vnzgwAH52LFj8tChQ+Xw8HD5/PnzFdzz4rj6LBYSExPl2rVry927d5fvvffeiulsKbj6HPn5+XKnTp3ke+65R96+fbucmJgob926VT548KBL7QrRKoOjR4/KgLxnzx7rvp9++kmWJEm+cOGC0/f56quvZI1GI+v1ek900yGdO3eWx4wZY/3baDTKsbGx8uzZsx2e/9BDD8n9+vWz29elSxf5mWee8Wg/ncHVZymKwWCQQ0ND5ZUrV3qqi05TnmcxGAxy165d5Y8++kh+8sknfUK0XH2OJUuWyA0aNJB1Ot0NtSuGh2Wwc+dOIiIi6NSpk3Vfr169UCgUDusuloQlnaxKVTHLPctTnXvnzp125wP06dOnxPMrivI8S1FycnLQ6/VUq1bNU910ivI+y4wZM4iKimL48OEV0c0yKc9zrF+/nltuuYUxY8YQHR1Nq1ateOuttzAajS61XekXTN8oly9fJioqym6fSqWiWrVqJVa5LsrVq1eZOXMmI0eO9EQXS2yzpOrcx48fd3jN5cuXXarmXVGU51mKMmnSJGJjY4uJckVTnmfZvn07H3/8MQcPHqyAHjpHeZ7j9OnT/PbbbwwZMoQNGzaQkJDA6NGj0ev1TJ8+3em2q6ylNXnyZCRJKvXl7BeiNDIyMujXrx8tWrTg9ddfv/GOC1xmzpw5rF69mrVr1xIQEODt7rhEZmYmTzzxBMuXL6dGjRre7s4NYTKZiIqKYtmyZXTs2JGHH36YV199laVLl7p0nyprab344osMHTq01HMaNGhArVq1SElJsdtvMBhITU0tscq1hczMTPr27UtoaChr165FrVbfaLedpjzVuWvVquXS+RVFeZ7Fwty5c5kzZw6bNm2iTZs2nuymU7j6LKdOneLMmTMMGDDAus9kMgFmi//EiRM0bNjQs512QHnek5iYGNRqNUql0rqvefPmXL58GZ1Oh0ajcartKmtp1axZk2bNmpX60mg03HLLLaSlpbFv3z7rtb/99hsmk4kuXbqUeP+MjAx69+6NRqNh/fr1Ff4LX57q3Lfccovd+QC//vqr16t5l7fS+DvvvMPMmTPZuHGjnU/Sm7j6LM2aNePw4cMcPHjQ+ho4cCA9evTg4MGDXsvIW573pFu3biQkJFhFF+DkyZPExMQ4LViACHlwhr59+8rt27eXd+/eLW/fvl1u3LixXcjD+fPn5aZNm8q7d++WZVmW09PT5S5dusitW7eWExIS5EuXLllfBoOhwvq9evVqWavVyp9++ql89OhReeTIkXJERIR8+fJlWZZl+YknnpAnT55sPX/Hjh2ySqWS586dKx87dkyePn26T4U8uPIsc+bMkTUajfzNN9/Y/f9nZmZ66xGsuPosRfGV2UNXn+PcuXNyaGioPHbsWPnEiRPyDz/8IEdFRcmzZs1yqV0hWk5w7do1+dFHH5VDQkLksLAwediwYXYf/sTERBmQt2zZIsuyLG/ZskUGHL4SExMrtO+LFi2S4+PjZY1GI3fu3FnetWuX9djtt98uP/nkk3bnf/XVV3KTJk1kjUYjt2zZUv7xxx8rtL+l4cqz1K1b1+H///Tp0yu+4w5w9X2xxVdES5Zdf44///xT7tKli6zVauUGDRrIb775pss/5CI1jUAg8CuqrE9LIBD4J0K0BAKBXyFESyAQ+BVCtAQCgV8hREsgEPgVQrQEAoFfIURLIBD4FUK0BAKBXyFES+AVhg4d6jCzRt++fb3dNYGPU2WzPAi8T9++ffnkk0/s9pVUjViv1xfLkuFKZgB3XCfwDYSlJfAaWq2WWrVq2b0iIyMBkCSJJUuWMHDgQIKDg3nzzTd5/fXXadeuHR999BH169e3Zs44d+4c9957LyEhIYSFhfHQQw/ZpUwp6TqBfyJES+CzvP766wwaNIjDhw/z1FNPAZCQkMCaNWv49ttvOXjwICaTiXvvvZfU1FS2bdvGr7/+yunTp3n44Yft7lX0OoH/IoaHAq/xww8/EBISYrfvlVde4ZVXXgHgscceY9iwYXbHdTodn332GTVr1gTM+b4OHz5MYmKiNbfUZ599RsuWLdmzZw833XSTw+sE/osQLYHX6NGjB0uWLLHbZ1t4wlHivrp169oJz7Fjx4iLi7NLhteiRQsiIiI4duyYVbSKXifwX4RoCbxGcHAwjRo1KvW4M/ucbUtQORA+LYFf07x5c5KSkkhKSrLuO3r0KGlpabRo0cKLPRN4CmFpCbxGfn5+sfJkKpXKpaozvXr1onXr1gwZMoQFCxZgMBgYPXo0t99+u8/khRe4F2FpCbzGxo0biYmJsXvdeuutLt1DkiTWrVtHZGQkt912G7169aJBgwZ8+eWXHuq1wNuIdMsCgcCvEJaWQCDwK4RoCQQCv0KIlkAg8CuEaAkEAr9CiJZAIPArhGgJBAK/QoiWQCDwK4RoCQQCv0KIlkAg8CuEaAkEAr9CiJZAIPArhGgJBAK/4v8BevYsS+1FeaMAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.018876, + "end_time": "2024-03-24T04:13:45.368020", + "exception": false, + "start_time": "2024-03-24T04:13:45.349144", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 5784.886195, + "end_time": "2024-03-24T04:13:48.109797", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/realtabformer/4/mlu-eval.ipynb", + "output_path": "eval/treatment/realtabformer/4/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/treatment/realtabformer/4", + "path_prefix": "../../../../", + "random_seed": 4, + "single_model": "realtabformer" + }, + "start_time": "2024-03-24T02:37:23.223602", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/realtabformer/model.pt b/treatment/realtabformer/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2d0670e70c58b05b269957db0dee4782d8f3210 --- /dev/null +++ b/treatment/realtabformer/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:975c41b72d75034c33164fbd50e39dbfc95bc53276b2426c7486aa545e464674 +size 80565037 diff --git a/treatment/realtabformer/params.json b/treatment/realtabformer/params.json new file mode 100644 index 0000000000000000000000000000000000000000..89043c64c6ec74cd99f7961a46c35eba034b2d1c --- /dev/null +++ b/treatment/realtabformer/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 1, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "torch", "grad_clip": 0.75, "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "loss_balancer_beta": 0.7999999999999999, "loss_balancer_r": 0.95, "tf_pma_low_exp_2": 3, "dataset_size_exp_2": 11, "batch_size_exp_2": 1, "epochs": 100, "lr_mul": 0.09, "n_warmup_steps": 100, "Optim": "diffgrad", "fixed_role_model": "realtabformer", "mse_mag_target": 0.2, "g_loss_mul": 0.1, "d_model_exp_2": 9, "attn_activation": "selu", "tf_d_inner_exp_2": 9, "tf_n_layers_enc": 4, "tf_n_head_exp_2": 6, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds_exp_2": 6, "ada_d_hid_exp_2": 10, "ada_n_layers": 8, "ada_activation": "leakyhardtanh", "ada_activation_final": "leakyhardsigmoid", "head_d_hid_exp_2": 7, "head_n_layers": 8, "head_n_head_exp_2": 6, "head_activation": "leakyhardsigmoid", "head_activation_final": "leakyhardsigmoid", "mse_mag": true, "mse_mag_multiply": true, "models": ["realtabformer"], "batch_size": 2, "max_seconds": 3600} \ No newline at end of file diff --git a/treatment/tab_ddpm_concat/eval.csv b/treatment/tab_ddpm_concat/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..f4e22be9d9c1e3a735a6ca48b1d9cffaa8192e3e --- /dev/null +++ b/treatment/tab_ddpm_concat/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tab_ddpm_concat,9.01566818356514e-06,0.019065888967249763,0.0021329918869256895,4.187764883041382,0.020785091444849968,0.7035645842552185,0.02979130670428276,1.004686600936111e-05,3.5432450771331787,0.03203897178173065,367951.71875,0.046184323728084564,0.2137211561203003,3.108413875452243e-05,7.7310099601745605 diff --git a/treatment/tab_ddpm_concat/history.csv b/treatment/tab_ddpm_concat/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..8caa67177839ffd4c53df25b6cc99bfc71ee6924 --- /dev/null +++ b/treatment/tab_ddpm_concat/history.csv @@ -0,0 +1,9 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.06340547758542622,6.714779690768261,0.0216546154567992,0.02457591868316134,0.0,0.0,0.0,0.0,0.25894568058539236,900,225,351.96785950660706,1.5643015978071424,0.3910753994517856,0.16249362598626046,0.01679980868561526,4.965276870460201,0.0022828475441168602,0.0,0.0,0.0,0.0,0.0,0.01679980868561526,450,113,98.52480149269104,0.8719008981654074,0.2189440033170912,0.09940910503458496 +1,0.010284998621031667,0.2255875137631098,0.0006483664865724725,0.03847181756566796,0.0,0.0,0.0,0.0,0.01054267091467207,900,225,345.0646252632141,1.533620556725396,0.383405139181349,0.2337058648798201,0.00841922588324514,3.2320097570229653,0.00070605612004593,0.0,0.0,0.0,0.0,0.0,0.00841922588324514,450,113,98.65367245674133,0.8730413491747021,0.21923038323720295,0.09933966914474768 +2,0.006607208507337469,0.5377882501872379,0.00014010620100567549,0.03052467739351818,0.0,0.0,0.0,0.0,0.0067294777951459155,900,225,346.46448493003845,1.5398421552446153,0.38496053881115383,0.23248913520145126,0.009977165092619644,2.5673703113514437,0.0019968906350478132,0.0,0.0,0.0,0.0,0.0,0.009977165092619644,450,113,97.69291520118713,0.8645390725768773,0.2170953671137492,0.10989512876503554 +3,0.005785522044221176,0.24223101447777884,0.0001330745897080286,0.030684991976660157,0.0,0.0,0.0,0.0,0.0061337225039238144,900,225,348.0978581905365,1.54710159195794,0.386775397989485,0.24560337937023077,0.007756494560071651,2.980153914541319,0.0005405202996384829,0.0,0.0,0.0,0.0,0.0,0.007756494560071651,450,113,98.20267009735107,0.8690501778526644,0.21822815577189128,0.10058078948723614 +4,0.007448349164959735,0.6328000489554128,0.0002071066759673005,0.02920709241492053,0.0,0.0,0.0,0.0,0.007665982826781854,900,225,346.766752243042,1.5411855655246312,0.3852963913811578,0.24027496062167403,0.0063575647280585046,3.3627717712450584,0.0004621822363243454,0.0,0.0,0.0,0.0,0.0,0.0063575647280585046,450,113,96.85975694656372,0.8571659906775551,0.21524390432569715,0.11672678085616316 +5,0.004543020532421198,0.5700210672009972,0.00011921863728762756,0.02387150165831877,0.0,0.0,0.0,0.0,0.0046356002659983155,900,225,346.64923548698425,1.540663268831041,0.3851658172077603,0.24926329110490364,0.007455925235068561,2.479032797534744,0.000639281961615226,0.0,0.0,0.0,0.0,0.0,0.007455925235068561,450,113,97.00784873962402,0.8584765375187967,0.2155729971991645,0.10956132564128933 +6,0.004305505793477601,0.2397287840495211,3.9819682542443306e-05,0.022900073562567638,0.0,0.0,0.0,0.0,0.0044476011980997205,900,225,346.15540647506714,1.5384684732225207,0.38461711830563017,0.23756346660614427,0.007061510610741443,3.6814213301790506,0.0005781446860156483,0.0,0.0,0.0,0.0,0.0,0.007061510610741443,450,113,96.51440978050232,0.8541098210663922,0.21447646617889404,0.1051591920189081 +7,0.004207614306922349,0.544365831900913,0.00016509564863483367,0.019844594347911578,0.0,0.0,0.0,0.0,0.004282006424441028,900,225,354.34157609939575,1.5748514493306478,0.39371286233266195,0.24139304114531518,0.006262556161943217,3.23558615521124,0.0006233526253580421,0.0,0.0,0.0,0.0,0.0,0.006262556161943217,450,113,99.24161124229431,0.8782443472769408,0.22053691387176513,0.1159596569104584 diff --git a/treatment/tab_ddpm_concat/mlu-eval-load.ipynb b/treatment/tab_ddpm_concat/mlu-eval-load.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..21841c37518b9ebd1cdb7ddf50a1eca6bb927277 --- /dev/null +++ b/treatment/tab_ddpm_concat/mlu-eval-load.ipynb @@ -0,0 +1,2000 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:29:40.916492Z", + "iopub.status.busy": "2024-03-26T13:29:40.915512Z", + "iopub.status.idle": "2024-03-26T13:29:40.956888Z", + "shell.execute_reply": "2024-03-26T13:29:40.955998Z" + }, + "papermill": { + "duration": 0.05794, + "end_time": "2024-03-26T13:29:40.959650", + "exception": false, + "start_time": "2024-03-26T13:29:40.901710", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:29:40.988772Z", + "iopub.status.busy": "2024-03-26T13:29:40.987896Z", + "iopub.status.idle": "2024-03-26T13:29:40.996380Z", + "shell.execute_reply": "2024-03-26T13:29:40.995302Z" + }, + "papermill": { + "duration": 0.026241, + "end_time": "2024-03-26T13:29:40.998908", + "exception": false, + "start_time": "2024-03-26T13:29:40.972667", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:29:41.024416Z", + "iopub.status.busy": "2024-03-26T13:29:41.023779Z", + "iopub.status.idle": "2024-03-26T13:29:41.028870Z", + "shell.execute_reply": "2024-03-26T13:29:41.027773Z" + }, + "papermill": { + "duration": 0.020416, + "end_time": "2024-03-26T13:29:41.031157", + "exception": false, + "start_time": "2024-03-26T13:29:41.010741", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:29:41.056109Z", + "iopub.status.busy": "2024-03-26T13:29:41.055766Z", + "iopub.status.idle": "2024-03-26T13:29:41.060804Z", + "shell.execute_reply": "2024-03-26T13:29:41.059607Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.020227, + "end_time": "2024-03-26T13:29:41.063218", + "exception": false, + "start_time": "2024-03-26T13:29:41.042991", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:29:41.090480Z", + "iopub.status.busy": "2024-03-26T13:29:41.089782Z", + "iopub.status.idle": "2024-03-26T13:29:41.096988Z", + "shell.execute_reply": "2024-03-26T13:29:41.095848Z" + }, + "papermill": { + "duration": 0.023716, + "end_time": "2024-03-26T13:29:41.099299", + "exception": false, + "start_time": "2024-03-26T13:29:41.075583", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "3b5141f3", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:29:41.124658Z", + "iopub.status.busy": "2024-03-26T13:29:41.124342Z", + "iopub.status.idle": "2024-03-26T13:29:41.130215Z", + "shell.execute_reply": "2024-03-26T13:29:41.129202Z" + }, + "papermill": { + "duration": 0.021494, + "end_time": "2024-03-26T13:29:41.132491", + "exception": false, + "start_time": "2024-03-26T13:29:41.110997", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"tab_ddpm_concat\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 0\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/tab_ddpm_concat/0\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011862, + "end_time": "2024-03-26T13:29:41.156318", + "exception": false, + "start_time": "2024-03-26T13:29:41.144456", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:29:41.182413Z", + "iopub.status.busy": "2024-03-26T13:29:41.181999Z", + "iopub.status.idle": "2024-03-26T13:29:41.193507Z", + "shell.execute_reply": "2024-03-26T13:29:41.192260Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.027349, + "end_time": "2024-03-26T13:29:41.195891", + "exception": false, + "start_time": "2024-03-26T13:29:41.168542", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/tab_ddpm_concat/0\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:29:41.223720Z", + "iopub.status.busy": "2024-03-26T13:29:41.223340Z", + "iopub.status.idle": "2024-03-26T13:29:43.343217Z", + "shell.execute_reply": "2024-03-26T13:29:43.342099Z" + }, + "papermill": { + "duration": 2.136533, + "end_time": "2024-03-26T13:29:43.345517", + "exception": false, + "start_time": "2024-03-26T13:29:41.208984", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:29:43.368565Z", + "iopub.status.busy": "2024-03-26T13:29:43.368125Z", + "iopub.status.idle": "2024-03-26T13:29:43.383790Z", + "shell.execute_reply": "2024-03-26T13:29:43.382966Z" + }, + "papermill": { + "duration": 0.028847, + "end_time": "2024-03-26T13:29:43.385760", + "exception": false, + "start_time": "2024-03-26T13:29:43.356913", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:29:43.408127Z", + "iopub.status.busy": "2024-03-26T13:29:43.407796Z", + "iopub.status.idle": "2024-03-26T13:29:43.416144Z", + "shell.execute_reply": "2024-03-26T13:29:43.415302Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.022101, + "end_time": "2024-03-26T13:29:43.418324", + "exception": false, + "start_time": "2024-03-26T13:29:43.396223", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:29:43.443994Z", + "iopub.status.busy": "2024-03-26T13:29:43.443645Z", + "iopub.status.idle": "2024-03-26T13:29:43.545341Z", + "shell.execute_reply": "2024-03-26T13:29:43.544327Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.116891, + "end_time": "2024-03-26T13:29:43.547605", + "exception": false, + "start_time": "2024-03-26T13:29:43.430714", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:29:43.569381Z", + "iopub.status.busy": "2024-03-26T13:29:43.569067Z", + "iopub.status.idle": "2024-03-26T13:29:48.415698Z", + "shell.execute_reply": "2024-03-26T13:29:48.414636Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.860445, + "end_time": "2024-03-26T13:29:48.418347", + "exception": false, + "start_time": "2024-03-26T13:29:43.557902", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-26 13:29:45.879410: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-26 13:29:45.879465: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-26 13:29:45.881169: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:29:48.445197Z", + "iopub.status.busy": "2024-03-26T13:29:48.444399Z", + "iopub.status.idle": "2024-03-26T13:29:48.451361Z", + "shell.execute_reply": "2024-03-26T13:29:48.450457Z" + }, + "papermill": { + "duration": 0.023358, + "end_time": "2024-03-26T13:29:48.453547", + "exception": false, + "start_time": "2024-03-26T13:29:48.430189", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:29:48.477375Z", + "iopub.status.busy": "2024-03-26T13:29:48.477069Z", + "iopub.status.idle": "2024-03-26T13:30:11.398548Z", + "shell.execute_reply": "2024-03-26T13:30:11.397495Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 22.936143, + "end_time": "2024-03-26T13:30:11.401060", + "exception": false, + "start_time": "2024-03-26T13:29:48.464917", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-26T13:30:11.428028Z", + "iopub.status.busy": "2024-03-26T13:30:11.427631Z", + "iopub.status.idle": "2024-03-26T13:30:11.434689Z", + "shell.execute_reply": "2024-03-26T13:30:11.433764Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.02279, + "end_time": "2024-03-26T13:30:11.436782", + "exception": false, + "start_time": "2024-03-26T13:30:11.413992", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 95,\n", + " 'realtabformer': (69, 281, Embedding(281, 768), True),\n", + " 'lct_gan': 75,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:30:11.461945Z", + "iopub.status.busy": "2024-03-26T13:30:11.461198Z", + "iopub.status.idle": "2024-03-26T13:30:40.397647Z", + "shell.execute_reply": "2024-03-26T13:30:40.396219Z" + }, + "papermill": { + "duration": 28.951677, + "end_time": "2024-03-26T13:30:40.400323", + "exception": false, + "start_time": "2024-03-26T13:30:11.448646", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/tab_ddpm_concat/all inf False\n", + "../../../../ml-utility-loss/aug_test/treatment 0\n", + "Caching in ../../../../treatment/_cache_bs_test/tab_ddpm_concat/all inf False\n", + "../../../../ml-utility-loss/bs_test/treatment 0\n", + "Caching in ../../../../treatment/_cache_synth_test/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/treatment 600\n", + "600\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:30:40.435565Z", + "iopub.status.busy": "2024-03-26T13:30:40.435183Z", + "iopub.status.idle": "2024-03-26T13:30:40.775853Z", + "shell.execute_reply": "2024-03-26T13:30:40.774858Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.360692, + "end_time": "2024-03-26T13:30:40.778139", + "exception": false, + "start_time": "2024-03-26T13:30:40.417447", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 4,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.75,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.1,\n", + " 'loss_balancer_beta': 0.7999999999999999,\n", + " 'loss_balancer_r': 0.96,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.04,\n", + " 'n_warmup_steps': 140,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'tab_ddpm_concat',\n", + " 'g_loss_mul': 0.1,\n", + " 'd_model': 512,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.SELU,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tab_ddpm_concat'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "# params = {\n", + "# **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "# }\n", + "params = getattr(PARAMS, dataset_name).BEST_DICT[gp][gp_multiply][single_model]\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).DEFAULTS,\n", + " **params,\n", + "}\n", + "if isinstance(params, (list, tuple)):\n", + " params = params[param_index]\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-26T13:30:40.807599Z", + "iopub.status.busy": "2024-03-26T13:30:40.807269Z", + "iopub.status.idle": "2024-03-26T13:30:41.349764Z", + "shell.execute_reply": "2024-03-26T13:30:41.348686Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.559832, + "end_time": "2024-03-26T13:30:41.352116", + "exception": false, + "start_time": "2024-03-26T13:30:40.792284", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n", + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['tab_ddpm_concat'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:30:41.380403Z", + "iopub.status.busy": "2024-03-26T13:30:41.380078Z", + "iopub.status.idle": "2024-03-26T13:30:41.384971Z", + "shell.execute_reply": "2024-03-26T13:30:41.384067Z" + }, + "papermill": { + "duration": 0.021186, + "end_time": "2024-03-26T13:30:41.387020", + "exception": false, + "start_time": "2024-03-26T13:30:41.365834", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:30:41.412625Z", + "iopub.status.busy": "2024-03-26T13:30:41.412278Z", + "iopub.status.idle": "2024-03-26T13:30:41.419620Z", + "shell.execute_reply": "2024-03-26T13:30:41.418751Z" + }, + "papermill": { + "duration": 0.022917, + "end_time": "2024-03-26T13:30:41.421681", + "exception": false, + "start_time": "2024-03-26T13:30:41.398764", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "18316289" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:30:41.448884Z", + "iopub.status.busy": "2024-03-26T13:30:41.448330Z", + "iopub.status.idle": "2024-03-26T13:30:41.546904Z", + "shell.execute_reply": "2024-03-26T13:30:41.545896Z" + }, + "papermill": { + "duration": 0.114874, + "end_time": "2024-03-26T13:30:41.549422", + "exception": false, + "start_time": "2024-03-26T13:30:41.434548", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 12] --\n", + "├─Adapter: 1-1 [2, 2648, 12] --\n", + "│ └─Sequential: 2-1 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 1024] 13,312\n", + "│ │ │ └─SELU: 4-2 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-4 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-6 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-8 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-10 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-12 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 2648, 512] 524,800\n", + "│ │ │ └─LeakyHardsigmoid: 4-14 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 12] (recursive)\n", + "│ └─Sequential: 2-2 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-16 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-18 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-20 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-22 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-24 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-26 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-28 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 2048] --\n", + "│ └─Encoder: 2-3 [2, 4, 512] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-6 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-18 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-30 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 4, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-42 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 262,656\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 4, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 4, 512] 2,048\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 4, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 4, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 4, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 4, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 4, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-54 [2, 4, 512] --\n", + "│ └─Encoder: 2-4 [2, 4, 512] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-60 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-31 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-72 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-40 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-84 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-42 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-43 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 4, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-45 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-46 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-96 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-48 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-49 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-18 [2, 4, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-51 [2, 4, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-52 [2, 4, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 4, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 4, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 4, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 4, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-108 [2, 4, 512] --\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 256] --\n", + "│ │ │ └─Linear: 4-37 [2, 256] 524,544\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 256] --\n", + "│ │ └─FeedForward: 3-18 [2, 256] --\n", + "│ │ │ └─Linear: 4-39 [2, 256] 65,792\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 256] --\n", + "│ │ └─FeedForward: 3-19 [2, 256] --\n", + "│ │ │ └─Linear: 4-41 [2, 256] 65,792\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 256] --\n", + "│ │ └─FeedForward: 3-20 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 65,792\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─LeakyHardsigmoid: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─LeakyHardsigmoid: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 1] --\n", + "│ │ │ └─Linear: 4-51 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-52 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 18,316,289\n", + "Trainable params: 18,316,289\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 71.16\n", + "========================================================================================================================\n", + "Input size (MB): 0.32\n", + "Forward/backward pass size (MB): 1067.90\n", + "Params size (MB): 73.27\n", + "Estimated Total Size (MB): 1141.48\n", + "========================================================================================================================" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = test_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "90576617", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:30:41.580652Z", + "iopub.status.busy": "2024-03-26T13:30:41.580247Z", + "iopub.status.idle": "2024-03-26T13:30:41.671044Z", + "shell.execute_reply": "2024-03-26T13:30:41.670123Z" + }, + "papermill": { + "duration": 0.108813, + "end_time": "2024-03-26T13:30:41.673207", + "exception": false, + "start_time": "2024-03-26T13:30:41.564394", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torch\n", + "model.load_state_dict(torch.load(\"model.pt\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:30:41.703510Z", + "iopub.status.busy": "2024-03-26T13:30:41.703187Z", + "iopub.status.idle": "2024-03-26T13:33:22.461795Z", + "shell.execute_reply": "2024-03-26T13:33:22.460959Z" + }, + "papermill": { + "duration": 160.776809, + "end_time": "2024-03-26T13:33:22.464382", + "exception": false, + "start_time": "2024-03-26T13:30:41.687573", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:33:22.495706Z", + "iopub.status.busy": "2024-03-26T13:33:22.494967Z", + "iopub.status.idle": "2024-03-26T13:33:22.516986Z", + "shell.execute_reply": "2024-03-26T13:33:22.516094Z" + }, + "papermill": { + "duration": 0.040116, + "end_time": "2024-03-26T13:33:22.519017", + "exception": false, + "start_time": "2024-03-26T13:33:22.478901", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tab_ddpm_concat0.0000090.0190660.0021334.1877650.0207850.7035650.0297910.000013.5432450.032039367951.718750.0461840.2137210.0000317.73101
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "tab_ddpm_concat 0.000009 0.019066 0.002133 4.187765 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss \\\n", + "tab_ddpm_concat 0.020785 0.703565 0.029791 0.00001 \n", + "\n", + " pred_duration pred_mae pred_mape pred_rmse pred_std \\\n", + "tab_ddpm_concat 3.543245 0.032039 367951.71875 0.046184 0.213721 \n", + "\n", + " std_loss total_duration \n", + "tab_ddpm_concat 0.000031 7.73101 " + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:33:22.546969Z", + "iopub.status.busy": "2024-03-26T13:33:22.546672Z", + "iopub.status.idle": "2024-03-26T13:33:23.106239Z", + "shell.execute_reply": "2024-03-26T13:33:23.105305Z" + }, + "papermill": { + "duration": 0.575915, + "end_time": "2024-03-26T13:33:23.108332", + "exception": false, + "start_time": "2024-03-26T13:33:22.532417", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:33:23.137946Z", + "iopub.status.busy": "2024-03-26T13:33:23.137516Z", + "iopub.status.idle": "2024-03-26T13:36:16.079281Z", + "shell.execute_reply": "2024-03-26T13:36:16.078262Z" + }, + "papermill": { + "duration": 172.95974, + "end_time": "2024-03-26T13:36:16.082155", + "exception": false, + "start_time": "2024-03-26T13:33:23.122415", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/tab_ddpm_concat/all inf False\n", + "Caching in ../../../../treatment/_cache_bs_test/tab_ddpm_concat/all inf False\n", + "Caching in ../../../../treatment/_cache_synth_test/tab_ddpm_concat/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:36:16.111951Z", + "iopub.status.busy": "2024-03-26T13:36:16.111615Z", + "iopub.status.idle": "2024-03-26T13:36:16.132510Z", + "shell.execute_reply": "2024-03-26T13:36:16.131643Z" + }, + "papermill": { + "duration": 0.038087, + "end_time": "2024-03-26T13:36:16.134739", + "exception": false, + "start_time": "2024-03-26T13:36:16.096652", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:36:16.163047Z", + "iopub.status.busy": "2024-03-26T13:36:16.162743Z", + "iopub.status.idle": "2024-03-26T13:36:16.168401Z", + "shell.execute_reply": "2024-03-26T13:36:16.167518Z" + }, + "papermill": { + "duration": 0.021896, + "end_time": "2024-03-26T13:36:16.170313", + "exception": false, + "start_time": "2024-03-26T13:36:16.148417", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tab_ddpm_concat': 0.46246694731147725}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:36:16.198406Z", + "iopub.status.busy": "2024-03-26T13:36:16.197645Z", + "iopub.status.idle": "2024-03-26T13:36:16.615320Z", + "shell.execute_reply": "2024-03-26T13:36:16.614361Z" + }, + "papermill": { + "duration": 0.434256, + "end_time": "2024-03-26T13:36:16.617978", + "exception": false, + "start_time": "2024-03-26T13:36:16.183722", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABFQUlEQVR4nO3deXhUVZ74//etfUlVZd9IQtgREIgINtqKqCMqY4P29LTaKqi09jSOrYwzLf19xrX7oZ1xbRfaX88Ibbu044zatrai4wi0u+yI7EsIJCF7JbWk1vP746YKygRIKpXcSuq8nidPUrdO7v1UJfnknHPPogghBJIkSRrSaR2AJEmSTESSJGlOJiJJkjQnE5EkSZqTiUiSJM3JRCRJkuZkIpIkSXMyEUmSpDmZiCRJ0pxMRENMZWUlf/u3fzug11AUhfvvv/+05e6//34URRnQWKTMIBPRIPv000+5//77aWtr0zoUaQh49tlnWb16tdZhDDiZiAbZp59+ygMPPCATkdQrMhFJkiQNEpmIBtH999/PP//zPwMwatQoFEVBURQOHTrEqlWruOiiiygsLMRsNjNp0iRWrlx50nO9//77TJ8+HYvFwqRJk3j99df7HE8gEOCuu+6ioKAAh8PB9773PY4cOdJj2Y8//piZM2disVgYM2YMzz33XI/lFEXh9ttv56WXXmLChAlYLBZmzJjB+vXru70XiqKwZ88err/+elwuFwUFBfzrv/4rQghqampYsGABTqeT4uJiHn300T6/PoB3332XOXPm4HA4cDqdzJw5k5dffjmhzGuvvcaMGTOwWq3k5+dz/fXXc/To0YQyixcvJisri6NHj7Jw4UKysrIoKCjg7rvvJhKJJJSNRqM8+eSTnHnmmVgsFgoKCrjsssvYsGFDvExvft6VlZXs2LGDdevWxX9XLrzwwqTeh7QnpEGzdetWce211wpAPP744+IPf/iD+MMf/iA8Ho+YOXOmWLx4sXj88cfFU089JS699FIBiKeffjrhHCNHjhTjx48X2dnZ4p577hGPPfaYOPPMM4VOpxPvv/9+n+K5/vrrBSCuu+468fTTT4urr75aTJ06VQDivvvui5fbtm2bsFqtoqKiQqxYsUI89NBDoqioKF72RICYMmWKyM/PFw8++KB4+OGHxciRI4XVahXbt2+Pl7vvvvsEIKZPny6uvfZa8eyzz4r58+cLQDz22GNiwoQJ4h/+4R/Es88+K8477zwBiHXr1vXp9a1atUooiiKmTJkifvWrX4lnnnlGLFmyRNxwww0JZQAxc+ZM8fjjj4t77rlHWK1WUVlZKVpbW+PlFi1aJCwWi5g8ebK4+eabxcqVK8X3v/99AYhnn3024bqLFy8WgLj88svFE088IR555BGxYMEC8dRTT8XL9Obn/cYbb4iysjIxceLE+O9KX3/GQ4VMRIPs3//93wUgDh48mHDc5/N1Kztv3jwxevTohGMjR44UgPif//mf+DG32y1KSkpEVVVVr+PYsmWLAMRPf/rThOPXXXddt0S0cOFCYbFYRHV1dfzYN998I/R6fY+JCBAbNmyIH6uurhYWi0VcddVV8WOxRHTrrbfGj4XDYVFWViYURRG//vWv48dbW1uF1WoVixYt6vXra2trEw6HQ5xzzjnC7/cnPBeNRoUQQgSDQVFYWCimTJmSUObtt98WgLj33nvjxxYtWiQA8eCDDyacq6qqSsyYMSP++P/+7/8EIO64445uMcWuK0Tvf96TJ08Wc+bM6cUrHtpk0yxNWK3W+Ndut5umpibmzJnDgQMHcLvdCWVLS0u56qqr4o+dTic33ngjmzdvpr6+vlfX+8tf/gLAHXfckXD8zjvvTHgciURYs2YNCxcupKKiIn78jDPOYN68eT2ee/bs2cyYMSP+uKKiggULFrBmzZpuzZglS5bEv9br9Zx99tkIIbjlllvix7Ozs5kwYQIHDhzo1WsD+OCDD+jo6OCee+7BYrEkPBcbcrBhwwYaGhr46U9/mlBm/vz5TJw4kXfeeafbeX/yk58kPD7//PMT4vqf//kfFEXhvvvu6/a9Jw516MvPOxPIRJQmPvnkEy655BLsdjvZ2dkUFBTwi1/8AqDbL+bYsWO7jd8ZP348AIcOHerV9aqrq9HpdIwZMybh+IQJExIeNzY24vf7GTduXLdzfLtsTE9lx48fj8/no7GxMeH4ickNwOVyYbFYyM/P73a8tbX15C/oW/bv3w/AlClTTlqmuroa6Pl1TJw4Mf58TKy/50Q5OTkJce3fv5/S0lJyc3NPGV9fft6ZwKB1AJL6y3vxxRczceJEHnvsMcrLyzGZTPzlL3/h8ccfJxqNah3igNHr9b06BiA0XtX4ZHH1VSb/vE9GJqJB1tNI5D//+c8EAgHeeuuthBrCRx991OM59u3bhxAi4Vx79uwB1DstvTFy5Eii0Sj79+9PqBHs3r07oVxBQQFWq5W9e/d2O8e3y8b0VHbPnj3YbLZuNYqBEqvpff3114wdO7bHMiNHjgTU13HRRRclPLd79+7483297po1a2hpaTlpragvP+9MGbkum2aDzG63AyQMaIz9pz3xP77b7WbVqlU9nqO2tpY33ngj/ri9vZ0XXniB6dOnU1xc3Ks4Lr/8cgB+85vfJBx/4oknEh7r9XrmzZvHm2++yeHDh+PHd+7cyZo1a3o892effcamTZvij2tqavjTn/7EpZdemrJaxelceumlOBwOVqxYQWdnZ8Jzsff57LPPprCwkN/+9rcEAoH48++++y47d+5k/vz5fb7u97//fYQQPPDAA92ei123Lz9vu92eEYNfZY1okMU6cf/f//t/XHPNNRiNRi644AJMJhNXXnklt912Gx6Ph9/97ncUFhZSV1fX7Rzjx4/nlltu4auvvqKoqIjnn3+eY8eOnTRx9WT69Olce+21PPvss7jdbs4991w+/PBD9u3b163sAw88wHvvvcf555/PT3/6U8LhME899RSTJ09m27Zt3cpPmTKFefPmcccdd2A2m3n22Wfj5xksTqeTxx9/nCVLljBz5kyuu+46cnJy2Lp1Kz6fj9///vcYjUYefvhhbrrpJubMmcO1117LsWPHePLJJ6msrOSuu+7q83Xnzp3LDTfcwG9+8xv27t3LZZddRjQa5a9//Stz587l9ttv59JLL+31z3vGjBmsXLmSX/7yl4wdO5bCwsJutbdhQbsbdpnroYceEiNGjBA6nS5+K/+tt94SU6dOFRaLRVRWVoqHH35YPP/8891u9Y8cOVLMnz9frFmzRkydOlWYzWYxceJE8dprr/U5Dr/fL+644w6Rl5cn7Ha7uPLKK0VNTU232/dCCLFu3ToxY8YMYTKZxOjRo8Vvf/vb+C34EwFi6dKl4sUXXxTjxo0TZrNZVFVViY8++iihXOx7GxsbE44vWrRI2O32brHOmTNHTJ48uc+v8a233hLnnnuusFqtwul0ilmzZolXXnklocyrr74qqqqqhNlsFrm5ueJHP/qROHLkSK/i6uk9CIfD4t///d/FxIkThclkEgUFBeLyyy8XGzduTIirNz/v+vp6MX/+fOFwOAQwbG/lK0LIfc2k1FEUhaVLl/L0009rHYo0hMg+IkmSNCf7iIah0w1qtFqtuFyuQYom9RobG7sNjDyRyWQ67TgeKb3IRDQMlZSUnPL5RYsWDemlJWbOnNltsOGJ5syZw9q1awcvIKnfZCIahj744INTPl9aWjpg1x6MLseXXnoJv99/0udzcnIGPAYptWRntSRJmpOd1ZIkaW5IN82i0Si1tbU4HI6MGQovSUOJEIKOjg5KS0vR6U5e7xnSiai2tpby8nKtw5Ak6TRqamooKys76fNDOhE5HA5AfZFOp1PjaCRJ+rb29nbKy8vjf6snM6QTUaw55nQ6ZSKSpDR2uq4T2VktSZLmZCKSJElzMhFJkqS5Id1H1BtCCMLh8CnnJkk90+v1GAwGOTRCGnDDOhEFg0Hq6urw+XxahzJk2Ww2SkpKMJlMWociDWPDNhFFo1EOHjyIXq+ntLQUk8kk/7P3gRCCYDBIY2MjBw8eZNy4cacckCZJ/TFsE1EwGCQajVJeXo7NZtM6nCHJarViNBqprq4mGAx22x9sODvQ6EEAYwqytA4lIwz7f3Hyv3j/ZOL75/aFeGdbHQ3tgdMXllIi837LJOk0NlS3EI4KWn1BzfdSyxQyEUnSCSJRwZ5jHgCmlLoIRwUHGj00eWTtaCDJRJThKisru+1llslq2/x0hiLYzXrKcqys293In7bU8vXRzNsGejDJRCRJJzjapq78WJZjQ6dTGJmn3uioaZFDQAaSTETDQDAY1DqEYeNIaywRWQEoyVY/N3uDBMJyUOxAychEFAxHT/oRjkR7XTbUi7LJuPDCC7n99tu5/fbbcblc5Ofn86//+q/xjtPKykoeeughbrzxRpxOJ7feeisAH3/8Meeffz5Wq5Xy8nLuuOMOvF5v/LwNDQ1ceeWVWK1WRo0axUsvvZRUfMNVNCqod6uJaERXAsoyG3BYDAiBvIs2gIbtOKJTeeaj7tsqx4zKt7OwakT88f+3fj+hSM93TspyrPzg7OMLsz3/yUH8wcT/mnf9zfikYvz973/PLbfcwpdffsmGDRu49dZbqaio4Mc//jEAjzzyCPfeey/33XcfAPv37+eyyy7jl7/8Jc8//zyNjY3xZBbbinrx4sXU1tby0UcfYTQaueOOO2hoaEgqvuFIAJdNKaHJEyDHdnwkebHLQkenh2PtnZTnyjFpAyEjE9FQUF5ezuOPP46iKEyYMIHt27fz+OOPxxPRRRddxD/90z/Fyy9ZsoQf/ehH3HnnnQCMGzeO3/zmN8yZM4eVK1dy+PBh3n33Xb788ktmzpwJwH/+539yxhlnDPprS1d6ncLYwizGFiYOYix0WNh7TN45G0gZmYiWzh170ud035oFcusFY05a9tszRm4+b1R/wkrwne98J2FKyuzZs3n00Ufjk3fPPvvshPJbt25l27ZtCc0tIUR8qsuePXswGAzMmDEj/vzEiRPJzs5OWczDVa5drR01e2Vf3EDJyERkMvS+a2ygyvaX3W5PeOzxeLjtttu44447upWtqKhgz549gxXakLW7vgODXmFEthWLUR8/XuKyMG9yMflZcuLvQMnIRDQUfPHFFwmPP//8c8aNG4der++x/FlnncU333zD2LE91/YmTpxIOBxm48aN8abZ7t27aWtrS2ncQ9nH+5po94f4uxllCX1BdrOBSaVyKeKBlJF3zYaCw4cPs2zZMnbv3s0rr7zCU089xc9+9rOTlv/5z3/Op59+yu23386WLVvYu3cvf/rTn7j99tsBmDBhApdddhm33XYbX3zxBRs3bmTJkiVYrdbBeklpLRyJ0tEZAo43xaTBIxNRmrrxxhvx+/3MmjWLpUuX8rOf/Sx+m74nU6dOZd26dezZs4fzzz+fqqoq7r333oTtpVetWkVpaSlz5szh6quv5tZbb6WwsHAwXk7aa+8MI4TavLaZutc6G9o72VrTFh/wKKWWbJqlKaPRyBNPPMHKlSu7PXfo0KEev2fmzJm8//77Jz1ncXExb7/9dsKxG264oV9xDhdtPrUj2mU19rhu1a76DjZWt1JVkR0fYySljqwRSRLQ5lebZdk2Y4/Pu6zq8fbO8KDFlElkIpIk1DWIALKtPfcPxRKRuythSaklm2ZpaO3atVqHkHHa/GrT7LQ1In8IIYRcdjjFZCKSJOD8cQVMLg1S5Ox5OVyHxYCiqPMJ/aEINpP800mltGma/frXv0ZRlPgUBUkaTPlZZsYXOeI1n28z6HVkmdXkI5tnqZcWieirr77iueeeY+rUqVqHIkkn5exKUh2ywzrlNE9EHo+HH/3oR/zud78jJydH63CkDNTmC7LpcCvVzd5Tlvvu2Hz+fmY5FXIGfsppnoiWLl3K/PnzueSSS05bNhAI0N7envAhSf1V5+5k3e5GvjzYkviEENC8Hxr3QDRCaba12zw0KTU07XH74x//yKZNm/jqq696VX7FihU88MADAxyVlGk8AbWp5bB8q39o3//CkQ3q13lj4MwfdF9yQUoJzWpENTU1/OxnP+Oll17q9cZ9y5cvx+12xz9qamoGOEopE8TmmDktJ/xfbj10PAkBNO/HV7OFLTVtbK1pG9T4MoFmNaKNGzfS0NDAWWedFT8WiURYv349Tz/9NIFAoNtMc7PZjNlsHuxQpWEu1vmcUCOKJaHSKrC44MBaQoc+46NOGw6rkWnl2YMf6DCmWSK6+OKL2b59e8Kxm266iYkTJ/Lzn//8pMtdSFKqxaZtZMVqRJ1uaO5aTrhsJpjscOhjLIEW7MFGvLoiolGB7tur6ElJ0ywRORwOpkyZknDMbreTl5fX7XjKCAERDcaA6I297lt44YUXuOuuu6itrU2o/S1cuBCHw8Ef/vCHgYoyY3niNaKuP4emverviqsM7HnqsbwxmBp2kdtZQ425EF8oEh9XJPVfZr2TkRD89dHBv+75/wSG3q1x84Mf/IA77riDt956ix/84AeAuvvGO++8c8qZ9VJyguEonSF1+d14Imo5oH7OH3e8YO5olMbdFIWPUsMMPJ1hmYhSKK3eSTnHCqxWK9dddx2rVq2KJ6IXX3yRiooKLrzwQm2DG4b0OoVrZpXjDYQxG/TqP6vWavXJ3BPWK88dDUB2pBl9pBNPIAT07iaLdHpplYgGnN6o1k60uG4f/PjHP2bmzJkcPXqUESNGsHr1ahYvXiwnWg4AvU6hxHXC+kIddRANgzkL7PnHj1ucYMvFrO/AEWygo7O8+8mkpGVWIlKUXjeRtFRVVcW0adN44YUXuPTSS9mxYwfvvPOO1mFlBvdR9bNzRPd+PUcJRkMNWcFGfEG562sqZVYiGkKWLFnCE088wdGjR7nkkksoL5f/gQdCTYuPRk+AUpeVYpcF2k9IRN/mHEGx08LFzghmefs+pTSf4iH17LrrruPIkSP87ne/4+abb9Y6nGFrX4OHdbsb2d/oUe+UtdeqTzhLuxd2lmA16skNN2HvYV1rKXkyEaUpl8vF97//fbKysli4cKHW4Qxb3qB6695m0qvjh4Je0OnBUdy9sL1Qba6F/BD0DHKkw5tMRGns6NGj/OhHP5KjyQeQL6D29djNBvA0qAdteT3fYNAbCJmyqXP72bpr3yBGOfzJRJSGWltbeeONN1i7di1Lly7VOpxhzXdijcjblYiyTr7FUsSWz6FmH7v2HyAaFYMRYkaQndVpqKqqitbWVh5++GEmTJigdTjDmrfr7pfdZABvo3rQfvJEZHQWogCWYAudYblkbKrIdzENnWzfMim1guEowXAUAJtZD56uRJRVcNLv0WcVYtAr2EIteAMyEaWKbJpJGcvfVRsy6hVMRMDftTDaKWpE2Asw6nVYQ234AnLt6lQZ9ulcCNmO74/h/P7ZzXqumVVOIBRF8TWrt++NVnW2/clYczAa9OiDQfweN+RnDV7Aw9iwrREZjepdD5/Pp3EkQ1vs/Yu9n8OJQa+jxGWlMt+e2FF9qqk0Oh2KJRuAYEfzwAeZIYZtjUiv15OdnU1Dg/oLZrPZ5FytPhBC4PP5aGhoIDs7e/ivD+XrapbZ8k9dDlDsOdDSQMgrE1GqDNtEBFBcrA5KiyUjqe+ys7Pj7+Nwc7jZR5M3wIhsK0Wx/iFb7mm/r6yklBGROgzZcr5ZqgzrRKQoCiUlJRQWFhIKyY7FvjIajcO6JrTnWAfbj7r5zug8imI1Iuvpt7TKyi6EJiOE5S4yqTKsE1GMXq8f1n9QUnJi0zvsJgX8berBXtSI4snK3zowgWWgjEhEktST2FIeWfjVNYh0ejC7Tvt9nUYnrW4/4Y46yoWQWwylgExEUsbydu1nlhXtamJZc0B3+hvJYZOTQ81+FHyUBb0oZnkLv7+G7e17SToVIUR8QKMtfEIi6gWr2UxAb0cAgXZ55ywVZCKSMlIgHCXcNWnVGm5TD/amfwh1edmI2QlAp6flNKWl3pCJSMpIsf4hs1GHvrNNPWjtXSICUCxqX1LQKzusU0H2EUkZyWExcM2sckJhAXs/UA/2skYExEdXh73uAYgu88hEJGUkY9f0DiJhdWVG6FONSGdTm2YhX9sARJd5ZNNMymydbnWyq8F06smu32KwZgMQ9csaUSrIGpGUkQ41eWn2BhlJA/kAFlefxgNNGlWGodmJ2RJUE5kcS9QvskYkZaTdxzpYv6eRY43H1ANdfT69lZObj8NiVNcxCgdSH2CGkYlIykixtart0a7dOCynH1GdQG9U1y6C431MUtJkIpIykrdr9w5bkonIGwhz2Gegzu2HgJz82l8yEUkZKVYjskQ61AN9TUTBMNuboLbND50yEfWXTERSxolGRXxAoyWcXI3IZjIQMGQRigiEbJr1m0xEUsbxhyIIAXoRxhjxqwf7mIisRj1BfRYCCHrbUh5jppGJSMo4sXWIXIpPXT7YYD7e8dxLep2CsMhBjakixxFJGSfHZuLaWRWI5n1QTd/vmHXRdSWisE82zfpLJiIp4xj1OopdFvB0jf9JMhEZrQ4AIp0eOaixn2TTTMpcsU7mPg5mjDHYnIBCOByGkNy2qj9kjUjKOAcaPbT6QoxtbcIFSdeIvjOmAEtzMVbRCYGOPs1VkxJpWiNauXIlU6dOxel04nQ6mT17Nu+++66WIUkZYHe9Or2jraVJPZBkIip0WHC6cjDqdRDwpDDCzKNpIiorK+PXv/41GzduZMOGDVx00UUsWLCAHTt2aBmWNMx5u8YQWZOd3nGirpUaCXb0M6rMpmnT7Morr0x4/Ktf/YqVK1fy+eefM3nyZI2ikoY7XzCMLhrCHO0EjEknIrcvRKMbnB2dFAZkIuqPtOkjikQivPbaa3i9XmbPnt1jmUAgQCBwfKZze7scWi/1nTcQwRzxYNTHxhBZkjpPqy/IpvoQE/ydFMqmWb9oftds+/btZGVlYTab+clPfsIbb7zBpEmTeiy7YsUKXC5X/KO8vHyQo5WGunAkSmcogjncjsmg61ezzGZSR1eHI0LtrJaSpnkimjBhAlu2bOGLL77gH/7hH1i0aBHffPNNj2WXL1+O2+2Of9TU1AxytNJQ5wsdn3Wv1yn9SkQWk56g3kY4GkXIRNQvmjfNTCYTY8eOBWDGjBl89dVXPPnkkzz33HPdyprNZsxm82CHKA0jvq7lP5z4UFCSHkME6nyzkN5GVEC4swNjimLMRJrXiL4tGo0m9ANJUirlZanTO2aV6NUD/agRGfU6hFkdXR3u9KoL8UtJ0bRGtHz5ci6//HIqKiro6Ojg5ZdfZu3ataxZs0bLsKRhLD69Q5/crPtu5zPbiCp6wpGoegu/l7vFSok0TUQNDQ3ceOON1NXV4XK5mDp1KmvWrOFv/uZvtAxLygTx6R39S0Q2s4Gg3k4oKtRBjTIRJUXTRPSf//mfWl5eykD7Gjy0e7xM8HZgNxn6nYjmTijEEi4jy18n75z1g+ad1ZI0mHbVt3Ok5jCl4TB2e1bSY4hiil0WcOVC4BgE5ViiZKVdZ7UkDSRfIII50qEOZuxnbSiuq8Na1oiSJxORlFG8wTDmsEedqJqCRNTkCbCrVdDsDchE1A8yEUkZxReMYA53YEpRIqp3d/Ll0SCNHQHZNOsHmYikjBEMRwmGo1giHV01oux+n9Nq0qt3zSJCLgXSD0klogMHDqQ6DkkacCfu7trf6R0x6m4eNkKRqFojEqLf58xESSWisWPHMnfuXF588UU6OztTHZMkDYjYOkRZwqseSEEispn0hPR2wlEBkRBEgv0+ZyZKKhFt2rSJqVOnsmzZMoqLi7ntttv48ssvUx2bJKVUQZaZ62YUMT6va9RKChKRxagnqjMQxEhEyOZZspJKRNOnT+fJJ5+ktraW559/nrq6Or773e8yZcoUHnvsMRobG1MdpyT1m8mgo8gUwGkxquOHDP2fQG026NDrFHUWfmyah9Rn/eqsNhgMXH311bz22ms8/PDD7Nu3j7vvvpvy8vL41A1JSispmtoRoyhKVz+RvWtdIlkjSka/EtGGDRv46U9/SklJCY899hh33303+/fv54MPPqC2tpYFCxakKk5J6re9xzrYeeiwutNrqgYzApefWcw5E0diMerlLfwkJTXF47HHHmPVqlXs3r2bK664ghdeeIErrrgCnU7Na6NGjWL16tVUVlamMlZJ6pdv6tqJHDhCrjmMPYWJqCzHBtk50K7IGlGSkkpEK1eu5Oabb2bx4sWUlJT0WKawsFBOapXSii8YwRnuwGhPzRiiBKYs9bPsI0pKUonogw8+oKKiIl4DihFCUFNTQ0VFBSaTiUWLFqUkSElKBW8gTEHEk7JR1TH17k6amyOM8AXJljWipCTVRzRmzBiampq6HW9paWHUqFH9DkqSUk0IEZ/ekdIJr8ChZi+fHQnS4g3KPqIkJZWIxElGj3o8HiyW/i2rIEkDoTMUhXAnhmggZRNeY+Kjq6NCjq5OUp+aZsuWLQPUW5b33nsvNpst/lwkEuGLL75g+vTpKQ1QklIhNuveoFPQmawpGUMUY4vt5hGJqutWhwP9Xuco0/QpEW3evBlQa0Tbt2/HZDLFnzOZTEybNo277747tRFKUgp4A2F11r0h9R3VFqMeoRjwi66/h6BHJqI+6lMi+uijjwC46aabePLJJ3E6nQMSlCSlWonLyvcm2DAesqe0WQbqDHwAn2JVDwQ6wJ6f0msMd0ndNVu1alWq45CkAWUy6MjVd4I5+b3uT8YWS0RYEcKHIjus+6zXiejqq69m9erVOJ1Orr766lOWff311/sdmCSlXGeb+tmandLTWgx6FIWuDmsvJnkLv896nYhcLheKosS/lqShZGddO7baWgoIY0txH5FOp3BV1QgcR0diaGqSt/CT0OtEdGJzTDbNpKHm6yNtlNTWYc83Y0tx0wxgZJ4d/HnQrMi1q5OQ1Dgiv9+Pz+eLP66uruaJJ57g/fffT1lgkpRKgU4v+mgo5WOIEsSnecgaUV8llYgWLFjACy+8AEBbWxuzZs3i0UcfZcGCBaxcuTKlAUpSKoR9bQAYrFmgN6b8/DUtPrY1hunoDMmJr0lIeoXG888/H4D//u//pri4mOrqal544QV+85vfpDRASeqvUCSK0rUOkck+MFtC767v4OPDAdr8IQh65ejqPkoqEfl8PhwOdVO5999/n6uvvhqdTsd3vvMdqqurUxqgJPWXL6DOMdMpYBigRKSOrraqi6NFwxCWa7n3RdKL57/55pvU1NSwZs0aLr30UgAaGhrkIEcp7XiDYSxhdQshJcW37mMsJnV0dUDpmjoim2d9klQiuvfee7n77ruprKzknHPOYfbs2YBaO6qqqkppgJLUX75gOKWbKvbEavzW6Gq5LlGfJDWy+u/+7u/47ne/S11dHdOmTYsfv/jii7nqqqtSFpwkpUJ5ro2iciOKz576BdG62BKmecgO675KKhEBFBcXU1xcnHBs1qxZ/Q5IklLNrNdhxgtmw8DXiIQFaJe38PsoqUTk9Xr59a9/zYcffkhDQwPRaDThebkTrJRWgl51eQ4ltQuinSg28dUjLAgEiqwR9UlSiWjJkiWsW7eOG264gZKSkvjUD0lKR7sOHcHZ5ic7Nx+bTj8g17CZDCysGoGzuQlqq2UfUR8llYjeffdd3nnnHc4777xUxyNJKbev5gh5LT4MeXZspy+eFL1OYVS+HUQe1MrdPPoqqbtmOTk55ObmpjoWSRoQomtU9UANZkwgp3kkJalE9NBDD3HvvfcmzDeTpHQUjQqEv2tUddbAJqIDjR42N4TwBcNydHUfJdU0e/TRR9m/fz9FRUVUVlZiNCbO3dm0aVOvzrNixQpef/11du3ahdVq5dxzz+Xhhx9mwoQJyYQlSd34QxFM4Q4UwOwY2Fr8tiNuDjX6GREIYzMZIOQH00A1BoeXpBLRwoULU3LxdevWsXTpUmbOnEk4HOYXv/gFl156Kd988w12uz0l15AymzcQxhJ2Y9Qr6G0DWyOymvQIRU9A17VeddAjE1EvJZWI7rvvvpRc/L333kt4vHr1agoLC9m4cSMXXHBBt/KBQIBAIBB/3N7enpI4pOGrwx/AHPZgMuvBOsCJqGsskV+xAT51XaKswgG95nCRVB8RqMt//Md//AfLly+npaUFUJtkR48eTToYt1tty5+sI3zFihW4XK74R3l5edLXkjKDv70FEBhMpuMdyQMkNpbIH5/mITuseyupRLRt2zbGjx/Pww8/zCOPPEJbWxugrlW9fPnypAKJRqPceeednHfeeUyZMqXHMsuXL8ftdsc/ampqkrqWlDnGu0JMHeGivLRUHdA4gOI1IrqaZvIWfq8llYiWLVvG4sWL2bt3b8LOrldccQXr169PKpClS5fy9ddf88c//vGkZcxmM06nM+FDkk7FFGzHbjaQlV0w4NeK1Yi8yBpRXyXVR/TVV1/x3HPPdTs+YsQI6uvr+3y+22+/nbfffpv169dTVlaWTEiS1DN/q/p5gPuH4PjEV4+I1Yjk6OreSioRmc3mHjuK9+zZQ0FB7//zCCH4x3/8R9544w3Wrl3LqFGjkglHkk5qX/VhbB1+ckY5Y/WUAZNrN6m7eXgCcPBrWSPqg6SaZt/73vd48MEHCYVCACiKwuHDh/n5z3/O97///V6fZ+nSpbz44ou8/PLLOBwO6uvrqa+vx+/3JxOWJHVTW19PdYuPgHHgt8AyG/RU5tvJy81TDwS9A37N4SKpRPToo4/i8XgoKCjA7/czZ84cxo4di8Ph4Fe/+lWvz7Ny5UrcbjcXXnghJSUl8Y9XX301mbAkKUEgFMIQVO/E2lyDuAV07O5cwCNHV/dSUk0zl8vFBx98wCeffMLWrVvxeDycddZZXHLJJX06j5A/JGkAed2tKCKKTm/AZM8elGvuPdZBuy/A5EgUix4I+cAkB+eeTp8TUTQaZfXq1bz++uscOnQIRVEYNWoUxcXFCCHkkiBS2vC3N6lfWLMH/NZ9zBcHW2jsCDBKmLHou1ZqlInotPrUNBNC8L3vfY8lS5Zw9OhRzjzzTCZPnkx1dTWLFy+Wy8RKaaWzQ01EyiDcMYuJjSUK6LumdsgO617pU41o9erVrF+/ng8//JC5c+cmPPd///d/LFy4kBdeeIEbb7wxpUFKUjJCHeqIf/1gLP/RJTaWqFOxAm55C7+X+lQjeuWVV/jFL37RLQkBXHTRRdxzzz289NJLKQtOkvoj7G0GwJiVN2jXTExEyBpRL/UpEW3bto3LLrvspM9ffvnlbN26td9BSVIqTHBGmFrmYlTF4A2SjTXNvLFEJKd59EqfmmYtLS0UFRWd9PmioiJaW1v7HZQk9Vs0ijHYhtFkgOzBmwFvk9M8ktKnGlEkEsFgOHnu0uv1hMPhfgclSf3W2QbRCOgMA7aXWU8StxVC9hH1Up9qREIIFi9ejNls7vH5E9cKkiQtBTuaONrkRecsogIYrEElpdlWrqoagTNshp2fyNHVvdSnRLRo0aLTlpF3zKR04Gs7Rn17J+3CxMhBHNtmNxuwmw0Q6NrrL+iFaBR0SS/9lRH6lIhWrVo1UHFIUkp1uhsB0NkH745ZAqNdHUQpouroavPALso21Mk0LQ1LwfYGAAzOQZxj1mVXfTtfHW4jpJcd1r2V1FwzSUprQhD2qKOqzY6BXxDt2z7Z10y7P8Q4i5VsfOotfMeghzGkyBqRNPyEfIQ7fYCC2TX4icgeH9QY281D3jk7HZmIpOHH10wgHCVgsOOyD/RyaN3ZzGpDQ93NAzmosRdkIpKGH18zgXAEvyEHp8V4+vIplmWODWo8YX8z6ZRkH5E0/PiamV6ejadgPFl206Bf3mZS/6w8Qk7z6C2ZiKThx9uMQacjO78Y9INf6bfHEhGyj6i3ZNNMGn686hgibIN/6x7A1tU0a4/EEpEcXX06skYkDS8hP62tzbT5Qgi/ldHZgx9CicvCVVUjyNJ1wlbk6OpekO+MNLx4G3H7Q1T7DBxuj2oSgs1koDLfTn52Dig6dQH9kKwVnYpMRNLw4mmkMxTBZ8wjxzb4HdUJdLrj61XLWfinJJtm0vDibSAQjuKz5JJtG/xb9zF7jnXg9oc4U2/DQodMRKchE5E0rAhPg1ojcuSQbdWuRvTFwRaaOgKMdNjUe2edbs1iGQpk00waPoQg4D5GVECnOR+HRbv/s7FpHj5dV9Oss/sW7dJxMhFJw4e/lUBnJ1FFj8mRh06n3R579q5pHj4llojaNItlKJCJSBo+vE3q1A5jDtl2i6ahxAY1thPrrJY1olORfUTS8OE5RqHDwgWjJ9E5dvAWzO9JfFBjbJqH7CM6JVkjkoaPjnoATNkjNJnseiJnV/9UW6RrBn6oE8JBDSNKbzIRScODENBRq37tKNY2FiDLrCbC9rAODF2bTcha0UnJppk0PATaiQa87Grw0FgL52VFMWgw4TUm125Sp3lYDLDTBZ4GNRFlDf5CbUOBrBFJw0NHPf5QhLqggx3H/Og1vGMGYDLo1GkeWWawuNSDAVkjOhmZiKThoaMOfzCCx5RPnt2EMohbCJ1WLBHJptlJyaaZNDx01Yi8pgIK7D1vADrYDjV5aegIMDZiIRfkoMZTkDUiaegT4oQaUQG5GqzK2JNv6tr5ZF8TdQHZWX06MhFJQ5+/FUKdeMMCnymXvDRJRLEpJu5obMlYWSM6GU0T0fr167nyyispLS1FURTefPNNLcORhir3EaJC0EwuQtGTm5UeiSira5pHW3x0tQciIQ0jSl+aJiKv18u0adN45plntAxjyOnoDPHpviY+3HmMg01ywS3cRwiGo3TaS7EY9TjM6dH1GasRtYcMx8cS+du0CyiNafoTu/zyy7n88su1DGFI2lXfwRcHWwDYdsTNtHIXcycUptedosHkPoLFqGf+d2fS6apMm/fB0TW6u6MzDLYcdeS3v1WOJepBevzr6KVAIEAgEIg/bm/PzDb3jEIF655PMHfUcMSro9YzlQ3m85g5Kk/r0AZf0Au+ZgAUVxnWruU30kGsaeYNhonk5qCPJSKpmyHVWb1ixQpcLlf8o7y8XOuQBp+vBd3mPzDFWM+4XCOTXCFGt6ynZuN7tHozcC6T+6j62Z4PJpu2sXyLzaRHr1MQAvwGh3pQJqIeDalEtHz5ctxud/yjpqZG65AGTU2Lj7U76+jc+t9qLcCeD2fdQOG0vyHbZqTEvZnNGz/TOszB565BCMH/1hp5a2stnaGI1hHFKYrCwukjuHH2SKzOrq2NZCLq0ZBqmpnNZszm9BisNti+ONhC9OAn1InDjCrJh2nXgNmB4ipjhK8T92fvUXxsHZHgLPSmwd/vXTNt1erUDvJpb/FhNqTX/9aKvK5aWjhX/SwXSOtRev3UpB7VuzupbWxhRMdWSrMtMPZiMDvizzsnzmXaxLFMyjeiP/KFhpEOsqAPPA14AmHc5hEUOMxp01HdjTVH/dzphmj61NrShaaJyOPxsGXLFrZs2QLAwYMH2bJlC4cPH9YyrLTz9VE3pR3bKbKBObsEiqYkFtDpsY6/SP36yFeZs7NoWzUIQQsuQgY7JS5tV2XsSbMnwJcHW/i6MQx6ozoKXI6w7kbTRLRhwwaqqqqoqqoCYNmyZVRVVXHvvfdqGVZaCYaj7K1rocjzDQUOC1R+F3r6r58/DhzFBAIBmvZkSK2o9RAAR5QiAEpc6dckbfYG+WRfEzvq2sGarR6U/UTdaNpHdOGFFyKE0DKEtLe3oYOsjv1k6UM4c0ohf3zPBRWFI1lTOLJ1B/rGv5J3xgUo+iHVBdh3rYcIR6PUCDURlWanX43IZVXHErn9IcjNAU8j+Fogb4zGkaUX2UeU5r6pbafIs5NChwWltKrn2lCXgtHTCBvthP0dtFRvH8QoNeBrAX8b7YEo7eYScmxGbKb0S7yxROQNRAhbusZ5dY17ko6TiSiNCSEo1reTF24gz2GB4qmnLG82mTCMmA5A494NgxChhpr2AhB1VVCU42BETnqNIYqxGPWYjeqfWYe+a10iX5OGEaWn9PsXIsUpisL5zgbEyByU/PFgzjrt9xSOm8GxQ5/grd+H6HSjxBblGm6a9gCQP3IK15RVpHUT32U10hAK4FZc5AB4G9VO63S9w6cBWSNKZ0JA424UFCic1KtvqRhRhs9WSjAcoWn/5gEOUCNBL7R3jajOHwuQvrftOd48ayVLTT6hTgj5NI4qvchElKYC4Qj1tdUIfwvoDZA3tlffZ9DrsJZPB6Bl/wY1mQ03zftACDot+XTqT19L1FosEbV1cnzZWK9snp1IJqI0dbDJyyef/pVvatshdwwYer/GTsm46UQVA97WBkR77QBGqZHG3QBs6yzkuXUH2Hw4vW+HTy3L5obZIzl/XD7YuqZ6yH6iBDIRpal9xzrI9R1UNwosmNin760oyGHkhOlMLXOhNHwzQBFqJOiFloNEheDrcBlRIdSdMtKYy2okP8usbm9k77pz5pV3zk4kE1EaCkWiNNQexhJuJ8dh7XWzLEavUyibOBODTgcN3wyvKQUNu0BEadXn4caBzaRnRHb6DWQ8KVkj6pFMRGmoutmH07Mfs0GHvXRin5plcTmj1GUxgr74CORh4djXABzUVwIwuiALncZ7mPXG9iNu3t9RTwtO9UDszpkEyESUlvY3dJDnO0Cu3YTSx2ZZnE7HAaWCbUfbaNq/MbUBasXbDO21RIFtgRIAxhWmf2c1wK76dnbUtlMfcYCiU/9BBD1ah5U2ZCJKM5GooP5otdosy+p7s+xENabReAMR3DXfQDhw+m9Id7WbAGgwleGOWrCb9VTkpudAxm/L61rQv8UvwNa1JEjHMQ0jSi8yEaWZ2jY/9vZ9GPUKjhFJNsu6jBw5Br8xm7YOL9GGXSmMUgPhINSr01Z2oCbnSSWuIdEsA8ixdSUiXxAcxepBT72GEaUXmYjSTFm2hb/Jb2VUvh1d4Rn9Old5np1253iCEUHrwSE+uLFhh1qrs+Zw7tkzuXBCAVNGOLWOqtdimz62eAKQ1ZWIOmQiipGJKM0o3kZceMhz2PvVLAP17plz5DQA3HX7INCRihAHXzQKNV+qX484C6vZQFVFDtm29Ni/rDdiQwza/CGC1q5dPDyyaRYjE1G6adypfs4b3a9mWczo8jI6zMW0egJE63f0+3yaaNwFvhaiBguiZJrW0STFbjaQZTYgBDSRrR7sbFc7rSWZiNLJVweb2b39K7zBMBT0r1kWU55ro905gWBE0HZoCDbPhIDDnwKwRYzllY3HONw8NP941aVsoS2oO95hPRxHvidBzr5PE0II9h3YR2XTMfJKcrD3s1kWo9cpFIyeRq7/S8z+JvA0QFZhSs49KI59DZ5GvFE9nwYrCQU6h+yk9YvPKMRs0GMy6KC9TF1Tqf1IfOJuJpM1ojTR6AlgatmDXgFX2RkpaZbFzJlSwYTJ07GbDfEBgUNCOAgH1iEQfBGZQEhnZkxhFuVD5Jb9tzksRjUJAThHqJ/dR7QLKI3IRJQm9tV3kOfbj8tmwljcuyU/+iS24P6xb4bOiN7Dn0GggyOdJrYxDqNe4YJx+VpHlRqurs1B2+uG1xScJMlElCZqjxzAHPaQ47APzHrGuWPwRg20tjaru1+ku45jcPhzOjpDrA1PRSgGLhhfMKTulPXkiwPNvPLlYepCVjBaIRqWt/GRiSgtNHR0omvchU6B7IpJ6rYzKXa0I8T/NuWwr8FDqDbN17OOhGHX2wRCIT73FNBkqWRMYRZnjhj6q002dASod3dytK0TXGXqwTa5fZZMRGlg19E28r37yLGZMJVMOf03JKHUZaEzbxLhqKDlwJb03vts3/+CpwGD2Yan4iIKHGYum1yc1qsw9lZp10oBR9v8kFOpHmw9qF1AaULeNUsDucEjBHUBcnMKIHf0gFxDURTGjJlAW10+De42io5uglHnD8i1+qVuK9RuBkVBP3kBf5s9ikA4eryTd4iLLVlS29aJGDcKBdQO63AwpTcohprh8dMd4qYoh6iqyCZ39Fmg0w/YdSaNcFHvmoYnEKb9wJcQCQ3YtZLStI/mTW9xqNkLI8+DvDEY9TqyzMPn/2WBw4xRr9AZitAUsaubLkYjQ6PfbgDJRKS1gAea96OgoCs59XZB/WUzGcgfNY2AIYu6xma15pEmgo0H2L/uRfbUu9keLmOfbWiOoD4dvU5hRI5aK6pu8R2vAbcc0DAq7clEpCFfMMzhnV8QjUbAWQJZBQN+zZmj8jjqOosWbxDvnnXqjhIaEkJweOcGtr/7OxraPLRaK3BMW8CoAoemcQ2kUfnqGkoHm7zH5xM27lbn1GWo4VPnHYK+rmnFv2U9PnOQiZNmDMo187LMuEadRcT7NaGATx2rM2buoFz72+pbvez5/M8Ya9XNIH3Z4zhzzrWU5w+dWfXJGJVn53OTHpfViMguQTFa1JsHbdWQO0rr8DQhE5FGolFBzZ5NlEa85GQXpmxuWW9cMKEIc9HVmHe+oc5qL5wEjqJBuz6A6Kjn4P++hLG9Dp0CWWPP46zZf4vZmPqhC+nGZTNy2wWjj98FLJgItVugYWfGJiLZNNPInmPtuJo2Y9Qr5E2Yre5dNkicFiPmoglQMB5EFHb9ecA7rj2BMF8dasHf0Qp71qBsXM14awd52U4mX3oTky+4KiOSUEzCUITY5pmNu9S7ZxlI1og0EI0Kdm3fSFGwiaJ8J4aywWmWJVAUGDePhiP78TQeZJT1bZTJC1O2DXIkKmjyBDjU5OVAowdvYzUFnt2MsNVidai3qfNGTSNv7CVgHr79QacihKDO3UmObQRWaw74W+HYdhihwe+DxmQi0sDuejeO+s8w6BSKJp2v7rahgbaIibfD32Gi5210uzYyUlFQzriyX0MI3L4Qf/m6jma3B7u/FlfnEYr96tZIDosBi94K2RVQ+V3IGZnCVzP0vLO9jr3HPFwwPp8ZZWfD3g/gyEYoPStl/xCGCpmIBlkgHGHPpnUUhlopLsjFVDlbs1iybSZmVU1j01ceaP6Izm1fUNneiGXyfHCW9vg9kaig2RugsSNAQ0eABrefcnuEc0sU8DZhc9dRtHs7lcE2DDpwWAzkZJvIdhRgLpkMJdMgu3yQX2l6qsyzs/eYh82H25h+zhT0B9eBr1m9g1aY5O4tQ5RMRIMs6HUz2rOJsEFHSdU8MFo0jWfKCBeKch4bNlkY3fgh7p27ya4+iDWvnIqxk8GaizBa+XhvI20dHvyedvRhL5ZwO5aQm7JwO04z0KDOAzMCVblhrEYX5qxslNzR6liZ3FFgSO8dWQfbxGIHn+5voqMzzK6mAJPLZsKhT+DAWsgfN6CDW9ONTESDKRrFceBdphSZCdgqMZSdpXVEAEwudVHgOJePd4zAcGgdEe9eskKHqDC0AaAA9sOtGMLqOBeDTsFm0mO3GbCbLWRZTGDLU1cddBSTk1Ws3oUzZWVcE6MvDHodZ1Xk8Ne9TXy2v5mxM8/GXLtZ7Ss6/JnafM0QMhENEn8gjOXwWpS2wyh6E5Yp30urP9JCh4WrzhlP46SRVNcdw+Q9DFY3BNoh5Ke4zIXeYCbL6cKW5UKxZKuJx5oDlmzQyRuwyZhWns22I27c/hCfVnuYO/YS+OYttWaUU3l8hv4wJxPRIGj3B/n0f99knH8ro/PtKBOuAHue1mF1oygKhU4Lhc6RQGJHcs89RlJ/GfU65k4s5M3NR9lyuI0RrnLGF06Ehl2w/b+h6oa0/F1JtbT4N/bMM89QWVmJxWLhnHPO4csvv9Q6pJQ50tjC5++sxtXwJW5/iM7KuVA0ACswSkPWqHw7Z1fmkGs3ke+wwIT56iaMIT9s/gO0Dv8JsYoQ2q4b+uqrr3LjjTfy29/+lnPOOYcnnniC1157jd27d1NYeOpF3tvb23G5XLjdbpzO9JoW0OzuYM+2zwjt/yvGiB+rycDYc68ia8w5WocmpaFoVBCKRjEb1A7qoK+D6LZXsfgb1QIl06DiO8d3/xgievs3qnkiOuecc5g5cyZPP/00ANFolPLycv7xH/+Re+6555Tfmw6JKBKJEgr4sBAEfysB9zG+2roVWg+jF+po5Zz8Ykad93eY8ys1iVEaejYdbuXT3fXMCH1FZXAfNpMBi1GHPnsEhrzRkFWk3iAw2cBgTds+ut7+jWraRxQMBtm4cSPLly+PH9PpdFxyySV89tln3coHAgECgUD8cXt7e+8u1HIQ9n+YuGi8EGysbiYSFRzPxQKEQAAOs57JJc748Y3VLYQiUfUcXWUQUQgHcZj1TOlaxtSEwNbeRlBEsecUUjLpPPLHnTOoUzikoa+hvZMQej43focd0UpK27aQ4z8Mh9vQKzs4uzIXXdfNjkPNXpo7FdDpEIoOgb7rs/r8jJE5GPRqTetAo4dGj/o3NKnEicNyimk1p7uZcubfgyU1FQBN/zqampqIRCIUFSVOuCwqKmLXrl3dyq9YsYIHHnig7xcKB8DT2O1w1NtCJNpzhVBEDRA4/oNQgl5EuOdlGsLRqLrOtMWFYs+nojAbW8k4bDkj0urOmDR0XDalhJmVuRxo8lLnzqK+o5zD3jYcvhocgXoUpwk6WyHUSTAcJRg4+Rw1xSdAp/4eRr0eol2JSPGFINKPFCBSt2zJkPo3vXz5cpYtWxZ/3N7eTnl5L0bpZpfDtGtOSAoKKAojRvnjxxRFAUWHAig6BaNeDzZT1/MKlZMCRIWCoihdZXQoKFisVsxWe8KC98NkwxtJY3lZZvKyjg8CFUIQjEylMxhFsXX9vkWj5LW1Yff71MQgoigiou4O0kVxmuO/53m+INagWrM3Okxw4hK8fe2lMdmTfm3fpmkiys/PR6/Xc+zYsYTjx44do7i4uFt5s9mM2ZzE6FyTvcflFUqze3+K3MyclymlEUVRMBv08Q5tAHQ68nJzgd51YufkQM7AhNcvmvZwmUwmZsyYwYcffhg/Fo1G+fDDD5k9W7s5WJIkDS7Nm2bLli1j0aJFnH322cyaNYsnnngCr9fLTTfdpHVokiQNEs0T0Q9/+EMaGxu59957qa+vZ/r06bz33nvdOrAlSRq+NB9H1B/pMI5IkqST6+3faHqOgpIkKaPIRCRJkuZkIpIkSXOad1b3R6x7q9dTPSRJGlSxv83TdUUP6UTU0dEB0LvR1ZIkaaajowOXy3XS54f0XbNoNEptbS0OhyNxn6gMEZviUlNTI+8a9oJ8v/quv++ZEIKOjg5KS0vRnWKFgCFdI9LpdJSVZcZSmqfidDrlH1YfyPer7/rznp2qJhQjO6slSdKcTESSJGlOJqIhzGw2c9999yW3IkEGku9X3w3WezakO6slSRoeZI1IkiTNyUQkSZLmZCKSJElzMhFJkqQ5mYjSWF93wH3ttdeYOHEiFouFM888k7/85S+DFGn66Mt7tnr1anUzhBM+LBbLIEarrfXr13PllVdSWlqKoii8+eabp/2etWvXctZZZ2E2mxk7diyrV69OSSwyEaWpV199lWXLlnHfffexadMmpk2bxrx582hoaOix/Keffsq1117LLbfcwubNm1m4cCELFy7k66+/HuTItdPX9wzUEcN1dXXxj+rq4b+9c4zX62XatGk888wzvSp/8OBB5s+fz9y5c9myZQt33nknS5YsYc2aNf0PRkhpadasWWLp0qXxx5FIRJSWlooVK1b0WP7v//7vxfz58xOOnXPOOeK2224b0DjTSV/fs1WrVgmXyzVI0aU3QLzxxhunLPMv//IvYvLkyQnHfvjDH4p58+b1+/qyRpSGYjvgXnLJJfFjp9oBF+Czzz5LKA8wb968k5YfbpJ5zwA8Hg8jR46kvLycBQsWsGPHjsEId0gayN8xmYjS0Kl2wK2vr+/xe+rr6/tUfrhJ5j2bMGECzz//PH/605948cUXiUajnHvuuRw5cmQwQh5yTvY71t7ejt/v79e5h/Tse0nqj9mzZyfsn3fuuedyxhln8Nxzz/HQQw9pGFnmkTWiNNTXHXABiouL+1R+uEnmPfs2o9FIVVUV+/btG4gQh7yT/Y45nU6sVmu/zi0TURpKZgfc2bNnJ5QH+OCDDzJmx9xU7BociUTYvn07JSUlAxXmkDagv2P97u6WBsQf//hHYTabxerVq8U333wjbr31VpGdnS3q6+uFEELccMMN4p577omX/+STT4TBYBCPPPKI2Llzp7jvvvuE0WgU27dv1+olDLq+vmcPPPCAWLNmjdi/f7/YuHGjuOaaa4TFYhE7duzQ6iUMqo6ODrF582axefNmAYjHHntMbN68WVRXVwshhLjnnnvEDTfcEC9/4MABYbPZxD//8z+LnTt3imeeeUbo9Xrx3nvv9TsWmYjS2FNPPSUqKiqEyWQSs2bNEp9//nn8uTlz5ohFixYllP+v//ovMX78eGEymcTkyZPFO++8M8gRa68v79mdd94ZL1tUVCSuuOIKsWnTJg2i1sZHH30kgG4fsfdo0aJFYs6cOd2+Z/r06cJkMonRo0eLVatWpSQWuQyIJEmak31EkiRpTiYiSZI0JxORJEmak4lIkiTNyUQkSZLmZCKSJElzMhFJkqQ5mYgkSdKcTETSkLJ69Wqys7Pjj++//36mT58ef7x48WIWLlw46HFJ/SMTkdSjxYsXoygKP/nJT7o9t3TpUhRFYfHixQnlU50AKisreeKJJxKO/fCHP2TPnj0n/Z4nn3wyYR3lCy+8kDvvvDOlcUmpJxORdFLl5eX88Y9/TFj0qrOzk5dffpmKigpNYrJarRQWFp70eZfLlVBjkoYGmYikkzrrrLMoLy/n9ddfjx97/fXXqaiooKqqql/n7qmmsnDhwngt68ILL6S6upq77rorvsMGdG+afduJNbPFixezbt06nnzyyfg5Dh48yNixY3nkkUcSvm/Lli0oiiLXItKITETSKd18882sWrUq/vj555/npptuGvDrvv7665SVlfHggw/Gd9joqyeffJLZs2fz4x//OH6OioqKbq8JYNWqVVxwwQWMHTs2VS9B6gOZiKRTuv766/n444+prq6murqaTz75hOuvv37Ar5ubm4ter8fhcFBcXJzUSpMulwuTyYTNZoufQ6/Xs3jxYnbv3h3f8ywUCvHyyy9z8803p/plSL0k16yWTqmgoID58+ezevVqhBDMnz+f/Px8rcPql9LSUubPn8/zzz/PrFmz+POf/0wgEOAHP/iB1qFlLFkjkk7r5ptvZvXq1fz+979PWa1Bp9Px7aWwQqFQSs7dG0uWLIl3xK9atYof/vCH2Gy2Qbu+lEgmIum0LrvsMoLBIKFQiHnz5qXknAUFBQn9PpFIpNuutCaTiUgk0q/rnOwcV1xxBXa7nZUrV/Lee+/JZpnGZNNMOi29Xs/OnTvjX5+M2+1my5YtCcfy8vIoLy/vVvaiiy5i2bJlvPPOO4wZM4bHHnuMtra2hDKVlZWsX7+ea665BrPZnFSTsLKyki+++IJDhw6RlZVFbm4uOp0u3le0fPlyxo0blzGbDKQrWSOSesXpdOJ0Ok9ZZu3atVRVVSV8PPDAAz2Wvfnmm1m0aBE33ngjc+bMYfTo0cydOzehzIMPPsihQ4cYM2YMBQUFScV99913o9frmTRpEgUFBRw+fDj+3C233EIwGByUu4DSqck1q6WM9de//pWLL76YmpqabjuYSoNLJiIp4wQCARobG1m0aBHFxcW89NJLWoeU8WTTTMo4r7zyCiNHjqStrY1/+7d/0zocCVkjkiQpDcgakSRJmpOJSJIkzclEJEmS5mQikiRJczIRSZKkOZmIJEnSnExEkiRpTiYiSZI09/8D67nUaLa/WNQAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:36:16.649373Z", + "iopub.status.busy": "2024-03-26T13:36:16.648561Z", + "iopub.status.idle": "2024-03-26T13:36:16.972430Z", + "shell.execute_reply": "2024-03-26T13:36:16.971432Z" + }, + "papermill": { + "duration": 0.341646, + "end_time": "2024-03-26T13:36:16.974554", + "exception": false, + "start_time": "2024-03-26T13:36:16.632908", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCIUlEQVR4nO3deXxU5b348c+ZfSazZN9IQthE2RGB4oYLt251qW3F3l4FxVat3l/VurYVxV6vXS3eWy+3L+8VrrV161W7eCtaqqjggiyKrBIgAZOQfSazL+f5/XGSgZAEkslyJpnn/XrlBXPmzDnfTDLfPPujCCEEkiRJOjLoHYAkSZJMRJIk6U4mIkmSdCcTkSRJupOJSJIk3clEJEmS7mQikiRJdzIRSZKkO5PeAQyEqqrU1tbicrlQFEXvcCRJOo4Qgvb2dkpLSzEYei/3jOhEVFtbS3l5ud5hSJJ0EocOHaKsrKzX50d0InK5XID2Tbrdbp2jkSTpeD6fj/Ly8uRntTcjOhF1VsfcbrdMRJKUxk7WdCIbqyVJ0p1MRJIk6U4mIkmSdCcTkSRJupOJSJIk3clEJEmS7mQikqQe7G/0U9Xo1zuMjCETkSQdxxuM8dqndTT4InqHkjFkIpKk43xc3UJcFbQGo8i9JYaHTESSdIyEKth7RKuSTSv1EFcF+xv9NPll6WgoyUQkSceobQsRjiXIshopy7Gzfk8jf9xWy2dfePUObVSTiUiSjvFFWwiAshwHBoPC2DwHAIdagnqGNerJRCRJxzjc2pmI7ACUZGv/NgeiROIJ3eIa7WQikqQOqiqo92qJaExHAnJaTbhsJoRA9qINIZmIJKmDAC6eVsKCCXnkOCzJ48UeGwBHfGGdIhv9ZCKSpA5Gg8LEQidfGp+HwXB0/ZxCl5aIZM/Z0JGJSJJOIjdLKx01B6I6RzJ6jegVGiVpMO2pb8dkVBiTbcdmNiaPl3hsXDS1mHyn5QSvlgZClogkqcN7+5r407ZaGtu7VsGyrCamlLopdNt0imz0k4lIkoB4QqU9HAOOVsWk4SOrZpIE+MJxhACLyYDDYuz2fIMvTJ03TL7LmuzalwaPLBFJEtAW1BqiPXZzjztO7K5v5++7G/j8SPtwh5YRZCKSJKAtpFXLsh3mHp/32LXjvnB82GLKJDIRSRLaGkQA2fae24c6E5G3I2FJg0smIkkC2kJa1eykJaJQTK5RNATSJhH95Cc/QVEU7rjjDr1DkTLQOZMKuGxGCeW5jh6fd9lMKApE4yqhmJz8OtjSotds06ZN/OY3v2HGjBl6hyJlqHynlXyntdfnTUYDTquJ9nAcbyiGw5IWH51RQ/cSkd/v51vf+hZPPfUUOTk5eocjSb1yd1TP2mWD9aDTPRHddtttXHbZZSxatEjvUKQM1RaMsqWmlermwAnPO3tiPtfMLaeil+qblDpdy5fPP/88W7ZsYdOmTX06PxKJEIkcHX7v8/mGKjQpg9R5w6zf00hZjp2xeVlHnxACWvaDmoC8CZTKgYxDRrdEdOjQIb73ve/x5ptvYrP1bQ7PY489xooVK4Y4MinT+CNaVctlO67HbN/f4PDH2v/zJsD0b0APgx2lgVOETn2Rr776Kl/96lcxGo8Op08kEiiKgsFgIBKJdHkOei4RlZeX4/V6cbvdwxa7NLr8ffcRPjnkZf64XM6cmK8dbD0I257rcl5wwsXsVcahADPLs4c7zBHJ5/Ph8XhO+hnVrUR04YUXsn379i7HbrjhBk499VTuu+++bkkIwGq1YrX23rMhSanobHzuUiLqLAmVzgabB/a/Tezg+7wVduCym2UiGmS6JSKXy8W0adO6HMvKyiIvL6/bcUkaSp3TNpy2jo9D2AvN+7T/l80FSxYcfA9bpIWsaCMBQxGqKrqs4igNjO69ZpKkN3+yRNSRiJo+1xqqPWWQlQdmG+RNwGI0kBs+hCoEQTmocVCl1aist99+W+8QpAwTjauEO5JKMhG17Nf+zZ909MTc8SiNeyiKf8Eh5uAPx3Fa0+rjM6LJd1LKaEaDwrXzyglE4lhNRkjEoLVaezJ3wtETc8cDkJ1oxpgI44/EALli42CRiUjKaEaDQonnmPFB7XWgxsHqhKz8o8dtbnDkYjW244o20B4uH/5gRzHZRiRJx/J+of3rHtN9zJCrBLPJgDPaSDAq24gGkywRSRntUEuQRn+EUo9d20jRd0wiOp57DMVuGxe6E1hl9/2gkiUiKaPta/Czfk8jVY1+rafMV6s94S7tfrK7BLvZSG68iawe1rWWUicTkZTRAlGt695hMWrjh6IBMBjBVdz95KxCrboWC0HUP8yRjm4yEUkZLRjR2nqyrCbwN2gHHXlg7GGlRqOJmCWbOm+IT3bvG8YoRz+ZiKSMFjy2RBToSETOwl7PTzjyOdgcZHfVflRVLhk7WGQikjJaoKP3K8tigkCjdjCr90RkdheiALZoC+G47DkbLDIRSRkrGleJxlUAHFYj+DsSkbOg19cYnYWYjAqOWAuBiExEg0UmIiljhTpKQ2ajgoUEhFq0J05QIiKrALPRgD3WRjAitxYaLHIckZSxsqxGrp1XTiSmogSbte57s12bbd8bew5mkxFjNErI74V85/AFPIrJEpGUsUxGAyUeO5X5WV0bqk+0CqPBgGLLBiDa3jz0QWYImYgkCSDYUS1z5J/4PEDJ0nabiQVkIhossmomZaya5iBNgQhjsu0UdbYPOXJP+rqyklLGJOowZcvG6sEiS0RSxtp7pJ31exrZ3xg4WiKyn3xvPWd2IS6bGXtc7iIzWGQikjJW5/SOLIsCoTbtYB9KRMlkFWodmsAykKyaSRmrcykPJyFtDSKDEayek74ubHbT6g0Rb6+jXAi5xdAgkIlIyliBjv3MnGpHFcueA4aTVxLiFjcHm0MoBCmLBlCssgt/oGTVTMpIQojkgEZH/JhE1Ad2q5WIMQsBRHyy52wwyEQkZaRIXCXeMWnVHm/TDvalfQhtedmEVdssMOxvGYrwMo5MRFJG6mwfspoNGMNt2kF73xIRgGLT2pKiAdlgPRhkG5GUkVw2E9fOKycWF/D5m9rBPpaIgOTo6njAOwTRZR6ZiKSMZO6Y3kEirq3MCP0qERkcWtUsFmwbgugyj6yaSZkt7NUmu5osJ57sehyTPRsANSRLRINBloikjHSwKUBzIMpYGsgHsHn6NR5oyrgyTM1urLaolsjkWKIBkSUiKSPtOdLOO3sbOdJ4RDvQ0ebTVzm5+bhsZm0do3hk8APMMDIRSRmpc63qLLVjNw7byUdUd2E0a2sXwdE2JillMhFJGalzmVdHiokoEIlTEzRR5w1BRE5+HSiZiKSM1FkisiXatQP9TUTRONuboLYtBGGZiAZKJiIp46iqSA5otMVTKxE5LCYiJiexhEDIqtmAyUQkZZxQLIEQYBRxzImQdrCfichuNhI1OhFANNA26DFmGpmIpIzTuQ6RRwmiKAqYrEcbnvvIaFAQNjmocbDIcURSxslxWPjmvApE8z6opv89Zh0MHYkoHpRVs4GSiUjKOGajgWKPDfwd439STERmuwuARNgvBzUOkKyaSZmrs5G5n4MZO5kcbkAhHo9DLDhoYWUiXRPRqlWrmDFjBm63G7fbzYIFC/jrX/+qZ0hSBtjf6GdzdSve1ibtQIoloi9NKGBqZTG5WRaItA9ihJlH10RUVlbGT37yEzZv3szHH3/MBRdcwJVXXsmOHTv0DEsa5fbUa9M72loGlogKXTbcnhzMRgNE/IMYYebRtY3o8ssv7/L40UcfZdWqVXzwwQdMnTpVp6ik0S7QMYbInur0jmNZ3dB+BKKyRDQQadNYnUgkeOmllwgEAixYsEDvcKRRLBiNY1BjWNUwYE45EXmDMRq94G4PUyirZgOieyLavn07CxYsIBwO43Q6eeWVV5gyZUqP50YiESKRozOdfT45tF7qv0AkgTXhx2zsHENkS+k6rcEoW+pjTA6FKZRVswHRvdds8uTJbNu2jQ8//JBbb72VJUuWsHPnzh7Pfeyxx/B4PMmv8vLyYY5WGuniCZVwLIE17sNiMgyoWuawaKOr4wkhG6sHSPdEZLFYmDhxInPmzOGxxx5j5syZPPHEEz2e+8ADD+D1epNfhw4dGuZopZEuGDs6695oUAaUiGwWI1Gjg7iqImQiGhDdq2bHU1W1S/XrWFarFavVOswRSaNJsGP5DzdBFJSUxxCBNt8sZnSgCoiH2zEPUoyZSNdE9MADD3DJJZdQUVFBe3s7v//973n77bdZu3atnmFJo1ieU5veYd69BYIMqERkNhoQVm10dTwcwJyIgzHt/raPCLq+aw0NDVx//fXU1dXh8XiYMWMGa9eu5R/+4R/0DEsaxZLTO4ypzbrvdj2rA1UxEk+oWhd+H3eLlbrSNRH993//t563lzJZcnrHwBKRw2oiaswipgptUKNMRCmR5Ugpo+xr8OPzB5gcaCfLYhpwIjp/ciG2eBnOUJ3sORsAmYikjLK73sfhQzWUxuNkZTlTHkPUqdhjA08uRI5AVI4lSpXu3feSNJyCkQTWRLs2mHGApaGkjgZrWSJKnUxEUkYJRONY435touogJKImf4TdrYLmQEQmogGQiUjKKMFoAmu8HcsgJaJ6b5iPvojS2B6RVbMBkG1EaSqRSBCLxfQOY1SJxlWMaowsQ4yE2UXYlA3h8ICuaVHiKFYn0aiTcCQ+4OuNNGazGaPROODrKEII0d8X7d+/n/Hjxw/45gPl8/nweDx4vV7cbrfe4QwKIQT19fW0tbXpHcqoo6oCfySORQ1iMwJmh7Zj6wAkVEEwEsWaCGIxGcE2On4P+yM7O5vi4mJtI4Lj9PUzmlKJaOLEiSxcuJBly5bx9a9/HZttYD0P0lGdSaiwsBCHw9HjD1dKTTSu4gvHcMTacJgVbczPABNRPKHSFoziiLfiMBshKx+UzGjxEEIQDAZpaGgAoKSkJOVrpZSItmzZwurVq7nrrru4/fbbWbx4McuWLWPevHkpByJp1bHOJJSXl6d3OKOORQjsNguGQACjooAja8BJQ1UFgYQBs2LGajGiWCwZNc3Dbte2YWpoaKCwsDDlalpKP4VZs2bxxBNPUFtby9NPP01dXR1nn30206ZN4/HHH6exsTGlYDJdZ5uQw+HQOZLRyaAomBWhJSHFMCgll84Cq8CAQIBIDPiaI03n7+tA2jQH9JMwmUxcffXVvPTSS/z0pz9l37593H333ZSXlyfnkEn9J6tjQ0jtSBSGgTewgvazMigKQjEiBCDUQbnuSDIYv68DSkQff/wx3/3udykpKeHxxx/n7rvvpqqqijfffJPa2lquvPLKAQcoSYMlHEsQikZJCDFoiQjAYzeRZbNgUJSjiU7ql5QS0eOPP8706dM588wzqa2t5ZlnnqG6upp/+Zd/Ydy4cZxzzjmsWbOGLVu2DHa80iiydOlSrrrqqkG95nnnnccdd9zR43PhWIJwJMqE6fNZ+evfDNo9LSYjJqMRBTKyRDQYUkpEq1at4h//8R+prq7m1Vdf5Stf+QoGQ9dLFRYWytn1GeRECSBdJIRA6UwUg92z1VnCGkVtRA8//DCzZs0alnul1Lz/5ptvUlFR0S35CCE4dOgQFRUVWCwWlixZMihBStJgUFUw05EoBrEdLhZXiccFZlVgUmWJKBUp/VmYMGECTU1N3Y63tLQwbty4AQcljSxLly5l/fr1PPHEEyiKgqIoVFVVsWzZMsaNG4fdbmfy5Mm9rkW+YsUKCgoKcLvd3HLLLUSj0T7dNxAIcP311+N0OikpKeGXv/xlt3MaGhq4/PLLsdvtzJk2mRdf+gModCkRKYrCqlWruOSSS7Db7YwfP54//OEPyecPHjyIoii8+OKLnHPOOdjtdubOncvevXvZtGkT8+fPJb+wmMu+/i0aG470+X17+umnmTp1KlarlZKSEm6//fbkczU1NVx55ZU4nU7cbjfXXHMNR44cvXZnaeW3v/0tlZWVeDwerr32Wtrbj853U1WVn/3sZ0ycOBGr1UpFRQWPPvpo8vn77ruPU045BYfDwfjx43nwwQeTPV9r1qxhxYoVfPLJJ8mf6Zo1a/r8vfWbSIGiKOLIkSPdjh88eFA4HI5ULpkSr9crAOH1eoftnkMpFAqJnTt3ilAo1O25SCzR61csnujzudE+nNtfbW1tYsGCBeLb3/62qKurE3V1dSIcDovly5eLTZs2if3794tnn31WOBwO8cILLyRft2TJEuF0OsXixYvFZ599Jv7yl7+IgoIC8YMf/KBP97311ltFRUWF+Nvf/iY+/fRT8ZWvfEW4XC7xve99L3nOJZdcImbOnCk2bNgo1r79npg/d46w223iV48/njwHEHl5eeKpp54Se/bsET/60Y+E0WgUO3fuFEIIceDAAQGIU089Vbz++uti586d4ktf+pKYM2eOOO+888Tf3npb/G39BjFhfKW4Zdn1QqjqSWP/j//4D2Gz2cTKlSvFnj17xEcffSR+9atfCSGESCQSYtasWeLss88WH3/8sfjggw/EnDlzxMKFC5Ovf+ihh4TT6RRXX3212L59u3jnnXdEcXFxl/fu3nvvFTk5OWLNmjVi37594t133xVPPfVU8vkf//jHYsOGDeLAgQPiT3/6kygqKhI//elPhRBCBINB8f3vf19MnTo1+TMNBoM9fi8n+r3t62e0X4nozjvvFHfeeacwGAzi5ptvTj6+8847xf/7f/9PzJ8/X5x55pn9ueSAZFIievyNPb1+vbLlcJdz/33d3l7PfXFTTZdzV729r9s5qVi4cGGXBNCT2267TXzta19LPl6yZInIzc0VgUDgaDyrVgmn0ykSiRMnxPb2dmGxWMSLL76YPNbc3Czsdnsyjj179ghAfPTRRyIaT4iG1naxeePfBZD80AuhJaJbbrmly/Xnz58vbr31ViHE0UT0X//1X8nnn3vuOQGIdevWiVA0LurbguKRB+8XkydNEOIksQshRGlpqfjhD3/Y43NvvPGGMBqNoqbm6M9qx44dye9FCC0RORwO4fP5kufcc889Yv78+UIIIXw+n7BarV0Sz8n8/Oc/F3PmzEk+fuihh8TMmTNP+rrBSET9aiPaunVrZymK7du3Y7FYks9ZLBZmzpzJ3XffPfBimjQqPPnkkzz99NPU1NQQCoWIRqPdGj9nzpzZZQDnggUL8Pv9HDp0iLFjx/Z67aqqKqLRKPPnz08ey83NZfLkycnHu3btwmQyMWfOHGKqQCHBqadMIju7+6z743cXXrBgAdu2betybMaMGcn/FxUVATB9+nQMCqAoFBQU0NDY3NFg3XurR0NDA7W1tVx44YU9Pr9r1y7Ky8u77Ns3ZcoUsrOz2bVrF3PnzgWgsrISl8uVPKekpCQ53WLXrl1EIpFe7wHwwgsv8G//9m9UVVXh9/uJx+O6zdnsVyJ66623ALjhhht44oknRs1E05HgtvMn9vqc4bh21++cO6HXc49vo73xrKFp03v++ee5++67+eUvf8mCBQtwuVz8/Oc/58MPPxyS+52M2Wgg22pAiabeW2Y2H52X1jmIz2w2Hx3QpxhQhXrSLvzOaREDdWw8nTGpHY3lJ7vH+++/z7e+9S1WrFjBRRddhMfj4fnnn++xnW04pPRTWb16tUxCw8xiMvT6ZTIa+nyuuQ/nphSfxUIicbTresOGDZx55pl897vfZfbs2UycOJGqqqpur/vkk08IhULJxx988AFOp/Oku/hOmDABs9ncJbG1trayd+/e5ONTTz2VeDzO5s2bMSgKJkWwb18VbW3ebtf74IMPuj0+7bTTTv6No00dAVDpnO9x4i58l8tFZWUl69at6/H50047jUOHDnXZQHTnzp20tbX1uh378SZNmoTdbu/1Hhs3bmTs2LH88Ic/5IwzzmDSpElUV1d3Oef4n+lQ6nOJ6Oqrr2bNmjW43W6uvvrqE5778ssvDzgwaWSprKzkww8/5ODBgzidTiZNmsQzzzzD2rVrGTduHL/97W/ZtGlTt17VaDTKsmXL+NGPfsTBgwd56KGHuP3227sNDTme0+lk2bJl3HPPPeTl5VFYWMgPf/jDLq+bPHkyF198MTfffDOrVq3CFA9wxz0P9FhaeOmllzjjjDM4++yz+d3vfsdHH33U53FwR0ukHffuQxf+ww8/zC233EJhYSGXXHIJ7e3tbNiwgX/+539m0aJFTJ8+nW9961usXLmSeDzOd7/7XRYuXMgZZ5zRp5hsNhv33Xcf9957LxaLhbPOOovGxkZ27NjBsmXLmDRpEjU1NTz//PPMnTuX1157jVdeeaXLNSorKzlw4ADbtm2jrKwMl8s1ZBuc9vnPn8fjSRZBj91/vqcvKfPcfffdGI1GpkyZQkFBARdddBFXX301ixcvZv78+TQ3N/Pd73632+suvPBCJk2axLnnnsvixYu54oorePjhh/t0z5///Oecc845XH755SxatIizzz6bOXPmdDln9erVlJaWsnDhQr567fXctORbFBYWdLvWihUreP7555kxYwbPPPMMzz33XJ9LH4qikO0wYzV3Dmo8eSJasmQJK1eu5D/+4z+YOnUqX/nKV/j888+T1/vjH/9ITk4O5557LosWLWL8+PG88MILfYqn04MPPsj3v/99li9fzmmnncbixYuTbUhXXHEFd955J7fffjuzZs1i48aNPPjgg11e/7WvfY2LL76Y888/n4KCAp577rl+3b8/UloYLV2MtoXRwuEwBw4cYNy4cXKNp0HWEohgDjViMxkwuwq6rEOkKAqvvPLKwKebRAMQ9mk7g2TQ/mYn+r3t62c0pQaBUChEMBhMPq6urmblypW88cYbqVxOkoacUFUUIbTG+kGc8NpF5yBJOfG131JKRFdeeSXPPPMMAG1tbcybN49f/vKXXHnllaxatWpQA5QyU01NDU6ns9evmpqafl1PdCQHZZDWITpeNJ4gGNfmsyHUE8b+7rvvDvr9R7qUV2j81a9+BcAf/vAHiouL2bp1K//7v//L8uXLufXWWwc1SCnzlJaWdhvHc/zzfaWKowuWKT2snjgYrRPhmEo4qmJSBUaDyratW3udzzZmzJgB32+0SSkRBYPB5ECqN954g6uvvhqDwcCXvvSlbl2AkpQKk8nExIm9j53qD1UVGEQCFFCGqFpmUDpWaRSAEEycMAFO0vMnHZXSOzVx4kReffVVDh06xNq1a/nyl78MaCNGR0OjsTS6qAIUoaKgoBiGZj1pRVFAUfo8lkjqKqVEtHz5cu6++24qKyuZP39+cnj8G2+8wezZswc1QEkaKFVoJSJtKsZQlYg6BjV2tj/JBdL6JaU/D1//+tc5++yzqaurY+bMmcnjF154IV/96lcHLThJGgwWkwGzRQHVMGQ9Zp21MBUDkJA9Z/2Ucjm1uLiY4uLiLsfkdkJSOtJyhKo1Hg91iShZNZMlov5IKREFAgF+8pOfsG7dOhoaGpIT7Trt379/UIKTpEEhVBBCWxBtCBurAVRhQACKbCPql5QS0U033cT69eu57rrrKCkpkdvfSGktFI1iTKgYTSZtT7MhYFAUsu1mjAkrRKOyRNRPKSWiv/71r7z22mucddZZgx2PNEKdd955zJo1i5UrV+odCtA1nkg0himuYjUZGKIx1SiK0jHXrOMjJduI+iWlXrOcnBxyc3MHOxYpw/V1rep+U+MAQ9Z134XsNUtJSonoxz/+McuXL+8y3ywVjz32GHPnzsXlclFYWMhVV13Fnj17BnRNafilunh+575mjz76KKWlpcnVFTdu3MisWbOw2WycccYZvPrqqyiK0mWk9WeffcYll1yC0+mkqKiI6667Lrmhw/Hx5OTkUV1zCCXFfdn7KhJLEIyL5DQPRu588mGX0p+IX/7yl1RVVVFUVERlZWW3leL6urHi+vXrue2225g7dy7xeJwf/OAHfPnLX2bnzp1kZWWlEtroIwQkUt9TPGVGc5+33HniiSfYu3cv06ZN45FHHgG0UnNZWRkvvfQSeXl5bNy4ke985zuUlJRwzTXXJF+7bt063G43b775JqDN1r788su59NJL+f3vf091dXW3/dLa2tq44IILuOmmm/jVr35FKBTivvvu45prruHvf/97l3geengFgbYminJdGIa4RBSMJYjGVMxCYOxMRkPUSzfapPSTGazdOV9//fUuj9esWUNhYSGbN2/m3HPPHZR7jHiJGLyrw/Kd53wfTJaTn4e2PpXFYsHhcHQZ0rFixYrk/8eNG8f777/Piy++2CURZWVl8V//9V/J9c//8z//E0VReOqpp7DZbEyZMoUvvviCb3/728nX/PrXv2b27Nn867/+a/LY008/TXl5OXv37uWUU05JxlNYVETYIjCQGLLpHZ0MHaOrhTi2eiYTUV+klIgeeuihwY4DAK9XW8Kzt/anSCRCJBJJPvb5fEMShzQ4+rJ4/vTp07tswrBnzx5mzJjRZV2b48enffLJJ7z11ls4nc5u96yqquKUU05JPk6oKopIoBgUGOISUbILHwOgdkzzMJ/oJVKHlH8ybW1t/OEPf6Cqqop77rmH3NxctmzZQlFRUUqzi1VV5Y477uCss85i2rRpPZ7z2GOPdfkrmxGMZq10osd9B6Cvi+enUgX3+/1cfvnl/PSnP+32XElJSZfHItG5/IcyJMt/HKvrNA+1T0vGSpqUEtGnn37KokWL8Hg8HDx4kG9/+9vk5uby8ssvU1NTk1yrqD9uu+02PvvsM957771ez3nggQe46667ko99Pt9JF1kf8RSlz1UkPZ1o8fxOPS2ef7zJkyfz7LPPEolEkusjb9q0qcs5p59+Ov/7v/9LZWUlJlPPv8Kd8ViNAmExaqWhIR7vdrREJEdX91dKfyLuuusuli5dyueff96lCH3ppZfyzjvv9Pt6t99+O3/5y1946623KCsr6/U8q9WK2+3u8iWlh2MXz29qamLSpEl8/PHHrF27lr179/Lggw92Syg9+cd//EdUVeU73/kOu3btYu3atfziF78Ajm7hc9ttt9HS0sI3v/lNNm3aRFVVFWvXruWGG25IJsPOeGoO7Ke1pWXI24eOjU/t/FjJ0dV9llIi2rRpEzfffHO342PGjKG+vr7P1xFCcPvtt/PKK6/w97//vdsOD9LIkeri+cdzu938+c9/Ztu2bcyaNYsf/vCHLF++HCD5R6+0tJQNGzaQSCT48pe/zPTp07njjjvIzs5O7uKRjGfWGRSMn07N4dqh++Y7JKtmQi4Z22993o/2GAUFBWLLli1CCCGcTqeoqqoSQmhb5ZaVlfX5OrfeeqvweDzi7bffTu6vfaI9to+XSVtOZ7Jnn31WmM3mPv9eHCvkbRCRlsMiEfYPQWRdJVRVhKNxEQsHhPDWCuFvHPJ7poPB2HI6pRLRFVdcwSOPPEIspo1vURSFmpoa7rvvPr72ta/1+TqrVq3C6/Vy3nnnUVJSkvzq77Yp0ujyzDPP8N5773HgwAFeffXV5BihVHZIjcXiROMq6jCM5zF0TPNItlvJNqI+S3lA49e//nUKCgoIhUIsXLiQ+vp6FixYwKOPPtrn6wg58lTqQX19PcuXL6e+vp6SkhK+8Y1v9Ov3qpPa0XUPYOhhreoh05n0RKJj1r+cFH4yKf10PB4Pb775Jhs2bOCTTz7B7/dz+umns2jRosGOT8pA9957L/fee++Ar6MmjiYCwzA0VgOEYwkSqoodMAjk6Oo+6nciUlWVNWvW8PLLL3Pw4EEURWHcuHEUFxcjhJBLgkhpQ+2Y7IrBOGylkkAkTlwVWIWCQRFydHUf9auNSAjBFVdcwU033cQXX3zB9OnTmTp1KtXV1SxdulQuEyulFTUxjLPuO3Rfu1r2nPVFv35Ca9as4Z133mHdunWcf/75XZ77+9//zlVXXcUzzzzD9ddfP6hBZprjV7yUUiMSx5SIhknnoEaRXLt69P8sB+P3tV+J6LnnnuMHP/hBtyQEcMEFF3D//ffzu9/9TiaiFFksFgwGA7W1tRQUFGCxWGRVdwAi4RCJWByDQcUYDg/LPWOxGLGYSog4ghgQ0hbtH4WEEESjURobGzEYDF3mDPZXvxLRp59+ys9+9rNen7/kkkv4t3/7t5SDyXQGg4Fx48ZRV1dHbe3QD8Ab7USkXdtq2tKOYYBz5/oqEk8QialYlRhWYmC0gLn/ww5GEofDQUVFRXIwaSr6lYhaWlooKirq9fmioiJaW1tTDkbSSkUVFRXE4/Euc7ekflJV+PhprY1m5rVg8wzLbffU+9ha1cwUcwPzxA7IqYRxFw3LvfVgNGrjpgZacu9XIkokEr1OMuwMKh6PDyggSRsgajabuy04J/VDsAUS7dpkV0/hsPWaOewxQqqRQMKCjSBEW+CY+ZhSz/qViIQQLF26NDkr+njHrhUkSXqKtjfxRVMAg7uICmC4WtpKs+18dfYY3HEr7NoA0cAw3Xlk61ciWrJkyUnPkQ3VUjoIth2h3hfGJyyMHcYG/yyriSyrCSIdPUnRgFZNHED7SSboVyJavXr1UMUhSYMq7G0EwJCVp08A5iytOihUiAXB2n01SekomaalUSnqawDA5M4f9nvvrvexqaaNmLGjtyzqH/YYRpphnAkoScNECOJ+bWshq6tg2G+/YV8zvlCMSTY72QQh4gfXsIcxosgSkTT6xILEw0FAweoZ/kSUZdFGcoeVjt6yaPuwxzDSyEQkjT7BZiJxlYgpC0/W8A8mdFi1ikZIcWgHIrJqdjIyEUmjT7CZSDxByJSD2zb8Y7GcVq1EFKCzRCQT0cnINiJp9Ak2M6s8G3/BKTizhn8HFIdF+1j5RUdpTJaITkomImn0CTRjMhjIzi8G4/AX+rM6ExGyjaivZNVMGn0C2hgiHMPfdQ/g6Kia+RKdiUiOrj4ZWSKSRpdYiNbWZtqCMUTIzvjs4Q+hxGPjq7PH4DSE4RPk6Oo+kO+MNLoEGvGGYlQHTdT49FmUzGExUZmfRX52jrbNtRAQk6WiE5GJSBpd/I2EYwmC5jxyHDpv1W0wgCVL+39EthOdiKyaSaNLoIFIXCVoyyXbod8yKnuPtOMNxZhudGCjXSaik5CJSBpVhL9BKxG5csi261ci+vBAC03tEca6HFrfWdirWywjgayaSaOHEES8R1AFhK35uGz6/Z3tnOYRNHRUzcI+3WIZCWQikkaPUCuRcBhVMWJx5WEw6LfxQFbHNI+g0pmI2nSLZSSQiUgaPQJN2tQOcw7ZWfouz9o5qNFHZ2O1LBGdiGwjkkYP/xEKXTbOHT+F8MRCXUNJDmrsnOYh24hOSJaIpNGjvR4AS/YYXSa7Hsvd0T7VluiYgR8LQzyqY0TpTSYiaXQQAto79oJzFesbC+C0aonQFzeAqWOzCVkq6pWsmkmjQ8SHGgmwu8FPYy2c5VQx6TDhtVNulkWb5mEzwS4P+Bu0ROQc/oXaRgJZIpJGh/Z6QrEEdVEXO46EMOrYYwZgMRm0aR5O69HNHSOyRNQbmYik0aG9jlA0gd+ST16WZcA7jw6qzkQkq2a9klUzaXToKBEFLAUUZPW8AehwO9gUoKE9wsSEjVyQgxpPQJaIpJFPiGNKRAXk6rAqY0921vnYsK+JuohsrD4ZmYikkS/UCrEwgbggaMklL00SUecUE6/auWSsLBH1RtdE9M4773D55ZdTWlqKoii8+uqreoYjjVTew6hC0EwuQjGS60yPROTsmObRlhxd7YdETMeI0peuiSgQCDBz5kyefPJJPcMYcdrDMTbua2LdriMcaJILbuE9TDSuEs4qxWY24rKmR9NnZ4nIFzMdHUsUatMvoDSm60/skksu4ZJLLtEzhBFpd307Hx5oAeDTw15mlns4f3JhevUUDSfvYWxmI5edPZewpzJt3gdXx+ju9nAcHDnayO9QqxxL1IP0+NPRR5FIhEgkknzs82VmnXtOoYJ97was7Yc4HDBQ65/Bx9azmDsuT+/Qhl80AMFmABRPGfaO5TfSQWfVLBCNk8jNwdiZiKRuRlRj9WOPPYbH40l+lZeX6x3S8Au2YNj6W6aZ65mUa2aKJ8b4lnc4tPl1WgMZOJfJ+4X2b1Y+WBz6xnIch8WI0aAgBIRMLu2gTEQ9GlGJ6IEHHsDr9Sa/Dh06pHdIw+ZQS5C3d9UR/uQPWikgKx9Ov47Cmf9AtsNMiXcrWze/r3eYw897CCEEf6s186dPagnHEnpHlKQoClfNGsP1C8Zid3dsbSQTUY9GVNXMarVitabHYLXh9uGBFtQDG6gTNYwryYeZ14LVheIpY0wwjPf91yk+sp5EdB5Gy/Dv966btmptagf5+FqCWE3p9be1Iq+jlBbP1f6VC6T1KL1+alKP6r1hahtbGNP+CaXZNph4IVhdyefdp57PzFMnMiXfjPHwhzpGOsyiQfA34I/E8VrHUOCypk1DdTf2HO3fsBfU9Cm1pQtdE5Hf72fbtm1s27YNgAMHDrBt2zZqamr0DCvtfPaFl9L27RQ5wJpdAkXTup5gMGI/5QLt/4c3Zc7Oom3VIAQteIiZsijx6LsqY0+a/RE+OtDCZ41xMJq1UeByhHU3uiaijz/+mNmzZzN79mwA7rrrLmbPns3y5cv1DCutROMqn9e1UOTfSYHLBpVnQ09/9fMngauYSCRC094MKRW1HgTgsFIEQIkn/aqkzYEoG/Y1saPOB/Zs7aBsJ+pG1zai8847DyGEniGkvc8b2nG2V+E0xnDnlEL+KT2fqCgcdk7j8Cc7MDa+S95p56IYR1QTYP+1HiSuqhwSWiIqzU6/EpHHro0l8oZikJsD/kYItkDeBJ0jSy+yjSjN7az1UeTfRaHLhlI6u+fSUIeC8TOJm7OIh9ppqd4+jFHqINgCoTZ8ERWftYQchxmHJf0Sb2ciCkQSxG0d47w6xj1JR8lElMaEEBQbfeTFG8hz2aB4xgnPt1osmMbMAqDx84+HIUIdNX0OgOqpoCjHxZic9BpD1MlmNmI1ax+zdmPHukTBJh0jSk/p9ydESlIUhXPcDYixOSj5p4DVedLXFE6aw5GDGwjU70OEvSidi3KNNk17AcgfO41ryyrSuorvsZtpiEXwKh5yAAKNWqN1uvbw6UCWiNKZENC4BwUFCqf06SUVY8oIOkqJxhM0VW0d4gB1Eg2Ar2NEdf5EgPTttudo9awVp5Z8YmGIBXWOKr3IRJSmIvEE9bXViFALGE2QN7FPrzMZDdjLZwHQUvWxlsxGm+Z9IARhWz5h48lLiXrrTERtYY4uGxuQ1bNjyUSUpg40Bdiw8V121vogdwKY+r7GTsmkWaiKiUBrA8JXO4RR6qRxDwCfhgv5zfr9bK1J7+7wGWXZXLdgLOdMygdHx1QP2U7UhUxEaWrfkXZygwe0jQILTu3XaysKchg7eRYzyjwoDTuHKEKdRAPQcgBVCD6Ll6EKoe2UkcY8djP5Tqu2vVFWR89ZQPacHUsmojQUS6g01NZgi/vIcdn7XC3rZDQolJ06F5PBAA07R9eUgobdIFRajXl4ceGwGBmTnX4DGXslS0Q9kokoDVU3B3H7q7CaDGSVntqvallSzjhtWYxoMDkCeVQ48hkAB4yVAIwvcGLQeQ+zvth+2MsbO+ppwa0d6Ow5kwCZiNJSVUM7ecH95GZZUPpZLUsyGNivVPDpF200VW0e3AD1EmgGXy0q8GmkBIBJhenfWA2wu97Hjlof9QkXKAbtD0TUr3dYaUMmojSTUAX1X1Rr1TJn/6tlxzpkGU8gksB7aCfEIyd/Qbqr3QJAg6UMr2ojy2qkIjc9BzIeL69jQf+WkABHx5Ig7Ud0jCi9yESUZmrbQmT59mE2KrjGpFgt6zB27ARC5mza2gOoDbsHMUodxKNQr01b2YGWnKeUeEZEtQwgx9GRiIJRcBVrB/31OkaUXmQiSjNl2Tb+Ib+VcflZGApPG9C1yvOy8LlPIZoQtB4Y4YMbG3ZopTp7DmeeMZfzJhcwbYxb76j6rHPTxxZ/BJwdiahdJqJOMhGlGSXQiAc/ea6sAVXLQOs9c4+dCYC3bh9E2gcjxOGnqnDoI+3/Y07HbjUxuyKHbEd67F/WF51DDNpCMaL2jl08/LJq1kkmonTTuEv7N2/8gKplncaXl9FuLabVH0Gt3zHg6+micTcEW1BNNkTJTL2jSUmW1YTTakIIaCJbOxj2aY3WkkxE6WTTgWb2bN9EIBqHgoFVyzqV5zrwuScTTQjaDo7A6pkQULMRgG1iIs9tPkJN88j88GpL2UJb1HC0wXo0jnxPgZx9nyaEEOzbv4/KpiPkleSQNcBqWSejQaFg/ExyQx9hDTWBvwGchYNy7WFx5DPwNxJQjWyMVhKLhEfspPULTyvEajJiMRnAV6atqeQ7nJy4m8lkiShNNPojWFr2YlTAU3baoFTLOi2cVsHkqbPIspqSAwJHhHgU9q9HIPgwMZmYwcqEQiflI6TL/ngum1lLQgDuMdq/3sP6BZRGZCJKE/vq28kLVuFxWDAX923Jj37pXHD/yM6RM6K35n2ItHM4bOFTJmE2Kpw7KV/vqAaHp2NzUF/d6JqCkyKZiNJE7eH9WON+clxZQ7Oece4EAqqJ1tZmbfeLdNd+BGo+oD0c4+34DIRi4txTCkZUT1lPPtzfzHMf1VAXs4PZDmpcduMjE1FaaGgPY2jcjUGB7Iop2rYzg+yL9hh/a8phX4OfWG2ar2ediMPuvxCJxfjAX0CTrZIJhU6mjxn5q002tEeo94b5oi0MnjLtYJvcPksmojSw+4s28gP7yHFYsJRMO/kLUlDqsRHOm0JcFbTs35bee5/t+xv4GzBZHfgrLqDAZeXiqcVpvQpjX5V2rBTwRVsIciq1g60H9AsoTcheszSQGz1M1BAhN6cAcscPyT0URWHChMm01eXT4G2j6IstMO6cIbnXgNR9ArVbQVEwTr2Sr2SPIxJXjzbyjnCdS5bUtoURk8ahgNZgHY8OagfFSDM6froj3DTlILMrsskdfzoYjEN2nyljPNR7ZuKPxPHt/wgSsSG7V0qa9tG85U8cbA7A2LMgbwJmowGndfT8vSxwWTEbFcKxBE2JLG3TRTUxMtrthpBMRHqL+KG5CgUFQ8mJtwsaKIfFRP64mURMTuoam7WSR5qINu6nav2z7K33sj1exj7HyBxBfTJGg8KYHK1UVN0SPFoCbtmvY1T6k4lIR8FonJpdH6KqCXCXgLNgyO85d1weX3hOpyUQJbB3vbajhI6EENTs+pjtf32KhjY/rfYKXDOvZFyBS9e4htK4fG0NpQNNgaPzCRv3aHPqMtToKfOOQJ8daiW07R2C1iinTpkzLPfMc1rxjDudROAzYpGgNlZnwvnDcu/j1bcG2PvBnzHXaptBBrMnMX3hNynPHzmz6lMxLi+LDyxGPHYzIrsExWzTOg/aqiF3nN7h6UImIp2oquDQ3i2UJgLkZBcO2tyyvjh3chHWoqux7npFm9VeOAVcRcN2fwDRXs+Bv/0Os68OgwLOiWdx+oKvYDUP/tCFdONxmLn53PFHewELToXabdCwK2MTkaya6WTvER+epq2YjQp5kxdoe5cNE7fNjLVoMhScAkKF3X8e8oZrfyTOpoMthNpbYe9alM1rOMXeTl62m6lfvoGp5341I5JQpy5DETo3z2zcrfWeZSBZItKBqgp2b99MUbSJonw3prLhqZZ1oSgw6SIaDlfhbzzAOPtfUKZeNWjbICdUQZM/wsGmAPsb/QQaqynw72GMoxa7S+umzhs3k7yJi8A6etuDTkQIQZ03TI5jDHZ7DoRa4ch2GKPD74POZCLSwZ56L6769zEZFIqmnKPttqGDtoSFv8S/xKn+v2DYvZmxioJy2uUDGkLgDcb4v8/qaPb6yQrV4gkfpjikbY3kspmwGe2QXQGVZ0PO2EH8bkae17bX8fkRP+eeks+csjPg8zfh8GYoPX3Q/iCMFDIRDbNIPMHeLespjLVSXJCLpXKBbrFkOyzMmz2TLZv80PwW4U8/pNLXiG3qZeAu7fE1CVXQHIjQ2B6hoT1CgzdEeVaCM0sUCDTh8NZRtGc7ldE2TAZw2UzkZFvIdhVgLZkKJTMhu3yYv9P0VJmXxedH/GytaWPW/GkYD6yHYLPWg1aY4u4tI5RMRMMsGvAy3r+FuMlAyeyLwGzTNZ5pYzwoyll8vMXG+MZ1eHftIbv6APa8ciomTgV7LsJs573PG2lr9xPy+zDGA9jiPmwxL2VxH24r0KDNAzMDs3Pj2M0erM5slNzx2liZ3HFgSu8dWYfbqcUuNlY10R6Os7spwtSyuXBwA+x/G/InDeng1nQjE9FwUlVc+//KtCIrEUclprLT9Y4IgKmlHgpcZ/LejjGYDq4nEfgcZ+wgFaY2ABQgq6YVU1wb52IyKDgsRrIcJrKsNpw2CzjytFUHXcXkOIu1XjiLM+OqGP1hMho4vSKHdz9v4v2qZibOPQNr7Vatrajmfa36miFkIhomoUgcW83bKG01KEYLtmlXpNWHtNBl46vzT6Fxyliq645gCdSA3QsRH8RCFJd5MJqsON0eHE4Pii1bSzz2HLBlg0F2wKZiZnk2nx724g3F2Fjt5/yJi2Dnn7SSUU7l0Rn6o5xMRMPAF4qy8W+vMin0CePzs1AmXwpZeXqH1Y2iKBS6bRS6xwJdG5J7bjGSBspsNHD+qYW8uvULttW0McZTzimFp0LDbtj+B5h9XVr+rgy2tPgz9uSTT1JZWYnNZmP+/Pl89NFHeoc0aA43tvDBa2vwNHyENxQjXHk+FA3BCozSiDUuP4szKnPIzbKQ77LB5Mu0TRhjIdj6W2gd/RNiFSH0XTf0hRde4Prrr+c///M/mT9/PitXruSll15iz549FBaeeJF3n8+Hx+PB6/XidqfXtIBmbzt7P32fWNW7mBMh7BYTE8/8Ks4J8/UOTUpDqiqIqSpWk9ZAHQ22o376ArZQo3ZCyUyo+NLR3T9GiL5+RnVPRPPnz2fu3Ln8+te/BkBVVcrLy/nnf/5n7r///hO+Nh0SUSKhEosEsRGFUCsR7xE2ffIJtNZgFNpo5Zz8Ysad9XWs+ZW6xCiNPFtqWtm4p545sU1URvfhsJiwmQ0Ys8dgyhsPziKtg8DiAJM9bdvo+voZ1bWNKBqNsnnzZh544IHkMYPBwKJFi3j//fe7nR+JRIhEIsnHPp+vbzdqOQBV67ouGi8Em6ubSaiCo7lYgBAIwGU1MrXEnTy+ubqFWELVrtFxDkKFeBSX1ci0jmVMLQgcvjaiQiUrp5CSKWeRP2n+sE7hkEa+Bl+YGEY+MH+JHWolpW3byAnVQE0bRmUHZ1TmYujo7DjYHKA5rIDBgFAMCIwd/2rPzxmbg8molbT2N/pp9GufoSklbly2E0yrOVlnyvRrwDY4BQBdPx1NTU0kEgmKirpOuCwqKmL37t3dzn/sscdYsWJF/28Uj4C/sdthNdBCQu25QChUE0SO/iCUaAAR73mZhriqautM2zwoWflUFGbjKJmEI2dMWvWMSSPHxdNKmFuZy/6mAHVeJ/Xt5dQE2nAFD+GK1KO4LRBuhViYaFwlGul9jpoSFGDQfg/VgB+1IxEpwRgkBpACxOAtWzKi/kw/8MAD3HXXXcnHPp+P8vI+jNLNLoeZ1x6TFBRQFMaMCyWPKYoCigEFUAwKZqMRHJaO5xUqp0RQhYKiKB3nGFBQsNntWO1ZXRa8HyUb3kg6y3NayXMeHQQqhCCamEE4qqI4On7fVJW8tjayQkEtMQgVRSS03UE6KG5r8vc8LxjFHtVK9maXBY5dgre/rTSWrJS/t+Ppmojy8/MxGo0cOXKky/EjR45QXFzc7Xyr1YrVmsLoXEtWj8srlGb3/RK5mTkvU0ojiqJgNRmTDdoAGAzk5eYCfWvEzsmBnKEJb0B0beGyWCzMmTOHdevWJY+pqsq6detYsEC/OViSJA0v3atmd911F0uWLOGMM85g3rx5rFy5kkAgwA033KB3aJIkDRPdE9HixYtpbGxk+fLl1NfXM2vWLF5//fVuDdiSJI1euo8jGoh0GEckSVLv+voZTc9RUJIkZRSZiCRJ0p1MRJIk6U73xuqB6Gze6vNUD0mShlXnZ/NkTdEjOhG1t7cD9G10tSRJumlvb8fj8fT6/IjuNVNVldraWlwuV9d9ojJE5xSXQ4cOyV7DPpDvV/8N9D0TQtDe3k5paSmGE6wQMKJLRAaDgbKyzFhK80Tcbrf8YPWDfL/6byDv2YlKQp1kY7UkSbqTiUiSJN3JRDSCWa1WHnroodRWJMhA8v3qv+F6z0Z0Y7UkSaODLBFJkqQ7mYgkSdKdTESSJOlOJiJJknQnE1Ea6+8OuC+99BKnnnoqNpuN6dOn83//93/DFGn66M97tmbNGm0zhGO+bDbbMEarr3feeYfLL7+c0tJSFEXh1VdfPelr3n77bU4//XSsVisTJ05kzZo1gxKLTERp6oUXXuCuu+7ioYceYsuWLcycOZOLLrqIhoaGHs/fuHEj3/zmN1m2bBlbt27lqquu4qqrruKzzz4b5sj109/3DLQRw3V1dcmv6urRv71zp0AgwMyZM3nyySf7dP6BAwe47LLLOP/889m2bRt33HEHN910E2vXrh14MEJKS/PmzRO33XZb8nEikRClpaXiscce6/H8a665Rlx22WVdjs2fP1/cfPPNQxpnOunve7Z69Wrh8XiGKbr0BohXXnnlhOfce++9YurUqV2OLV68WFx00UUDvr8sEaWhzh1wFy1alDx2oh1wAd5///0u5wNcdNFFvZ4/2qTyngH4/X7Gjh1LeXk5V155JTt27BiOcEekofwdk4koDZ1oB9z6+voeX1NfX9+v80ebVN6zyZMn8/TTT/PHP/6RZ599FlVVOfPMMzl8+PBwhDzi9PY75vP5CIVCA7r2iJ59L0kDsWDBgi7755155pmcdtpp/OY3v+HHP/6xjpFlHlkiSkP93QEXoLi4uF/njzapvGfHM5vNzJ49m3379g1FiCNeb79jbrcbu90+oGvLRJSGUtkBd8GCBV3OB3jzzTczZsfcwdg1OJFIsH37dkpKSoYqzBFtSH/HBtzcLQ2J559/XlitVrFmzRqxc+dO8Z3vfEdkZ2eL+vp6IYQQ1113nbj//vuT52/YsEGYTCbxi1/8QuzatUs89NBDwmw2i+3bt+v1LQy7/r5nK1asEGvXrhVVVVVi8+bN4tprrxU2m03s2LFDr29hWLW3t4utW7eKrVu3CkA8/vjjYuvWraK6uloIIcT9998vrrvuuuT5+/fvFw6HQ9xzzz1i165d4sknnxRGo1G8/vrrA45FJqI09u///u+ioqJCWCwWMW/ePPHBBx8kn1u4cKFYsmRJl/NffPFFccoppwiLxSKmTp0qXnvttWGOWH/9ec/uuOOO5LlFRUXi0ksvFVu2bNEhan289dZbAuj21fkeLVmyRCxcuLDba2bNmiUsFosYP368WL169aDEIpcBkSRJd7KNSJIk3clEJEmS7mQikiRJdzIRSZKkO5mIJEnSnUxEkiTpTiYiSZJ0JxORNKKsWbOG7Ozs5OOHH36YWbNmJR8vXbqUq666atjjkgZGJiKpR0uXLkVRFG655ZZuz912220oisLSpUu7nD/YCaCyspKVK1d2ObZ48WL27t3b62ueeOKJLsuXnnfeedxxxx2DGpc0+GQiknpVXl7O888/32WtmXA4zO9//3sqKip0iclut1NYWNjr8x6Pp0uJSRoZZCKSenX66adTXl7Oyy+/nDz28ssvU1FRwezZswd07Z5KKldddVWylHXeeedRXV3NnXfemVzYHrpXzY53bMls6dKlrF+/nieeeCJ5jQMHDjBx4kR+8YtfdHndtm3bUBRFLgGiE5mIpBO68cYbWb16dfLx008/zQ033DDk93355ZcpKyvjkUceSS5s319PPPEECxYs4Nvf/nbyGhUVFd2+J4DVq1dz7rnnMnHixMH6FqR+kIlIOqF/+qd/4r333qO6uprq6mo2bNjAP/3TPw35fXNzczEajbhcLoqLi1Na4M3j8WCxWHA4HMlrGI1Gli5dyp49e5JbDcViMX7/+99z4403Dva3IfWRXCpWOqGCggIuu+wy1qxZgxCCyy67jPz8fL3DGpDS0lIuu+wynn76aebNm8ef//xnIpEI3/jGN/QOLWPJEpF0UjfeeCNr1qzhf/7nfwat1GAwGDh+BZpYLDYo1+6Lm266KdkQv3r1ahYvXozD4Ri2+0tdyUQkndTFF19MNBolFotx0UUXDco1CwoKurT7JBKJbptBWiwWEonEgO7T2zUuvfRSsrKyWLVqFa+//rqslulMVs2kkzIajezatSv5/954vV62bdvW5VheXh7l5eXdzr3gggu46667eO2115gwYQKPP/44bW1tXc6prKzknXfe4dprr8VqtaZUJaysrOTDDz/k4MGDOJ1OcnNzMRgMybaiBx54gEmTJmXM2t7pSpaIpD5xu9243e4TnvP2228ze/bsLl8rVqzo8dwbb7yRJUuWcP3117Nw4ULGjx/P+eef3+WcRx55hIMHDzJhwgQKCgpSivvuu+/GaDQyZcoUCgoKqKmpST63bNkyotHosPQCSicml4qVMta7777LhRdeyKFDh7ptHCgNL5mIpIwTiURobGxkyZIlFBcX87vf/U7vkDKerJpJGee5555j7NixtLW18bOf/UzvcCRkiUiSpDQgS0SSJOlOJiJJknQnE5EkSbqTiUiSJN3JRCRJku5kIpIkSXcyEUmSpDuZiCRJ0p1MRJIk6e7/A3xyZ+wNCl20AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:36:17.007685Z", + "iopub.status.busy": "2024-03-26T13:36:17.006853Z", + "iopub.status.idle": "2024-03-26T13:36:17.191818Z", + "shell.execute_reply": "2024-03-26T13:36:17.190886Z" + }, + "papermill": { + "duration": 0.20384, + "end_time": "2024-03-26T13:36:17.194053", + "exception": false, + "start_time": "2024-03-26T13:36:16.990213", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA6VElEQVR4nO3de1wU1f8/8NcuC8vdG3IRwVUxL6lgoIifTC0uSZpkFqkFklmJaB83TekCmX2jCyKaJmkSaXnJJO2hphBeU/ACaaiIgSimXL2AAu4uu+f3B7+djysLLArMXt7Px4OHzpkzO+9hZt/MnDlzRsAYYyCEEAMj5DsAQgh5GJS8CCEGiZIXIcQgUfIihBgkSl6EEINEyYsQYpAoeRFCDBIlL0KIQRLxHUBHU6lUuH79Ouzs7CAQCPgOhxDyAMYY7ty5gx49ekAobPr8yuSS1/Xr1+Hm5sZ3GISQFly9ehU9e/Zscr7JJS87OzsADb8Ye3t7nqNpPwqFAmlpaQgMDIS5uTnf4ZA2YCr7tLq6Gm5ubtx3tSkml7zUl4r29vZGn7ysra1hb29v1Ae6KTG1fdpSs45eNNivXr0aEokElpaW8PX1xYkTJ5qsO3bsWAgEgkY/zz33XAdGTAjhG+/Ja+vWrZBKpYiNjUVOTg48PT0RFBSE8vJyrfVTU1NRUlLC/Zw9exZmZmZ46aWXOjhyQgifeE9eCQkJmDVrFiIiIjBo0CAkJSXB2toaycnJWut37doVzs7O3E96ejqsra0peRFiYnht85LL5cjOzkZ0dDRXJhQK4e/vj8zMTJ0+Y/369XjllVdgY2Ojdb5MJoNMJuOmq6urATS0HygUikeIXr+pt82Yt9HUmMo+1XX7eE1elZWVUCqVcHJy0ih3cnLChQsXWlz+xIkTOHv2LNavX99knbi4OCxZsqRReVpaGqytrVsftIFJT0/nOwTSxox9n9bW1upUz6DvNq5fvx5DhgzBiBEjmqwTHR0NqVTKTatvwwYGBhr93cb09HQEBASYxJ0pU2Aq+1R9ddQSXpOXg4MDzMzMUFZWplFeVlYGZ2fnZpetqanBli1b8MknnzRbTywWQywWNyo3Nzc36gNAzVS209jU1tY2uvq4WyfDsdxCdHFwhK3V/47pAQMGGNVVhK7HK6/Jy8LCAt7e3sjIyEBISAiAhsd3MjIyEBUV1eyy27Ztg0wmw6uvvtoBkRLSsS5cuABvb2+t8758YDo7OxtPPPFE+welZ3i/bJRKpQgPD4ePjw9GjBiBxMRE1NTUICIiAgAQFhYGV1dXxMXFaSy3fv16hISEoFu3bnyETUi7GjBgALKzszXK8ktuQ7otFwkvDUF/l84adU0R78krNDQUFRUViImJQWlpKby8vLB3716uEb+4uLjRw5n5+fn4888/kZaWxkfIhLQ7a2vrRmdTwis3ID5Sh4GDPeHVi/5o8568ACAqKqrJy8SDBw82Kuvfvz/ojW2EmDbeO6kSQsjDoORFCDFIlLwIIQaJkhchxCBR8iKEGCRKXoQQg0TJixBikCh5EUIMEiUvQohBouRFCDFIlLwIIQaJkhchxCBR8iKEGCRKXoQQg0TJixBikPRiPC/y6B4c8/x/452f0hjvHDC+Mc+JaaLkZSSaGvP8wfHOAdMd85wYF0peRuLBMc+bGu9cXZcQQ0fJy0g8OOY5jXdOjB3vDfarV6+GRCKBpaUlfH19ceLEiWbr3759G3PmzIGLiwvEYjEee+wx7Nmzp4OiJYToC17PvLZu3QqpVIqkpCT4+voiMTERQUFByM/Ph6OjY6P6crkcAQEBcHR0xC+//AJXV1dcuXIFnTt37vjgCSG84jV5JSQkYNasWdw7GpOSkrB7924kJydj8eLFjeonJyfj5s2bOHbsGPdWXYlE0pEhE0L0BG/JSy6XIzs7G9HR0VyZUCiEv78/MjMztS7z22+/wc/PD3PmzMHOnTvRvXt3TJs2DYsWLYKZmVlHhU5IuyiqrEGNrL7J+YUVNdy/IpH2r66NWITeDjbtEp++4S15VVZWQqlUci+XVXNyctLor3S/S5cuYf/+/Zg+fTr27NmDgoICREZGQqFQIDY2VusyMpkMMpmMm66urgYAKBQKKBSKNtoa/VNfX8/9a8zbaSwu36hBQOJRneq++0tus/PT//sfSLoZbgLT9Xg1qLuNKpUKjo6OWLt2LczMzODt7Y1r167hq6++ajJ5xcXFYcmSJY3K09LSjLqj5tW7ACBCVlYWrp3lOxrSEvX+es1DCScr7S9UVqiAmzKgqxgw13KrraxOgI0FZtiXcQhutu0abruqra3VqR5vycvBwQFmZmYoKyvTKC8rK4Ozs7PWZVxcXGBubq5xiThw4ECUlpZCLpfDwsKi0TLR0dGQSqXcdHV1Ndzc3BAYGAh7e/s22hp+XL5RgxqZUus8WWkVkJsHR48h6OXcqcnPsBGbGfRfaWNx7no14nOz8GLAf/B4D+3HpUKhQHp6OgICArg23wc/Y2NBFp588skmP8MQqK+OWsJb8rKwsIC3tzcyMjIQEhICoOHMKiMjA1FRUVqX+c9//oNNmzZBpVJBKGz403Px4kW4uLhoTVwAIBaLIRaLG5Wbm5trPQAMRVGlbpcZ7/2a12KdAwvGmkw7ib5St2GJRKIWj8umjt3WfIY+0zV2Xi8bpVIpwsPD4ePjgxEjRiAxMRE1NTXc3cewsDC4uroiLi4OADB79mysWrUK77zzDubOnYt//vkHn332GebNm8fnZvBC3bCbGOoFD8fG1wg1dTLsOpiJCWP9YGPVOHkDQEH5Xfx36+lmG4kJ0Ve8Jq/Q0FBUVFQgJiYGpaWl8PLywt69e7lG/OLiYu4MCwDc3Nywb98+zJ8/H0OHDoWrqyveeecdLFq0iK9N4J2Hoy0Guza+LFQoFCjtDjzRq4tB/xUmpCm8N9hHRUU1eZl48ODBRmV+fn7Iyspq56gIIfqO98eDCCHkYVDyIoQYJEpehBCDxHubFyEEkCnvQWh5DUXV+RBaau9hWl9fj+v115F3M0/r40FF1XchtLwGmfIegKb79hkLSl4GqqWDvaUDHTC9g12fXa+5ApveX+P95keEAgB8s/ebJufZ9Aau13jBG05N1jEWlLwMlK4He3MHOmBaB7s+62HTCzVFc7Ei1At9tfTbAxr+IB398yj+8+R/tP5BKiy/i3e2nkaPcb3aO1y9QMnLQLV0sLd0oAOmd7DrM7GZJVT3XNHbvj8GddN+FqxQKFAkKsLArgO19t1T3auC6l4FxGaW7R2uXqDkZaBaOthbOtAB0zvYiXGhu42EEINEyYsQYpAoeRFCDBIlL0KIQaLkRQgxSJS8CCEGiZIXIcQgUfIihBgkSl6EEINEyYsQYpAoeRFCDBIlL0KIQdKL5LV69WpIJBJYWlrC19cXJ040Pc5LSkoKBAKBxo+lJT1YTIip4T15bd26FVKpFLGxscjJyYGnpyeCgoJQXl7e5DL29vYoKSnhfq5cudKBERNC9AHvySshIQGzZs1CREQEBg0ahKSkJFhbWyM5ObnJZQQCAZydnbkf9XseCSGmg9fxvORyObKzsxEdHc2VCYVC+Pv7IzMzs8nl7t69i169ekGlUuGJJ57AZ599hscff1xrXZlMBplMxk1XV1cDaBjvSqFQtNGWdLz6+nruX23boS5rbhtb+gzScXTZFy3tU2PZn7rGzmvyqqyshFKpbHTm5OTkhAsXLmhdpn///khOTsbQoUNRVVWF+Ph4jBo1CufOnUPPnj0b1Y+Li8OSJUsalaelpcHa2rptNoQHV+8CgAh//vknrmgfNRgAkJ6e/sifQdpfa/ZFU/vUWPZnbW2tTvUMbiRVPz8/+Pn5cdOjRo3CwIED8e2332Lp0qWN6kdHR0MqlXLT1dXVcHNzQ2BgIOzt7Tsk5vZw7no14nOz8OSTT+LxHo23Q6FQID09HQEBAU2OpNrSZ5COo8u+aGmfGsv+VF8dtYTX5OXg4AAzMzOUlZVplJeVlcHZ2VmnzzA3N8ewYcNQUFCgdb5YLIZYLNa6XFNfakOgHpdeJBI1ux3Nbaeun0HaX2v2RVP71Fj2p66x89pgb2FhAW9vb2RkZHBlKpUKGRkZGmdXzVEqlcjNzYWLi0t7hUkI0UO8XzZKpVKEh4fDx8cHI0aMQGJiImpqahAREQEACAsLg6urK+Li4gAAn3zyCUaOHAkPDw/cvn0bX331Fa5cuYI33niDz80ghHQw3pNXaGgoKioqEBMTg9LSUnh5eWHv3r1cI35xcTGEwv+dIN66dQuzZs1CaWkpunTpAm9vbxw7dgyDBg3iaxMIITzgPXkBQFRUFKKiorTOO3jwoMb08uXLsXz58g6IihCiz3jvpEoIIQ+DkhchxCBR8iKEGCS9aPMixNTVKZQAgLPXqpqsU1Mnw6kKwPnKLdhYNe67WFB+t93i00eUvAxUSwd7Swc6YHoHuz4r/P/7YnFqbgs1RdhYcLLZGjZi0/ham8ZWGiHdDvaWD3TAdA52fRb4eMMTJX0dbWFlbqa1Tn5JFd79JRfLpgxBf5dOWuvYiEXo7WDTbnHqEzpqDVRLB7suBzpgWge7PutqY4FXRrg3W0c9akTf7jYY7Nr0PjUVlLwMVEsHOx3oxNjR3UZCiEFqdfK6dOlSe8RBCCGt0urk5eHhgXHjxuHHH3/EvXv32iMmQghpUauTV05ODoYOHQqpVApnZ2e89dZbzb7thxBC2kOrk5eXlxdWrFiB69evIzk5GSUlJXjyyScxePBgJCQkoKKioj3iJIQQDQ/dYC8SiTB58mRs27YNX3zxBQoKCrBgwQK4ubkhLCwMJSUlbRknIYRoeOjkderUKURGRsLFxQUJCQlYsGABCgsLkZ6ejuvXr2PSpEltGSchhGhodT+vhIQEfP/998jPz0dwcDA2bNiA4OBgbsDA3r17IyUlBRKJpK1jJYQQTquT15o1a/D6669jxowZTY4b7+joiPXr1z9ycIQQ0pRWJ6/09HS4u7trDM0MAIwxXL16Fe7u7rCwsEB4eHibBUkIIQ9qdZtX3759UVlZ2aj85s2b6N27d5sERQghLWl18mKMaS2/e/cuLC0tHyqI1atXQyKRwNLSEr6+vjr3G9uyZQsEAgFCQkIear2EEMOl82Wj+q3TAoEAMTExsLa25uYplUocP34cXl5erQ5g69atkEqlSEpKgq+vLxITExEUFIT8/Hw4Ojo2udzly5exYMECjB49utXrJIQYPp2T119//QWg4cwrNzcXFhYW3DwLCwt4enpiwYIFrQ4gISEBs2bN4t7TmJSUhN27dyM5ORmLFy/WuoxSqcT06dOxZMkSHDlyBLdv3271egkhhk3n5HXgwAEAQEREBFasWAF7e/tHXrlcLkd2djaio6O5MqFQCH9/f2RmZja53CeffAJHR0fMnDkTR44caXYdMpkMMpmMm66urgYAKBQKKBSKR9wC/aUeEqe+vt6ot9OUmMo+1XXbWn238fvvv291ME2prKyEUqnkXjCr5uTkhAsXLmhd5s8//8T69etx+vRpndYRFxeHJUuWNCpPS0vTuPQ1NlfvAoAIWVlZuHaW72hIWzCVfVpbW6tTPZ2S1+TJk5GSkgJ7e3tMnjy52bqpqak6rfhh3LlzB6+99hrWrVsHBwcHnZaJjo7m2uuAhjMvNzc3BAYGtsnZo746U3wTyD2FkSNHwtO9K9/hkDZgKvtUfXXUEp2SV6dOnSAQCLj/txUHBweYmZmhrKxMo7ysrAzOzs6N6hcWFuLy5cuYOHEiV6ZSqQA0PGuZn5+Pvn37aiwjFoshFjd+AYW5uTnMzc3bYjP0kkgk4v415u00JaayT3XdNp2S1/2Xim152WhhYQFvb29kZGRw3R1UKhUyMjIQFRXVqP6AAQOQm6v5wokPP/wQd+7cwYoVK+Dm5tZmsRFC9BvvY9hLpVKEh4fDx8cHI0aMQGJiImpqari7j2FhYXB1dUVcXBwsLS0xePBgjeU7d+4MAI3KCSHGTafkNWzYMO6ysSU5OTmtCiA0NBQVFRWIiYlBaWkpvLy8sHfvXq4Rv7i4uNGjSIQQolPyau8e7FFRUVovEwHg4MGDzS6bkpLS9gERQvSeTskrNja2veMghJBWoesxQohB0unMq2vXrrh48SIcHBzQpUuXZtu/bt682WbBEUJIU3RKXsuXL4ednR33f10b7wkhpL3olLzuH1hwxowZ7RULIYTorNVtXmZmZigvL29UfuPGDZiZmbVJUIQQ0pI2G4xQJpNpDJNDCCHtSece9itXrgTQMBjhd999B1tbW26eUqnE4cOHMWDAgLaPkBBCtNA5eS1fvhxAw5lXUlKSxiWihYUFJBIJkpKS2j5CQgjRQufkVVRUBAAYN24cUlNT0aVLl3YLihBCWtLqB7PVI6oSQgifWp28Xn/99WbnJycnP3QwhBCiq1Ynr1u3bmlMKxQKnD17Frdv38bTTz/dZoERQkhzWp28fv3110ZlKpUKs2fPbjSKKSGEtJc2eTBbKBRCKpVydyQJIaS9tdmoEoWFhdyrmQghpL21+rLx/jfxAA39vkpKSrB7926NZyAJIaQ9tTp5qd+crSYUCtG9e3csW7asxTuRhBDSVqifFyHEIOnFSKqrV6+GRCKBpaUlfH19ceLEiSbrpqamwsfHB507d4aNjQ28vLywcePGDoyWEKIPeE9eW7duhVQqRWxsLHJycuDp6YmgoCCtw+4ADaO6fvDBB8jMzMTff/+NiIgIREREYN++fR0cOSGET7wnr4SEBMyaNQsREREYNGgQkpKSYG1t3WRP/bFjx+KFF17AwIED0bdvX7zzzjsYOnQo/vzzzw6OnBDCJ15fOiuXy5GdnY3o6GiuTCgUwt/fH5mZmS0uzxjD/v37kZ+fjy+++EJrHZlMBplMxk1XV1cDaHgyQKFQPOIW6C91t5X6+nqj3k5TYir7VNdta7Pk9e+//+KTTz7B2rVrdV6msrISSqWSe8GsmpOTEy5cuNDkclVVVXB1dYVMJoOZmRm++eYbBAQEaK0bFxeHJUuWNCpPS0uDtbW1zrEamqt3AUCErKwsXDvLdzSkLZjKPq2trdWpXpslrxs3bmD9+vWtSl4Py87ODqdPn8bdu3eRkZEBqVSKPn36YOzYsY3qRkdHa/RNq66uhpubGwIDA2Fvb9/usfLlTPFNIPcURo4cCU/3rnyHQ9qAqexT9dVRS3i9bHRwcICZmRnKyso0ysvKyuDs7NzkckKhEB4eHgAALy8v5OXlIS4uTmvyEovFEIvFjcrNzc1hbm7+aBugx0QiEfevMW+nKTGVfarrtvHaYG9hYQFvb29kZGRwZSqVChkZGfDz89P5c1QqlUa7FiHE+PF65gU0PG4UHh4OHx8fjBgxAomJiaipqUFERAQAICwsDK6uroiLiwPQ0Ibl4+ODvn37QiaTYc+ePdi4cSPWrFnD52YQQjqYzslr8uTJzc6/ffv2QwUQGhqKiooKxMTEoLS0FF5eXti7dy/XiF9cXAyh8H8niDU1NYiMjMS///4LKysrDBgwAD/++CNCQ0Mfav2EEMOkc/Lq1KlTi/PDwsIeKoioqChERUVpnXfw4EGN6U8//RSffvrpQ62HEGI8dE5e33//fXvGQQghrcJ7D3tCCHkYOp956TrcDb2Ag5BHV1tb26ijdn7JbchKC5B31gqqG5258gEDBhh1h+um6Jy8UlJS0KtXLwwbNgyMsfaMiTyEBw/2pg50wHQPdkNy4cIFeHt7a5037QfN6ezsbDzxxBMdEJV+0Tl5zZ49G5s3b0ZRUREiIiLw6quvomtX4+3la2iaOtgfPNAB0z3YDcmAAQOQnZ2tUXa3TobdBzLx3Dg/2FqJNeqaIgFrxWmUTCZDamoqkpOTcezYMTz33HOYOXMmAgMDIRAI2jPONlNdXY1OnTqhqqrKqB4PevDMq6kDHaAzL0OlUCiwZ88eBAcHG3UPe12/o63qpCoWizF16lRMnToVV65cQUpKCiIjI1FfX49z587B1tb2kQMnD8fa2lrjbEqhUOBWZTn8RvgY9YFOTNdD320UCoUQCARgjEGpVLZlTIQQ0qJWJS+ZTIbNmzcjICAAjz32GHJzc7Fq1SoUFxfTWRchpEPpfNkYGRmJLVu2wM3NDa+//jo2b94MBweH9oyNEEKapHODvVAohLu7O4YNG9Zs43xqamqbBdcejLXB/n5KpRIHDhzA77//jvHjx2PcuHEwMzPjOyzyCOrq6iCVSpGVlYWRI0ciISEBVlZWfIfVLnT9juqcvGbMmKHTHUV9f4zI2JNXamoqpFIprly5wpX16tULCQkJLT5cT/RTSEgIdu7c2ah80qRJ2LFjR8cH1M50/o4yE1NVVcUAsKqqKr5DaXPbt29nAJilpSUDwP2op7dv3853iKSVJk2apLEvH/yZNGkS3yG2OV2/o63q52UMjPXMS6lUwsXFBRUVFbCyskJdXR03Tz3t6OiI69ev0yWkgairq+P64zW1T4GGPn7GdAmp63eUHsw2EgcPHkRFRQUA4JlnnsGRI0ewefNmHDlyBM888wwAoLy8vNEQQ0R/3f/uhab26YP1TAklLyOxf/9+AICfnx+2bduG48ePY+PGjTh+/Di2bduGkSNHatQj+u/kyZMAAG9vb+zcuRO+vr6wsrKCr68vdu7cyXVKVtczNbwPA03aRnFxMYCGnvZ2dnbcO/727NmDxYsXY/To0Rr1iP5Tt+j069cPjDEcOnQIhw8fho2NDcaNGwcPDw/k5OSY7EAJlLyMhLu7OwAgIyMDjo6OmD59OmpqamBjY4OffvoJBw4c0KhH9J+/vz9ycnLw888/4+jRo7h69SqAhrfMu7m54dq1a1w9U0TJy0ioz6yAhgbP5cuXc9OWlpZa6xH9FhgYiC+//BIqlQr//vsvpk2bBm9vb2RnZ2Pz5s3cGVdgYCDPkfKDkpeROHfuHPf/B18DJ5fLNeo9++yzHRYXeXijR4+GUCiESqUCYwybNm3Cpk2bNOoIhUKT/YOkFw32q1evhkQigaWlJXx9fXHixIkm665btw6jR49Gly5d0KVLF/j7+zdb31RcunSJ+//9Z1oANF66e389ot+OHTsGlUoFgUDQaJ9aWVlBIBBApVLh2LFjPEXIL96T19atWyGVShEbG4ucnBx4enoiKCgI5eXlWusfPHgQU6dOxYEDB5CZmQk3NzcEBgZy1/+mSv30wwsvvABHR0eNeU5OTpg0aZJGPaL/SkpKAADz5s3jbsCoKRQKzJs3T6OeyWnv3rItGTFiBJszZw43rVQqWY8ePVhcXJxOy9fX1zM7Ozv2ww8/6FTfWHvYb9iwgQFg9vb2zN3dXaMXtru7O7O3t2cA2IYNG/gOlejowIEDDAATCARswoQJbOXKlSwqKoqtXLmSTZgwgQkEAgaAHThwgO9Q25Su31Fe27zkcjmys7MRHR3NlQmFQvj7+yMzM1Onz6itrYVCoWhySGqZTKbRBlRdXQ2g4S+XQqF4hOj1i4uLC4CG7bt37x6kUik8PDxQUFCAVatWce1eLi4uRrXdxmz48OEQiUTo2rUrfv75ZzDGkJ6ejoCAALzxxhuQSCS4efMmhg8fblT7VNdt4TV5VVZWQqlUcm/HVnNycmr05pSmLFq0CD169GjydnFcXByWLFnSqDwtLc2ohkKWy+UQCoUQiURQKBRISEjg5gkEAlhYWECpVOLWrVvYs2cPj5ESXeXm5qK+vh7l5eV46qmnMGzYMIjFYuzevRt//fUX17SyfPlyDBkyhOdo205tba1O9Qz6buPnn3+OLVu24ODBg40aNNWio6M1Hp+orq7m2smM6dnGQ4cOQaVSQS6Xa23XUp95denSBWPGjOno8MhDUF8lzJ07F2vWrMGpU6e4eSKRCFFRUVi1ahV69eqF4OBgvsJsc+rtbgmvycvBwQFmZmYoKyvTKC8rK4Ozs3Ozy8bHx+Pzzz/HH3/8gaFDhzZZTywWa9xtUzM3Nzeqsd3VzzUC4IbnVhMKhdxQ3RUVFUa13cbMzc0NALBq1So899xzCAgIwD///IN+/fohPT0dq1ev5uoZ0z7VeVs6pAWuGSNGjGBRUVHctFKpZK6urs022H/xxRfM3t6eZWZmtnp9xtpgv2/fPgaAde3ald27d4+lp6czqVTK0tPT2b1791iXLl0YALZv3z6+QyU6kslkTCQSMScnJ6ZQKJhcLmc7duxgcrmcKRQK5uTkxEQiEZPJZHyH2qYMosEeaHgiPjw8HD4+PhgxYgQSExNRU1ODiIgIAEBYWBhcXV0RFxcHAPjiiy8QExODTZs2QSKRoLS0FABga2tr0uPo5+bmAgB69uwJc3NzjBkzBjU1NRgzZgzMzMzg5uaGW7duITc312R7ZBuaY8eOob6+HmVlZQgJCUHv3r1x8eJF/PHHHygqKuKuWI4dO4axY8fyGywPeE9eoaGhqKioQExMDEpLS+Hl5YW9e/dyjfjFxcUQCv/XHW3NmjWQy+WYMmWKxufExsbi448/7sjQ9crly5cBAH///Teef/75Rgf633//rVGP6D91/63nnnsOu3fv5srT0tIAAMHBwdizZ4/J9vPiPXkBQFRUFKKiorTOe3D8Kfryade3b18AgKenp9YD3dPTE2fOnOHqEf2n7v6ye/duODk5Ydq0adzD9ps2beLuGqvrmRoaSdVIyOVyWFpagjGG7t2749VXX0VtbS2sra3x448/oqKiAgKBAPfu3YOFhQXf4RIdqEdStbCwwJ07dyAQCLg3ZjPGYGdnB7lcTiOpEuMhEAjQr18/+Pr6ol+/fvRIkIH69ttvATR02pwyZQqysrJQV1eHrKwsTJkyhevMqa5naih5GYlvvvkGjDEEBQXh5s2biIyMxOuvv47IyEjcvHkTgYGBYIzhm2++4TtUoqPCwkIADYMR5Obm4qmnnsLUqVPx1FNP4ezZs1i7dq1GPVNDyctIqA/glJQUVFdX4+2334aXlxfefvttVFdXc6+kM9UD3RCp2ycZYygoKEB6ejqkUinS09Pxzz//QKVSadQzOe3dZ0PfGGs/r+XLlzMAbPbs2UwikWg8mC2RSNhbb73FALDly5fzHSrREfXzav47SmdeRiIyMhJCoRBr1qzB4MGDNd40M3jwYHz77bcQCoWIjIzkO1SiIwsLC8yfPx9lZWXo2bMnvvvuO9y8eRPfffcdevbsibKyMsyfP99kb8DoRVcJ8ujMzMxgZ2eHqqoqnDhxArm5ubC0tERubi43WKOdnR29s9HAfPnllwAaHr6+/w+PSCTCwoULufkmqYPOBPWGsV42qsd+mj59OhOJRBqXjSKRiE2bNs0ox34yFTKZjMXHx7Pg4GAWHx9vdJeK96PLRhOj7mWdlJSEmpoaxMfHIzg4GPHx8aipqUFSUpJGPWJYLCwsMG/ePLz55puYN2+eyV4q3o8uG42Eupf12bNnMXLkSMybNw8eHh4IDg6Gubk5srOzNeoRYujozMtIjB49GhKJBJ999hl3C11NpVIhLi4OvXv3Ntk3zRDjQ8nLSJiZmWHZsmXYtWsXQkJCNHpjh4SEYNeuXYiPj6cGewMll8uxcuVKrF27FitXrtR4nZ3J6qA2OL1hrA32atu3b2/Uz6t3795s+/btfIdGHtLChQu13oRZuHAh36G1C12/o/RgthFSKpU4cOAAfv/9d4wfPx7jxo2jMy4D9d577+Grr76Ck5MTlixZArFYDJlMhtjYWJSVlRlldwldv6OUvIyUQqHgRiAwpiGCTYlcLoeNjQ26deuGf//9F4wxbp8KBAL07NkTN27cQE1NjVHdfaRRJQgxcN988w3q6+vx6aefQiTS7BggEonwySefoL6+3mQftqfkRYieUj9EP2HCBK3z1eWm+rA9JS9C9JR6tIhdu3Zpna8uN9VRJSh5EaKnIiMjIRKJ8OGHH6K+vl5jXn19PWJiYiASiUz2YXvek9fq1ashkUhgaWkJX19f7iFibc6dO4cXX3wREokEAoEAiYmJHRcoIR2MRpVoHq+PB23duhVSqRRJSUnw9fVFYmIigoKCkJ+fD0dHx0b1a2tr0adPH7z00kuYP38+DxET0rHU3SASEhI0zrDMzMyMsptEa/B65pWQkIBZs2YhIiICgwYNQlJSEqytrZGcnKy1/vDhw/HVV1/hlVde0foWbEKM0ciRI9GzZ0+Nsp49e2LkyJE8RaQfeDvzksvlyM7ORnR0NFcmFArh7++PzMzMNluPTCaDTCbjpqurqwE09INSv8DAGKm3zZi30RT8+uuveOWVVxAcHIyUlBSUlpbC2dkZ8fHxmDJlCrZs2YIXXniB7zDblK7HLG/Jq7KyEkqlknu5rJqTkxMuXLjQZuuJi4vDkiVLGpWnpaXB2tq6zdajr9LT0/kOgTwkpVKJuXPnwsfHBzNnzkRVVRWsrKxQVVWFmTNnory8HPPmzYNIJDKqJyhqa2t1qmf0Q+JER0dDKpVy09XV1XBzc0NgYKDR97BPT09HQEAA9bA3UIcOHUJ5eTm2b98OX1/fRvvUwcEBTz31FOzt7TFmzBi+w20z6qujlvCWvBwcHGBmZoaysjKN8rKyMjg7O7fZesRisdb2MXNzc5P4UpvKdhqjiooKAICXl5fGPlTvUy8vL66eMe1jXbeFtwZ7CwsLeHt7IyMjgytTqVTIyMiAn58fX2ERojfuH2BSG3W5qQ4wyevdRqlUinXr1uGHH35AXl4eZs+ejZqaGkRERAAAwsLCNBr05XI5Tp8+jdOnT0Mul+PatWs4ffo0CgoK+NoEQtoNDTDZgvYfnad5X3/9NXN3d2cWFhZsxIgRLCsri5s3ZswYFh4ezk0XFRVpjGmk/hkzZozO6zP28bzU7n/HHzFc27dvZwKBgE2cOJEdPnyYbd68mR0+fJhNnDiRCQQCoxynjcbzagINiUMMTWpqKt59911cvnyZK+vduzfi4+MxefJk/gJrJ7p+R43+biMhhm7y5MmYNGkSDTD5AEpehBgAMzMzjBkzBjU1NRgzZozJJy5ADx7MJoSQh0HJixBikCh5EUIMEiUvQohBouRFCDFIlLwIIQaJkhchxCBR8iKEGCRKXoQQg0TJixBikCh5EWIAlEolDh06hMOHD+PQoUNQKpV8h8Q7Sl6E6LnU1FR4eHggICAACQkJCAgIgIeHB1JTU/kOjVeUvAjRY6mpqZgyZQqGDBmCI0eOYPPmzThy5AiGDBmCKVOmmHQCo+RFiJ5SKpV49913MWHCBOzYsQO+vr6wsrKCr68vduzYgQkTJmDBggUmewlJyYsQPXXkyBFcvnwZ77//Purr67Fy5UqsXbsWK1euRH19PaKjo1FUVIQjR47wHSovaDwvQvRUSUkJAGDLli0YPXo06uvrAQB79uzB4sWLMWfOHI16pobOvIxQXV0d5s2bh48//hjz5s1DXV0d3yGRh6B+K9CKFSu4xKVWX1+PFStWaNQzNXqRvFavXg2JRAJLS0v4+vrixIkTzdbftm0bBgwYAEtLSwwZMgR79uzpoEj1X0hICKytrZGUlITTp08jKSkJ1tbWCAkJ4Ts00kq+vr5tWs/Y8J68tm7dCqlUitjYWOTk5MDT0xNBQUEoLy/XWv/YsWOYOnUqZs6cib/++gshISEICQlp8t12piQkJAQ7d+7UOm/nzp2UwAzMt99+26b1jA3vbw/y9fXF8OHDsWrVKgAN76Nzc3PD3LlzsXjx4kb1Q0NDUVNTg127dnFlI0eOhJeXF5KSklpcn7G+Paiurg7W1tYt1qutrYWVlVUHREQelaurK65fv95ivR49euDatWsdEFHHMIi3B8nlcmRnZ2u8WFYoFMLf3x+ZmZlal8nMzIRUKtUoCwoKwo4dO7TWl8lkkMlk3HR1dTWAhleDKRSKR9wC/eHq6sr9//XXX8fXX3+N9PR0BAQEYO7cuUhOTubqlZWV8RUmaYX7E1ddXR0OHjzI7dOxY8dyf4SuX79uVMeyrtvCa/KqrKyEUqmEk5OTRrmTkxMuXLigdZnS0lKt9UtLS7XWj4uLw5IlSxqVp6Wl6XSmYihu3brF/f/5559Heno6ACA9PR3PP/88l7xu3bpFbYQGaPfu3RCJRHjqqacgk8mwe/dujfnGtE9ra2t1qmf0XSWio6M1ztSqq6vh5uaGwMBAo7psvF9wcDAUCgX3V/rBl84GBwfzFBl5WJGRkfjoo49gY2ODmpoaLF26VGO+Me1T9dVRS3hNXg4ODjAzM2t0GVNWVgZnZ2etyzg7O7eqvlgshlgsblRubm5utG+SZoxx22Zubo4HmzWNdbuNjb+/P/744w8AQHl5OebOndtkPWPap7puC693Gy0sLODt7Y2MjAyuTKVSISMjA35+flqX8fPz06gPNFwaNVXfVIwbN477v1gsxvvvv49r167h/fff10je99cj+k196d9W9YwO49mWLVuYWCxmKSkp7Pz58+zNN99knTt3ZqWlpYwxxl577TW2ePFirv7Ro0eZSCRi8fHxLC8vj8XGxjJzc3OWm5ur0/qqqqoYAFZVVdUu28MnAC3+EMNjavtT1+8o7/28QkNDER8fj5iYGHh5eeH06dPYu3cv1yhfXFys8fjDqFGjsGnTJqxduxaenp745ZdfsGPHDgwePJivTdAbrIVeLy3NJ/qJMQZ/f3+NMn9/f5Pfn7z38+poxtrP635PP/00Dhw4wE2PGzcO+/fv5zEi0hYUCgX27NmD4OBgo2rjepCu31Hez7xI29u/fz/kcjl27NgBuVxOiYsYJUpehBCDRMmLEGKQKHkRQgyS0fewf5D6/oSuvXgNlUKhQG1tLaqrq426cdeUmMo+VX83W7qXaHLJ686dOwAANzc3niMhhDTnzp076NSpU5PzTa6rhEqlwvXr12FnZweBQMB3OO1G/Qzn1atXjbZLiKkxlX3KGMOdO3fQo0cPCIVNt2yZ3JmXUChEz549+Q6jw9jb2xv1gW6KTGGfNnfGpUYN9oQQg0TJixBikCh5GSmxWIzY2FitwwERw0T7VJPJNdgTQowDnXkRQgwSJS9CiEGi5EUIMUiUvFowY8aMNn9Z69ixY/Hf//632ToSiQSJiYltul5CjIlJJS9dkgYxLB9//DG8vLz4DqNJ+nbM6Vs8j8Kkkhchhkgul/Mdgl4ymeQ1Y8YMHDp0CCtWrIBAIIBAIEBhYSFmzpyJ3r17w8rKCv3798eKFSu0Lr9kyRJ0794d9vb2ePvtt3U+oGpqahAWFgZbW1u4uLhg2bJljeqUl5dj4sSJsLKyQu/evfHTTz81qiMQCLBmzRqMHz8eVlZW6NOnD3755Rdu/uXLlyEQCPDzzz9j9OjRsLKywvDhw3Hx4kWcPHkSPj4+sLW1xfjx41FRUaHjbw1ITk7G448/DrFYDBcXF0RFRXHziouLMWnSJNja2sLe3h4vv/yyxmvp1GdFGzduhEQiQadOnfDKK69wD8cDDc+afvnll/Dw8IBYLIa7uzv+7//+j5u/aNEiPPbYY7C2tkafPn3w0UcfcW9UTklJwZIlS3DmzBlun6akpOi8be3tYY85dVPF//3f/6FHjx7o378/AODYsWPw8vKCpaUlfHx8sGPHDggEApw+fZpb9uzZsxg/fjxsbW3h5OSE1157DZWVlU3Gc/ny5Y76dbS99nsHiH65ffs28/PzY7NmzWIlJSWspKSE3bt3j8XExLCTJ0+yS5cusR9//JFZW1uzrVu3csuFh4czW1tbFhoays6ePct27drFunfvzt5//32d1jt79mzm7u7O/vjjD/b333+zCRMmMDs7O/bOO+9wdcaPH888PT1ZZmYmO3XqFBs1ahSzsrJiy5cv5+oAYN26dWPr1q1j+fn57MMPP2RmZmbs/PnzjDHGioqKGAA2YMAAtnfvXnb+/Hk2cuRI5u3tzcaOHcv+/PNPlpOTwzw8PNjbb7+tU+zffPMNs7S0ZImJiSw/P5+dOHGCi0mpVDIvLy/25JNPslOnTrGsrCzm7e3NxowZwy0fGxvLbG1t2eTJk1lubi47fPgwc3Z21vjdvffee6xLly4sJSWFFRQUsCNHjrB169Zx85cuXcqOHj3KioqK2G+//cacnJzYF198wRhjrLa2lr377rvs8ccf5/ZpbW2tTtvWER71mHvttdfY2bNn2dmzZ1lVVRXr2rUre/XVV9m5c+fYnj172GOPPcYAsL/++osxxtitW7dY9+7dWXR0NMvLy2M5OTksICCAjRs3rsl46uvr+fjVtAmTSV6MMTZmzBiNpKHNnDlz2IsvvshNh4eHs65du7KamhqubM2aNczW1pYplcpmP+vOnTvMwsKC/fzzz1zZjRs3mJWVFRdHfn4+A8BOnDjB1cnLy2MAGiWvB5OOr68vmz17NmPsf8nru+++4+Zv3ryZAWAZGRlcWVxcHOvfv3+zcav16NGDffDBB1rnpaWlMTMzM1ZcXMyVnTt3TmNbYmNjmbW1NauurubqLFy4kPn6+jLGGKuurmZisVgjWbXkq6++Yt7e3tx0bGws8/T01Hn5jvawx5yTkxOTyWRc2Zo1a1i3bt1YXV0dV7Zu3TqN5LV06VIWGBio8dlXr15lAFh+fr7O8RgKkxtV4kGrV69GcnIyiouLUVdXB7lc3qgB2NPTE9bW1ty0n58f7t69i6tXr6JXr15NfnZhYSHkcjl8fX25sq5du3KXAQCQl5cHkUgEb29vrmzAgAHo3Llzo8978MW6fn5+GpcMADB06FDu/+rXxw0ZMkSjrLy8vMmY1crLy3H9+nU888wzWufn5eXBzc1NY1y0QYMGoXPnzsjLy8Pw4cMBNNw1tbOz4+q4uLhw68/Ly4NMJmtyHQCwdetWrFy5EoWFhbh79y7q6+sNfkQFXY65IUOGwMLCgpvOz8/H0KFDYWlpyZWNGDFCY5kzZ87gwIEDsLW1bbTOwsJCPPbYY227ITwzmTYvbbZs2YIFCxZg5syZSEtLw+nTpxEREWHQDaT3j7CpHq/swTKVStXi51hZWbV5PA+uv6V1ZGZmYvr06QgODsauXbvw119/4YMPPjDo/aPrMWdjY9Pqz7579y4mTpyI06dPa/z8888/eOqpp9pqE/SGSSUvCwsLKJVKbvro0aMYNWoUIiMjMWzYMHh4eKCwsLDRcmfOnEFdXR03nZWVBVtb2xZHY+3bty/Mzc1x/PhxruzWrVu4ePEiNz1gwADU19cjOzubK8vPz8ft27cbfV5WVlaj6YEDBzYbw8Oys7ODRCJBRkaG1vkDBw7E1atXcfXqVa7s/PnzuH37NgYNGqTTOvr16wcrK6sm13Hs2DH06tULH3zwAXx8fNCvXz9cuXJFo86D+1TfPOwx96D+/fsjNzcXMpmMKzt58qRGnSeeeALnzp2DRCKBh4eHxo86Ger776s1TCp5SSQSHD9+HJcvX0ZlZSX69euHU6dOYd++fbh48SI++uijRgcE0HCreubMmTh//jz27NmD2NhYREVFNTvKIwDY2tpi5syZWLhwIfbv34+zZ89ixowZGsv1798fzz77LN566y0cP34c2dnZeOONN7SelWzbtg3Jycm4ePEiYmNjceLECY27f23t448/xrJly7By5Ur8888/yMnJwddffw2g4Y3NQ4YMwfTp05GTk4MTJ04gLCwMY8aMgY+Pj06fb2lpiUWLFuG9997Dhg0bUFhYiKysLKxfvx5AQ3IrLi7Gli1bUFhYiJUrV+LXX3/V+AyJRIKioiKcPn0alZWVGl9uffCwx9yDpk2bBpVKhTfffBN5eXnYt28f4uPjAfzvDHvOnDm4efMmpk6dipMnT6KwsBD79u1DREQEl7AejEeXs3C9xXejW0fKz89nI0eOZFZWVgwAu3DhApsxYwbr1KkT69y5M5s9ezZbvHixRgNweHg4mzRpEouJiWHdunVjtra2bNasWezevXs6rfPOnTvs1VdfZdbW1szJyYl9+eWXjRpNS0pK2HPPPcfEYjFzd3dnGzZsYL169WrUYL969WoWEBDAxGIxk0gkGneo1A326sZbxhg7cOAAA8Bu3brFlX3//fesU6dOOv/OkpKSWP/+/Zm5uTlzcXFhc+fO5eZduXKFPf/888zGxobZ2dmxl156iZWWlnLztTWmL1++nPXq1YubViqV7NNPP2W9evVi5ubmzN3dnX322Wfc/IULF3K/99DQULZ8+XKN+O/du8defPFF1rlzZwaAff/99zpvW0d4lGPuQUePHmVDhw5lFhYWzNvbm23atIn7TLWLFy+yF154gXXu3JlZWVmxAQMGsP/+979MpVJpjaeoqKidfwPth4bEMRACgQC//vprmz+qRAzXTz/9hIiICFRVVbVZG6UhMfm7jYQYig0bNqBPnz5wdXXFmTNnsGjRIrz88ssmmbgASl6PpLi4uNnG6fPnz8Pd3b0DI2odbbfU1X7//XeMHj26A6MhLSktLUVMTAxKS0vh4uKCl156SeNpBFNDl42PoL6+vtnHKyQSCUQi/f37UFBQ0OQ8V1dXk/2LTgwDJS9CiEEyqa4ShBDjQcmLEGKQKHkRQgwSJS9CiEGi5EXa1YwZM7iB78zNzeHk5ISAgAAkJye36tGUlJQUrSNttLf2eIcBaRuUvEi7e/bZZ1FSUoLLly/j999/x7hx4/DOO+9gwoQJqK+v5zs8Yqj4fDaJGL+mntPLyMhgALiBCJctW8YGDx7MrK2tWc+ePdns2bPZnTt3GGP/e0bz/p/Y2FjGGGMbNmxg3t7ezNbWljk5ObGpU6eysrIybj03b95k06ZNYw4ODszS0pJ5eHiw5ORkbn5xcTF76aWXWKdOnViXLl3Y888/zz3vFxsb22i9Bw4caJffE2k9OvMivHj66afh6emJ1NRUAIBQKMTKlStx7tw5/PDDD9i/fz/ee+89AMCoUaOQmJgIe3t7lJSUoKSkBAsWLAAAKBQKLF26FGfOnMGOHTtw+fJlzJgxg1vPRx99hPPnz+P3339HXl4e1qxZAwcHB27ZoKAg2NnZ4ciRIzh69ChsbW3x7LPPQi6XY8GCBXj55Ze5M8eSkhKMGjWqY39RpGl8Z09i3Jo682KMsdDQUDZw4ECt87Zt28a6devGTes6GsbJkycZAO6sbeLEiSwiIkJr3Y0bN7L+/ftzIy4wxphMJmNWVlZs3759LcZP+EVnXoQ3jDFuLKo//vgDzzzzDFxdXWFnZ4fXXnsNN27cQG1tbbOfkZ2djYkTJ8Ld3R12dnYYM2YMgIbnTgFg9uzZ2LJlC7y8vPDee+/h2LFj3LJnzpxBQUEB7OzsYGtrC1tbW3Tt2hX37t3TaYBAwi9KXoQ3eXl56N27Ny5fvowJEyZg6NCh2L59O7Kzs7F69WoAzb+zsKamBkFBQbC3t8dPP/2EkydPcoMVqpcbP348rly5gvnz53Nj8qsvOe/evQtvb+9GwyZfvHgR06ZNa+etJ49Kf58aJkZt//79yM3Nxfz585GdnQ2VSoVly5Zxo8z+/PPPGvW1DV984cIF3LhxA59//jk3JPepU6carat79+4IDw9HeHg4Ro8ejYULFyI+Ph5PPPEEtm7dCkdHxyZf6mFMwyYbGzrzIu1OJpOhtLQU165dQ05ODj777DNMmjQJEyZMQFhYGDw8PKBQKPD111/j0qVL2LhxI5KSkjQ+QyKR4O7du8jIyEBlZSVqa2vh7u4OCwsLbrnffvsNS5cu1VguJiYGO3fuREFBAc6dO4ddu3Zx4/5Pnz4dDg4OmDRpEo4cOYKioiIcPHgQ8+bNw7///sut9++//0Z+fj4qKyu5F94SPcB3oxsxbuHh4Vw3A5FIxLp37878/f1ZcnKyxnsvExISmIuLC7OysmJBQUFsw4YNjYawfvvtt1m3bt00ukps2rSJSSQSJhaLmZ+fH/vtt98avctw4MCBzMrKinXt2pVNmjSJXbp0ifvMkpISFhYWxhwcHJhYLGZ9+vRhs2bNYlVVVYwxxsrLy1lAQACztbWlrhJ6hobEIYQYJLpsJIQYJEpehBCDRMmLEGKQKHkRQgwSJS9CiEGi5EUIMUiUvAghBomSFyHEIFHyIoQYJEpehBCDRMmLEGKQKHkRQgzS/wMujMrQMmvYjAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T13:36:17.226316Z", + "iopub.status.busy": "2024-03-26T13:36:17.226023Z", + "iopub.status.idle": "2024-03-26T13:36:17.468762Z", + "shell.execute_reply": "2024-03-26T13:36:17.467886Z" + }, + "papermill": { + "duration": 0.26112, + "end_time": "2024-03-26T13:36:17.470745", + "exception": false, + "start_time": "2024-03-26T13:36:17.209625", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABg/klEQVR4nO2dd3hU1daH36npjRoCgQAivTdBFJFosAHiJ4iIgFxBJaIEkU7EBlcBQVBQvIgFpaiAFxEvREClE0LvNSGQBAjpZdr+/phkyCSTMskkk4T9Ps88zJyzzz7rkMwva++99loKIYRAIpFIqiFKZxsgkUgk5YUUOIlEUm2RAieRSKotUuAkEkm1RQqcRCKptkiBk0gk1RYpcBKJpNoiBU4ikVRb1M42oDJiMpm4du0aXl5eKBQKZ5sjkUjyIYQgNTWVgIAAlMrC/TQpcDa4du0agYGBzjZDIpEUQ0xMDA0aNCj0vBQ4G3h5eQHm/zxvb28nWyORSPKTkpJCYGCg5btaKMLJLFmyRDRq1Ei4uLiIbt26iX379hXa9vjx42LQoEGiUaNGAhCffPJJgTYffvih6NKli/D09BS1a9cWAwYMEKdPn7bLpuTkZAGI5ORkex9HIpFUACX9jjp1kWHNmjWEhYURHh7OoUOHaN++PSEhISQkJNhsn5GRQZMmTZg7dy7+/v422+zcuZNx48axd+9etm7dil6v59FHHyU9Pb08H0UikVRGKkhwbdKtWzcxbtw4y2ej0SgCAgLEnDlzir22UaNGNj24/CQkJAhA7Ny5s8R2SQ9OIqncVHoPTqfTERkZSXBwsOWYUqkkODiYPXv2OOw+ycnJANSoUaPQNtnZ2aSkpFi9JBJJ1cdpiww3b97EaDRSt25dq+N169bl9OnTDrmHyWTizTff5P7776dNmzaFtpszZw6zZ892yD2NRiN6vd4hfUkkdysqlQq1Wl3mMK1qvYo6btw4jh8/zj///FNku6lTpxIWFmb5nLtCYy9paWlcvXoVIXOISiRlxt3dnXr16qHVakvdh9MErlatWqhUKuLj462Ox8fHF7qAYA+hoaFs2rSJv/76q8g4GQAXFxdcXFxKfS+TSSCEiatXr+Lu7k7t2rVlgLBEUkqEEOh0Om7cuMGlS5do1qxZkcG8ReE0gdNqtXTu3JmIiAgGDhwImIeUERERhIaGlrpfIQSvv/4669evZ8eOHTRu3NhBFhdk7cEYvvzrIk93rM/oHg0QQlC7dm3c3NzK7Z4Syd2Am5sbGo2GK1euoNPpcHV1LVU/Th2ihoWFMWLECLp06UK3bt1YuHAh6enpjBo1CoAXX3yR+vXrM2fOHMC8MHHy5EnL+9jYWA4fPoynpyf33HMPYB6W/vDDD2zcuBEvLy/i4uIA8PHxcbjwZBtMnE9I46+zNxjdw+wlSs9NInEMpfXa8uJUgRsyZAg3btxg1qxZxMXF0aFDB7Zs2WJZeIiOjrZ6yGvXrtGxY0fL53nz5jFv3jx69+7Njh07AFi6dCkADz30kNW9vv76a0aOHOlQ+3vdUwuAQ9G3ydQZHNq3RCIpO05fZAgNDS10SJorWrkEBQUVO4FfkRP8QTXdqe/rRmxSJkdjk6lTYXeWSCQlQaZLKgMKhYIHmuV4cZdvO9maimXkyJGWuVNH8dBDD/Hmm28W2SYoKIiFCxc69L4S51CSn3dZkQJXRu7PGaZGRlddgauIXzRJxfLOO+/QoUMHZ5vhdKTAlZH776mFQgGXbqZjNMn4N8ndRWUPapcCV0ZqeGhpHWBOqZRtMFqOCyHI0Bmc8rJnHnLkyJHs3LmTRYsWoVAoUCgUXLhwgdGjR9O4cWPc3Nxo3rw5ixYtsnn97NmzqV27Nt7e3rzyyivodLoS3Tc9PZ0XX3wRT09P6tWrx/z58wu0SUhI4KmnnsLNzY3GjRuzatWqAm0UCgVLly7lsccew83NjSZNmvDTTz9Zzl++fBmFQsHatWt54IEHcHNzo2vXrpw9e5YDBw7QpUsXPD09eeyxx7hx40YJ/9dgxYoVtG7dGhcXF+rVq2c1jxwdHc2AAQPw9PTE29ubwYMHW8V75npX3333HUFBQfj4+PDcc8+RmppqaWMymfjoo4+45557cHFxoWHDhnzwwQeW85MnT+bee+/F3d2dJk2aMHPmTIvYrFy5ktmzZ3PkyBHLz3TlypXFPtPp06fp1asXrq6utGrVim3btqFQKNiwYYPV/+WaNWvo3bs3rq6urFq1ilu3bjF06FDq16+Pu7s7bdu25ccff7TquyQ/7/LA6YsM1YH776nFpkPpZOtNlmOZeiOtZv3hFHtOvhuCu7ZkP9pFixZx9uxZ2rRpw7vvvguAn58fDRo0YN26ddSsWZPdu3czZswY6tWrx+DBgy3XRkRE4Orqyo4dO7h8+TKjRo2iZs2aVl/Ewpg0aRI7d+5k48aN1KlTh2nTpnHo0CGrYdXIkSO5du0a27dvR6PRMH78eJuZZmbOnMncuXNZtGgR3333Hc899xzHjh2jZcuWljbh4eEsXLiQhg0b8tJLL/H888/j5eXFokWLcHd3Z/DgwcyaNcuyCl8US5cuJSwsjLlz5/LYY4+RnJzMrl27ALMw5Yrbzp07MRgMjBs3jiFDhlgtml24cIENGzawadMmbt++zeDBg5k7d67l/27q1KksX76cTz75hF69enH9+nWrLYxeXl6sXLmSgIAAjh07xssvv4yXlxdvv/02Q4YM4fjx42zZsoVt27YB5jCpojAajQwcOJCGDRuyb98+UlNTmThxos22U6ZMYf78+XTs2BFXV1eysrLo3LkzkydPxtvbm99++43hw4fTtGlTunXrBpTs510ulPOm/yqJvdlE/j57Q/R8/3cRseeQyMjIEEIIkZ6tF40mb3LKKz1bb9fz9u7dW7zxxhtFthk3bpx45plnLJ9HjBghatSoIdLT0y3Hli5dKjw9PYXRaCyyr9TUVKHVasXatWstx27duiXc3Nwsdpw5c0YAYv/+/ZY2p06dKpAHEBCvvPKKVf/du3cXr776qhBCiEuXLglAfPXVV5bzP/74owBERESE5dicOXNE8+bNi7Q7l4CAADF9+nSb5/73v/8JlUoloqOjLcdOnDhh9Szh4eHC3d1dpKSkWNpMmjRJdO/eXQghREpKinBxcRHLly8vkT1CCPHxxx+Lzp07Wz6Hh4eL9u3bl/j633//XajVanH9+nXLsa1btwpArF+/Xghx5/9y4cKFxfb3xBNPiIkTJwohSvbztkVmZqY4efKkyMzMLHCupN9R6cE5gC5BfmhUSowmgc5gwg1w06g4+W6IU+xx06jK3Mdnn33GihUriI6OJjMzE51OV+Cvbfv27XF3d7d87tGjB2lpacTExNCoUaNC+75w4QI6nY7u3btbjtWoUYPmzZtbPp86dQq1Wk3nzp0tx1q0aIGvr2+B/nr06FHg8+HDh62OtWvXzvI+N86ybdu2VscKy0OYl4SEBK5du0bfvn1tnj916hSBgYFWe5lbtWqFr68vp06domvXroB5NThvNtp69epZ7n/q1Cmys7MLvQeYcyl++umnXLhwgbS0NAwGQ5myT585c4bAwECrbZK53ld+unTpYvXZaDTy4YcfsnbtWmJjY9HpdGRnZ1t+N0ry8y4vpMA5AFeNirb1zUOADJ0RH8xzQyUdJlY2Vq9ezVtvvcX8+fPp0aMHXl5efPzxx+zbt8/ZppUajUZjeZ+72yT/MZPJVOC6/DhqN0zee+e/f3H32LNnD8OGDWP27NmEhITg4+PD6tWrK2xey8PDw+rzxx9/zKJFi1i4cCFt27bFw8ODN998s8TzseWJXGRwEJ0b+QGQUQV3NGi1WozGOwsku3btomfPnrz22mt07NiRe+65hwsXLhS47siRI2RmZlo+7927F09Pz2IzsTRt2hSNRmMlmLdv3+bs2bOWzy1atMBgMBAZGWk5dubMGZKSkgr0t3fv3gKf886/ORIvLy+CgoKIiIiweb5ly5bExMQQExNjOXby5EmSkpJo1apVie7RrFkz3NzcCr3H7t27adSoEdOnT6dLly40a9aMK1euWLXJ/zMtjubNmxMTE2O1GHLgwIESXbtr1y4GDBjACy+8QPv27WnSpInVz7IkP+/yomq6GJWQTo38iI9NI11nxCQEyiq0JzUoKIh9+/Zx+fJlPD09adasGd9++y1//PEHjRs35rvvvuPAgQMFEhfodDpGjx7NjBkzuHz5MuHh4YSGhha7h9DT05PRo0czadIkatasSZ06dZg+fbrVdc2bN6dfv36MHTuWpUuXolarefPNN216N+vWraNLly706tWLVatWsX//fv7zn/845j/HBu+88w6vvPIKderU4bHHHiM1NZVdu3bx+uuvExwcTNu2bRk2bBgLFy7EYDDw2muv0bt37wJDu8JwdXVl8uTJvP3222i1Wu6//35u3LjBiRMnGD16NM2aNSM6OprVq1fTtWtXfvvtN9avX2/VR1BQEJcuXeLw4cM0aNAALy+vIjPmPPLIIzRt2pQRI0bw0UcfkZqayowZM4Di91c3a9aMn376id27d+Pn58eCBQuIj4+3CHpJft7lhfTgHMQ9tT1RKczhIZm6kv/lrAy89dZbqFQqWrVqRe3atQkJCWHQoEEMGTKE7t27c+vWLV577bUC1/Xt25dmzZrx4IMPMmTIEPr3788777xTont+/PHHPPDAAzz11FMEBwfTq1cvq/k2MO8fDggIoHfv3gwaNIgxY8ZQp07BDXGzZ89m9erVtGvXjm+//ZYff/yxxN5SaRgxYgQLFy7k888/p3Xr1jz55JOcO3cOMIvBxo0b8fPz48EHHyQ4OJgmTZqwZs0au+4xc+ZMJk6cyKxZs2jZsiVDhgyxzNH179+fCRMmEBoaSocOHdi9ezczZ860uv6ZZ56hX79+9OnTh9q1axcI28iPSqViw4YNpKWl0bVrV/71r38xffp0gGIzecyYMYNOnToREhLCQw89hL+/f4FdLiX5eZcHCiFkdsb8pKSk4OPjQ3JycoknbrOysog8fgbPWvXwr+FNXe/SpXeR2IdCoWD9+vUO3zYmMQ89e/Xqxfnz52natGmF3z8rK4tLly7RuHHjAiJb0u+oHKI6EBe12SFOzTJQV5ZTlVQx1q9fb5miOH/+PG+88Qb333+/U8TNUcghqgNx0Zj/OzN1RowlWJGrrkRHR+Pp6VnoKzo62tkmFklRtv/999/ONq9UrFq1qtBnat26NQCpqamMGzeOFi1aMHLkSLp27crGjRudbHnZkENUG5R2iHrp0iXwrI1eoSaopgfebpriL6yGGAwGLl++XOj5oKAg1OrKO3g4f/58oefq169fJTM2p6amFigPkItGoykybtFZyCFqJcRdqyJZD6nZhrtW4NRqtSXDclWkKtteGF5eXlaBxXcLcojqYNxdzH8z0rKqXjycRFLdkALnYNy0KhSYM4voDHfvPJxEUhmQAudg1EolbjlbtNKypRcnkTgTKXDlgGfOMDVdCpxE4lSkwJUDufFweqMcokokzkQKXDlQhbahlpn8RWDyZoCVSJyNDBMpB3L17W6MMLx+/Tp+fn7ONkMiAaTAlQ85LtxdqG9WCROdhRACo9FYqYOJJRWDHKKWAwowu2/6dNA54WWn65iamsqwYcPw8PCgXr16fPLJJ6UuJWirSMkvv/xCnz59cHd3p3379uzZs8fqmn/++cdSECYwMJDx48eTnp5uOf/dd9/RpUsXvLy88Pf35/nnn7fKvrtjxw4UCgW///47nTt3xsXFhX/++cdu2yXVD6f/ifvss8/4+OOPiYuLo3379ixevLjQVMknTpxg1qxZREZGcuXKFT755BObX0J7+iwPFApQGDK555vySbpYLNOugdaj+HY5hIWFsWvXLn799Vfq1q3LrFmzHFoQZPr06cybN49mzZoxffp0hg4dyvnz51Gr1Vy4cIF+/frx/vvvs2LFCm7cuEFoaCihoaF8/fXXgLk03XvvvUfz5s1JSEggLCyMkSNHsnnzZqv7TJkyhXnz5tGkSRM5TJYATvbg1qxZQ1hYGOHh4Rw6dIj27dsTEhJSaG78jIwMmjRpwty5cwsdCtnbZ3lQldYYUlNT+eabb5g3bx59+/alTZs2fP3113Zlgy2Ot956iyeeeIJ7772X2bNnc+XKFct+zzlz5jBs2DDefPNNmjVrRs+ePfn000/59ttvycrKAuCll17iscceo0mTJtx33318+umn/P7776SlpVnd591337UkbqxRo4bD7JdUYYotj1OOdOvWTYwbN87y2Wg0ioCAADFnzpxir23UqJFVdSVH9JmLvVW1hLCuAJSapRdHohPFmeg4IbLTKv5lMpXY7sOHDwtAXLlyxep4x44di620JUTBnwM2qjDlrYyVmJgoALFz504hhBBdunQRWq1WeHh4WF7u7u4CECdPnhRCCHHw4EHx5JNPisDAQOHp6Wk5f+LECSGEENu3bxeAuHr1aomfW1L5qdJVtXQ6HZGRkUydOtVyTKlUEhwcXGCOprz7zM7OJjs72/I5JSWlVPfPRQGgUGBSu9k1VKyu2Cr4kltgJS0tjbFjxzJ+/PgC1zVs2JD09HRCQkIICQlh1apV1K5dm+joaEJCQgoUNclfDEUicZrA3bx5E6PRaCnhlkvdunWtCtxWRJ9z5sxh9uzZpbqnLZQ5Y1STybyiV1xOe2fSpEkTNBoNBw4coGHDhgAkJydz9uxZHnzwwXK/f6dOnTh58mShGTyOHTvGrVu3mDt3rqWYzcGDB8vdLkn1QK6iYq4inpycbHnlrYhUGlzUKhQKBQaTCV0l383g5eXFiBEjmDRpEtu3b7cUNlEqlRUizJMnT2b37t2EhoZy+PBhzp07x8aNGwkNDQXMXpxWq2Xx4sVcvHiRX3/9lffee6/c7ZJUD5wmcLVq1UKlUhVIwhcfH1/qWKrS9uni4oK3t7fVqywolQpL8eWM7MpfgGbBggX06NGDJ598kuDgYO6//35atmxZbLERR9CuXTt27tzJ2bNneeCBB+jYsSOzZs0iICAAgNq1a7Ny5UrWrVtHq1atmDt3LvPmzSt3uyTVhHKaHywR3bp1E6GhoZbPRqNR1K9fv8yLDKXtM5eyLjIIIcS1pAxxJOa2iElML3EflYW0tDTh4+MjvvrqK2ebIrmLqdKLDGCOvxoxYgRdunShW7duLFy4kPT0dEaNGgXAiy++SP369ZkzZw5gXkQ4efKk5X1sbCyHDx/G09PTModTXJ8VhYdWzQ2ySa8CHlxUVBSnT5+mW7duJCcn8+677wIwYMAAJ1smkZQNpwrckCFDuHHjBrNmzSIuLo4OHTqwZcsWyyJBdHS0VXHYa9eu0bFjR8vnefPmMW/ePHr37s2OHTtK1GdF4a41D1GzDUYMRhNqVeWe7pw3bx5nzpxBq9XSuXNn/v77b06dOsVjjz1W6DX549AkksqGLDpjg7IUnclbIONsXCpZBiONanrgUwXrM2RmZhIbG1vo+epYu0BSeZBFZyo57i4qsgxGMnSGKilwbm5uUsQkVZrKPW6qguR1iD20uZl9K/88nERS2XDE4FIKnINQqcxzbnmj6z1czMcy9UZMJjkTIJHYQ0ZGBmC9E8Ze5BDVQajVatzd3blx4wYajQalUokQApXJgMFk4nZausWjk0gkhSOEICMjg4SEBHx9fS3OQ2mQ3zgHoVAoqFevHpcuXeLKlSuW40npOjJ0RrJvq/FyrXrzcBKJs/D19S1zAlUpcA5Eq9XSrFkzq2Hq4UNXWbL9PF2DajD3mXudaJ1EUnXQaDRl8txykQLnYJRKpdWSdttGtYlNPUPK2dtotC6olJV3471EUt2QiwzlTAt/Lzxd1KRmGzgTl+pscySSuwopcOWMWqWkY0NfAA5eSXSuMRLJXYYUuAqga5A5ffaBy7edbIlEcnchBa4C6BJkLoBy4FKiQ4IXJRJJyZACVwF0CPRFrVQQl5JFbFKms82RSO4apMBVAO5aNa3r+wBwUA5TJZIKQwpcBdG1Uc4w9bJcaJBIKgopcBVEl5yFBunBSSQVhxS4CiJ3oeFMfCrJGXonWyOR3B1Igasganm60KSWuW5nZLQcpkokFUGZBC4tLY2UlBSrl6RwLOEicpgqkVQIdgvcpUuXeOKJJ/Dw8MDHxwc/Pz/8/Pzw9fXFz8+vPGysNtyZh5MenERSgMzb5mrpDsTuzfYvvPACQghWrFhB3bp1K3XV9spG7o6GIzHJZOmNuGrKni1BIqkW6LPgu0HgWReeXgZuvg7p1m6BO3LkCJGRkTRv3twhBtxNBNV0p5anlptpOo7HJls8Oonkruf3SXDtELj5QVaywwTO7iFq165diYmJccjN7zYUCgVdGsl9qRKJFZEr4dC3oFDC/60Av0YO69puD+6rr77ilVdeITY2ljZt2hTIl96uXTuHGVcd6RLkx5YTcTnzcE2dbY5E4lyuHoTNk8zvH54JTR92aPd2C9yNGze4cOGCVaV4hUKBEAKFQoHRKCtIFUXuPNzBK7cxmQRKmQBTcreSlgBrhoNRBy2fgl4THH4Lu4eoL730Eh07dmTPnj1cvHiRS5cuWf1rL5999hlBQUG4urrSvXt39u/fX2T7devW0aJFC1xdXWnbti2bN2+2Op+WlkZoaCgNGjTAzc2NVq1asWzZMrvtKi9aBXjjplGRnKnn/A1ZGV5yl2I0wLpRkHoNat0LA5dCeSxYCjtxd3cX586ds/cym6xevVpotVqxYsUKceLECfHyyy8LX19fER8fb7P9rl27hEqlEh999JE4efKkmDFjhtBoNOLYsWOWNi+//LJo2rSp2L59u7h06ZL44osvhEqlEhs3biyxXcnJyQIQycnJZX5GWwz9co9oNHmT+H7v5XLpXyKp9Pw+VYhwbyE+qC9Ewhm7Ly/pd9RugXvyySfFTz/9ZLdBtujWrZsYN26c5bPRaBQBAQFizpw5NtsPHjxYPPHEE1bHunfvLsaOHWv53Lp1a/Huu+9atenUqZOYPn16ie0qb4Gb/78zotHkTeLN1VHl0r9EUqk5us4sbuHeQpz8tVRdlPQ7avcc3FNPPcWECRM4duwYbdu2LbDI0L9//xL1o9PpiIyMZOrUqZZjSqWS4OBg9uzZY/OaPXv2EBYWZnUsJCSEDRs2WD737NmTX3/9lZdeeomAgAB27NjB2bNn+eSTT0r4hOVPN0uGXxnwK7nLiDsOG0PN7x+YaJ57K0fsFrhXXnkFgHfffbfAOXsWGW7evInRaKRu3bpWx+vWrcvp06dtXhMXF2ezfVxcnOXz4sWLGTNmDA0aNECtVqNUKlm+fDkPPvhgobZkZ2eTnZ1t+VzeW846NPRFpVRw9XYm15MzqefjVq73k0gqBZm3Yc0LYMg0r5b2mV7ut7R7kcFkMhX6qgwrqIsXL2bv3r38+uuvREZGMn/+fMaNG8e2bdsKvWbOnDn4+PhYXoGBgeVqo6eLmlb1vAGZPklyl2AywS9j4PYl8G0Iz/wHlOW/k8cugdPr9ajVao4fP17mG9eqVQuVSkV8fLzV8fj4+EKrWfv7+xfZPjMzk2nTprFgwQKeeuop2rVrR2hoKEOGDGHevHmF2jJ16lSSk5Mtr4oIZM7deC/3pUruCnb+G879D9SuMGQVuFfMLh67BE6j0dCwYUOHeGparZbOnTsTERFhOWYymYiIiKBHjx42r+nRo4dVe4CtW7da2uv1evR6PUql9WOpVCpMRWzidXFxwdvb2+pV3shKW5K7hjO/w8655vdPfQr1KnAzgL2rF1999ZV4/PHHxa1bt0q1+pGX1atXCxcXF7Fy5Upx8uRJMWbMGOHr6yvi4uKEEEIMHz5cTJkyxdJ+165dQq1Wi3nz5olTp06J8PDwAmEivXv3Fq1btxbbt28XFy9eFF9//bVwdXUVn3/+eYntKu9VVCGEiE/OFI0mbxKNp2wSyZm6cruPROJUbp4X4sMG5hXT3yY5rNtyCxPp0KGD8PT0FC4uLuLee+8VHTt2tHrZy+LFi0XDhg2FVqsV3bp1E3v37rWc6927txgxYoRV+7Vr14p7771XaLVa0bp1a/Hbb79Znb9+/boYOXKkCAgIEK6urqJ58+Zi/vz5wmQyldimihA4IYR48KM/RaPJm8T207bj/iSSKk1WqhBLupvF7T8hQuizHdZ1Sb+jCiHsK9Q5e/bsIs+Hh4eX2pusLKSkpODj40NycnK5Dlcnrj3Cz4euEtrnHt4KkdlZJNUIIeCnUXBiPXj6w9id4GV7br00lPQ7aneYSHUQsMpC1yA/fj50VcbDSaofe5aYxU2pgcHfOlTc7MFugcslMjKSU6dOAdC6dWs6duzoMKPuFnJXUo9eTcZgNKFWyRIZkmrAxZ2wdZb5/WNzoWF3p5lit8AlJCTw3HPPsWPHDnx9fQFISkqiT58+rF69mtq1azvaxmpLk1qeeGhVpOuMXLiRTnN/L2ebJJGUjeSr5qGpMEGHYdBltFPNsdtleP3110lNTeXEiRMkJiaSmJjI8ePHSUlJYfz48eVhY7VFqVTQJqfi/ZGrSc41RiIpK/os806FjFtQrz08Mb98MoTYgd0Ct2XLFj7//HNatmxpOdaqVSs+++wzfv/9d4cadzfQroFZ4I5dTXayJRJJGRACNk+Ea1HgVgOGfA8a529BLNVWrfwb7MEcBFxUMK3ENu0a+AJwNFYKnKQKE7kSor7PSTv+H/N2rEqA3QL38MMP88Ybb3Dt2jXLsdjYWCZMmEDfvn0datzdQK4Hd+p6CjqD/AMhqYLEHLiTdrzvLIenHS8LdgvckiVLSElJISgoiKZNm9K0aVMaN25MSkoKixcvLg8bqzUNa7jj7apGZzBxNj7V2eZIJPaRlgBrXwSTHlr2h/vfdLZFVti9ihoYGMihQ4fYtm2bJa1Ry5YtCQ4OdrhxdwMKhYJ2DXz55/xNjl5Ntiw6SCSVHqM+T9rx5jDwc6cvKuSnVHFwCoWCRx55hEceecTR9tyVtGvgwz/nb3IsNgmoHHMXEkmxbJ0FV/4BrRc8twpcKl+YU6kELiIigoiICBISEgosLKxYscIhht1N5M7DHZUrqZKqwtF1sPdz8/unl0GtZs61pxDsFrjZs2fz7rvv0qVLF+rVq4eikrmkVZG2OSupZ+JSydIbcdWUfyJAiaTUxB2HX183v3/gLWj5pHPtKQK7BW7ZsmWsXLmS4cOHl4c9dyUBPq7U8tRyM03HqespdGzo52yTJBLbZN6GNcNy0o73hT7TnG1Rkdi9iqrT6ejZs2d52HLXolAoaJuzuHBMxsNJKismE/z8Mty+DL6N4JmvKiTteFmwW+D+9a9/8cMPP5SHLXc1ucPUIzFS4CSVlJ1z4fxWULuZFxUqKO14WbB7iJqVlcWXX37Jtm3baNeuXYFdDQsWLHCYcXcT7XO3bMUmOdcQicQWpzeb6yoAPLUI/Ns6154SYrfAHT16lA4dOgAUKD4jFxxKT+4Q9XxCGunZBjxcSp3JSiJxLDfPw/qx5vfdX4H2Q5xrjx3Y/S3avn17edhx11PH2xV/b1fiUrI4eT3FUpSmJGw5HseeCzeZ+WQr1kfFcig6ifcHtkGllH9wJGUkO828qJCdAg17wqPvO9siu5AZFstCdpq5SnfiRYd01zZnmHokJsmu6175PpJv9lxh7cGrTPrpKD/uj2brybjiL5RIikII2DgObpw2px1/diWo7kxJxSRm8NfZGySm65xnYzHIcVBZ2DwJjvxgThEzeito3cvUXfsGPmw9GV/qldTYpAzL+5QsQ5lskUjYvRhObjCnHR/yHXjVtZyKvpXBU0v+ITlTD5j3VHcI9KV9oC8dAn1pHeBdKeI5pcCVhYdnmFeV4o/Df8fDoOVl2ouXu5Ja2txwebORaGX6c0lZuLgTtuXUX3ns3xDYzXIqS2/k1VWRJGfq8XRRk5ZtIDoxg+jEDH49Ys4ypFYqaFnPm/aBPnQI9KNDoA9NanmirOBpEylwZcGnvtlt/6Y/HFsH9bvAfa+UurvchYaLN9NJztTj41Yw715R5BU4tUrOv0lKSVJMnrTjL0CXl6xOz9p4nBPXUqjpoWXT+F64a9QcjU3icHQSR64mcTgmiZtpOo7FJnMsNpnv90YD4OWipl2gj9nTa+BLh4a+1PFyLddHsVvg/vrrL3r27IlabX2pwWBg9+7dPPjggw4zrkoQ1Ms88frHVPjfdPPyedD9peqqhoeWwBpuxCRmciI2mZ731LLr+ix9HoFTSg9OUgqs0o53gCfmWY1KVu+PZu3BqygVsHhoR+r5mLP2PtCsNg80M9djEUIQm5TJ4Zg7oncsNpnUbAO7zt9i1/lblv4CfFzp0DBH8AJ9advAB3et4/wuu3vq06cP169fp06dOlbHk5OT6dOnD0aj0WHGVRnuexWuHTJ7cetGmmtAegeUqqt29X2JSczkaCkELk13Z95Nq5YenMROhIDfJsL1wzlpx7+zSjt+7Goys349AcBbIc0L/f1UKBQ08HOngZ87T7Yzfw/0RnO+w8MxSRyJSeJITDJnE1K5lpzFtWNxbD5mXhRTKmDvtL4O8+zsFjghhM14t1u3buHh4eEQo6ocCoU5+DHhlHk+bu2LMPI3ULvY3VXbBj78dux6qebh0rPvCJxGzsFJ7CXyazicm3Z8hVXa8dvpOl75PhKdwcQjreryau+mdnWtUSlpHeBD6wAfhnVvBEBatoGjV81idzjmNkdikjEKQW1P+783hVHib8GgQYMYNGgQCoWCkSNHWj4PGjSIAQMGEBISUqo9qp999hlBQUG4urrSvXt39u/fX2T7devW0aJFC1xdXWnbti2bN28u0ObUqVP0798fHx8fPDw86Nq1K9HR0XbbZhdaD/NfPFcfuHoAtkwpVTftylBlKzXPymlxMXBZeiPf7b1CTGJGke0kdwkxB2Dz2+b3fcOhaR/LKaNJ8Oaaw8QmZdKopjvznm3vkKB+Txc1PZvW4tWHmvLF8C7sndaXbRN6O3TDQIkFzsfHBx8fH4QQeHl5WT77+Pjg7+/PmDFj+P777+26+Zo1awgLCyM8PJxDhw7Rvn17QkJCSEhIsNl+9+7dDB06lNGjRxMVFcXAgQMZOHCg1Y6KCxcu0KtXL1q0aMGOHTs4evQoM2fOxNW1fCczAajRBJ75D6CAgyvg0Hd2d9EmJxbu6u1Mu+OL0vKGhoii236+4wIzNxzn4fk77LRQUu1IS4C1w81px1sNgPvfsDq9+M9z7Dx7A1eNkmUvdLZ78csefNwd27dCCFHMV8Ga2bNn89ZbbzlkONq9e3e6du3KkiVLAHPFrsDAQF5//XWmTCnoAQ0ZMoT09HQ2bdpkOXbffffRoUMHli1bBsBzzz2HRqPhu+/sF5dcUlJS8PHxITk5GW9vb/s72PkRbP8AVC7w0hao38muyx+et4OLN9P55qVu9L63+ELaQVN+A6C+rxuxSZkAfD+6O72aFT6HN3jZHvZfTgTg8twn7LJPUo0w6s1RANG7oXYL+Nc2q8y8O84kMGrlAYSABYPbM6hTAycae4eSfkftnqgJDw93iLjpdDoiIyOtajkolUqCg4PZs2ePzWv27NlToPZDSEiIpb3JZOK3337j3nvvJSQkhDp16tC9e3c2bNhQpC3Z2dmkpKRYvcrEA29B88fBmA1rhkP6Tbsuz93RcNTOHQ2pWXrLe1GMCycXWSUA/G+mWdxcvM21TPOIW0xiBm+sPowQ8MJ9DSuNuNmD3b/m8fHxDB8+nICAANRqNSqVyupVUm7evInRaKRu3bpWx+vWrUtcnO1tRnFxcUW2T0hIIC0tjblz59KvXz/+97//8fTTTzNo0CB27txZqC1z5syxGnIHBgaW+DlsolSa0zjXaAopV+Gnl8BY8p0Fpa2Vmq67s4JtKsYvl/tUJRxdC/uWmt/nSzuepTfy2qpDJGfqaR/oy8wnWznJyLJh9yrqyJEjiY6OZubMmZUuZXlufYgBAwYwYcIEADp06MDu3btZtmwZvXv3tnnd1KlTCQsLs3xOSUkpu8i5+phzZi3vC5d2wp/vwiPvlujS0la7N+ZRNVMxMw/KSvRzkziBuGPw63jz+wcnQQvraYp3fj3Bsdhk/Nw1fD6sEy5q52+7Kg12C9w///zD33//bUmZVFpq1aqFSqUiPj7e6nh8fDz+/v42r/H39y+yfa1atVCr1bRqZf3XpmXLlvzzzz+F2uLi4oKLi+OWpi3UaQkDPzPHxu1aBAEdofXTxV7WOsAbpQLiUrJISMmijrf9CyTFTa1KD+4uJiMRVuekHb8nGB6aanV67YEYVh+IQaGAT4d2pL6vWyEdVX7sHqIGBgYW++UpCVqtls6dOxMREWE5ZjKZiIiIoEePHjav6dGjh1V7gK1bt1raa7VaunbtypkzZ6zanD17lkaNGpXZ5lLR+mnomfOXcsM4c6xcMbhr1TSrY54LKW2lreJ+RCrpwd2dmIzwy8uQdAX8gsz7p/OkHT8em8yMjeaohImP3GvZnVBVsVvgFi5cyJQpU7h8+XKZbx4WFsby5cv55ptvOHXqFK+++irp6emMGjUKgBdffJGpU+/8dXnjjTfYsmUL8+fP5/Tp07zzzjscPHiQ0NBQS5tJkyaxZs0ali9fzvnz51myZAn//e9/ee2118psb6npGw6NHwR9uvkvZ1bxomVZaChlZpHi5uAqetOzpJKwYw6c32ZOOz7ke6u048kZel5dZQ7mDW5Zh9ceuseJhjoGu4eoQ4YMISMjg6ZNm+Lu7l4gZXliYqJdfd24cYNZs2YRFxdHhw4d2LJli2UhITo6GmWe5b6ePXvyww8/MGPGDKZNm0azZs3YsGEDbdq0sbR5+umnWbZsGXPmzGH8+PE0b96cn3/+mV69etn7qI5DpYb/+xq+fAgSL8D6V2DIqiKXMts18OGnyKscK0XALxQ/Byc9OOey+/xNLt1Kt0T1Vwinf4O/Pja/7/+pVdpxk0nw5pooYhIzaVjDnfmDO1SLP4J2C9zChQsdakBoaKiVB5aXHTt2FDj27LPP8uyzzxbZ50svvcRLL71UZJsKx6MWDP4WVvSDM5vh7/nQe1KhzfNW2Spse1xRyDm4ys2kn44Sm5RJ98Y1uKdOBVSEv3kOfslNO/4qtBtsdXrJ9vNsP3MDF7WSpS90Ktdg3orEboEbMWJEedhxd1C/Ezy5wJwldfsHENABmj1is2nLet6olQpupum4lpxl90RvcXNw1eGvc1UmJSdR5KWbGeUvcNmp5gwhutSctOPvWZ3eefYGn2w7C8D7A9vQOsCnfO2pQEoV7nnhwgVmzJjB0KFDLduqfv/9d06cOOFQ46olHXPzawn4eXSh6c5dNSqa+5t/8UszTC02Dk7qm1PJNppDmq7l7DwpN/KmHfeqVyDt+NXbGbyxOgohYGi3hjzbpYzhUZUMuwVu586dtG3bln379vHLL7+QlpYGwJEjRwgPD3e4gdWSfnOhQVfzYsPqF0CXbrNZbjxcaVZSi42Dkx6c0xBCWJKTlrvA7f4UTm40px0f/K1V2vFsg5Fxqw6RlKGnXQMfwp+qmsG8RWG3wE2ZMoX333+frVu3otVqLccffvhh9u7d61Djqi1qF/Mvm0dtSDgB/33D5pjSsqOhHAROLjI4D53xTmLS2PIUuIs7YNs75vePf2SVdhxg9n9PcuRqMr45wbyVoYaCo7Fb4I4dO8bTTxcMVq1Tpw43b9q35/KuxjsAnv0GlGpzosx9ywo0yV1oOHo1ySGxh3mRiwzOI29q+XITuKRoWJeTdrzjC9B5lNXpnyKv8sO+aHMw73MdaeBXtoJJlRW7Bc7X15fr168XOB4VFUX9+vUdYtRdQ9D98OgH5vd/TIfL1rstmvt7oVUrSckyF/Wwh7IMUf85d5MHPvqT3eflH6zyIK/AlcsQVZ9pXlTITDTvnnl8vlXa8RPXkpm+/hgAE4Lv5cESZKypqtgtcM899xyTJ08mLi4OhUKByWRi165dvPXWW7z44ovlYWP1pvtYaDsYhNG8pSvlmuWURqWkVT1zKhh7h6kmU9HnbQ1R/3cijnl/nOHP0wnEJGby52nbefkkZSM7j8AlpGZbCV6ZsaQdPwLuNWHwd6Axb/XTGUzsuXCLV78/RLbBRJ/mtQntU/WDeYvC7jCRDz/8kHHjxhEYGIjRaKRVq1YYjUaef/55ZsyYUR42Vm+s0p0fM6dXGrXZku68XQMfDsckcfRqEk+1L3mdh2Ln4PJ4cCaTQKlUMOa7SABLSEqW4S6sr1EB5BU0ISAuOYuGNR00RDy4Ag6vAoUS8cwKLur9+HvXJf46d5O9F2+RkZNxpoGfG58MqR7BvEVht8BptVqWL1/OzJkzOX78OGlpaXTs2JFmzZoVf7HENlp3c7rzL3tD7EH4fTI8tRDIOw9XuAenViow5IsLKW7GLm82EYNJoM3zi34t2TxsylulS+I4svN5bLFJmY4RuJj95t8dYHPdsXywzkRsknWasFqeLjzYrBbj+zbD111rq5dqRanrczVs2JCGDRsW31BSMmo0Nqc7X/WsufhH/U7Q6UXLSurx2GSLp5UfrVqJQWftbRW/k+HOe4PJhDbPbEXupfm/iBLHkH9I6pB5uNR4c7Ejk55Nxu6EXu4FZKJVKena2I8HmtXmwWa1aeHvVe29tryUSODCwsJ477338PDwsMqbZosFCxY4xLC7kmaPQJ/psP19+O0tqNuae+p1wk2jIl1n5OLNNJtR71q10jL0yKX4hJd3BE1vtN04Sy+HqOWBLl9pzTILnFFvnr9NvU6yZ1PevjmWprU9mfFkK7o3ruHQOqNVjRI9eVRUFHq93vK+MCpT8ssqywMT4VoUnPkN1ryIauxO2tT35sDl2xyLTbYpcLZKBBaf8PLOe2MhaigFrnzI1hccopaJ/82wpB1f5v8uGTcVhLT2p0/zOsVfW80pkcBt377d5ntJOaBUwtNLYfnDcOs8/DSKQJ9ZHABupdmusqW1IXD2hM0ZjLaHovm/iBLHkG10oMAdWWOJodT1X8ofWzyBdLoE+ZXBwuqDLD1SGXH1MadT0njApb8YkPgfADJ1tj0qF7UtgSta4W5n3ClQoy/Mg5OrqOWCw+bgrh8174IBfvV5gXu/U3DxhnnbX6eGUuCghB7coEGDStzhL7/8UmpjJHmo0wIGfg7rRtD7xo88oaxNpt52NXFbMwNFzcEt3HaWH/ffKYRtlHNwFUru4k2AjyvXkrOITcq0PyVWRqI5mNeQibFpMG+e6Gc5dU8dz7tihbQklMiDy1txytvbm4iICA4ePGg5HxkZSUREBD4+1SfNSqWg9UBLEd6PNF/gmXK2xJcWNQe3cNs5q8/6QqKCZZhI+ZDrwTWqaS6/maU3WXnUxWIyws//sqQdP9ljAaY8X+XO0nuzUCIP7uuvv7a8nzx5MoMHD2bZsmWWMoFGo5HXXnutdEWSJUXz8CxiTuwhMGk/g89PhcwHwc232Mvsm4Oz3ThbDlHLhdz/Vy9XNbW9XLiRms21pExqeJTQ69r+IVyIyEk7vopjV6w9v85y/s2C3XNwK1as4K233rKqgapSqQgLC2PFihUONU4CqNTsaDuXq6IWtXRXzenOi9uHRfGrqHkxSA+uQsn14LRqJQE5u0au3i7hPNypTfD3PPP7/ovBvw0nr1sHgXdpJAUuF7sFzmAwcPr06QLHT58+balLKnEwHrV4RfcmeoUGzv5+5xe8CBzhwck5uPIhr8DV9zXvEy3RQsPNc+Y/cAD3vQbtzKn7T15LsWrWuJaH44yt4tgdAThq1ChGjx7NhQsX6NbNnF9q3759zJ0711INS+JY3DQqjosmrPR7g5cT55mHKPU6wL2PFnqNIzy4bIOpVPUgJEWTu8jgolZRI2cxoFiBy041V2TTpUKj+y1FxE0mwem4VEuz1x5qKn9eebBb4ObNm4e/vz/z58+3pE2qV68ekyZNYuLEiQ43UGIWOICtLsG83OU2HPwP/PIveHk71LS9sppX3kwmwcWb6TSt7WHzl78wDw7MX8bqmAjRmegsAqekvp95iJq7/zc/p+NSeO37SBarF9I66Qx4BVilHb+SmEGGzoirRsmJ2f1knr982D1EVSqVvP3228TGxpKUlERSUhKxsbG8/fbbVvNyEsfhpjX/mLL0Rut052uGF5ruPK8Ht3L3ZYIX7OSHPKEhecm/UT8vcpjqeHIz+uadg4stZA5u4dZzPHJ7Da2TdqBHTfxjy8Hzzg6F3OFpc39vKW42KFOgr7e3t1w5rQByPahMnRHUWnOOL4865nTnv463OeGW99DpOPOX4NIN22JYlMDJDfeOJzvnj4aLWmlJTRWblFWgXWxSJhmntvK2ejUAs/QjeGRtOttOxlva5C4w5OYNlFhTKoH76aefGDx4MPfddx+dOnWyekkcT+4Q1bKzwLseDM5Jd378Jwbp/1vgmrw7GXJjrAoTq8K2aoH04MoDiwenuiNwN9OyC/xf/3fHHhZqFqNSCNJbP8/pgEGkZBn417cHmfv7aQxGk8WDaxUgBc4Wdgvcp59+yqhRo6hbty5RUVF069aNmjVrcvHiRR577LHysPGux0WdI3B5wzYa9YSQDwF4JftruitOWV2T1ylLyjDvYS0srq2wbCIF7ilxCLl7fLVqJb7uGssfsOvJd7y4rIw0eh+eSA1FGsl+bfAY+AlrxvZk1P1BACzbeYHnv9rHsVjpwRWF3QL3+eef8+WXX7J48WK0Wi1vv/02W7duZfz48SQn21/9CeCzzz4jKCgIV1dXunfvzv79+4tsv27dOlq0aIGrqytt27Zl8+bNhbZ95ZVXUCgULFy4sFS2VQY0OUVMC2T96DYG2g1BhYkl2kX4c8tyymSHB1dYNhGQHlx5kLvZ3kWtRKFQEJA/VEQI4n54jZZc5DbeeAz/ATSuaNVKwp9qzWfPd8JDq2L/pURupulQKKCFfzkXj66i2C1w0dHR9OzZEwA3NzdSU81L1MOHD+fHH3+024A1a9YQFhZGeHg4hw4don379oSEhFgKSudn9+7dDB06lNGjRxMVFcXAgQMZOHAgx48fL9B2/fr17N27l4CAkqf6rozkTh7r8w8lFQp4ciFnFUHUVqSwVLsILWYxs+nBFeKNFRYmAuaq5xPWHOZWWnYZnkCSlztxcGbPzbLQkCNw4sB/CLq6EaNQsLPdv1HXaGR1/RPt6vHr671oXtcsak1re+LhcvfmfCsKuwXO39+fxMREwJzVN7cW6qVLl0pV2m7BggW8/PLLjBo1ilatWrFs2TLc3d0L3RWxaNEi+vXrx6RJk2jZsiXvvfcenTp1YsmSJVbtYmNjef3111m1ahUajcZmX1UFdU5ySlueltC48ap+AknCg47K87yj/ibnhMj5R5Bk8eDsH6Iu2HqW9VGxzP7vybI8giQP2XkCfcFcHwFyPLjofYgtUwD42PQ8Dzz6jM0+mtb2ZMO4+5n6WAv+/Uy7CrC6amK3wD388MP8+uuvgDnod8KECTzyyCMMGTLEZr3UotDpdERGRhIcHHzHIKWS4OBg9uzZY/OaPXv2WLUHCAkJsWpvMpkYPnw4kyZNonXr1sXakZ2dTUpKitWrMqHOGaLaildL1xm5YKjNG/pQTELB8+o/GaLabvHgUrMNllXSwoeoxc+zXbGzbKGkcHSGO6uoAAE+ZoFLSYiBtS+iNOnZZLyPG23GUNPTpdB+3LQqxvZuSme5NatQ7PZrv/zyS8uWrHHjxlGzZk12795N//79GTt2rF193bx5E6PRSN26da2O161b1+Z2MIC4uDib7ePi4iyf//3vf6NWqxk/fnyJ7JgzZw6zZ8+2y/aKRJ0zRLU1lMwdOu40tWee4Vne1qzlXfXXrEntATQnKf1OlorC5tOK8uAsOLjw9N2MLp8HF+DrhgYDQy7PBF0cZ0UD3taPYc39jZ1pZrXALg/OYDDw/vvvW4nJc889x6effsrrr7+OVuv8HFSRkZEsWrSIlStXlnjLytSpU0lOTra8YmJiytlK+1DnZOw1CfOuhLzcSr+T5XepsT9/GLvgojAw8OwUSLvB7Yw750sTJiJxPPmHqPX93JimXkVz3QmyVR6M0U2gRUN/2jaQ6cfKil0Cp1ar+eijjzAYDA65ea1atVCpVMTHx1sdj4+Px9/f3+Y1/v7+Rbb/+++/SUhIoGHDhqjVatRqNVeuXGHixIkEBQXZ7NPFxcUStFwZg5fzRqjnD8pNzJPGXKBkov4VLpjq4a1LgJ9GkZR2Z2hZqMAVV6GG4ssQSkpO3q1aAM2ub2KU+g8ApvI6l0U9RvQMcpZ51Qq75+D69u3Lzp07i29YArRaLZ07dyYiIsJyzGQyERERQY8ePWxe06NHD6v2AFu3brW0Hz58OEePHuXw4cOWV0BAAJMmTeKPP/5wiN0VTW6YCBRcaEhMt67TkIY7Y/UTyFa6w+W/CYj8yHKusEWGEgmcVDiHocsTJsL1I9TY/jYAiwyD+CW9HV4uah5rU8+ZJlYb7J6De+yxx5gyZQrHjh2jc+fOeHhYp2bp37+/Xf2FhYUxYsQIunTpQrdu3Vi4cCHp6emWzCQvvvgi9evXZ86cOQC88cYb9O7dm/nz5/PEE0+wevVqDh48yJdffglAzZo1qVmzptU9NBoN/v7+NG/e3N7HrRTk9eD0JhNu3Nnzeyu9YCGa86IB/208g/+7MI1m57/mSaUnm0w9Cg8TKcEQ1Z7sJJKiyf05uOlTYf0LKAxZ7FJ0YqHBXBrA201jGb5KyobdAvfaa68BtuufKhQKjEb7AkOHDBnCjRs3mDVrFnFxcXTo0IEtW7ZYFhKio6NR5qnh2bNnT3744QdmzJjBtGnTaNasGRs2bKBNmzb2PkqVQZ3n+fPXT8hdZMhf3f6kbx+4/03YtZCPNF9yVteAeEMTm/1LD65i0RlNKDHRaOfrkBQNfo1ZppqMuGr+7qhVctO8o7Bb4MojqWVoaCihoaE2z+3YsaPAsWeffZZnn322xP1fvny5lJZVDlRKBQqFWWTy10/IHaLW9Xa1Kj8nENB3FheO/kPT1IN8oVnAYMOHNvsvKl2SxPFk642EqdfhEbMTNO4w5Ht8/tTBVXP6MbXMCuIwpB9cRcj9pc8/B5c7RH20dV383DX4e5u3/Vy8kQ5KFcvrzOCqqEVjZTxzWIIwFfSw5SJDxdLbtI9Q9Ubzh5y047mb7sF2IW9J6SixB5eZmUlERARPPvkkYA6tyM6+s31HpVLx3nvv4erq6ngrJaiVSvRGYwFvK9eDe6BZLWY92YortzLoM38HO8/e4HxCKrE6d8bqJvCz9h2CVVFc+uUd4D6rPkoyB1eaXSqSgpgSzvBv1VIAMjuPxa3t/wFYEl+CFDhHUuL/yW+++YYvvvjC8nnJkiXs3r2bqKgooqKi+P7771m6dGm5GCnJG+xrW+BqeLigUCgIquXBo63M85df/X2J2xk6TojGTNePBqDRscX0UUZZ9VESD07iALJTYc0wvBSZ7DW1xPDwO5ZTubsZQM7BOZISC9yqVasYM2aM1bEffviB7du3s337dj7++GPWrl3rcAMlZtSWjCJ3vC0hBDdzFhlq5ik59/ID5sWE9VGx3Eg1n//Z9CDfGh5BqRAs0nxGI8WdYO2iNtvnIldRy4gQsOFVlLfOcV3UIFQ3HheXO6OdgLxDVKX04BxFif8nz58/T9u2bS2fXV1drVY3u3XrxsmTckN2eaHK+b/Ou60qQ2e0BO/mranZuZEfLmol2QYT8Sl3phHeMwznoOlevBUZfKH5BDfM+ccMRmHXEFQIwbCv9jL2u4PFN5aY2bUQTv0Xk1LLq7o3wbO2VShI3iFqJ7m31GGUWOCSkpKs5txu3LhhtTPAZDJZnZc4FluLDLnDUxe1Enftndg4hUJBwxruBfrQo+Y13RskCF9aKGP4t2Y5IMjUG23G0+Ulr/5dvZ3JrvO3+ONEvMwXVxIu/AkR5ipYR9tN57C4p0BpP29XNU+2q8d9TWowvu89zrCyWlJigWvQoIHNnGu5HD16lAYNGjjEKElBcoeoeXPC5YpSTQ9tgX23eQUub9RBAn68phuPXqjor9rDaNXvbDx8jS7vbyvy/nn9O2URW8ck+bh9BX56CYQJOr3I9/o+ADTPl6BSoVCw5PlOrB7TA3etzO3mKEoscI8//jizZs0iK6tgcYzMzExmz57NE0884VDjJHew7cHlzL/ZSKkTmEfgfNys8+EdFC14z/ACAFPVP3CfsviphbxDWE1egZMb9QtHnwlrXoDM2xDQiZSHP2TT0WsAPN1ROgMVQYkFbtq0aSQmJtK8eXM+/vhjNm7cyMaNG/noo49o3rw5t2/fZtq0aeVp611NbkaRvB7TzbTcFdSCWVzyenB+7toCOcO+NT7Kz8ZeqBUmlmg+tUp3bou8fprV1jEZJGwbIWDTBIg7Cu61YMh3/Ho8kSy9iWZ1POnU0NfZFt4VlNgXrlu3Lrt37+bVV19lypQplr/oCoWCRx55hM8//7xAnjaJ47CEiRgLzsHVLEbgfN01jOgZROSV23laKJiuH00LRQytlVdYpl3IYN0sdBSS/TiPjuUVuKLqOdzVHPgKjvwIChU8+zX4NGDNgX8AGNI1UFafryDsGuw3btyYLVu2kJiYyPnz5wG45557qFGjRrkYJ7mDJatvnpCOOzFwBQWuUU1rD87W7p8sXBirn8Am7XQ6KC/wjnol0wwv27x/YTJWoE6ExJJ2XAGcaPMW9wTez/I/z3EsNhmtSsmgTnJ4WlGUajazRo0adOvWzdG2SIogN0wkrwd3K3eI6llQ4Br45fXgtCgL8RiuijqM14eyUvMRz6u3c1Q0ZbXx4QLtCgsjkYsM+UiNg7XDUZgM/Nd4H68faEfjS39z6aa56PbQboE2/yBJygcZUVhFsLWTIXeRoZZHwUUGN62KOl7m477umkIFDqBlr6eZZxgMwGz1StorzhdoU5iMyUWGPBh0sHYEpMUTrQ7ibf0YQMGlm+nU9nJhweD2hD9VfI0QieOQAldFsFWX4VYRQ1S4Mw/n566xOUQF+GtSH7oG1eBzY3+2GLviojCwVLuQmljXuM0fB5eLXGTIw/+mQ8xecPFhYc1ZZHJnp8KfE3szqFMDqxAbSfkjBa6KoLZR/LmoISpgWTltWc/byoMb0CGAsEfuZf+0vjSs6Y6fhwZQ8JZ+LBdM9QhQJLJEsxgVd4J4RY4Pl5yp58nF/1iOF7XIsPvCTZ77cg8XbqTZ+bRVkMM/wn5z0lUGfUlk2p15aQ+tCi/Xql26sqoiBa6KoLYxB1fUKirApJDm7Jz0EH1b1iXv9kZ/b1fG921GnZzUSj5u5uvTcGeMPow04UoP1Ukmq1dbrsn14KJvWZcPzJ+fLi/PL9/H3ouJvPp9ZAmfsopy/QhsetP8vvcUzvnez5Wc/yc/dw1fjejqPNvucqTAVRHyD1EzdUYyc7ZJFTZEVauUNKpp3hJkNQeXb5Tk637Hu7gg6jNR/woAY9S/8ZRyN3BH4PLvAy9JsszcDf/VkoxEczCvIQuahUDvyayLvApAcMu6RM16lB5NaxbTiaS8kAJXRbgTJmIWlFs5CwxatRJPl+IXw/MKXP4Fh/w7Hf4wdeNzg7m2xr81y2muiAZgys9HCVtzxKptSRYZqlrM1+GYJNYeiCk+AYHJCD+PtqQdZ9CX6AX8cigWgGe7yHAQZyMFroqQf4iaO/9max+qLawFzvqcRqVkSJdA+raoYzk2zzCY/Yr2uCuy+ULzCcaM26w+EMOZ+FSra/UlCBOpWvIGAz/bxds/H+Wf8zeLbvjn+3DhTwwqN8SQ78HNl9+Px3EzLZtanloezvP/KXEOUuCqCKp8YSJFBfnaIu/Q0lbIyL//rx3/GXlnrsiEkg/d3+KqqEWQMp4PxGIUFPTWjCXIJVfVPLhcLiQUvjiiP74B/jEXXpqQOZq1Md7cTtfx7n/N+3qHdW8kM/NWAuRPoIpgGaLmDAmLCxHJT15RK6ncZGv9GKubQJbQ0FcVxXjV+gJtShImUkX1rXBhvnEW4y/mecrlhsf5r6knW47H8e6mk9xMy+aeOp68+lDTCrRUUhhS4KoI+QN9LZlESiNwJVCcjg19iU/Jskp3PkHzMw8rD1m1K8kiQ2F3E0JwPDa50ILUlQ0hBDNW7+bmf/4PV1Mme4ytmGsYCsD2MzdYHxWLUgEf/187XDWqYnqTVARS4KoIudlEjJZFhju1GEpC3nm3onY1/PJaTx5v68+nz3W0DIN/Nj3IN4ZHAFio+ZwgxXVL+5KkOy/sdt/vi+bJxf/w8reVM4zkdoZ1EtCYWxn0OjGLWllXuCZqEKp/HSPWQvbyA03o2FBm5K0sSIGrItzJJpIzRM1dZCgkyDc/eSPoiwqm79TQj8+HdbbKJwfwvmE4B3LSnS/TLMQ9J915iYaohfhwX++6BMBfZ28U24czWLjtnNVn1/2L6Kc6QLZQ86ruTW7hY3W+YQ13Jjxyb0WaKCmGSiFwn332GUFBQbi6utK9e3f2799fZPt169bRokULXF1dadu2LZs3b7ac0+v1TJ48mbZt2+Lh4UFAQAAvvvgi165dK+/HKFcsq6j5FhlKN0S1//4F051/CYgSLjLYf79Kx/kIau//CIBZhlEcEQXTin8ypL0cmlYynC5wa9asISwsjPDwcA4dOkT79u0JCQkhISHBZvvdu3czdOhQRo8eTVRUFAMHDmTgwIGWdOoZGRkcOnSImTNncujQIX755RfOnDlD//79K/KxHE7BODh7FxnuvC/pquaGcfdbfb6BH6/q3kAvVDyl2sto1eYSeXCFDomryDZW461LpP0wEgWCHwx9WGPsY3W+rrcL28J607mRTBtW2XC6wC1YsICXX36ZUaNG0apVK5YtW4a7uzsrVqyw2X7RokX069ePSZMm0bJlS9577z06derEkiVLAPDx8WHr1q0MHjyY5s2bc99997FkyRIiIyOJjo6uyEdzKKp8CS/vpCu334Mrag4uLx0CfVn0XAerY5GiOe8ahgMwVf0jtW7sK7af2KRMNh6OLXC8SuibLgPdD8PwNKVw2NSUdwwjCzT58eX7uKeOZ8XbJikWpwqcTqcjMjKS4OBgyzGlUklwcDB79uyxec2ePXus2gOEhIQU2h4gOTkZhUKBr6+vQ+x2Bpp8W7US0+xdZCjZHFx+BnSoz6j7g6yOfWd8hJ+ND6BWmOh1ZBIkXy22nzdWHy75TSsN5rTjbrdOcFN486ruTXRo6NTQl5fubwxA6wBvy3Y4SeXDqeV7bt68idFoLJDqvG7dupw+fdrmNXFxcTbbx8XF2WyflZXF5MmTGTp0KN7e3jbbZGdnW5U8TElJsecxKgRVnjm4LL2RdJ05tKLkiwx33ts7Jzbx0eaWhYIVuy4BCqbpR9NCEU1rwxVuf/0cfuO2gca16I7yYU8tVmcwXLUVjq7GhIpQ/XiuY95T2rS2J7OeasWsp1ohhKiygcx3A04fopYner2ewYMHI4Rg6dKlhbabM2cOPj4+lldgYGAFWlkyLOmSjMIy/6ZRKfAqwT5UKN0QNRdPFzWznmrFsPsaWo5lo2WsPozbwhO/pGOIzZPs6jM/yZl60rMNZerDEeSKbhfFaWapvwPgA/1Q9ppaWdoM7X7n/0GKW+XGqQJXq1YtVCoV8fHxVsfj4+Px9/e3eY2/v3+J2ueK25UrV9i6dWuh3hvA1KlTSU5OtrxiYmJK+UTlR26YiN5kyjM8Ldk+VLA/0NcWTfIVK74qavO6/nWMQoEi6luIXAlATGIGqVn6YvvL67+1n/0/Wof/4VSvbvf5m8QkZlKH23yu/RSNwsivxh78x/iYpc3/dW5AJxnnVmVwqsBptVo6d+5MRESE5ZjJZCIiIoIePXrYvKZHjx5W7QG2bt1q1T5X3M6dO8e2bduoWbPodDUuLi54e3tbvSobeQN9czOJlHT+DfIH+pbOBlvC+I+pLfMMQ8wfNk8i/uQ/PPDRdvov2VWqezirxMOBy4k8/9U++n68lc+1i6ijSOK0KZDJ+pfJ3Yuxf1pf5j3b3jkGSkqF04eoYWFhLF++nG+++YZTp07x6quvkp6ezqhRowB48cUXmTp1qqX9G2+8wZYtW5g/fz6nT5/mnXfe4eDBg4SGhgJmcfu///s/Dh48yKpVqzAajcTFxREXF4dOp7NpQ1Ugb9lAe2PgoHR7UW3x0ysF//AsNT7FKZ/eYNSh/XkEtUi2FFkpClvOmrM8uP2XEgGYof6OLsqzpAh3xuonkIkrQTXd+eDpNpYEoZKqg9MFbsiQIcybN49Zs2bRoUMHDh8+zJYtWywLCdHR0Vy/fmdrUM+ePfnhhx/48ssvad++PT/99BMbNmygTZs2AMTGxvLrr79y9epVOnToQL169Syv3bt3O+UZHUHesoEWgSvhAgPkm4MrQ12ALkE1OP1eP3zdNTSv60V9XzdAwbPxwzlvCsDPeJMl2k+t0p3bg7MGqEaT4BnlX4xQbwXgDf04rgjztMcXw7swrHsjJ1kmKQtOXUXNJTQ01OKB5WfHjh0Fjj377LM8++yzNtsHBQVV+tW50pDXgyuqon1hWK+ilm1i3FWjImqmeW/qhRvpBC/YSRpmj2eDdhb3KU8xRf0jHxheKLIfYUPOnPWjq5lyijGa/wDwif4Ztps6Ws41kzFuVRane3CSkpE3TMTeTCJQ+ji4wlAoFCgUCqsA1wuiPm/lpDt/Wb2Z/srCPeZJ644Qk5hZ4Lgt0XMEeqOJESv2s3Db2YIn028RcuItXBV6thk78qnxaavTshJW1UUKXBVBk6eqVqKdmUQg/xycY7+wERN7W97/YerKEsMAAP6t+ZIWCtu7R3LrFuSnvDy4/52IZ+fZGwU20JvTjr9ELUM8l0x1CdO/xuNt6/P7Gw+w7IVOHJ8dUj4GSSoEKXBVhNytWnqjye59qOCYVdTCaFrbkwPT7+wuWWB4lr+MbXFT6Fim+QRvnF82MLdATwH+fA8u7iBDuDBWH0YKHjzSqi4t63nTr029EtW7kFRepMBVEXKzieT14OxaZFDmjYNzrG1gFtvGOXFyJpSM14cSY6pNkDKehZrPUWDi0s104lOyiuynIubgkjP0bIiKJfvoL/DPJwC8rR/DWWEO8FbJIWm1Qf55qiLcCfQVVgVnSkp5DlHBLAq/v/EASoWCub+fZsWuS7yin8DP2nAeVh3mDfELfeaZRfry3CcK7ae85uDyLjyF/niIa+eP8IjLTFyALw1PsMl0J/zlgWa1ysUGScUjPbgqQm6YSKbOQFrOlqaapQz0LS8RcdWo0KqVjOtjrkdwQgQxVf8vAN5U/2JJdz7n91OF9lFeHlzebqPORfOlZgEeZLHb2Ip/G54DoHEtD/6c2Btf95L/4ZBUbqTAVRFyh6gJOUWU1UoF3m4ld8DzDlHLexhY09OFET3McWPrTQ+w0vAocCfd+Rc7LxZ6bXmZtiHKnK5JgYn5mmU0VV4nVtQkVD8eIyrGPNiE7W89RJPaMiSkOiEFroqQOy+UlGHe4+lnxz5UsH+DfVmZPaCNZU7uA8ML7Dc1x1uRwReaTyzpzm1RHjGMR2KS2H3hFgCvqn4lRHXQknY8EfO2vHEPFczQK6n6SIGrIuSGieRiz/wb5B+iVgx/TuzNwRnB6FEzTjeeeOFLc+VVPspJd26L8rDterJZUB9UHuEt9ToAZhpGcVSYh9Jfj+yKj7umHO4scTZS4KoI+Vf27FlBBWsPrqJ2CygUCmp5uvB4W/+cdOdvohMqnlTt5V+qzTavKQ/bfNw0NFAk8KlmCUqF4AfDw6zNSTt+YnYIfWQF+mqLFLgqQv4q6fYE+ULFD1Hz8tnznQA4JO7lXcOLAExV/0AP5YmCjctB4LzVer7QfIKvIj0n7fgItGolf7/dBw8Z51atkQJXRSjgwZVhiFrRKBQKSyDw98ZgfjI+iEohWKL5lHrcsmrr8BVeIQj4eyqtlVe4Kbx5JSfteNTMRwqURpRUP6TAVRHyz8HZs4sB8g1RnZCzo7ZXrsepYLr+JY6ZgqipSGWZ9hNcuJPGSgjzosA3uy87ZMHBtO9L/M7/gkEoGad7g7ictOPSc7s7kAJXRVAp8w9R7RO4vCNUZ2XsyA3wzUbLK7oJJApP2isv8q56JbljUwEM+GwX4b+eYNPR64X2VSKu7Mb4uzmX4IeGYewTLQH434QHy9avpMogBa6KoM43xqxl5yJDZakdcOHDxwGIpTbjc9KdD1HvYKjqTwB2nLlTD/dMXGrpb5RyHbF2BBqFkY3Gnqww9uPvt/tw8cPHubeuV5meQVJ1kAJXRVAXGKLat8hQWVApFVzMEbl/TG35OCfd+Wz1SjoqznE5TyZgYylczSy9kRU7z5C5ahiK9AROmQKZov8XoCCwhrtMfXSXIQWuipB/kcHeIWpenJ0OVKlUWIary4xPsdnYDa3CyOfaRXgZb1vamewQOJNJIIRgxa5LqLdOxy0+kmThzlh9GJm4sv61ng5/DknlRwpcFUGTbw7O3lVUKypVxmMFk/RjOW8KoJ4ikcdOT0WNea+tqYQVaPRGE48u/IsRXx/AGPk9L6q3YhIK3tCP45rCnz1TH6ajrIR1VyIFroqgyjdE9XGrPpH36bgxRh9GqnCjQfIhpqh/BMBoKtn1h67c5nxCGrfO7WdMyhIAFhqeQdOiH+c+eIx6Pm7lZbqkkiMFroqQ14Nz16qq1VySUgExyvpMzEl3/i/17/RX7irxEFWlVOBHCl9oP8FFoWersROLjQNZ9FyHSrO4InEOUuCqCHnn4MrqvVWmASqYY/TOffA4/zN1ZbFhIAD/1ixHEX+8RNdfTkhmsWYxDRQ3uWjyJ0z/GnumPoK7Vsa63e3I34AqgtqRAlfJFC7XnO9Hd+fF/5hop7hIb9VRRsbMICKqA1kqbwJ8XWlYw52antarx0kZOm78OpP/U58gXbjwin4Cu94ZiLdr9RnCS0qPFLgqQt4hqXc1mn+DOymSejWrxc63+/LkRxn8VzGdRsoELv7yMqH6SQiUuKiVnHn/Mct1mTojf29cwavq/wLwtn4sE1+Q4ia5gxyiVkHK7sFVMhcuD4E13Plr1iBe0U8gS2joozrCm+pfAMg2WK86fLByPX1OhwPwheEJ6vd6npDW/hVus6TyIgWuClJWD6WyyVt+e3zcNZwUQUzRvwzAG+pf6KuMtGpz40YCo67OwFORxS5ja060nMC0x1tWkMWSqkKlELjPPvuMoKAgXF1d6d69O/v37y+y/bp162jRogWurq60bduWzZutc4sJIZg1axb16tXDzc2N4OBgzp07V0hvVY/qFCICtucEG9ZwZ4OpF18bzHVJP9F8TmOFeW/qugNXiPr0OUva8df1r/PpsK4VabKkiuB0gVuzZg1hYWGEh4dz6NAh2rdvT0hICAkJCTbb7969m6FDhzJ69GiioqIYOHAgAwcO5PjxOytuH330EZ9++inLli1j3759eHh4EBISQlZW0SXrqgrVTeBskTvl+IFhGPtMLfBWZPKFZgEiO5XLG97nUVUk2ULDK7oJ7P9giHONlVRanC5wCxYs4OWXX2bUqFG0atWKZcuW4e7uzooVK2y2X7RoEf369WPSpEm0bNmS9957j06dOrFkiTnAUwjBwoULmTFjBgMGDKBdu3Z8++23XLt2jQ0bNlTgk5UfrQK8y3S9M5NflpRcGw2oWd/0feKFL/cqYzn8wUNMzEk7PsMwil8/DEWtcvqvsaSS4tTfDJ1OR2RkJMHBd6qiK5VKgoOD2bNnj81r9uzZY9UeICQkxNL+0qVLxMXFWbXx8fGhe/fuhfaZnZ1NSkqK1asysnJUV2Y+2YrglqVLsf2vXo1pWtuDZzo3cLBljievBs8d8Ygl3XlH5XmUCkGExxPMefffMpBXUiROFbibN29iNBqpW7eu1fG6desSFxdn85q4uLgi2+f+a0+fc+bMwcfHx/IKDAws1fOUNw81r8PoXo1L/aWe8WQrIiY+hGclSfb42fOd8HZV8/3o7gXOTe7XAoDh95nLDx4S9zLbMAKAIzTnvteWS89NUiyV4zfdyUydOpWwsDDL55SUlEorctWJJ9rV4/G2/jYF+9HW/hya+Qh+OdWujoQ/StiaOuxv9wJd2rVDqa7+85CSsuNUgatVqxYqlYr4+Hir4/Hx8fj7245n8vf3L7J97r/x8fHUq1fPqk2HDh1s9uni4oKLS9XMr1bVKcobzZsSysdNw39GypVSiX041cfXarV07tyZiIgIyzGTyURERAQ9evSweU2PHj2s2gNs3brV0r5x48b4+/tbtUlJSWHfvn2F9imRSKopwsmsXr1auLi4iJUrV4qTJ0+KMWPGCF9fXxEXFyeEEGL48OFiypQplva7du0SarVazJs3T5w6dUqEh4cLjUYjjh07Zmkzd+5c4evrKzZu3CiOHj0qBgwYIBo3biwyMzNLZFNycrIARHJysmMfViKROISSfkedPgc3ZMgQbty4waxZs4iLi6NDhw5s2bLFskgQHR2NMk+qoJ49e/LDDz8wY8YMpk2bRrNmzdiwYQNt2rSxtHn77bdJT09nzJgxJCUl0atXL7Zs2YKrq2uFP59EInEeCiEq8cZEJ5GSkoKPjw/Jycl4e5ct5kwikTiekn5H5Tq7RCKptkiBk0gk1RYpcBKJpNri9EWGykjutGRl3bIlkdzt5H43i1tCkAJng9RUc0V1uZtBIqncpKam4uPjU+h5uYpqA5PJxLVr1/Dy8qrym7lzt53FxMRUixXh6vQ81elZoGKfRwhBamoqAQEBVmFk+ZEenA2USiUNGlT+jBv24O3tXS2+RLlUp+epTs8CFfc8RXluuchFBolEUm2RAieRSKotUuCqOS4uLoSHh1ebbCnV6Xmq07NA5XweucggkUiqLdKDk0gk1RYpcBKJpNoiBU4ikVRbpMBJJJJqixS4akhiYiLDhg3D29sbX19fRo8eTVpaWpHXfPnllzz00EN4e3ujUChISkqqGGNt8NlnnxEUFISrqyvdu3dn//79RbZft24dLVq0wNXVlbZt27J58+YKsrR47HmWEydO8MwzzxAUFIRCoWDhwoUVZ2gJsed5li9fzgMPPICfnx9+fn4EBwcX+7N0NFLgqiHDhg3jxIkTbN26lU2bNvHXX38xZsyYIq/JyMigX79+TJs2rYKstM2aNWsICwsjPDycQ4cO0b59e0JCQkhISLDZfvfu3QwdOpTRo0cTFRXFwIEDGThwIMePH69gywti77NkZGTQpEkT5s6dW2jRJWdi7/Ps2LGDoUOHsn37dvbs2UNgYCCPPvoosbGxFWd0+WZOl1Q0J0+eFIA4cOCA5djvv/8uFAqFiI2NLfb67du3C0Dcvn27HK0snG7duolx48ZZPhuNRhEQECDmzJljs/3gwYPFE088YXWse/fuYuzYseVqZ0mw91ny0qhRI/HJJ5+Uo3X2U5bnEUIIg8EgvLy8xDfffFNeJhZAenDVjD179uDr60uXLl0sx4KDg1Eqlezbt8+JlhWPTqcjMjKS4OBgyzGlUklwcDB79uyxec2ePXus2gOEhIQU2r6iKM2zVGYc8TwZGRno9Xpq1KhRXmYWQApcNSMuLo46depYHVOr1dSoUYO4uDgnWVUybt68idFotBQcyqVu3bqF2h4XF2dX+4qiNM9SmXHE80yePJmAgIACf5DKEylwVYQpU6agUCiKfJ0+fdrZZkokNpk7dy6rV69m/fr1FVrdTqZLqiJMnDiRkSNHFtmmSZMm+Pv7F5j0NRgMJCYmVsqJ67zUqlULlUpFfHy81fH4+PhCbff397erfUVRmmepzJTleebNm8fcuXPZtm0b7dq1K08zCyA9uCpC7dq1adGiRZEvrVZLjx49SEpKIjIy0nLtn3/+iclkonv37k58guLRarV07tyZiIgIyzGTyURERAQ9evSweU2PHj2s2gNs3bq10PYVRWmepTJT2uf56KOPeO+999iyZYvVvHCFUWHLGZIKo1+/fqJjx45i37594p9//hHNmjUTQ4cOtZy/evWqaN68udi3b5/l2PXr10VUVJRYvny5AMRff/0loqKixK1btyrU9tWrVwsXFxexcuVKcfLkSTFmzBjh6+sr4uLihBBCDB8+XEyZMsXSfteuXUKtVot58+aJU6dOifDwcKHRaMSxY8cq1G5b2Pss2dnZIioqSkRFRYl69eqJt956S0RFRYlz58456xGssPd55s6dK7Rarfjpp5/E9evXLa/U1NQKs1kKXDXk1q1bYujQocLT01N4e3uLUaNGWf1SXbp0SQBi+/btlmPh4eECKPD6+uuvK9z+xYsXi4YNGwqtViu6desm9u7daznXu3dvMWLECKv2a9euFffee6/QarWidevW4rfffqtgiwvHnmfJ/bnkf/Xu3bviDS8Ee56nUaNGNp8nPDy8wuyV6ZIkEkm1Rc7BSSSSaosUOIlEUm2RAieRSKotUuAkEkm1RQqcRCKptkiBk0gk1RYpcBKJpNoiBU4ikVRbpMBJqgwjR460mUWlX79+zjZNUkmR2UQkVYp+/frx9ddfWx0rrJK6Xq9Ho9FYHdPpdGi1WrvvW9rrJM5FenCSKoWLiwv+/v5WLz8/PwAUCgVLly6lf//+eHh48MEHH/DOO+/QoUMHvvrqKxo3bmzJRRYdHc2AAQPw9PTE29ubwYMHW6UCKuw6SdVCCpykWvHOO+/w9NNPc+zYMV566SUAzp8/z88//8wvv/zC4cOHMZlMDBgwgMTERHbu3MnWrVu5ePEiQ4YMseor/3WSqoccokqqFJs2bcLT09Pq2LRp0yzVwJ5//nlGjRpldV6n0/Htt99Su3ZtwJwv7tixY1y6dInAwEAAvv32W1q3bs2BAwfo2rWrzeskVQ8pcJIqRZ8+fVi6dKnVsbxFTGwlVWzUqJGVSJ06dYrAwECLuAG0atUKX19fTp06ZRG4/NdJqh5S4CRVCg8PD+65554iz5fkWEnvJanayDk4yV1Hy5YtiYmJISYmxnLs5MmTJCUl0apVKydaJnE00oOTVCmys7MLlKlTq9XUqlWrxH0EBwfTtm1bhg0bxsKFCzEYDLz22mv07t3bOXUDJOWG9OAkVYotW7ZQr149q1evXr3s6kOhULBx40b8/Px48MEHCQ4OpkmTJqxZs6acrJY4C5myXCKRVFukByeRSKotUuAkEkm1RQqcRCKptkiBk0gk1RYpcBKJpNoiBU4ikVRbpMBJJJJqixQ4iURSbZECJ5FIqi1S4CQSSbVFCpxEIqm2SIGTSCTVlv8H8FztMhzCXbsAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.01583, + "end_time": "2024-03-26T13:36:17.502526", + "exception": false, + "start_time": "2024-03-26T13:36:17.486696", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 400.849255, + "end_time": "2024-03-26T13:36:20.239902", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/tab_ddpm_concat/0/mlu-eval-load.ipynb", + "output_path": "eval/treatment/tab_ddpm_concat/0/mlu-eval-load.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/treatment/tab_ddpm_concat/0", + "path_prefix": "../../../../", + "random_seed": 0, + "single_model": "tab_ddpm_concat" + }, + "start_time": "2024-03-26T13:29:39.390647", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/tab_ddpm_concat/mlu-eval.ipynb b/treatment/tab_ddpm_concat/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..593e94cab2015f76493740dccb63f1e19cd8f667 --- /dev/null +++ b/treatment/tab_ddpm_concat/mlu-eval.ipynb @@ -0,0 +1,2351 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:14:51.183805Z", + "iopub.status.busy": "2024-03-24T19:14:51.183455Z", + "iopub.status.idle": "2024-03-24T19:14:51.217489Z", + "shell.execute_reply": "2024-03-24T19:14:51.216565Z" + }, + "papermill": { + "duration": 0.049313, + "end_time": "2024-03-24T19:14:51.219837", + "exception": false, + "start_time": "2024-03-24T19:14:51.170524", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:14:51.245362Z", + "iopub.status.busy": "2024-03-24T19:14:51.244972Z", + "iopub.status.idle": "2024-03-24T19:14:51.251506Z", + "shell.execute_reply": "2024-03-24T19:14:51.250709Z" + }, + "papermill": { + "duration": 0.021513, + "end_time": "2024-03-24T19:14:51.253647", + "exception": false, + "start_time": "2024-03-24T19:14:51.232134", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:14:51.277670Z", + "iopub.status.busy": "2024-03-24T19:14:51.277400Z", + "iopub.status.idle": "2024-03-24T19:14:51.281351Z", + "shell.execute_reply": "2024-03-24T19:14:51.280530Z" + }, + "papermill": { + "duration": 0.018362, + "end_time": "2024-03-24T19:14:51.283277", + "exception": false, + "start_time": "2024-03-24T19:14:51.264915", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:14:51.307344Z", + "iopub.status.busy": "2024-03-24T19:14:51.306877Z", + "iopub.status.idle": "2024-03-24T19:14:51.311012Z", + "shell.execute_reply": "2024-03-24T19:14:51.310155Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.01818, + "end_time": "2024-03-24T19:14:51.312926", + "exception": false, + "start_time": "2024-03-24T19:14:51.294746", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:14:51.336444Z", + "iopub.status.busy": "2024-03-24T19:14:51.336189Z", + "iopub.status.idle": "2024-03-24T19:14:51.341702Z", + "shell.execute_reply": "2024-03-24T19:14:51.340870Z" + }, + "papermill": { + "duration": 0.019347, + "end_time": "2024-03-24T19:14:51.343559", + "exception": false, + "start_time": "2024-03-24T19:14:51.324212", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "7865219f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:14:51.369008Z", + "iopub.status.busy": "2024-03-24T19:14:51.368392Z", + "iopub.status.idle": "2024-03-24T19:14:51.373646Z", + "shell.execute_reply": "2024-03-24T19:14:51.372866Z" + }, + "papermill": { + "duration": 0.020004, + "end_time": "2024-03-24T19:14:51.375487", + "exception": false, + "start_time": "2024-03-24T19:14:51.355483", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"tab_ddpm_concat\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 0\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/tab_ddpm_concat/0\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011016, + "end_time": "2024-03-24T19:14:51.397570", + "exception": false, + "start_time": "2024-03-24T19:14:51.386554", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:14:51.421046Z", + "iopub.status.busy": "2024-03-24T19:14:51.420753Z", + "iopub.status.idle": "2024-03-24T19:14:51.430209Z", + "shell.execute_reply": "2024-03-24T19:14:51.429390Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023378, + "end_time": "2024-03-24T19:14:51.432059", + "exception": false, + "start_time": "2024-03-24T19:14:51.408681", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/tab_ddpm_concat/0\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:14:51.455890Z", + "iopub.status.busy": "2024-03-24T19:14:51.455623Z", + "iopub.status.idle": "2024-03-24T19:14:53.517498Z", + "shell.execute_reply": "2024-03-24T19:14:53.516524Z" + }, + "papermill": { + "duration": 2.076369, + "end_time": "2024-03-24T19:14:53.519758", + "exception": false, + "start_time": "2024-03-24T19:14:51.443389", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:14:53.586802Z", + "iopub.status.busy": "2024-03-24T19:14:53.585263Z", + "iopub.status.idle": "2024-03-24T19:14:53.604027Z", + "shell.execute_reply": "2024-03-24T19:14:53.603023Z" + }, + "papermill": { + "duration": 0.065143, + "end_time": "2024-03-24T19:14:53.607823", + "exception": false, + "start_time": "2024-03-24T19:14:53.542680", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:14:53.653784Z", + "iopub.status.busy": "2024-03-24T19:14:53.653091Z", + "iopub.status.idle": "2024-03-24T19:14:53.663991Z", + "shell.execute_reply": "2024-03-24T19:14:53.663087Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.029918, + "end_time": "2024-03-24T19:14:53.666263", + "exception": false, + "start_time": "2024-03-24T19:14:53.636345", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:14:53.694496Z", + "iopub.status.busy": "2024-03-24T19:14:53.694147Z", + "iopub.status.idle": "2024-03-24T19:14:53.792345Z", + "shell.execute_reply": "2024-03-24T19:14:53.791481Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.115226, + "end_time": "2024-03-24T19:14:53.795012", + "exception": false, + "start_time": "2024-03-24T19:14:53.679786", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:14:53.820847Z", + "iopub.status.busy": "2024-03-24T19:14:53.820502Z", + "iopub.status.idle": "2024-03-24T19:14:58.650964Z", + "shell.execute_reply": "2024-03-24T19:14:58.650015Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.84571, + "end_time": "2024-03-24T19:14:58.653514", + "exception": false, + "start_time": "2024-03-24T19:14:53.807804", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-24 19:14:56.170540: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-24 19:14:56.170596: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-24 19:14:56.172318: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:14:58.682395Z", + "iopub.status.busy": "2024-03-24T19:14:58.681225Z", + "iopub.status.idle": "2024-03-24T19:14:58.688477Z", + "shell.execute_reply": "2024-03-24T19:14:58.687610Z" + }, + "papermill": { + "duration": 0.023065, + "end_time": "2024-03-24T19:14:58.690605", + "exception": false, + "start_time": "2024-03-24T19:14:58.667540", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:14:58.714981Z", + "iopub.status.busy": "2024-03-24T19:14:58.714676Z", + "iopub.status.idle": "2024-03-24T19:15:21.690731Z", + "shell.execute_reply": "2024-03-24T19:15:21.689606Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 22.991296, + "end_time": "2024-03-24T19:15:21.693316", + "exception": false, + "start_time": "2024-03-24T19:14:58.702020", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-24T19:15:21.720994Z", + "iopub.status.busy": "2024-03-24T19:15:21.720672Z", + "iopub.status.idle": "2024-03-24T19:15:21.727236Z", + "shell.execute_reply": "2024-03-24T19:15:21.726364Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.022483, + "end_time": "2024-03-24T19:15:21.729107", + "exception": false, + "start_time": "2024-03-24T19:15:21.706624", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 95,\n", + " 'realtabformer': (69, 281, Embedding(281, 768), True),\n", + " 'lct_gan': 75,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:15:21.755866Z", + "iopub.status.busy": "2024-03-24T19:15:21.755606Z", + "iopub.status.idle": "2024-03-24T19:15:21.760207Z", + "shell.execute_reply": "2024-03-24T19:15:21.759359Z" + }, + "papermill": { + "duration": 0.020967, + "end_time": "2024-03-24T19:15:21.762221", + "exception": false, + "start_time": "2024-03-24T19:15:21.741254", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:15:21.787742Z", + "iopub.status.busy": "2024-03-24T19:15:21.787495Z", + "iopub.status.idle": "2024-03-24T19:15:22.261264Z", + "shell.execute_reply": "2024-03-24T19:15:22.260347Z" + }, + "papermill": { + "duration": 0.489103, + "end_time": "2024-03-24T19:15:22.263462", + "exception": false, + "start_time": "2024-03-24T19:15:21.774359", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/tab_ddpm_concat/all inf False\n", + "../../../../ml-utility-loss/aug_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_bs_test/tab_ddpm_concat/all inf False\n", + "../../../../ml-utility-loss/bs_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_synth_test/tab_ddpm_concat/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:15:22.292557Z", + "iopub.status.busy": "2024-03-24T19:15:22.292022Z", + "iopub.status.idle": "2024-03-24T19:15:22.608247Z", + "shell.execute_reply": "2024-03-24T19:15:22.607312Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.33341, + "end_time": "2024-03-24T19:15:22.610405", + "exception": false, + "start_time": "2024-03-24T19:15:22.276995", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'loss_balancer_beta': 0.7999999999999999,\n", + " 'loss_balancer_r': 0.96,\n", + " 'tf_pma_low': 4,\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.75,\n", + " 'bias_weight_decay': 0.1,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 4,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.04,\n", + " 'n_warmup_steps': 140,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'tab_ddpm_concat',\n", + " 'g_loss_mul': 0.1,\n", + " 'd_model': 512,\n", + " 'attn_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 64,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 7,\n", + " 'ada_activation': torch.nn.modules.activation.SELU,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 8,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'single_model': True,\n", + " 'models': ['tab_ddpm_concat'],\n", + " 'max_seconds': 3600,\n", + " 'Body': 'twin_encoder',\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "# params = {\n", + "# **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "# }\n", + "params = getattr(PARAMS, dataset_name).BEST_DICT[gp][gp_multiply][single_model]\n", + "if isinstance(params, (list, tuple)):\n", + " params = params[param_index]\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:15:22.638956Z", + "iopub.status.busy": "2024-03-24T19:15:22.638653Z", + "iopub.status.idle": "2024-03-24T19:15:22.738183Z", + "shell.execute_reply": "2024-03-24T19:15:22.737290Z" + }, + "papermill": { + "duration": 0.11607, + "end_time": "2024-03-24T19:15:22.740212", + "exception": false, + "start_time": "2024-03-24T19:15:22.624142", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_train/tab_ddpm_concat/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/treatment [400, 0]\n", + "Caching in ../../../../treatment/_cache_aug_val/tab_ddpm_concat/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/treatment [0, 200]\n", + "Caching in ../../../../treatment/_cache_bs_train/tab_ddpm_concat/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/treatment [100, 0]\n", + "Caching in ../../../../treatment/_cache_bs_val/tab_ddpm_concat/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/treatment [0, 50]\n", + "Caching in ../../../../treatment/_cache_synth/tab_ddpm_concat/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/treatment [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-24T19:15:22.772197Z", + "iopub.status.busy": "2024-03-24T19:15:22.771885Z", + "iopub.status.idle": "2024-03-24T19:15:23.291631Z", + "shell.execute_reply": "2024-03-24T19:15:23.290687Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.539764, + "end_time": "2024-03-24T19:15:23.293743", + "exception": false, + "start_time": "2024-03-24T19:15:22.753979", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['tab_ddpm_concat'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:15:23.323957Z", + "iopub.status.busy": "2024-03-24T19:15:23.323212Z", + "iopub.status.idle": "2024-03-24T19:15:23.327965Z", + "shell.execute_reply": "2024-03-24T19:15:23.327073Z" + }, + "papermill": { + "duration": 0.021999, + "end_time": "2024-03-24T19:15:23.329888", + "exception": false, + "start_time": "2024-03-24T19:15:23.307889", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:15:23.357169Z", + "iopub.status.busy": "2024-03-24T19:15:23.356887Z", + "iopub.status.idle": "2024-03-24T19:15:23.363930Z", + "shell.execute_reply": "2024-03-24T19:15:23.363104Z" + }, + "papermill": { + "duration": 0.022997, + "end_time": "2024-03-24T19:15:23.365801", + "exception": false, + "start_time": "2024-03-24T19:15:23.342804", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "18316289" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:15:23.393426Z", + "iopub.status.busy": "2024-03-24T19:15:23.393155Z", + "iopub.status.idle": "2024-03-24T19:15:23.485401Z", + "shell.execute_reply": "2024-03-24T19:15:23.484490Z" + }, + "papermill": { + "duration": 0.108868, + "end_time": "2024-03-24T19:15:23.487972", + "exception": false, + "start_time": "2024-03-24T19:15:23.379104", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 12] --\n", + "├─Adapter: 1-1 [2, 2648, 12] --\n", + "│ └─Sequential: 2-1 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 1024] 13,312\n", + "│ │ │ └─SELU: 4-2 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-4 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-6 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-8 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-10 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 1024] 1,049,600\n", + "│ │ │ └─SELU: 4-12 [2, 2648, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-13 [2, 2648, 512] 524,800\n", + "│ │ │ └─LeakyHardsigmoid: 4-14 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 12] (recursive)\n", + "│ └─Sequential: 2-2 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-8 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-16 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-18 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-20 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-22 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-24 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 661, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 661, 1024] (recursive)\n", + "│ │ │ └─SELU: 4-26 [2, 661, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-28 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 2048] --\n", + "│ └─Encoder: 2-3 [2, 4, 512] --\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 64, 32, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 64, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-6 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 64, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 64, 32, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 64, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-18 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 64, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 64, 32, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 64, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-30 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 64, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 262,656\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 4, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 64, 32, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 64, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-42 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 64, 2648, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 64, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 262,656\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 4, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 4, 512] 2,048\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 4, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 4, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 64, 4, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 64, 4, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 4, 512] 262,656\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-54 [2, 4, 512] --\n", + "│ └─Encoder: 2-4 [2, 4, 512] (recursive)\n", + "│ │ └─ModuleList: 3-16 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 64, 32, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 64, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-60 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 64, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-31 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 64, 32, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 64, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-72 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 64, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-40 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 64, 32, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 64, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-84 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 64, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-42 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-43 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 4, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-45 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-46 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 64, 32, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 64, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-96 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 64, 661, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 64, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-48 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-49 [2, 661, 512] --\n", + "│ │ │ │ │ └─Linear: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-18 [2, 4, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-51 [2, 4, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-52 [2, 4, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 4, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 64, 4, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 64, 4, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 4, 512] (recursive)\n", + "│ │ │ │ │ │ └─LeakyHardtanh: 7-108 [2, 4, 512] --\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-17 [2, 256] --\n", + "│ │ │ └─Linear: 4-37 [2, 256] 524,544\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 256] --\n", + "│ │ └─FeedForward: 3-18 [2, 256] --\n", + "│ │ │ └─Linear: 4-39 [2, 256] 65,792\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 256] --\n", + "│ │ └─FeedForward: 3-19 [2, 256] --\n", + "│ │ │ └─Linear: 4-41 [2, 256] 65,792\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 256] --\n", + "│ │ └─FeedForward: 3-20 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 65,792\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─LeakyHardsigmoid: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─LeakyHardsigmoid: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 1] --\n", + "│ │ │ └─Linear: 4-51 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-52 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 18,316,289\n", + "Trainable params: 18,316,289\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 71.16\n", + "========================================================================================================================\n", + "Input size (MB): 0.32\n", + "Forward/backward pass size (MB): 1067.90\n", + "Params size (MB): 73.27\n", + "Estimated Total Size (MB): 1141.48\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T19:15:23.519640Z", + "iopub.status.busy": "2024-03-24T19:15:23.519325Z", + "iopub.status.idle": "2024-03-24T20:25:21.381875Z", + "shell.execute_reply": "2024-03-24T20:25:21.380883Z" + }, + "papermill": { + "duration": 4197.895813, + "end_time": "2024-03-24T20:25:21.399098", + "exception": false, + "start_time": "2024-03-24T19:15:23.503285", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.06340547758542622, 'avg_role_model_std_loss': 6.714779690768261, 'avg_role_model_mean_pred_loss': 0.0216546154567992, 'avg_role_model_g_mag_loss': 0.02457591868316134, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.25894568058539236, 'n_size': 900, 'n_batch': 225, 'duration': 351.96785950660706, 'duration_batch': 1.5643015978071424, 'duration_size': 0.3910753994517856, 'avg_pred_std': 0.16249362598626046}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.01679980868561526, 'avg_role_model_std_loss': 4.965276870460201, 'avg_role_model_mean_pred_loss': 0.0022828475441168602, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01679980868561526, 'n_size': 450, 'n_batch': 113, 'duration': 98.52480149269104, 'duration_batch': 0.8719008981654074, 'duration_size': 0.2189440033170912, 'avg_pred_std': 0.09940910503458496}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.010284998621031667, 'avg_role_model_std_loss': 0.2255875137631098, 'avg_role_model_mean_pred_loss': 0.0006483664865724725, 'avg_role_model_g_mag_loss': 0.03847181756566796, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.01054267091467207, 'n_size': 900, 'n_batch': 225, 'duration': 345.0646252632141, 'duration_batch': 1.533620556725396, 'duration_size': 0.383405139181349, 'avg_pred_std': 0.2337058648798201}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00841922588324514, 'avg_role_model_std_loss': 3.2320097570229653, 'avg_role_model_mean_pred_loss': 0.00070605612004593, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00841922588324514, 'n_size': 450, 'n_batch': 113, 'duration': 98.65367245674133, 'duration_batch': 0.8730413491747021, 'duration_size': 0.21923038323720295, 'avg_pred_std': 0.09933966914474768}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.006607208507337469, 'avg_role_model_std_loss': 0.5377882501872379, 'avg_role_model_mean_pred_loss': 0.00014010620100567549, 'avg_role_model_g_mag_loss': 0.03052467739351818, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0067294777951459155, 'n_size': 900, 'n_batch': 225, 'duration': 346.46448493003845, 'duration_batch': 1.5398421552446153, 'duration_size': 0.38496053881115383, 'avg_pred_std': 0.23248913520145126}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.009977165092619644, 'avg_role_model_std_loss': 2.5673703113514437, 'avg_role_model_mean_pred_loss': 0.0019968906350478132, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.009977165092619644, 'n_size': 450, 'n_batch': 113, 'duration': 97.69291520118713, 'duration_batch': 0.8645390725768773, 'duration_size': 0.2170953671137492, 'avg_pred_std': 0.10989512876503554}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005785522044221176, 'avg_role_model_std_loss': 0.24223101447777884, 'avg_role_model_mean_pred_loss': 0.0001330745897080286, 'avg_role_model_g_mag_loss': 0.030684991976660157, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0061337225039238144, 'n_size': 900, 'n_batch': 225, 'duration': 348.0978581905365, 'duration_batch': 1.54710159195794, 'duration_size': 0.386775397989485, 'avg_pred_std': 0.24560337937023077}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007756494560071651, 'avg_role_model_std_loss': 2.980153914541319, 'avg_role_model_mean_pred_loss': 0.0005405202996384829, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007756494560071651, 'n_size': 450, 'n_batch': 113, 'duration': 98.20267009735107, 'duration_batch': 0.8690501778526644, 'duration_size': 0.21822815577189128, 'avg_pred_std': 0.10058078948723614}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.007448349164959735, 'avg_role_model_std_loss': 0.6328000489554128, 'avg_role_model_mean_pred_loss': 0.0002071066759673005, 'avg_role_model_g_mag_loss': 0.02920709241492053, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007665982826781854, 'n_size': 900, 'n_batch': 225, 'duration': 346.766752243042, 'duration_batch': 1.5411855655246312, 'duration_size': 0.3852963913811578, 'avg_pred_std': 0.24027496062167403}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0063575647280585046, 'avg_role_model_std_loss': 3.3627717712450584, 'avg_role_model_mean_pred_loss': 0.0004621822363243454, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0063575647280585046, 'n_size': 450, 'n_batch': 113, 'duration': 96.85975694656372, 'duration_batch': 0.8571659906775551, 'duration_size': 0.21524390432569715, 'avg_pred_std': 0.11672678085616316}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004543020532421198, 'avg_role_model_std_loss': 0.5700210672009972, 'avg_role_model_mean_pred_loss': 0.00011921863728762756, 'avg_role_model_g_mag_loss': 0.02387150165831877, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0046356002659983155, 'n_size': 900, 'n_batch': 225, 'duration': 346.64923548698425, 'duration_batch': 1.540663268831041, 'duration_size': 0.3851658172077603, 'avg_pred_std': 0.24926329110490364}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007455925235068561, 'avg_role_model_std_loss': 2.479032797534744, 'avg_role_model_mean_pred_loss': 0.000639281961615226, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007455925235068561, 'n_size': 450, 'n_batch': 113, 'duration': 97.00784873962402, 'duration_batch': 0.8584765375187967, 'duration_size': 0.2155729971991645, 'avg_pred_std': 0.10956132564128933}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004305505793477601, 'avg_role_model_std_loss': 0.2397287840495211, 'avg_role_model_mean_pred_loss': 3.9819682542443306e-05, 'avg_role_model_g_mag_loss': 0.022900073562567638, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0044476011980997205, 'n_size': 900, 'n_batch': 225, 'duration': 346.15540647506714, 'duration_batch': 1.5384684732225207, 'duration_size': 0.38461711830563017, 'avg_pred_std': 0.23756346660614427}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007061510610741443, 'avg_role_model_std_loss': 3.6814213301790506, 'avg_role_model_mean_pred_loss': 0.0005781446860156483, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007061510610741443, 'n_size': 450, 'n_batch': 113, 'duration': 96.51440978050232, 'duration_batch': 0.8541098210663922, 'duration_size': 0.21447646617889404, 'avg_pred_std': 0.1051591920189081}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004207614306922349, 'avg_role_model_std_loss': 0.544365831900913, 'avg_role_model_mean_pred_loss': 0.00016509564863483367, 'avg_role_model_g_mag_loss': 0.019844594347911578, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004282006424441028, 'n_size': 900, 'n_batch': 225, 'duration': 354.34157609939575, 'duration_batch': 1.5748514493306478, 'duration_size': 0.39371286233266195, 'avg_pred_std': 0.24139304114531518}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006262556161943217, 'avg_role_model_std_loss': 3.23558615521124, 'avg_role_model_mean_pred_loss': 0.0006233526253580421, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006262556161943217, 'n_size': 450, 'n_batch': 113, 'duration': 99.24161124229431, 'duration_batch': 0.8782443472769408, 'duration_size': 0.22053691387176513, 'avg_pred_std': 0.1159596569104584}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00405201113243998, 'avg_role_model_std_loss': 0.15210635947048287, 'avg_role_model_mean_pred_loss': 0.0001132995204278981, 'avg_role_model_g_mag_loss': 0.022547589465361347, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004117198001800312, 'n_size': 900, 'n_batch': 225, 'duration': 347.5573856830597, 'duration_batch': 1.5446994919247097, 'duration_size': 0.3861748729811774, 'avg_pred_std': 0.24259670994554958}\n", + "Time out: 3924.808913230896/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tab_ddpm_concat', 'n_size': 1050, 'n_batch': 263, 'role_model_metrics': {'avg_loss': 0.004023964082093915, 'avg_g_mag_loss': 0.008524223082743249, 'avg_g_cos_loss': 0.0, 'pred_duration': 5.916023254394531, 'grad_duration': 7.633917808532715, 'total_duration': 13.549941062927246, 'pred_std': 0.2330770343542099, 'std_loss': 0.0016544610261917114, 'mean_pred_loss': 0.0001295258553000167, 'pred_rmse': 0.06343472748994827, 'pred_mae': 0.03694874420762062, 'pred_mape': 5749574.0, 'grad_rmse': 0.0330444797873497, 'grad_mae': 0.02197916805744171, 'grad_mape': 0.7060891389846802}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.004023964082093915, 'avg_g_mag_loss': 0.008524223082743249, 'avg_g_cos_loss': 0.0, 'avg_pred_duration': 5.916023254394531, 'avg_grad_duration': 7.633917808532715, 'avg_total_duration': 13.549941062927246, 'avg_pred_std': 0.2330770343542099, 'avg_std_loss': 0.0016544610261917114, 'avg_mean_pred_loss': 0.0001295258553000167}, 'min_metrics': {'avg_loss': 0.004023964082093915, 'avg_g_mag_loss': 0.008524223082743249, 'avg_g_cos_loss': 0.0, 'pred_duration': 5.916023254394531, 'grad_duration': 7.633917808532715, 'total_duration': 13.549941062927246, 'pred_std': 0.2330770343542099, 'std_loss': 0.0016544610261917114, 'mean_pred_loss': 0.0001295258553000167, 'pred_rmse': 0.06343472748994827, 'pred_mae': 0.03694874420762062, 'pred_mape': 5749574.0, 'grad_rmse': 0.0330444797873497, 'grad_mae': 0.02197916805744171, 'grad_mape': 0.7060891389846802}, 'model_metrics': {'tab_ddpm_concat': {'avg_loss': 0.004023964082093915, 'avg_g_mag_loss': 0.008524223082743249, 'avg_g_cos_loss': 0.0, 'pred_duration': 5.916023254394531, 'grad_duration': 7.633917808532715, 'total_duration': 13.549941062927246, 'pred_std': 0.2330770343542099, 'std_loss': 0.0016544610261917114, 'mean_pred_loss': 0.0001295258553000167, 'pred_rmse': 0.06343472748994827, 'pred_mae': 0.03694874420762062, 'pred_mape': 5749574.0, 'grad_rmse': 0.0330444797873497, 'grad_mae': 0.02197916805744171, 'grad_mape': 0.7060891389846802}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T20:25:21.433164Z", + "iopub.status.busy": "2024-03-24T20:25:21.432299Z", + "iopub.status.idle": "2024-03-24T20:25:21.437107Z", + "shell.execute_reply": "2024-03-24T20:25:21.436236Z" + }, + "papermill": { + "duration": 0.023771, + "end_time": "2024-03-24T20:25:21.439148", + "exception": false, + "start_time": "2024-03-24T20:25:21.415377", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T20:25:21.471519Z", + "iopub.status.busy": "2024-03-24T20:25:21.471251Z", + "iopub.status.idle": "2024-03-24T20:25:21.592421Z", + "shell.execute_reply": "2024-03-24T20:25:21.591411Z" + }, + "papermill": { + "duration": 0.140283, + "end_time": "2024-03-24T20:25:21.594807", + "exception": false, + "start_time": "2024-03-24T20:25:21.454524", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T20:25:21.630449Z", + "iopub.status.busy": "2024-03-24T20:25:21.630112Z", + "iopub.status.idle": "2024-03-24T20:25:21.894209Z", + "shell.execute_reply": "2024-03-24T20:25:21.893327Z" + }, + "papermill": { + "duration": 0.285012, + "end_time": "2024-03-24T20:25:21.896303", + "exception": false, + "start_time": "2024-03-24T20:25:21.611291", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAESCAYAAABU2qhcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvTklEQVR4nO3deXQT570//vdotFleZGPjjTiWHUzYbYJrlyXJSeNiKEkgpQnwo2E5Cdxy4TZcF8J1CnZuaWMWNyELB1ryZUnLdntvk9ubpm6IwWmTGJywBIgJAWKwWeQNLNmSrW2e3x+yxghkW5IlS7Y+r3PmWBo9Gj1jrDczz8x8hmOMMRBCSBCQBLoDhBDiQIFECAkaFEiEkKBBgUQICRoUSISQoEGBRAgJGhRIhJCgIQ10B3xBEATcuHEDkZGR4Dgu0N0hhNyFMYbW1lYkJydDIul+O2hQBNKNGzeQkpIS6G4QQnpRV1eH++67r9vXB0UgRUZGArCvbFRUVIB7Qwi5m16vR0pKivhd7c6gCCTHblpUVBQFEiFBrLchFRrUJoQEDQokQkjQoEAihASNQTGGRPrGZrPBYrEEuhtkAJPJZOB5vs/LoUAKYYwxaLVatLS0BLorZBCIjo5GYmJin84FpEAKYY4wio+Ph0qlopNKiVcYYzAajWhoaAAAJCUleb2skAokq03A+ZutuHbbiBnjvP+lDQY2m00Mo9jY2EB3hwxwYWFhAICGhgbEx8d7vfsWUoPaRosNT779KZbvO4nWjtAeM3GMGalUqgD3hAwWjr+lvoxHhlQgRSlliItQAACuNBkD3JvgQLtpxFd88bcUUoEEAOlx4QCAmmZDgHtCCLlbyAWSJs6+WVnTSIFESLDxKpC2bdsGjUYDpVKJ3NxcVFVVddt2586dePjhhxETE4OYmBjk5eXd037x4sXgOM5pmj59ujdd61VaXAQAoKapzS/LJ4Mfx3F4//33A90Nn3rllVeQlZUV6G54HkiHDh1CQUEBiouLcfLkSWRmZiI/P1885He3iooKzJ8/H0ePHkVlZSVSUlIwbdo0XL9+3and9OnTcfPmTXE6cOCAd2vUizTHLlsTbSGRgWvPnj2Ijo722fJWr16N8vJyny3PWx4H0muvvYalS5diyZIlGD16NHbs2AGVSoVdu3a5bL9v3z7867/+K7KysjBy5Ei88847EAThnpVXKBRITEwUp5iYGO/WqBd3BhLdI5MMdmaz2a12ERERQXH6h0eBZDabceLECeTl5XUtQCJBXl4eKisr3VqG0WiExWLBkCFDnOZXVFQgPj4eDz74IJYvX47m5uZul2EymaDX650md6XGqsBxgL7DilsG9/6xQgVjDEaztd8nT/9jKCsrw9SpUxEdHY3Y2Fg88cQTuHz5MgBg8uTJWLt2rVP7xsZGyGQy/OMf/wAA3Lx5EzNnzkRYWBjS0tKwf/9+aDQabN261avf29mzZ/GDH/wAYWFhiI2NxbJly9DW1jUkUFFRgZycHISHhyM6OhpTpkzB1atXAQBfffUVHnvsMURGRiIqKgoTJ07El19+2ePnVVRUYMmSJdDpdOIQxyuvvAIA0Gg02LBhAxYuXIioqCgsW7YMALB27VqMGDECKpUK6enpWL9+vdPh+bt32RYvXozZs2ejtLQUSUlJiI2NxYoVK/x+iZFHJ0Y2NTXBZrMhISHBaX5CQgK++eYbt5axdu1aJCcnO4Xa9OnT8eMf/xhpaWm4fPkyXn75ZcyYMQOVlZUuT7AqKSnBf/7nf3rSdZFSxiNZHYbrLe2oaTIgtvM0AAK0W2wYXfT3fv/c6l/lQyV3/0/RYDCgoKAA48ePR1tbG4qKivD000/j9OnTWLBgATZv3oyNGzeKh6EPHTqE5ORkPPzwwwCAhQsXoqmpCRUVFZDJZCgoKOh2yMGdvuTn52PSpEn44osv0NDQgBdeeAErV67Enj17YLVaMXv2bCxduhQHDhyA2WxGVVWV2LcFCxZgwoQJ2L59O3iex+nTpyGTyXr8zMmTJ2Pr1q0oKirChQsXANi3cBxKS0tRVFSE4uJicV5kZCT27NmD5ORknD17FkuXLkVkZCReeumlbj/n6NGjSEpKwtGjR3Hp0iXMnTsXWVlZWLp0qVe/K3f065naGzduxMGDB1FRUQGlUinOnzdvnvh43LhxGD9+PB544AFUVFTg8ccfv2c5hYWFKCgoEJ87qtG5K31ouBhI2Zohvb+BBJU5c+Y4Pd+1axeGDh2K6upqPPvss1i1ahU+/fRTMYD279+P+fPng+M4fPPNN/j444/xxRdfIDs7GwDwzjvvICMjw6u+7N+/Hx0dHXj33XcRHm4fDnj77bfx5JNPYtOmTZDJZNDpdHjiiSfwwAMPAABGjRolvr+2thZr1qzByJEjAcCtfsjlcqjVanAch8TExHte/8EPfoBf/OIXTvPWrVsnPtZoNFi9ejUOHjzYYyDFxMTg7bffBs/zGDlyJGbOnIny8vLgCaS4uDjwPI/6+nqn+fX19S5/MXcqLS3Fxo0b8fHHH2P8+PE9tk1PT0dcXBwuXbrkMpAUCgUUCu+3bDSx4fjnxSYa2L5LmIxH9a/yA/K5nrh48SKKiopw/PhxNDU1QRAEAPYv99ixYzFt2jTs27cPDz/8MGpqalBZWYnf/e53AIALFy5AKpXioYceEpc3fPhwr8csz58/j8zMTDGMAGDKlCkQBAEXLlzAI488gsWLFyM/Px8//OEPkZeXh2effVa83qugoAAvvPAC/vCHPyAvLw/PPPOMGFzecgTtnQ4dOoQ333wTly9fRltbG6xWa6/VVceMGeO0h5KUlISzZ8/2qW+98WgMSS6XY+LEiU4D0o4B6kmTJnX7vs2bN2PDhg0oKytz+cu627Vr19Dc3Nyni/R6QkfaXOM4Diq5tN8nT8/wffLJJ3Hr1i3s3LkTx48fx/HjxwF0DeAuWLAA//3f/w2LxYL9+/dj3LhxGDdunM9/X+7avXs3KisrMXnyZBw6dAgjRozAsWPHANjHbr7++mvMnDkTR44cwejRo/Hee+/16fPuDEcAqKysxIIFC/CjH/0IH3zwAU6dOoVf/vKXvQ54373ryHGcGP7+4vFRtoKCAuzcuRN79+7F+fPnsXz5chgMBixZsgSAff+8sLBQbL9p0yasX78eu3btgkajgVarhVarFQf92trasGbNGhw7dgxXrlxBeXk5Zs2aheHDhyM/3z//W6cNpUAaqJqbm3HhwgWsW7cOjz/+OEaNGoXbt287tZk1axY6OjpQVlaG/fv3Y8GCBeJrDz74IKxWK06dOiXOu3Tp0j3LcNeoUaPw1VdfwWDo+lv67LPPIJFI8OCDD4rzJkyYgMLCQnz++ecYO3Ys9u/fL742YsQI/Pu//zs++ugj/PjHP8bu3bt7/Vy5XA6bzeZWHz///HOkpqbil7/8JbKzs5GRkSEOqgcbjwNp7ty54qBZVlYWTp8+jbKyMnGgu7a2Fjdv3hTbb9++HWazGT/5yU+QlJQkTqWlpQAAnudx5swZPPXUUxgxYgSef/55TJw4Ef/85z/7tFvWE8flI1eaDRAEOvQ/kMTExCA2Nha///3vcenSJRw5csRpPBGwbyHMnj0b69evx/nz5zF//nzxtZEjRyIvLw/Lli1DVVUVTp06hWXLliEsLMyra7EWLFgApVKJRYsW4dy5czh69Cj+7d/+Dc899xwSEhJQU1ODwsJCVFZW4urVq/joo49w8eJFjBo1Cu3t7Vi5ciUqKipw9epVfPbZZ/jiiy+cxpi6o9Fo0NbWhvLycjQ1NcFo7P7azIyMDNTW1uLgwYO4fPky3nzzzT5vhfkNGwR0Oh0DwHQ6nVvtLVYbe6Dwryx17Qfs+m2jn3sXnNrb21l1dTVrb28PdFc8dvjwYTZq1CimUCjY+PHjWUVFBQPA3nvvPbHNhx9+yACwRx555J7337hxg82YMYMpFAqWmprK9u/fz+Lj49mOHTvc+vy7P+vMmTPsscceY0qlkg0ZMoQtXbqUtba2MsYY02q1bPbs2SwpKYnJ5XKWmprKioqKmM1mYyaTic2bN4+lpKQwuVzOkpOT2cqVK93+N/nZz37GYmNjGQBWXFzMGGMsNTWVvf766/e0XbNmDYuNjWURERFs7ty57PXXX2dqtVp8vbi4mGVmZorPFy1axGbNmuW0jBdffJE9+uij3fanp78pd7+jHGMD/+xAvV4PtVoNnU7n9m2QfvDbCnzXaMC+F3IxZXicn3sYfDo6OlBTU4O0tDSnI56h6Nq1a0hJScHHH3/s8iAKcU9Pf1PufkdD7uJaB8du23c0jhRyjhw5gr/85S+oqanB559/jnnz5kGj0eCRRx4JdNdCXsgGkuNI2xUKpJBjsVjw8ssvY8yYMXj66acxdOhQ8STJffv2ISIiwuU0ZsyYfuvjjBkzuu3Hq6++2m/96G8hVcL2Tho69B+y8vPzuz2C+9RTTyE3N9fla72dQe1L77zzDtrb212+dvdlV4NJyAYSnYtEXImMjOz1/vP9YdiwYYHuQkCE7C5bemddpNpbRlhs/j3ZixDinpANpIQoBcJkPGwCw7XbrjeNCSH9K2QDieO4O8aRqHokIcEgZAMJuOPQP9XXJiQohHQg0cA2IcElpANJc8c1bYS4azAW+Q8WIR1I4hYS7bKRAcbXRf4Be2lcjuPQ0tLi0+V6IqQDyTGGdEPXgXaze6UcCCH+E9KBFBMuR7TKfvYt7bYBYAwwG/p/oiL/PivybzKZsHr1agwbNgzh4eHIzc1FRUWF+N6rV6/iySefRExMDMLDwzFmzBh8+OGHuHLlCh577DEA9hIvHMdh8eLFXv0++iJkz9R20MSG47SxBVeaDBiV5F6lgEHLYgReTe7/z335BiAP771dJyry332R/5UrV6K6uhoHDx5EcnIy3nvvPUyfPh1nz55FRkYGVqxYAbPZjH/84x8IDw9HdXU1IiIikJKSgv/5n//BnDlzcOHCBURFRSEsLMyr30lfhHwgpceF43RdC131P4BQkX/XRf5ra2uxe/du1NbWIjnZ/h/L6tWrUVZWht27d+PVV19FbW0t5syZI5b0TU9PF9/vuEYuPj7e5+NT7gr5QKJD/3eQqexbK4H4XA9QkX/Xzp49C5vNhhEjRjjNN5lM4k0gf/7zn2P58uX46KOPkJeXhzlz5vR6043+FNJjSEBXfW0qQwKA4+y7Tv09UZF/nxT5b2trA8/zOHHiBE6fPi1O58+fxxtvvAEAeOGFF/Ddd9/hueeew9mzZ5GdnY233nrLZ+vaVyEfSJpY2kIaSKjIv52rIv8TJkyAzWZDQ0MDhg8f7jTduWuXkpKCn/3sZ/jzn/+MX/ziF9i5c6e4TABu3zzAH0I+kBy7bM0GM3RG/94mmPQdFfm3c1Xkf8SIEViwYAEWLlyIP//5z6ipqUFVVRVKSkrw17/+FQCwatUq/P3vf0dNTQ1OnjyJo0ePip+XmpoKjuPwwQcfoLGx0elIYb/pseL2AOFpkf+75fzmMEtd+wE7VXvbtx0LYlTkf3AW+TebzayoqIhpNBomk8lYUlISe/rpp9mZM2cYY4ytXLmSPfDAA0yhULChQ4ey5557jjU1NYnL/NWvfsUSExMZx3Fs0aJFbvXDgYr8d/KmyP+d5v6uEsdrbmHr3CzMnhAahbGoyH8XKvLvG74o8h/yR9kAIH1oOI7X3KJD/yHiyJEjaGtrw7hx43Dz5k289NJLVOQ/SIT8GBJAh/5DDRX5D160hQQgrbOcLRVqCw1U5D94USABSIuzn5h3pckIxphXR1vI4EBF/gOLdtkApAxRQcIBbSYrGttMge5OvxoExzRIkPDF3xIFEgCFlMd9MfatpFCpjeTY/TAajQHuCRksHH9Lfdm1pV22Tmlx4ai9ZURNkwG56bGB7o7f8TyP6Oho8Sp3lUpFu6rEK4wxGI1GNDQ0IDo6GjzPe70sCqROaXHh+OTbRtSEUF0kx+UE3pbeIORO0dHRTpeoeIMCqVMolrPlOA5JSUmIj4+HxUKXzRDvyWSyPm0ZOXgVSNu2bcOWLVug1WqRmZmJt956Czk5OS7b7ty5E++++y7OnTsHAJg4cSJeffVVp/aMMRQXF2Pnzp1oaWnBlClTsH37dq9r1HgjlM9F4nneJ39MhPSVx4Pahw4dQkFBAYqLi3Hy5ElkZmYiPz+/283+iooKzJ8/H0ePHkVlZSVSUlIwbdo0XL9+XWyzefNmvPnmm9ixYweOHz+O8PBw5Ofno6Ojw/s185AjkK42G2ET6MgTIQHh0dVzjLGcnBy2YsUK8bnNZmPJycmspKTErfdbrVYWGRnJ9u7dyxhjTBAElpiYyLZs2SK2aWlpYQqFgh04cMDlMjo6OphOpxOnurq6Pl1cyxhjVpvAMl7+kKWu/YDVNhu8Xg4h5F7uXlzr0RaS2WzGiRMnkJeXJ86TSCTIy8tDZWWlW8swGo2wWCzi2aY1NTXQarVOy1Sr1cjNze12mSUlJVCr1eKUkpLiyWq4xEs4pMZ2HvoPwd02QoKBR4HU1NQEm82GhIQEp/kJCQnQarVuLWPt2rVITk4WA8jxPk+WWVhYCJ1OJ051dXWerEa3QnkciZBg0K9H2TZu3IiDBw+ioqKiTyUvFAoFFAqFD3tmR4FESGB5tIUUFxcHnudRX1/vNL++vr7X8w9KS0uxceNGfPTRR05FxR3v82aZvkaBREhgeRRIcrkcEydORHl5uThPEASUl5dj0qRJ3b5v8+bN2LBhA8rKysRbzzikpaUhMTHRaZl6vR7Hjx/vcZn+QIFESGB5vMtWUFCARYsWITs7Gzk5Odi6dSsMBgOWLFkCwH4TvmHDhqGkpAQAsGnTJhQVFYl3B3WMCzlqu3Ach1WrVuHXv/41MjIykJaWhvXr1yM5ORmzZ8/23Zq6wXEHkmu3jTBbBcildKkfIf3J40CaO3cuGhsbUVRUBK1Wi6ysLJSVlYmD0rW1tZBIur7I27dvh9lsxk9+8hOn5RQXF4u3/33ppZdgMBiwbNkytLS0YOrUqSgrK+v30qpDIxQIl/MwmG2ovWXE8PiIfv18QkId1dS+yxNv/RPnruuxc2E2fjg6ofc3EEJ65e53lPZJ7kLVIwkJHAqku9DANiGBQ4F0F0c5WwokQvofBdJdunbZKJAI6W8USHdJi7XvstXrTTCYrAHuDSGhhQLpLmqVDLHhcgC0lURIf6NAckHTObB9JYTK2RISDCiQXAjFcraEBAMKJBfo0D8hgUGB5IIjkL6jQCKkX1EguZBGY0iEBAQFkguazkP/LUYLbhvMAe4NIaGDAsmFMDmPZLW90gDtthHSfyiQuqGhgW1C+h0FUjfEcSQKJEL6DQVSN+jQPyH9jwKpG+lD6dA/If2NAqkbjiNtV5oMGARFNQkZECiQupEyRAVewqHdYkO93hTo7hASEiiQuiHjJbh/iL1Y23dUzpaQfkGB1AMa2Cakf1Eg9eDOcSRCiP9RIPXAceNI2kIipH9QIPUgna76J6RfUSD1wHH5SG2zEVabEODeEDL4USD1IClKCYVUAqvAcL2lPdDdIWTQo0DqgUTCUbE2QvoRBVIvqL42If2HAqkXVIaEkP5DgdQLKmdLSP/xKpC2bdsGjUYDpVKJ3NxcVFVVddv266+/xpw5c6DRaMBxHLZu3XpPm1deeQUcxzlNI0eO9KZrPice+qddNkL8zuNAOnToEAoKClBcXIyTJ08iMzMT+fn5aGhocNneaDQiPT0dGzduRGJiYrfLHTNmDG7evClOn376qadd8wvHFtINXTs6LLYA94aQwc3jQHrttdewdOlSLFmyBKNHj8aOHTugUqmwa9cul+2/973vYcuWLZg3bx4UCkW3y5VKpUhMTBSnuLg4T7vmF0PC5YhUSsEYcLXZGOjuEDKoeRRIZrMZJ06cQF5eXtcCJBLk5eWhsrKyTx25ePEikpOTkZ6ejgULFqC2trbbtiaTCXq93mnyF47jxN02GtgmxL88CqSmpibYbDYkJCQ4zU9ISIBWq/W6E7m5udizZw/Kysqwfft21NTU4OGHH0Zra6vL9iUlJVCr1eKUkpLi9We7g676J6R/BMVRthkzZuCZZ57B+PHjkZ+fjw8//BAtLS34r//6L5ftCwsLodPpxKmurs6v/UuLiwAA1FBdJEL8SupJ47i4OPA8j/r6eqf59fX1PQ5Yeyo6OhojRozApUuXXL6uUCh6HI/yNU2cvVDblSYaQyLEnzzaQpLL5Zg4cSLKy8vFeYIgoLy8HJMmTfJZp9ra2nD58mUkJSX5bJl9kd65hUSXjxDiXx5tIQFAQUEBFi1ahOzsbOTk5GDr1q0wGAxYsmQJAGDhwoUYNmwYSkpKANgHwqurq8XH169fx+nTpxEREYHhw4cDAFavXo0nn3wSqampuHHjBoqLi8HzPObPn++r9ewTxxZSU5sJ+g4LopSyAPeIkMHJ40CaO3cuGhsbUVRUBK1Wi6ysLJSVlYkD3bW1tZBIuja8bty4gQkTJojPS0tLUVpaikcffRQVFRUAgGvXrmH+/Plobm7G0KFDMXXqVBw7dgxDhw7t4+r5RqRShrgIBZraTLjSZMD4+6ID3SVCBiWODYJ7/Oj1eqjVauh0OkRFRfnlM57dUYmqK7fwxrwszMoa5pfPIGSwcvc7GhRH2QYCOvRPiP9RILmJ6msT4n8USG5y3IGEAokQ/6FAclP6HVtIg2DYjZCgRIHkpvuHqMBxQGuHFc0Gc6C7Q8igRIHkJqWMx7DoMAC020aIv1AgeYDqaxPiXxRIHhADicrZEuIXFEgeoC0kQvyLAskDdHIkIf5FgeSBO+9AIgh06J8QX6NA8sCw6DDIeA4mq4Cb+o5Ad4eQQYcCyQNSXoL7h9hLkdA4EiG+R4Hkoa5xJCpnS4ivUSB5yBFIVD2SEN+jQPKQo+D/FQokQnyOAslDdOifEP+hQPKQI5DqbrfDYhMC3BtCBhcKJA8lRCkQJuNhExjqbtFtkQjxJQokD3EcR7tthPgJBZIXqJwtIf5BgeSFtFg69E+IP1AgeUG8po0CiRCfokDyAu2yEeIfFEhecOyy3dR1wGi2Brg3hAweFEheiAmXI1olAwBcaaJD/4T4CgWSl+6sjUQI8Q0KJC/RuUiE+B4FkpfEQ/9UF4kQn6FA8lLXkTaqi0SIr3gVSNu2bYNGo4FSqURubi6qqqq6bfv1119jzpw50Gg04DgOW7du7fMyg0HXGBINahPiKx4H0qFDh1BQUIDi4mKcPHkSmZmZyM/PR0NDg8v2RqMR6enp2LhxIxITE32yzGCg6dxlu2Uwo8VIt9YmxCeYh3JyctiKFSvE5zabjSUnJ7OSkpJe35uamspef/11ny6TMcZ0Oh0DwHQ6nVvtfSXnN4dZ6toP2Mmrt/r1cwkZaNz9jnq0hWQ2m3HixAnk5eWJ8yQSCfLy8lBZWelVIHqzTJPJBL1e7zQFAh1pI8S3PAqkpqYm2Gw2JCQkOM1PSEiAVqv1qgPeLLOkpARqtVqcUlJSvPrsvqJytoT41oA8ylZYWAidTidOdXV1AelHOhX8J8SnpJ40jouLA8/zqK+vd5pfX1/f7YC1P5apUCigUCi8+jxf0tAuGyE+5dEWklwux8SJE1FeXi7OEwQB5eXlmDRpklcd8Mcy+8udZUgYo1trE9JXHm0hAUBBQQEWLVqE7Oxs5OTkYOvWrTAYDFiyZAkAYOHChRg2bBhKSkoA2Aetq6urxcfXr1/H6dOnERERgeHDh7u1zGB1/xAVJBxgMNvQ2GpCfJQy0F0iZEDzOJDmzp2LxsZGFBUVQavVIisrC2VlZeKgdG1tLSSSrg2vGzduYMKECeLz0tJSlJaW4tFHH0VFRYVbywxWcqkE98WoUHvLiO+aDBRIhPQRxwbBvoZer4darYZOp0NUVFS/fvaiXVX45NtGlPx4HObn3N+vn03IQOHud3RAHmULJlTOlhDfoUDqo/ShdOifEF+hQOojxzVtdOifkL6jQOojxy5bbbMRNmHAD8cRElAUSH2UHB0GuVQCs03AjZb2QHeHkAGNAqmPeAkHTawKAI0jEdJXFEg+II4jNVL1SEL6ggLJB+jGkYT4BgWSDziu+q+hcraE9AkFkg846iJRwX9C+oYCyQc0cfZB7Wu322Gy2gLcG0IGLgokHxgaoUCEQgrG7OcjEUK8Q4HkAxzHUX1tQnyAAslHqHokIX1HgeQjtIVESN9RIPkIFfwnpO8okHyE6iIR0ncUSD7iGENqaDWhzWQNcG8IGZgokHxEHSZDbLgcAG0lEeItCiQfooFtQvqGAsmHKJAI6RsKJB+ic5EI6RsKJB+iQ/+E9A0Fkg+JdZEa2+jW2oR4gQLJhxyVI/UdVtw2WgLcG0IGHgokH1LKeCSr7bfTptpIhHiOAsnHHLtt3zXSOBIhnqJA8jHxEpJmCiRCPEWB5GN0J1tCvEeB5GPptMtGiNe8CqRt27ZBo9FAqVQiNzcXVVVVPbb/05/+hJEjR0KpVGLcuHH48MMPnV5fvHgxOI5zmqZPn+5N1wLOUfD/SrMBAt1amxCPeBxIhw4dQkFBAYqLi3Hy5ElkZmYiPz8fDQ0NLtt//vnnmD9/Pp5//nmcOnUKs2fPxuzZs3Hu3DmndtOnT8fNmzfF6cCBA96tUYDdFxMGqYRDh0VAfWtHoLtDyIDicSC99tprWLp0KZYsWYLRo0djx44dUKlU2LVrl8v2b7zxBqZPn441a9Zg1KhR2LBhAx566CG8/fbbTu0UCgUSExPFKSYmxrs1CjAZL0HKEPtdSGpot40Qj3gUSGazGSdOnEBeXl7XAiQS5OXlobKy0uV7KisrndoDQH5+/j3tKyoqEB8fjwcffBDLly9Hc3Nzt/0wmUzQ6/VOUzBJo0tICPGKR4HU1NQEm82GhIQEp/kJCQnQarUu36PVanttP336dLz77rsoLy/Hpk2b8Mknn2DGjBmw2Vzf46ykpARqtVqcUlJSPFkNv6Or/gnxjjTQHQCAefPmiY/HjRuH8ePH44EHHkBFRQUef/zxe9oXFhaioKBAfK7X64MqlKicLSHe8WgLKS4uDjzPo76+3ml+fX09EhMTXb4nMTHRo/YAkJ6ejri4OFy6dMnl6wqFAlFRUU5TMKEtJEK841EgyeVyTJw4EeXl5eI8QRBQXl6OSZMmuXzPpEmTnNoDwOHDh7ttDwDXrl1Dc3MzkpKSPOle0HAEUu0tIyw2IcC9IWTg8PgoW0FBAXbu3Im9e/fi/PnzWL58OQwGA5YsWQIAWLhwIQoLC8X2L774IsrKyvDb3/4W33zzDV555RV8+eWXWLlyJQCgra0Na9aswbFjx3DlyhWUl5dj1qxZGD58OPLz8320mv0rMUoJpUwCq8Bw7XZ7oLtDyIDh8RjS3Llz0djYiKKiImi1WmRlZaGsrEwcuK6trYVE0pVzkydPxv79+7Fu3Tq8/PLLyMjIwPvvv4+xY8cCAHiex5kzZ7B37160tLQgOTkZ06ZNw4YNG6BQKHy0mv1LIuGgiQ3HN9pWXGkyiFtMhJCecWwQVBLT6/VQq9XQ6XRBM560/I8n8LdzWqx/YjSen5oW6O4QElDufkfpWjY/6RrYprpIhLiLAslPug79GwPcE0IGDgokP6FD/4R4jgLJTxyBdL2lHR0W12ecE0KcUSD5yZBwOaKU9oOYVD2SEPdQIPkJx3FIG9pZG4l22whxCwWSH6XF2suQ0FX/hLiHAsmPHNUjqS4SIe6hQPIj8U62tIVEiFsokPwonW6JRIhHKJD8SNMZSE1tZuja6dbahPSGAsmPIhRSDI20XyBMR9oI6R0Fkp/RGduEuI8Cyc/SKZAIcRsFkp9pKJAIcRsFkp/RLhsh7qNA8rM7d9kGQS08QvyKAsnPUoaowHFAm8mKpjZzoLtDSFCjQPIzpYzHsOgwALTbRkhvQi+Q/vJz4MivgYsfA+0t/fKRVM6WEPcExZ1r+02HHjj1B4A57pXGAfGjgJRc+3R/LhCTBnCcTz82PS4c/7zYhBoqZ0tIj0IrkDgOmPkaUHccqD0G3K4BGqrt04nd9jbh8UBKTmdAfR9IygSkfbsdk4a2kAhxS2gFkiISyF5inwCgrcEeTnXHgdrjwM3TgKEB+OYD+wQAvAJInmDfekr5vj2swuM8+lg69E+Ie0IrkO4WEQ+MetI+AYClwx5KtceAuiqg7hhgbLb/rDsG4A17u9jhXbt5KblA3AhA0v1wXHpnXaQrzUYIAoNE4ttdQkIGi9AOpLvJlPbdtPu/b3/OGNB8uXMrqjOkGr8Bmi/Zp9P77O2U0Z3hlGN/b/JDgFwlLnZYTBhkPAezVcB7p65jREIkEtQKxIUrKJwIuQPdudZTxlvAtS+6dvOunwCs7c5tJFIgcbw9nFJygJTvY9r/+xbf1juPIUklHIZGKpAQpURCpBz3RUqQHMGQpBSQqBIwVGFDrNwCFcyAxQBY2gGzAbAYAbPRPs9s7Hx+13xLe9djmwWQRwDKKECpBhSdP+95rnb9ujzC5wP9JLS4+x2lQOormwXQnrGHk2M8qvXmPc06wu/Dt2wYBEsHeGs7ZLZ2KGGGijMhDCao0AGeC9J/Ck7SGVKOAFP3EGh3zZOpAFmY/Scv83uwCQJDh9UGOS+BlA+9s1qCFQVSoDAG6Oo6A+qYPaDqv77jVIPeWSBDOxQwMgUMTA4jFDBCiXamQDvsz9tZ57zOdu1QQKIIh0IVCZUqAmERUYiKVEOtViM6MhxKoQNyayvk1lbIrK2QWbom3tIK3qwHb24Fb9KDM+shMenBmXTgBKvvfjccDyZTAVIlBGkYBKkSNj4MVl4Jq0QBs0QBM6eEiVPADDnaYV/fdiaHkclhEGRos8nRJsigt8nQapVBb5NCZ5HitoVHi9X+2AQZOA6IVwIJ4RwSwjgMDWOIUwFxCoYhCgHRCiBaLiBaZkOkTECExApeMAM2M2Dt6JxMd0wdd7x29/w7HguCfTyR4wEJb//JSTofS1zM4zvb3/UeCW8Pb3eXI5HaJ17eOXU+lsjs/xHwss75ss557raROT/vYay0JxRIwaRDb9+1a7naucWgso8xycI7f6oAeXjXa3zX0J7BZEW9vgNafQca9CZo9R3Q6jrQ0Gr/Wa83oaG1AxabP/4ZGZQwIxJGRHH2KUZihFrSjhiuHVGSdkTBgCiuHVEwIhIGRHBGRDIDwpkR4cwABUzg4X4Y+4LAOEiCdWtzoOP4ruByBOCjLwHfe77Ht7n7HaVB7f6gjAIeeMyrt4YrpEgfGoH0znu8uSIIDLeMZtTrO+zhpTN1PdZ3oLnNDItNgE1gsAoMVkGAzWZ/LM6zCU7P7Th0QIEOKNDIYgAGeJ4tDDLYEAYTlDAjjDMhDGYoYUaU1AK11AK11IpI3oII3ooIiRkREgvCJWaoODPCOHtbJUxQMBPkzAyZ0AEZ64DM1gHe1gHeZoLEagTH7HcIdhVGNokcFs4+mSGFicnQzmRoF6QwClJ0MJl9PmQwQQ4z63psgszpueP9ErkSSqUKCqUKyjAVVCoVFHIZeCZAAhskHOw/GbP/BAMPAZLOieuc75jHQYAEDBJm63zOwMMmtrP/FDofd7XlIEDCbOCZFRJmgUSwgu/8KWGdk2CxT47Hd/8ULOCYFRLBDIlgBeeYh7t+l8wGWG32rcFOLa1tiPb0z6IbFEiDgETCIS5CgbgIBcYkq/u8PMYYBAZYBQFWp+DqDDVxntAZZs7B5nhsExgUUgnC5DxUcilUch5KGQ+VnEeYjPf9EUabxT6wb2m3/08ulQNSJcDLwXMceABKV28TGFqMZtwymNFsMKO5zQyDwYTbhq55t9rMaDaYcKtznsAAdHROg5gEAmSwipMUNshhhZSzQQYr5LDi/2O5+KmPPs+rQNq2bRu2bNkCrVaLzMxMvPXWW8jJyem2/Z/+9CesX78eV65cQUZGBjZt2oQf/ehH4uuMMRQXF2Pnzp1oaWnBlClTsH37dmRkZHjTPdJHHMeB5wBewkMxkP7L4mUA3zm47snbJBxiIxSIjVDAnb84QWDQtVvsQWUwo7nNJD42mm1gYAADBMbA7A/BOp8DXYHPwDrnAxAfd71H6HzgeHzvfAZB6FoO61y20+POz3Z+rfOnYxkM4vLufq/QuYCu5XT1mTEGAYAqJtGj33ePmIcOHjzI5HI527VrF/v666/Z0qVLWXR0NKuvr3fZ/rPPPmM8z7PNmzez6upqtm7dOiaTydjZs2fFNhs3bmRqtZq9//777KuvvmJPPfUUS0tLY+3t7W71SafTMQBMp9N5ujqEkH7g7nfU40DKyclhK1asEJ/bbDaWnJzMSkpKXLZ/9tln2cyZM53m5ebmsn/5l39hjDEmCAJLTExkW7ZsEV9vaWlhCoWCHThwwOUyOzo6mE6nE6e6ujoKJEKCmLuB5NExPLPZjBMnTiAvL0+cJ5FIkJeXh8rKSpfvqaysdGoPAPn5+WL7mpoaaLVapzZqtRq5ubndLrOkpARqtVqcUlJSPFkNQkiQ8iiQmpqaYLPZkJCQ4DQ/ISEBWq3W5Xu0Wm2P7R0/PVlmYWEhdDqdONXV1XmyGoSQIDWQhixFCoUCCkXfSoIQQoKPR1tIcXFx4Hke9fX1TvPr6+uRmOh6pD0xMbHH9o6fniyTEDI4eRRIcrkcEydORHl5uThPEASUl5dj0qRJLt8zadIkp/YAcPjwYbF9WloaEhMTndro9XocP36822USQgYpT0fLDx48yBQKBduzZw+rrq5my5YtY9HR0Uyr1TLGGHvuuefYf/zHf4jtP/vsMyaVSllpaSk7f/48Ky4udnnYPzo6mv3v//4vO3PmDJs1axYd9idkEHH3O+rxGNLcuXPR2NiIoqIiaLVaZGVloaysTByUrq2theSOC/AmT56M/fv3Y926dXj55ZeRkZGB999/H2PHjhXbvPTSSzAYDFi2bBlaWlowdepUlJWVQal0dV6ty1AFYN+yIoQEH8d30/Fd7c6guLj22rVrdOifkAGgrq4O9913X7evD4pAEgQBN27cQGRkJLhe6u3o9XqkpKSgrq4uOCsD+FiorS8Qeus8ENaXMYbW1lYkJyc77UHdbUAe9r+bRCLpMXVdiYqKCtp/PH8ItfUFQm+dg3191ererzGkknqEkKBBgUQICRohF0gKhQLFxcUhc6Z3qK0vEHrrPJjWd1AMahNCBoeQ20IihAQvCiRCSNCgQCKEBA0KJEJI0KBAIoQEjZALpG3btkGj0UCpVCI3NxdVVVWB7pJflJSU4Hvf+x4iIyMRHx+P2bNn48KFC4HuVr/ZuHEjOI7DqlWrAt0Vv7p+/Tp++tOfIjY2FmFhYRg3bhy+/PLLQHfLayEVSIcOHUJBQQGKi4tx8uRJZGZmIj8/Hw0NDYHums998sknWLFiBY4dO4bDhw/DYrFg2rRpMBgMge6a333xxRf43e9+h/Hjxwe6K351+/ZtTJkyBTKZDH/7299QXV2N3/72t4iJiQl017zn3yoowcXTO6YMJg0NDQwA++STTwLdFb9qbW1lGRkZ7PDhw+zRRx9lL774YqC75Ddr165lU6dODXQ3fCpktpC8uWPKYKLT6QAAQ4YMCXBP/GvFihWYOXPmPXe6GYz+8pe/IDs7G8888wzi4+MxYcIE7Ny5M9Dd6pOQCSRv7pgyWAiCgFWrVmHKlClOhfEGm4MHD+LkyZMoKSkJdFf6xXfffSfe4fnvf/87li9fjp///OfYu3dvoLvmtUFRfoT0bMWKFTh37hw+/fTTQHfFb+rq6vDiiy/i8OHDblcaHegEQUB2djZeffVVAMCECRNw7tw57NixA4sWLQpw77wTMltI3twxZTBYuXIlPvjgAxw9etTjmlEDyYkTJ9DQ0ICHHnoIUqkUUqkUn3zyCd58801IpVLYbLZAd9HnkpKSMHr0aKd5o0aNQm1tbYB61HchE0je3DFlIGOMYeXKlXjvvfdw5MgRpKWlBbpLfvX444/j7NmzOH36tDhlZ2djwYIFOH36NHieD3QXfW7KlCn3nMrx7bffIjU1NUA98oFAj6r3p97umDKYLF++nKnValZRUcFu3rwpTkajMdBd6zeD/ShbVVUVk0ql7De/+Q27ePEi27dvH1OpVOyPf/xjoLvmtZAKJMYYe+utt9j999/P5HI5y8nJYceOHQt0l/wCgMtp9+7dge5avxnsgcQYY//3f//Hxo4dyxQKBRs5ciT7/e9/H+gu9QnVQyKEBI2QGUMihAQ/CiRCSNCgQCKEBA0KJEJI0KBAIoQEDQokQkjQoEAihAQNCiRCSNCgQCKEBA0KJEJI0KBAIoQEjf8flNqFtUyihywAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T20:25:21.932384Z", + "iopub.status.busy": "2024-03-24T20:25:21.931852Z", + "iopub.status.idle": "2024-03-24T20:29:53.243583Z", + "shell.execute_reply": "2024-03-24T20:29:53.242517Z" + }, + "papermill": { + "duration": 271.33273, + "end_time": "2024-03-24T20:29:53.246334", + "exception": false, + "start_time": "2024-03-24T20:25:21.913604", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T20:29:53.282945Z", + "iopub.status.busy": "2024-03-24T20:29:53.282116Z", + "iopub.status.idle": "2024-03-24T20:29:53.303078Z", + "shell.execute_reply": "2024-03-24T20:29:53.302223Z" + }, + "papermill": { + "duration": 0.041128, + "end_time": "2024-03-24T20:29:53.305033", + "exception": false, + "start_time": "2024-03-24T20:29:53.263905", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tab_ddpm_concat0.0003990.01130.0040247.6495470.0219790.7060890.0330440.000135.8710770.0369495749573.50.0634350.2330770.00165413.520624
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration \\\n", + "tab_ddpm_concat 0.000399 0.0113 0.004024 7.649547 \n", + "\n", + " grad_mae grad_mape grad_rmse mean_pred_loss \\\n", + "tab_ddpm_concat 0.021979 0.706089 0.033044 0.00013 \n", + "\n", + " pred_duration pred_mae pred_mape pred_rmse pred_std \\\n", + "tab_ddpm_concat 5.871077 0.036949 5749573.5 0.063435 0.233077 \n", + "\n", + " std_loss total_duration \n", + "tab_ddpm_concat 0.001654 13.520624 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T20:29:53.338505Z", + "iopub.status.busy": "2024-03-24T20:29:53.338018Z", + "iopub.status.idle": "2024-03-24T20:29:53.791522Z", + "shell.execute_reply": "2024-03-24T20:29:53.790684Z" + }, + "papermill": { + "duration": 0.47262, + "end_time": "2024-03-24T20:29:53.793612", + "exception": false, + "start_time": "2024-03-24T20:29:53.320992", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T20:29:53.830320Z", + "iopub.status.busy": "2024-03-24T20:29:53.829454Z", + "iopub.status.idle": "2024-03-24T20:34:51.158066Z", + "shell.execute_reply": "2024-03-24T20:34:51.157049Z" + }, + "papermill": { + "duration": 297.349954, + "end_time": "2024-03-24T20:34:51.160731", + "exception": false, + "start_time": "2024-03-24T20:29:53.810777", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/tab_ddpm_concat/all inf False\n", + "Caching in ../../../../treatment/_cache_bs_test/tab_ddpm_concat/all inf False\n", + "Caching in ../../../../treatment/_cache_synth_test/tab_ddpm_concat/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T20:34:51.198102Z", + "iopub.status.busy": "2024-03-24T20:34:51.197781Z", + "iopub.status.idle": "2024-03-24T20:34:51.223837Z", + "shell.execute_reply": "2024-03-24T20:34:51.223088Z" + }, + "papermill": { + "duration": 0.047041, + "end_time": "2024-03-24T20:34:51.225841", + "exception": false, + "start_time": "2024-03-24T20:34:51.178800", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T20:34:51.259911Z", + "iopub.status.busy": "2024-03-24T20:34:51.259178Z", + "iopub.status.idle": "2024-03-24T20:34:51.264470Z", + "shell.execute_reply": "2024-03-24T20:34:51.263600Z" + }, + "papermill": { + "duration": 0.02428, + "end_time": "2024-03-24T20:34:51.266467", + "exception": false, + "start_time": "2024-03-24T20:34:51.242187", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tab_ddpm_concat': 0.42585911273292454}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T20:34:51.302060Z", + "iopub.status.busy": "2024-03-24T20:34:51.301768Z", + "iopub.status.idle": "2024-03-24T20:34:51.713689Z", + "shell.execute_reply": "2024-03-24T20:34:51.712723Z" + }, + "papermill": { + "duration": 0.432473, + "end_time": "2024-03-24T20:34:51.715734", + "exception": false, + "start_time": "2024-03-24T20:34:51.283261", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABH1klEQVR4nO2deZhU1bX231PzXNXzPAANtMwNAqLRBiUgchWi12scu0XURLzEEBMhuRGH5EETRXEifrkBYpxyvUFj4oBcRXACmQQEAgJNzyPdNc919vfH6aruoruhurqqTlWzfs9TT/c5Z9c569Tw1tp7r7U2xxhjIAiCEBGJ2AYQBEGQEBEEITokRARBiA4JEUEQokNCRBCE6JAQEQQhOiREBEGIDgkRQRCiQ0JEEITokBClGKWlpfi3f/u3uF6D4zg88sgj5233yCOPgOO4uNpCXBiQECWYL7/8Eo888gjMZrPYphApwEsvvYRNmzaJbUbcISFKMF9++SUeffRREiIiIkiICIIgEgQJUQJ55JFH8POf/xwAMGLECHAcB47jcPr0aWzcuBFXXnklsrOzoVQqMW7cOKxfv37Ac3300UeYMmUKVCoVxo0bh82bNw/aHo/Hg5/+9KfIysqCXq/Hddddh4aGhn7bfv7555g+fTpUKhVGjRqFl19+ud92HMfh/vvvx2uvvYaxY8dCpVJh2rRp2LFjR5/XguM4HD9+HLfddhuMRiOysrLw61//Gowx1NfXY9GiRTAYDMjNzcXTTz896PsDgA8++ACVlZXQ6/UwGAyYPn06Xn/99bA2b731FqZNmwa1Wo3MzEzcdtttaGxsDGtTXV0NnU6HxsZGLF68GDqdDllZWXjwwQcRCATC2vI8j3Xr1mHixIlQqVTIysrC1VdfjT179oTaRPJ+l5aW4vDhw9i+fXvoszJ79uyoXoekhxEJ48CBA+zmm29mANgzzzzD/vKXv7C//OUvzG63s+nTp7Pq6mr2zDPPsOeff57NmzePAWAvvPBC2DlKSkrYmDFjmMlkYitXrmRr165lEydOZBKJhH300UeDsue2225jANgtt9zCXnjhBXb99dezSZMmMQBs9erVoXYHDx5karWaFRcXszVr1rDHH3+c5eTkhNr2BgCbMGECy8zMZI899hh78sknWUlJCVOr1ezQoUOhdqtXr2YA2JQpU9jNN9/MXnrpJbZw4UIGgK1du5aNHTuW/fjHP2YvvfQSu+yyyxgAtn379kHd38aNGxnHcWzChAnst7/9LXvxxRfZ0qVL2e233x7WBgCbPn06e+aZZ9jKlSuZWq1mpaWlrKurK9SuqqqKqVQqNn78eLZkyRK2fv16dsMNNzAA7KWXXgq7bnV1NQPAFixYwJ599ln21FNPsUWLFrHnn38+1CaS9/vtt99mhYWFrLy8PPRZGex7nCqQECWY3//+9wwAq6mpCdvvdDr7tJ0/fz4bOXJk2L6SkhIGgP3tb38L7bNYLCwvL49VVFREbMc333zDALD77rsvbP8tt9zSR4gWL17MVCoVq62tDe07cuQIk0ql/QoRALZnz57QvtraWqZSqdgPfvCD0L6gEN1zzz2hfX6/nxUWFjKO49gTTzwR2t/V1cXUajWrqqqK+P7MZjPT6/Vs5syZzOVyhR3jeZ4xxpjX62XZ2dlswoQJYW3++c9/MgDs4YcfDu2rqqpiANhjjz0Wdq6Kigo2bdq00PYnn3zCALDly5f3sSl4XcYif7/Hjx/PKisrI7jj1Ia6ZkmCWq0O/W+xWNDR0YHKykqcOnUKFoslrG1+fj5+8IMfhLYNBgPuuOMO7N+/Hy0tLRFd7/333wcALF++PGz/Aw88ELYdCASwZcsWLF68GMXFxaH9F110EebPn9/vuWfNmoVp06aFtouLi7Fo0SJs2bKlTzdm6dKlof+lUikuvvhiMMZw1113hfabTCaMHTsWp06diujeAGDr1q2w2WxYuXIlVCpV2LFgyMGePXvQ1taG++67L6zNwoULUV5ejvfee6/PeX/0ox+FbV9++eVhdv3tb38Dx3FYvXp1n+f2DnUYzPt9IUBClCR88cUXmDt3LrRaLUwmE7KysvDLX/4SAPp8MMvKyvrE74wZMwYAcPr06YiuV1tbC4lEglGjRoXtHzt2bNh2e3s7XC4XRo8e3eccZ7cN0l/bMWPGwOl0or29PWx/b3EDAKPRCJVKhczMzD77u7q6Br6hszh58iQAYMKECQO2qa2tBdD/fZSXl4eOBwmO9/QmLS0tzK6TJ08iPz8f6enp57RvMO/3hYBMbAMI4cN71VVXoby8HGvXrkVRUREUCgXef/99PPPMM+B5XmwT44ZUKo1oHwAwkasaD2TXYLmQ3++BICFKMP1FIv/jH/+Ax+PBu+++G+YhbNu2rd9znDhxAoyxsHMdP34cgDDTEgklJSXgeR4nT54M8wiOHTsW1i4rKwtqtRrfffddn3Oc3TZIf22PHz8OjUbTx6OIF0FP79tvv0VZWVm/bUpKSgAI93HllVeGHTt27Fjo+GCvu2XLFnR2dg7oFQ3m/b5QItepa5ZgtFotAIQFNAZ/aXv/4lssFmzcuLHfczQ1NeHtt98ObVutVrzyyiuYMmUKcnNzI7JjwYIFAIDnnnsubP+zzz4bti2VSjF//ny88847qKurC+0/evQotmzZ0u+5v/rqK+zbty+0XV9fj7///e+YN29ezLyK8zFv3jzo9XqsWbMGbrc77Fjwdb744ouRnZ2NP/zhD/B4PKHjH3zwAY4ePYqFCxcO+ro33HADGGN49NFH+xwLXncw77dWq70ggl/JI0owwUHcX/3qV/jhD38IuVyOK664AgqFAtdeey3uvfde2O12/PGPf0R2djaam5v7nGPMmDG46667sHv3buTk5GDDhg1obW0dULj6Y8qUKbj55pvx0ksvwWKx4NJLL8XHH3+MEydO9Gn76KOP4sMPP8Tll1+O++67D36/H88//zzGjx+PgwcP9mk/YcIEzJ8/H8uXL4dSqcRLL70UOk+iMBgMeOaZZ7B06VJMnz4dt9xyC9LS0nDgwAE4nU78+c9/hlwux5NPPok777wTlZWVuPnmm9Ha2op169ahtLQUP/3pTwd93Tlz5uD222/Hc889h++++w5XX301eJ7HZ599hjlz5uD+++/HvHnzIn6/p02bhvXr1+M3v/kNysrKkJ2d3cd7GxaIN2F34fL444+zgoICJpFIQlP57777Lps0aRJTqVSstLSUPfnkk2zDhg19pvpLSkrYwoUL2ZYtW9ikSZOYUqlk5eXl7K233hq0HS6Xiy1fvpxlZGQwrVbLrr32WlZfX99n+p4xxrZv386mTZvGFAoFGzlyJPvDH/4QmoLvDQC2bNky9uqrr7LRo0czpVLJKioq2LZt28LaBZ/b3t4etr+qqopptdo+tlZWVrLx48cP+h7fffdddumllzK1Ws0MBgObMWMGe+ONN8La/PWvf2UVFRVMqVSy9PR0duutt7KGhoaI7OrvNfD7/ez3v/89Ky8vZwqFgmVlZbEFCxawvXv3htkVyfvd0tLCFi5cyPR6PQMwbKfyOcZoXTMidnAch2XLluGFF14Q2xQihaAxIoIgRIfGiIYh5wtqVKvVMBqNCbIm9rS3t/cJjOyNQqE4bxwPkVyQEA1D8vLyznm8qqoqpUtLTJ8+vU+wYW8qKyvx6aefJs4gYsiQEA1Dtm7des7j+fn5cbt2IoYcX3vtNbhcrgGPp6Wlxd0GIrbQYDVBEKJDg9UEQYhOSnfNeJ5HU1MT9Hr9BRMKTxCpBGMMNpsN+fn5kEgG9ntSWoiamppQVFQkthkEQZyH+vp6FBYWDng8pYVIr9cDEG7SYDCIbA1BEGdjtVpRVFQU+q4OREoLUbA7ZjAYSIgIIok539AJDVYTBCE6JEQEQYgOCRFBEKKT0mNEBBELGGPw+/3nzF8j+kcqlUImkw05fIaEiLig8Xq9aG5uhtPpFNuUlEWj0SAvLw8KhSLqc5AQERcsPM+jpqYGUqkU+fn5UCgUFBg7CBhj8Hq9aG9vR01NDUaPHn3OoMVzQUJEDEs6HV58eqwNGTolLi/LhETSV2C8Xi94nkdRURE0Go0IVqY+arUacrkctbW18Hq9fdaQixQSImLYwRjDB982o83qQe0ZJ0xqOSYXmQZsH+2vOCEQi9eP3gFi2NFh96LN2rMqx6HGC2/BwlQjaYToiSeeAMdxfZY8JojBcqLNDgAoSFOD4wBfgIfXf+EtWphKJIUQ7d69Gy+//DImTZoktinEMKDJLBRNK8/V467vjcCdl42AQpYUH/WUpLS0tM96d7FG9HfHbrfj1ltvxR//+EeqrEfEhMqxWZh7UQ5K0rXQq+Rim0NEgOhCtGzZMixcuBBz5849b1uPxwOr1Rr2IIizydQpMbHQCKOGRCiI1+sV24RzIqoQvfnmm9i3bx/WrFkTUfs1a9bAaDSGHlSLiDgfNrcPm/c14C87By623x9ePz/gwx/gI27ri6BtNMyePRv3338/7r//fhiNRmRmZuLXv/51qGZ4aWkpHn/8cdxxxx0wGAy45557AACff/45Lr/8cqjVahQVFWH58uVwOByh87a1teHaa6+FWq3GiBEj8Nprr0Vl32ARbfq+vr4eP/nJT7B169aIYw9WrVqFFStWhLaDtU4IIkjdGSe6nF4UpKmRqVNCLpWg9owQNe3xB6CUSSM6z4vb+i69HWREphaLKwpC2/9vx0n4Av2Xfi9MU+PGi3s+oxu+qIHLG55K8tPvj4nIprP585//jLvuugtff/019uzZg3vuuQfFxcW4++67AQBPPfUUHn74YaxevRoAcPLkSVx99dX4zW9+gw0bNqC9vT0kZsHlyqurq9HU1IRt27ZBLpdj+fLlaGtri8q+wSCaEO3duxdtbW2YOnVqaF8gEMCOHTvwwgsvwOPxQCoN/9AolUoolcpEm0qkEMdabfi20YJLRmYgU6eESi6FRiGF0xuAxelDtiEyIUoFioqK8Mwzz4DjOIwdOxaHDh3CM888ExKiK6+8Ej/72c9C7ZcuXYpbb701NDM9evRoPPfcc6isrMT69etRV1eHDz74AF9//TWmT58OAPjTn/6Eiy66KO73IpoQXXXVVTh06FDYvjvvvBPl5eV46KGH+ogQQURCl0MYC0nX9uQ9mTRyOL0BmF0+ZBsi876XzSkb8NjZQdr3XDFqwLZnZ4wsuWxERNePhEsuuSQsJWXWrFl4+umnQ8m7F198cVj7AwcO4ODBg2HdLcZYKNXl+PHjkMlkmDZtWuh4eXk5TCZTzGweCNGESK/XY8KECWH7tFotMjIy+uwniEjpcgpClNZroNqoVqDJ7IbZ6Yv4PIOZ7o9X26Gi1WrDtu12O+69914sX768T9vi4mIcP348Uab1gVI8iGGDxx+As3v8xRgmRML/VlfkQpQK7Nq1K2x7586dGD169IC9ialTp+LIkSMoK+vf2ysvL4ff78fevXtDXbNjx47BbDbH1O7+SCohomWCiaFgc/sBACq5NGxQWq8SPuZ2j18Uu+JFXV0dVqxYgXvvvRf79u3D888/j6effnrA9g899BAuueQS3H///Vi6dCm0Wi2OHDmCrVu34oUXXsDYsWNx9dVX495778X69eshk8nwwAMPQK1Wx/1eRI8jIohYYe8WIp0q/PfVoJJDp5RBOcyiq++44w64XC7MmDEDy5Ytw09+8pPQNH1/TJo0Cdu3b8fx48dx+eWXo6KiAg8//HDYEuQbN25Efn4+Kisrcf311+Oee+5BdnZ23O8lpZectlqtMBqNsFgstIoHgUMNFvzf0VaMzNJi0ZSC87Z3u92oqanBiBEjoi5fIRazZ8/GlClT4p56EQnneh0j/Y4mVdeMIIbC6Bwd0nUKSKm4WcpBQkQMG1RyKQpM8R/PIGIPCRFxQbDlcAuazC7MvSgHRempX41xuE3skBARw4a9tZ2QSSQYk6OHWhE+he3w+GF2+mB1D68p/OECCRExLGCM4csTZ+DnGUoyNH2ESKsUPupOLy0ZlIwMr/lM4oLF7ePh54UJYJ2y7++rpluYHMMslmi4QEJEDAscXkFg1AopZNK+H+ugEJ2d+U4kByRExLDA6REERqPoP71Bo6CuWTJDQkQMC5w+wSMKCs7ZBAXK6aWuWTJCQkQMCxzdHpF2AI9Iq5RBr5KFBq2J5ILeFWJY4Ow1RtQfmTolll4+MpEmEYOAhIgYFkwtTsPILB00ciqol4pQ14wYFmiVMhSY1EjrVZkxKhgD/N7EPwaRe/7KK68gIyMDHo8nbP/ixYtx++23D+3+RYI8IuKC4cNvW9BsOU+aR8AHfDZwTZ+4cfnPAFlkInrjjTdi+fLlePfdd3HjjTcCEFbfeO+99/DRRx/F08q4QR4RMSzYfboT++u64PYNPD1v707zSPUCaWq1Grfcckto5Q0AePXVV1FcXIzZs2eLZ9gQII+ISHmC6R08YyjL1kHVe5zI3g6c+D9AroJeMhnAeWKJpHLBO0k00sEtBnn33Xdj+vTpaGxsREFBATZt2oTq6uqwYvqpBAkRkfK4fAHw3WMsYXFEAT9w6C3AbQEAFPrMOMJddk6vCRwXcRdJTCoqKjB58mS88sormDdvHg4fPoz33ntPbLOihoSISHmCMURqhRTS3mv9tB8NiRAAGOynoFKNh9tnTLSJcWHp0qV49tln0djYiLlz56b0YqM0RkSkPMH8sT7BjM0HhL8jK4GMUZBLJch0nIDrXB5RCnHLLbegoaEBf/zjH7FkyRKxzRkSJEREytOT8NrLwfe5AUuj8H/2RUDWWMgkHEzuhmGT+Go0GnHDDTdAp9Nh8eLFYpszJKhrRqQ8QQ8nLOHVXAswHtBkAOo0ABzkUgnS+TPwS3lxDI0DjY2NuPXWW1N+KXYSIiLlcXd7OCp5LwffUi/8TSsR/qpNMKZnYZpaDhSn9vQ9AHR1deHTTz/Fp59+ipdeeklsc4YMCRGR8kwuMqE0Uxs+bW9tFv4aetbsgj4PcJkBexuQMfB69alARUUFurq68OSTT2Ls2LFimzNkSIiIlEerPCurnucBe4vwv763EOUCbUd7jqUwp0+fFtuEmEKD1cTww9khxBDJFIAmvWe/LhuHmyzYefAIbFREP6kgj4hIeb6pN4NnDGNy9EK9anurcECXIwQoBtHlwO3jwQJmuFxuyNVCNHMKL3acFMTi9SOPiEh5dtd0YvuxdjiDOWTOM8JfbVZ4Q7kGnFwJgMFr74RcLgiR0+lMnLHDkODrF3w9o4E8IiKlYYyFpu9Vwel7R4fwV5MR3pjjwKvSAXcjfLYOSKUlMJlMaGtrE5prNCmbqyUGjDE4nU60tbXBZDJBKo2+FhQJEZHSeAM8At3LCKmDs2ZBj+hsIQLANOmAuREBuyBWubm5ABASI2LwmEym0OsYLSRERErj9grBiXKpELAIPiBM0QP9ChHXvS/QLVYcxyEvLw/Z2dnw+WgAe7DI5fIheUJBSIiIlCbULQt5Q51CRLVMASj1fdpz2oyedr2QSqUx+UIR0UFCRKQ0fYUoOD6UGT5j1o1Snw5OJoHSZ02UiUQEkBARKU0wgVXd2yMC+u2WAcCEkcVAU5qwEfANuiAZER9IiIiUZmSWFjdNL+qpQ+Q2C3/Vaf0/QaYSxCfgA9xWQNu/YBGJheKIiJRGJZci36RGjkEl7AgOVKtN/T+B4wBVd2G0oGgRokNCRAwvguKiMvV72OL04ctGP75ttAAeGidKFqhrRqQ0/2qxwukNoDRDi3SVBPDYhAOq/svBchKgzauE1OsHc1lA4YvJAQkRkdIcarCgocsF7UQZ0jmfsFChVAYotP22V8ok8Mj04BkQcJnpC5AkiNo1W79+PSZNmgSDwQCDwYBZs2bhgw8+ENMkIsUIrsihlksBV5ewU2Xqd+oeABRSCbwyIb7I5zAnwEIiEkQVosLCQjzxxBPYu3cv9uzZgyuvvBKLFi3C4cOHxTSLSCF68swk558xgxBJDZUBABBwWQZsRyQWUT3Ta6+9Nmz7t7/9LdavX4+dO3di/PjxIllFpAqMMbi6UzzUcmnP0kEDDFQH4ZQ6AADvtgldOUp0FZ2k6SIHAgG89dZbcDgcmDVrVr9tPB4PPB5PaNtqpVmPCxmPnw8trKiSS88/dd+NTK0HwMEfCAA+54DjSUTiEH36/tChQ9DpdFAqlfjRj36Et99+G+PGjeu37Zo1a2A0GkOPVF5Qjhg6wfEhhUwiJLyeZ+o+iFalhETVLT7BWTZCVEQXorFjx+Kbb77Brl278OMf/xhVVVU4cuRIv21XrVoFi8USetTX1yfYWiKZcPuEblkoz8zd7SF3jwENxLWT8zFzbDHSNQrAY4+niUSEiN41UygUKCsrAwBMmzYNu3fvxrp16/Dyyy/3aatUKlN+/SYidqRrFbhpepHQPQv4AJ9LOKA8txABABR6AK2AlzyiZEB0ITobnufDxoEIYiAUMgnyTWphI5jsKpUDsgh+rLoHrMkjSg5EFaJVq1ZhwYIFKC4uhs1mw+uvv45PP/0UW7ZsEdMsIhUJpmsoDeedBTveakPdKSdGOx0oySOPKBkQVYja2tpwxx13oLm5GUajEZMmTcKWLVvw/e9/X0yziBSh9owDZxxe5BvVyI1wfAgAPD4erW45Cr0BwEseUTIgqhD96U9/EvPyRIrzXasdhxotmDUqA7mSbs+mn6qMZ6OUS+CVaoRa1zRrlhSIPmtGENHi6p3e4YlciFQyKbxSDfw8I48oSSAhIlKWkBAppOFjROdB8Ih0gkfkdQoF9wlRISEiUpZgQKNK1luIIvOI/BIlusOQAK8jThYSkUJCRKQs7rCE18F5ROA4uDmVEINEQiQ6JEREStI74VUlCQD+7tizCGbNFFIJVHIppCq9IEQ+WnJabJIuoJEgIsEb6El4VfPdQiJTRBTMKJFw+PHsUcCBYqDzFA1YJwEkRERKIpNI8MMZRXB5A5D7upeLjiS1ozfBrHsveURiQ0JEpCRSCYc8Y3d6R/NJ4e+ghUgj/KUxItEhISJSn0HEEAX57Lt2uE7ZMd3vRZqPhEhsSIiIlKTN5kZDlwuZWiWKB5HeEcTs9MHsksLD8eQRJQE0a0akJI1dLmw/1o5DjZaoPCKlTAKfVA1/gIQoGSAhIlKSnqhqyaCCGYOo5FL4JOru6GoSIrEhISJSkv6jqvtfVLE/lDJJT76Z3w3w/PmfRMQNEiIiJQmWiVVL/YDfK+wcpEfklyjh5yGs5EED1qJCQkSkJC6v4BFp+O7ysDKlENAYIUKahwQeiUrYQbFEokJCRKQkbn+3ELFuT2YQM2aA0KVTyiVg8mAsEUVXiwkJEZGSBD2iUHrHIIMZSzO1uG92GSaPLBB2UL6ZqFAcEZGSXDs5H05vAAZzk7BjEONDYcgpujoZiMojOnXqVKztIIhBkWNQYUSmFgp/t4AMNr0jSCjfjIRITKISorKyMsyZMwevvvoq3G53rG0iiMiJIpgRAHieYfO+Bmw9YYefp6BGsYlKiPbt24dJkyZhxYoVyM3Nxb333ouvv/461rYRRL/YPX7sre3C8VZbr4JogxMiiYRDk9mFFrcU/gDVJBKbqIRoypQpWLduHZqamrBhwwY0Nzfje9/7HiZMmIC1a9eivb091nYSRIguhxc7jrdj58mOnmBGVeTBjEGUMoquThaGNGsmk8lw/fXX46233sKTTz6JEydO4MEHH0RRUVFovTKCiDXB9A6NJCAsNQ1ENVitlEvgl6qF6GryiERlSEK0Z88e3HfffcjLy8PatWvx4IMP4uTJk9i6dSuampqwaNGiWNlJECGC6R16rls85GphqelBopJJ4ZOouseInEKENSEKUU3fr127Fhs3bsSxY8dwzTXX4JVXXsE111wDiUTQtREjRmDTpk0oLS2Npa0EAaAnhkiH7qjqKKfulXIhAz/gYwDjAZ+rp1gakVCiEqL169djyZIlqK6uRl5eXr9tsrOzaSVXIi6EumYsumDGIEqZBIyTwovu1BCfk4RIJKISoq1bt6K4uDjkAQVhjKG+vh7FxcVQKBSoqqqKiZEE0ZtgwmuPEEXrEQlpHgG/BoBbGLDWZsbISmIwRDVGNGrUKHR0dPTZ39nZiREjRgzZKII4F6ESIHx0eWZBZo/Jwn2zy1CSmyXsoAFr0YhKiNgAg3p2ux0qlWpIBhHE+fje6EwsrihAtmLw5T96w3Gc8E8ozYOESCwG1TVbsWIFAOENfPjhh6HR9PSnA4EAdu3ahSlTpsTUQII4m0ydEpk6JXByaF2zEME0D6pJJBqDEqL9+/cDEDyiQ4cOQaHoqf+iUCgwefJkPPjgg7G1kCD6g7FelRmj65q1Wt344kQHiixuTAfIIxKRQQnRtm3bAAB33nkn1q1bB4MhykRDgogSnmfYX2+GmvOi3O+FhOOi9oh8AR61Z5yQ+ySAGuQRiUhUs2YbN26MtR0EERFufwA7jrdD4z2Di9SIOpgREFI8AMAJqtIoNhEL0fXXX49NmzbBYDDg+uuvP2fbzZs3D9kwguiPUDAj5xIGm4cwPqSUC3M1TqYEAwNHs2aiEbEQGY3G0CyD0Tj4BEOCiAVuvxBDpEe3aESR7BpE1e0RuTkVeAZIKfFVNCIWot7dMeqaEWLh8voBAHpuaOkdACCXcpBwnJBvFuAh5TxAwA9IqXBpookqjsjlcsHp7HFja2tr8eyzz+Kjjz6KmWEE0R8ub2yiqgEhDEUplyAgUcLPumOKqHsmClEJ0aJFi/DKK68AAMxmM2bMmIGnn34aixYtwvr162NqIEH0xtntEWmHmPAaRCWTQKmQIiDrjokjIRKFqCs0Xn755QCA//3f/0Vubi5qa2vxyiuv4LnnnoupgQTRG2cw4ZUfYq3qbqouLcV9s8tg0Hefh8aJRCGqzrDT6YReL/wSffTRR7j++ushkUhwySWXoLa2NqYGEkRvKopMKE3XIOdA9AXRehNK86Ai+qISdfH8d955B/X19diyZQvmzZsHAGhraxtUkOOaNWswffp06PV6ZGdnY/HixTh27Fg0JhEXCCaNAiNMUmik3fmOQ/SIQoTSPKhrJgZRCdHDDz+MBx98EKWlpZg5cyZmzZoFQPCOKioqIj7P9u3bsWzZMuzcuRNbt26Fz+fDvHnz4HDQrxJxDoIrdyg0Q57h+rbRgs37GnC6O1uEPCJxiOpd/Pd//3d873vfQ3NzMyZPnhzaf9VVV+EHP/hBxOf58MMPw7Y3bdqE7Oxs7N27F1dccUU0phHDnAP1ZmhtTSjleciGmuwKwOz0ofaME6VqGUoB8ohEIuqfk9zcXOTm5obtmzFjxpCMsVgsAID09PR+j3s8Hng8ntC21Wrttx0xPGGM4dNj7ciynkaRnkEWg25ZMLraRWkeohKVEDkcDjzxxBP4+OOP0dbWBp7nw45HsxIsz/N44IEHcNlll2HChAn9tlmzZg0effTRaEwmhgEePw+eMSgCDsilkqGX/0BPdLUrVC6WumZiEJUQLV26FNu3b8ftt9+OvLy8npmHIbBs2TJ8++23+Pzzzwdss2rVqlBNJEDwiIqKioZ8bSI1cHbnmWngHFLWfW9C+WbkEYlKVEL0wQcf4L333sNll10WEyPuv/9+/POf/8SOHTtQWFg4YDulUgmlUhmTaxKpR7Bovh7BErFDz3lUyrqFiO9VQJ8xIAY/rkTkRDVrlpaWNuA4zmBgjOH+++/H22+/jU8++YTqXRPnJJhnpuVjJ0QqudA1c7DuHzg+APg953gGEQ+iEqLHH38cDz/8cFi+WTQsW7YMr776Kl5//XXo9Xq0tLSgpaUFLpdrSOclhidObwBgPNRDXEaoN0GPiOdkgKyXV0QklKi6Zk8//TROnjyJnJwclJaWQi4PL0y1b9++iM4TzEubPXt22P6NGzeiuro6GtOIYYzLG4Ai4IRCAkAijckYkVEtx0+uGg2JhAN2agG/V4gl0gzd4yciJyohWrx4cUwuPtBqIATRH+V5BuRDi/TjKsEbisE4DsdxPadRaABXF3lEIhCVEK1evTrWdhDEeTGq5TBqvIBSFpPxoT6ElhWyx/7cxDmJaowIEMp//Pd//zdWrVqFzs5OAEKXrLGxMWbGEUQf3N1BrFEuqtgf2461YfO+BliDM2c0hZ9wovKIDh48iLlz58JoNOL06dO4++67kZ6ejs2bN6Ouri5Uq4ggYsm3jRYYmpqQF+Ahj6FH1NjlQrvNA2eWEgaAumYiEJVHtGLFClRXV+O7774LW9n1mmuuwY4dO2JmHEH05suTHThS0wCvn49p1yw4c+bh1MIOSnxNOFEJ0e7du3Hvvff22V9QUICWlpYhG0UQZ8PzDE5vAEq/DXKZJLZC1B1L5JZ0/6iSR5RwohIipVLZb8Lp8ePHkZWVNWSjCOJsXL4AGM+gDNghl3Cxq0MEoVwsQImvYhKVEF133XV47LHH4PMJVfI4jkNdXR0eeugh3HDDDTE1kCAAwOH1Qx5wQiFh4CTSmApRyCOixFfRiEqInn76adjtdmRlZcHlcqGyshJlZWXQ6/X47W9/G2sbCQJOTwDKgK0n614S9YRvH0L5ZizYNXMLqR5Ewohq1sxoNGLr1q344osvcODAAdjtdkydOhVz586NtX0EAQCwe/xQ+u1QxHh8COjJN/NySiFIkjFhnCgGkdtEZAxaiHiex6ZNm7B582acPn0aHMdhxIgRyM3NBWMsJiVBCOJsnN6AMD4klcQ0hggAJhYYManAKKR5fKERZs28JESJZFD+LWMM1113HZYuXYrGxkZMnDgR48ePR21tLaqrqwdVJpYgBsNFeXrMLpYj16CKuUcklXCCCAFCmgdA40QJZlAe0aZNm7Bjxw58/PHHmDNnTtixTz75BIsXL8Yrr7yCO+64I6ZGEoReJYde7hLSO9Rp8buQXAugnWbOEsygPKI33ngDv/zlL/uIEABceeWVWLlyJV577bWYGUcQYbi6hL8xFiKHx4/3DjbjHweaenlEJESJZFBCdPDgQVx99dUDHl+wYAEOHDgwZKMI4mwO1nWgraMdfp6PuRAxAMdbbTjZbgcLJb5S1yyRDEqIOjs7kZOTM+DxnJwcdHV1Ddkogjib3UdrcLLNDh8n78mSjxHBgEbGAK+E0jzEYFBCFAgEIJMNPKwklUrh9/uHbBRB9MbjD0DiMQMA5NqMmNeTlkklQlgAuqfwAeqaJZhBDVYzxlBdXT1gAfvea44RRKxwegJQ+a2QSjjIdPGpnKiSS+H183BxKugB8ogSzKCEqKqq6rxtaMaMiDUOrx8qnxUKKQeoTHG5hkougdXVK9+MPKKEMigh2rhxY7zsIIgBcXoDUPktQjBjnKbu1fLgQou9El9pWaGEEbuEHYKIEza3Hyq/Ne5CxHGAJzhGxPuBgDcu1yL6ElWuGZEEeGxAzQ5hDa7iSwBDvtgWxQ27ywOl3walVhk3IZo7Lgfzx+cKEdZ1ciDgE8aJZLSgZyIgIUpFAn7gwJuAo0PY7jwFTLsT0GaIa1ecqMiRgM/VQa5QxC3/Sy7t1TlQaAGXuXuciJYVSgTUNUtFGnYLIqTQALps4df75CdiWxU3DLwNJo0CWmNmYsZsQkGNNGCdKEiIUg2eBxr3ws/z+FY1Dd+mXQUGAGdO9HhIww3nGeFvHBc9bLW68f6hZuw43i54RAAlviYQEqJUo/MU/C4LDrT68H9n0mGBDlzmaOFYyyFxbYsDPM9w7NRptNncCKjj1/V0+wI41mJD7RkHeUQiQEKUanQcR7PZjTppCTQqJWaMSAdyxgvH2v8lTDkPI+xeP07X16GmwwFJHMfAgtP3bh9Pia8iQEKUSvA8Au3H0Wx1o0tTgivLs4VB1vRRgEQmDLDa28S2MqbY3H5ofGbIpRJw2vgtzKBSdMcR+QKU+CoCJESphL0VXWYzvJABpmKMytIBAGotPuy1GWFz+4CuGpGNjC0Omw0y3i3UlVbHb4xIJROEKMAz+KSU+JpoSIhSCUs9OuweWJV5KM83hcryHm224TtPOs44vIC5TmQjY4vb0goAkKiNgEwRt+vIpRxk3VUa3ZTmkXBIiFKIQOdpWF0+2JR5KMvWhfaXZWthUeWjy+kFM9cNqxUoPLZ2AIBEF9/18jiOg7q7exaKriaPKGGQEKUKjIE3NyDboIImewSydD0Rv8XpWniUmbAH5HC73YCtWURDY4vfJoQkyPSZcb+WqjvNw43urpnfLYRLEHGHIqtTBUcH5LwHpdkmlF5WERbYp5BJUJCmgbUtDxZXO9TWJsBYKKKxsSNg74AUgMqQHfdr3XhxIeQSCSRgwLHeywrpzv9kYkiQR5QqOLpnw3Q5gETa53BBmhoORZYwYG1tTLBx8WNahh/luXpk5MQ/l04pkwq5ZhIJIO/2imicKCGQEKUIAVsbrG4f/Jr+uygFJjXsyixYXX4wa1OCrYsTPhfUvANpGgVUxtzEXpum8BMKCVGKYOtoxOEmK9494QfrJ2gx16iCW5UNmUyCgNM8PL5AwZgolRGQq+J+uZoOB94/1Iy9tV290jzII0oEJEQpgqNT8HK06Xn9rqYrl0qwdM44TB4zCjKJBLCm/oB1e0sDGrqc6EBsF1QcCKvLh2MtNjSaXT1C5LEn5NoXOiREqYDXCZfNAgAwZg48VqKQSQB9nrBhS/3uWWdrPeq7XKhxaxNyPa2yO7ra6+8pN+KxJeTaFzokRKmAow12jx8emR456efxDgx5YGDDwiPy24RgRmWCxofUCmES2eEJAIpuIfKSECUCEqIUwGtpgcsXgEOegRzDwBUDXd4A3j0RwN7aLvD21gRaGAd4HrxdCGZUpecl5JIaeU++GXlEiYWEKAWwdXR3s3TZ0CgGDv1SySVo9OvhCwAuuyW1xzdcXfB6vAhwchhM8Y8hAgBNd9fM6+fhkwfHiEiIEoGoQrRjxw5ce+21yM/PB8dxeOedd8Q0J2lxdgndLKXp3F0UjuOQadTBJTfC4fH3xB6lIF5LE7wBHi55Gkza+OWY9UYhlYTyzZzonr732IddaZVkRFQhcjgcmDx5Ml588UUxzUhueB4ZsKI0Q4MRJaXnbZ5jUMEpzxCEKIVLgtjbGwAAfm0OVPK+AZzxIJhvJuE4uDmVEL3O+OERCpHkiJrisWDBAixYsEBME5IfVyc0MkCTbgCKi87bPMegQrMiA3ZXLZDC40TuM/UAAJmpIKHXve2SEihlEiFEQqEVPCKPjdI84kxK5Zp5PJ6wZa2tVquI1iSIoFejy46ocHyOQQmHPANOix+8rTU1BwF5HrlSC9KKTLCPvyihlw7zvhQ6QYi8KTzWliKk1Od0zZo1MBqNoUdR0fk9hFTHY25Gu90DizSy9byMajn8mizwDHCZ24QVPlINRzskvB9qtRpZ2SKu1xaaObsAfvBEJqWEaNWqVbBYLKFHfX292CbFHXN7I0602fFFS2RvFcdxKMrJgsFoAsAAR3tc7YsLwWBMfX7Cl3w+2W7Hewebsb+uC1AahJ00cxZ3UqprplQqoVReWCtveswtAAD1eWbMerNgYh7AjxYWXrS1pNwqsH5zI0632eFX6jCaZ5BKEidGFpcPx1tt4DigwhD0iKhrFm9SyiO64PA64XGYAQD6c6R29IuuO/YmBWfOHB0NaLd78I1ZhQRqEABA012l0eHx9wxQk0cUd0T1iOx2O06cOBHarqmpwTfffIP09HQUFxeLaFmS4GiH0xuAR6ZHrtEwqKcyXTa8/gDkqTZg7XPBHfQCM4v7TfCNJ9pQmgflmyUSUYVoz549mDNnTmh7xYoVAICqqips2rRJJKuSB7+1BS5vAA51BjJ0kQf1Mcbw2mEPRtWZMZmTQcPzQrGvVMDSAIfHB7fMiPS0xK87r1MKXwm7xw+mSAcHCIPVjCV8vOpCQlQhmj17dr+1dQgBe2cTGACfOgN6ZeRvFcdxkOvSwXMyOF1uaFydgDb+NZ9jgrkOTk8AVmUuyvXxr0F0NjqV8Dr7AgwemU5YzyPgA3yunoUXiZiTIj+TFybOTqGLojL1X4PoXGQbNHAoMmD3+FMqsJE318Pp9cOqykeWPvETE3KpJBRLZPehZ5zIbUm4LRcSJETJCs8ji7OiPFePcWWjBv30LL0SDnmmMNZha4mDgXHA74WzsxEBBri1+TCp5aKYoVMKaR4ub0CoDgkAbrMotlwopNT0/QWFqwtyjkeaXoO0gsGnOWQblDigyIDDGQCztSAlRjesDfB4ffDJ9cjIyBIK2YvAjRcX9aR5tBgBSyN5RHGGhChZCXantNlRDTRnaJVwq7IQ4BncXU1Qp8Jga9dpZGiVuGzEBDhH5YhmRliah8ok/CUhiivUNUtSvJYW1Hc50RTQRzWgL5Vw0KbngnESOJ2O1PgidZ4CAEgzR0GvEqdb1odQ1ywFXr8UhjyiJMV2pgkNXS60KWVYFKUnMybPBH1HIVQyi+BhqU2xNTKWuK1g9jZwnARIKxXVlCazC/vrzDCoZbg8g4QoEZBHlKSEgvrSoq/XPLU4DaNHjRKC9JJ9wLrzFNptHnzVrsCBVq+oprh9ARxvtaGu0xk+WE2hJnGDhCgZ8bngsXUBAHQZQ6zHo+sWsmRP9eg8BYvLhwYuDw6vX1RTgrFEdrdfECKOAwJ+KpAWR0iIkhF7G5y+ADwyHTIGmdpxNm51JiwuH3yWJF5eiA+A76yB2emDWVWIkozELB80EHqlMD7l9AbgZ5xQlwig7lkcISFKQpi9FU5vAE754FI7+mPzMR+ONNtgs5iTN4u86zQcDgecnBpeTS5yDYmPqO6NSi6BXCqMy9nc/p6xNVeXeEYNc0iIkhBnZ7Mw7a7MRJpmaEIUKqbvTeLAxrajOOPwolM9AqOydQkt+9EfHMfB2B1MaXH5AHV3zpurU0SrhjckREmIs7MRACA15g35S5ltUMGuyBYirK2NsTAvtvABsI7j6HR40akZgbJsvdgWAQAM3UJkdfsATYaw00lCFC9o+j7Z4API4GzQFZlgmVg+5NNl65U4rMyBw/JdcgpR12nY7HbYmRJubT5KM5IjsdSglkMq4eDx84Cu2yNynhHXqGEMCVGy4eiAhPFQqzVQZw89ujhTp4RDmQ1vgMHT1QBlspUEaT0MuVQCbeF4jMs3QSZNDtu+V5aJ2WOyhDQPR/e0vauTyoHECRKiZMPePY6jy4nJB14hk0BtykGgVQ6Hwwmlox3Qi5c+EYbPDbQfg1ouxbRpVwCGxKzoGgny3oKoNgGcRJjC99gA1dBmMom+JMfPDxHCZ2nGd202fOfSxKxWU36aFnZFDqxuP2BtiMk5Y0LbYYD3C7WS9IlZ3z4qJNKemTPqnsUFEqIkw9bRhA67F4fMqpiVSb0oz4Cy0WORY1AJmeTJAGPwN3yDU+12mE3jkq67w/MM7x9qxhtf18HtC9DMWZwhIUomGIOrSwg81KTHbuWNfJMaZWVjoZZLAUt9cqQqmOvQ1lSLZnsA/2hJS7pKnRIJh4YuJ1osbmEKX9s9c+boENewYQoJUTLh7ITL6QTPyWDKjHFXxVgkdDHc1qQIzHPX7ESD2YV27WhMKytMeJH8SAjGcJ2xe4VyLEDyp8qkKCREyYS1EQ5vAHZFJrKNsZ3Gdvg51AfS0WZzA501MT334I3pQP13BxDgAb5gOi7KS47YobNJ1wpC1OX0CpMHAOBoSw6PcphBQpRE+MwNcHkDsCuyhfGcGNJsceHLLgMazS6w7ro/YtGw70N02L3o0pTgskljktIbAnqEqNPhBTTpgkfp91LZ2DhAQpRE2NtqwQDw+jxoB7FqRyQUp2thVxfC7ePhaq8B+EBMzx8pXc2n0XTiAAAO2ZPnxVxwY0mYEEmkPRHW1D2LOSREyYLfC5+1FRIO0GWVxPz0CpkE6blF8EtUMFvtwqB1ouF5tOz9BwI8QyBnPKaWj068DYMgrVuIzE4fAjzr6Z6REMUcCmhMFmzNyNYpkZGeAdfEkXG5xIgsPWrVJUhznkR+278SXwmxaT/KNTbUZRiRVXmdaMXxI0WvlEEpl0CnlMHlC0AXWsY7dZZnShVIiJIFixBoKDUVhlYbjTVl2Trs145CdtsxOJuOQDN6XkLSPSxOH7T+LshObQPHcSiZfg2gT/wqroOF4zjcc/nInrQTQ3dIhbWRUj1iDHXNkgVzrfDXVBy3S+iUMqQVlMEvUeFMZ1fPNePIyXY73tj5Hb7b9hdhxdS0EiB/atyvGyvCct90ucJYkdeZFCEQwwkSomQg4Edr/UkcbDTjkDMtrpcaV5CGM9qRCDAGNH8Tt+u4fQF8fLQV/9hfhxHNHyJgb0dArgUuui4lPQnGGCCVAfru0rvJWMkghSEhSgasDbA6XejyK2Hj4htTMypLhzlz5qE0Qwu0HxcCHGNIgGc41GDBX76qxeG6NpS3b8EYZSfGFWVBOunGniWcUwSn14+/7q7D/9txCjzPAEN3DfFkSZUZJpAQJQF852mYnT5YVfkoyYzvF1Uq4WDMKgRMRQDjgfqvY3buM3YP/vzlafzf0VYwazNmdP4Ts9JsGJFtgmTivwOGJE5sHQCVTIp2mwdOb0AIbDQWCgfEmHUcxpAQJQH2pn/BzzM4tMWJq9dccimcXj8aj3wOuMxRncLp9aPF4g5tG9RyBDwOjLHtwr8FtmJGDoPRlA5U3C6MDaUgEgmHXKMaANBodglCxHFCzpnHJrJ1wweaNRMbVxes7Y1g4KArKE9YvWaruhA7z+igdzXC+9XfMGLOknOO3fA8Q4fDgxaLG01mN1osLnQ5fTBp5Ki+tBScox3ypm9wU2AftEYeUokSyC4HRs8DFOKuyjFUCtPUqO90or7ThUmFJmGcyNosrEybN1ls84YFJERi03ECnU4vbKo8lGRnJOyyBrUCGVMWwr3zT2ipOYpG/u8oqvg+0rQKMAYYVLJQ6sW2f7XhSLMVXj/fcwLGoPF1osDXBP/Xn0DuFIL8DHIItYVGXAFkjErY/cSTonQNvjp5BvVdTjDGwKWPFITozEkSohhBQiQyzsbDcHgCMKeV4orsxA7kTr1oNI46F8B64J+Q1n6OvZ0W1Bung5fI8OPZo6CSSwEAPGPw+nloOTdGys6ggGtHtr8ZBoldqGTohFDBMHM0kD8FSBuRkjNjA5FrUEEhk8DlDaDd7kF2+ijg9BdAV40QkiCVi21iykNCJCbOTnDWBmTpVbCUTIBaIU3o5TmOw7iLZ8OiBToObYXSdRjZrhPoUpfAV9MKlUoB+F2Y6WzFTGU7tAErOMYBDMLookQBpI8AMsoEEUrxLthASCUcCtPUONXuwMk2B7JH5gvlYt1WwSvKHvoiBxc6JERi0nIQarkUZeWTUDZpjGhmGC+aDWNmPnDi/8DcZgBt4BrbQ8dDfhonAbRZgKlEGHw2lQCyoa27liqU5xrAGJBtUAreXvZFQN0uodwtCdGQISESi4APaD4g/J87UVxbACBrDJAxClzXacBcB3jtwvS+TC3Ua9ZmCYO0crXYlorC2Fw9xub2ivHKmSAIUccJwTOigvpDgoRILFoOorHtDIxpmdBliucNhSGRCgPMw2SQOa7osoVYLHM90LgHGHWl2BalNBRHJAZ+L6zHdqCu04l/dhXB7qOKf6mCw+PHlyc6hJVzi2YKO5v2Ax67uIalOCREIsCf/hJ1za3wyPRIL7s4btn2ROx571AzdtV04quTZ4RBen2OULWxZrvYpqU0JESJpus0mr/dBpvbj8bMS3HJ6FyxLSIGwaWjhFivQ40W1He5hIBNAGg+CLQfE9Gy1CYphOjFF19EaWkpVCoVZs6cia+/jl3+U1JhbUbHrjdRf8aBNu1YTJo8AwYVxaCkEoVpGkwoMAIA/nmwGW2STKBohnDw6D+EgX5i0IguRH/961+xYsUKrF69Gvv27cPkyZMxf/58tLUNo3KcfABo2IvGbX/EicYOWJU50E1YgAkFNNOSiswem4U8owpuXwD/s7se+2ST4TeVCDOhB94E6nYKy1MTEcMxkVe2mzlzJqZPn44XXngBAMDzPIqKivCf//mfWLly5Tmfa7VaYTQaYbFYYDAk0Zc64IPfZYPL3AKlowmKzmOA24qGLie+dWVAPukGzBlfmPSlUomBcfsCeP9QM2rPOAEAlaNNmGr/DGg/DqfXDwdU4DPGgjcVg9NmglMZIJUpIJVIoFfJQgXXAjxDgGeQSjhIOCTtiibREul3VNRRUq/Xi71792LVqlWhfRKJBHPnzsVXX33Vp73H44HH4wltW62R19JpNLvwydFWBFU3KL9BHZ4xIgPj8g3A4bdhOdOCw43mXg0ZODAhohgMxelqFBiFLHmbx4dvG8wAGMAYJCwAju8utg6gNEODPKMaUGiQe/FcOBVjMTpHP+w+cBcaKrkUi6cU4EizFbtPd2JMXjqguB5oOYiOvVvQ2NoKNITXtmacBH5Oickl6TCqlQAnQWOXGyfa7WAQPg8cB0jAgesWpYkFRqRphO57q82D0x2OPracLFiEgFSF2WOzUZQurId3st2OL08MvCrt90ZnYUSmEAlfe8aBHcfbB2w7a1QGyrKFGKpGswuf/KsN/z61MKaZAKIKUUdHBwKBAHJycsL25+Tk4F//+lef9mvWrMGjjz4a1bV8fh4ddu+Axz3+7uV1XF3gHO3gnOcQObcPUPkAAJzHD3h7PhzBtFCek8InN+KMvhR546cCGaMgk8qRJBFDRAyQSDhMKDBifL6h54clbzKsk4tgPvEtVLYaqNztUHg6wfF+8IxBxtyQeu0AJ5RPkbldUPucA1/D6QOYIETM7gZv7ytEZ+xe+KUSeAM9Scke37k/770TmL3n+W64fWe1tXnAENuOlKhds6amJhQUFODLL7/ErFmzQvt/8YtfYPv27di1a1dY+/48oqKiooi6Zi5vAO02T1guZvB/juNgVMuFaXRLIzxeDywuLzhIwDhO+JBxHDhwADholDJoFDKA4+DjGaxuv3CM48BJZIBcA7VaDaVcSp4PIXjVAR/gdwF+jxCx3v3gAwH4AzwYGHjGwDMhyZjxwv8ahQxBx8Pu9cPq6jv2FNAXAhIpMnXKkJdi9/jR5RhYXNK1itDaeU6vHx22c7TVKUIhJsHvUUGaOqKSNSnRNcvMzIRUKkVra7gL29raitzcvtPaSqUSSqUyqmupFVIUZ0SwjLOxAEoA2RGeVw4gwxiVScSFAscJOXn95OVJAESaradDr7y/87VVyiKOT9MoZCjOiKxtxN+jQSLqrJlCocC0adPw8ccfh/bxPI+PP/44zEMiCGJ4I3pI74oVK1BVVYWLL74YM2bMwLPPPguHw4E777xTbNMIgkgQogvRTTfdhPb2djz88MNoaWnBlClT8OGHH/YZwCYIYvgiehzRUEjaOCKCIABE/h0VPbKaIAiChIggCNEhISIIQnREH6weCsHhrcGkehAEkTiC383zDUWntBDZbMJKm0VFRSJbQhDEubDZbDAaB478TelZM57ncezYMYwbNw719fUX3MxZMMWF7v3CufdUu2/GGGw2G/Lz8yGRDDwSlNIekUQiQUFBAQDAYDCkxBsTD+jeL7x7T6X7PpcnFIQGqwmCEB0SIoIgRCflhUipVGL16tVRZ+WnMnTvF969D9f7TunBaoIghgcp7xERBJH6kBARBCE6JEQEQYgOCRFBEKKTkkLU2dmJW2+9FQaDASaTCXfddRfsdvs5nzN79mxw3YXwg48f/ehHCbI4ega7Cu5bb72F8vJyqFQqTJw4Ee+//36CLI09g7n3TZs29Xl/VSpVAq2NDTt27MC1116L/Px8cByHd95557zP+fTTTzF16lQolUqUlZVh06ZNcbcz5rAU5Oqrr2aTJ09mO3fuZJ999hkrKytjN9988zmfU1lZye6++27W3NwcelgslgRZHB1vvvkmUygUbMOGDezw4cPs7rvvZiaTibW2tvbb/osvvmBSqZT97ne/Y0eOHGH/9V//xeRyOTt06FCCLR86g733jRs3MoPBEPb+trS0JNjqofP++++zX/3qV2zz5s0MAHv77bfP2f7UqVNMo9GwFStWsCNHjrDnn3+eSaVS9uGHHybG4BiRckJ05MgRBoDt3r07tO+DDz5gHMexxsbGAZ9XWVnJfvKTnyTAwtgxY8YMtmzZstB2IBBg+fn5bM2aNf22/4//+A+2cOHCsH0zZ85k9957b1ztjAeDvfeNGzcyo9GYIOsSQyRC9Itf/IKNHz8+bN9NN93E5s+fH0fLYk/Kdc2++uormEwmXHzxxaF9c+fOhUQi6bMO2tm89tpryMzMxIQJE7Bq1So4nQMvbCc2wVVw586dG9p3rlVwAeG16d0eAObPnz9g+2QlmnsHALvdjpKSEhQVFWHRokU4fPhwIswVleHynqdc0mtLSwuys8NXHZPJZEhPT0dLS8uAz7vllltQUlKC/Px8HDx4EA899BCOHTuGzZs3x9vkqBjsKriA8Nr01/5cr0syEs29jx07Fhs2bMCkSZNgsVjw1FNP4dJLL8Xhw4dRWFiYCLNFYaD33Gq1wuVyQa1Wi2TZ4EgaIVq5ciWefPLJc7Y5evRo1Oe/5557Qv9PnDgReXl5uOqqq3Dy5EmMGjUq6vMSycGsWbPC1sK79NJLcdFFF+Hll1/G448/LqJlRCQkjRD97Gc/Q3V19TnbjBw5Erm5uWhrawvb7/f70dnZ2e/qsAMxc+ZMAMCJEyeSUogGuwouAOTm5g6qfbISzb2fjVwuR0VFBU6cOBEPE5OGgd5zg8GQMt4QkETT91lZWSgvLz/nQ6FQYNasWTCbzdi7d2/ouZ988gl4ng+JSyR88803AIC8vLxY30pMiGYV3FmzZoW1B4CtW7em3Kq5sVgBOBAI4NChQ0n7/saK4fKep9ysGWPC9H1FRQXbtWsX+/zzz9no0aPDpu8bGhrY2LFj2a5duxhjjJ04cYI99thjbM+ePaympob9/e9/ZyNHjmRXXHGFWLcQEW+++SZTKpVs06ZN7MiRI+yee+5hJpMpNC19++23s5UrV4baf/HFF0wmk7GnnnqKHT16lK1evTqlp+8Hc++PPvoo27JlCzt58iTbu3cv++EPf8hUKhU7fPiwWLcQFTabje3fv5/t37+fAWBr165l+/fvZ7W1tYwxxlauXMluv/32UPvg9P3Pf/5zdvToUfbiiy/S9H2iOHPmDLv55puZTqdjBoOB3Xnnncxms4WO19TUMABs27ZtjDHG6urq2BVXXMHS09OZUqlkZWVl7Oc//3nSxxExxtjzzz/PiouLmUKhYDNmzGA7d+4MHausrGRVVVVh7f/nf/6HjRkzhikUCjZ+/Hj23nvvJdji2DGYe3/ggQdCbXNyctg111zD9u3bJ4LVQ2Pbtm0MQJ9H8F6rqqpYZWVln+dMmTKFKRQKNnLkSLZx48aE2z1UqAwIQRCikzRjRARBXLiQEBEEITokRARBiA4JEUEQokNCRBCE6JAQEQQhOiREBEGIDgkRQRCiQ0JEpBSbNm2CyWQKbT/yyCOYMmVKaLu6uhqLFy9OuF3E0CAhIvqlurp6wLrey5YtA8dxYdUS4iEApaWlePbZZ8P23XTTTTh+/PiAz1m3bl1YzebZs2fjgQceiKldROwhISIGpKioCG+++SZcLldon9vtxuuvv47i4mJRbFKr1X0K4/XGaDSGeUxEakBCRAzI1KlTUVRUFFbFcvPmzSguLkZFRcWQzt2fp7J48eKQlzV79mzU1tbipz/9aWhVDqBv1+xsentm1dXV2L59O9atWxc6R01NDcrKyvDUU0+FPe+bb74Bx3HDvn5RskJCRJyTJUuWYOPGjaHtDRs24M4774z7dTdv3ozCwkI89thjaG5uRnNz86DPsW7dOsyaNQt333136BzFxcV97gkANm7ciCuuuAJlZWWxugViEJAQEefktttuw+eff47a2lrU1tbiiy++wG233Rb366anp0MqlUKv1yM3NzeqKpNGoxEKhQIajSZ0DqlUiurqahw7diy0TprP58Prr7+OJUuWxPo2iAhJmlKxRHKSlZWFhQsXYtOmTWCMYeHChcjMzBTbrCGRn5+PhQsXYsOGDZgxYwb+8Y9/wOPx4MYbbxTbtAsW8oiI87JkyRJs2rQJf/7zn2PmNUgkEpxdCsvn88Xk3JGwdOnS0ED8xo0bcdNNN0Gj0STs+kQ4JETEebn66qvh9Xrh8/kwf/78mJwzKysrbNwnEAjg22+/DWujUCgQCASGdJ2BznHNNddAq9Vi/fr1+PDDD6lbJjLUNSPOi1QqDS3lJJVKB2xnsVhCixIEycjIQFFRUZ+2V155JVasWIH33nsPo0aNwtq1a2E2m8PalJaWYseOHfjhD38IpVIZVZewtLQUu3btwunTp6HT6ZCeng6JRBIaK1q1ahVGjx6desXmhxnkERERYTAYYDAYztnm008/RUVFRdjj0Ucf7bftkiVLUFVVhTvuuAOVlZUYOXIk5syZE9bmsccew+nTpzFq1ChkZWVFZfeDDz4IqVSKcePGISsrC3V1daFjd911F7xeb0JmAYlzQzWriQuWzz77DFdddRXq6+v7rJZKJBYSIuKCw+PxoL29HVVVVcjNzcVrr70mtkkXPNQ1Iy443njjDZSUlMBsNuN3v/ud2OYQII+IIIgkgDwigiBEh4SIIAjRISEiCEJ0SIgIghAdEiKCIESHhIggCNEhISIIQnRIiAiCEJ3/DzFaMb8EVfiMAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T20:34:51.751809Z", + "iopub.status.busy": "2024-03-24T20:34:51.751134Z", + "iopub.status.idle": "2024-03-24T20:34:52.108635Z", + "shell.execute_reply": "2024-03-24T20:34:52.107796Z" + }, + "papermill": { + "duration": 0.377391, + "end_time": "2024-03-24T20:34:52.110682", + "exception": false, + "start_time": "2024-03-24T20:34:51.733291", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABEbElEQVR4nO29eXzU9bX///p8Zp/Mkn0jCQmEfZdNUAGVq6KillZpaxUQd+wtUhSpVUSvP1qrFtpruf3ZQq51weUirVpFiwsKgqzKVpCQjeyQzL5/Pu/vH5/MJEMWkslkPjPJeT4eQzKf9Qwz88p5n/c5580xxhgIgiBkhJfbAIIgCBIigiBkh4SIIAjZISEiCEJ2SIgIgpAdEiKCIGSHhIggCNkhISIIQnaUchvQG0RRRE1NDYxGIziOk9scgiAugDEGu92O3Nxc8Hznfk9CC1FNTQ3y8/PlNoMgiItQVVWFvLy8TvcntBAZjUYA0os0mUwyW0MQxIXYbDbk5+eHvqudkdBCFByOmUwmEiKCiGMuFjqhYDVBELJDQkQQhOyQEBEEITskRARByA4JEUEQskNCRBCE7CT09D1BdEaT04fPTzYgzaDBFcXp4HnKvI9nSIiIfgdjDB8erUWDzYuK8y4k61SYkJ8st1lEF8TN0Ow3v/kNOI7D8uXL5TaFSHDOOXxosHlDz49UW2W0hugOcSFE+/btw5///GeMHz9eblOIfsDpBgcAYFCKDhwH+AURvoAos1VEV8guRA6HA7fffjtefvllpKSkyG0O0Q+osbgBACOzjVh6eRGWXFYEtVL2jzrRBbK/O8uWLcMNN9yAuXPnym0K0U+YPSIDc0dlYXBqEoxaldzmEN1A1mD1li1bcPDgQezbt69bx3u9Xni9rWN/m83WV6YRCUy6QYN0g0ZuM4geIJtHVFVVhV/84hd47bXXoNVqu3XOunXrYDabQw/qRURcDLvHj60Hz+JveyrkNoXoAk6uJae3bduGH/zgB1AoFKFtgiCA4zjwPA+v1xu2D+jYI8rPz4fVaqU2IAQAoPK8C80uHwal6JBu0MDjF7Dx81IAwINXDoVGqbjIFYhoYrPZYDabL/odlW1odvXVV+PIkSNh25YsWYKRI0di1apV7UQIADQaDTQacrmJzjlZb8fRaisuHZKGdIMGWpUCerUCLp8Aq8uPTBMJUTwimxAZjUaMHTs2bFtSUhLS0tLabSeI7tLs9AEAUpPUoW3JehVcPgEWtx+Zpu6FAYjYIvusGUFEk2aXJEQp+tbZMrNOEiWLyy+LTcTFiasSj88//1xuE4gExhsQ4PIJAABzmBBJv9vcJETxCnlERL/B7gkAALQqRVhQ2qiV/t46vAFZ7CIuDgkR0W9wtAiRQRvu6Ju0Khg0SmgouzpuiauhGUH0hqBHZLpAiArS9Lhn1hA5TCK6CQkR0W8YlmVAqkENBa36m3CQEBH9Bq1KgUHJOrnNICKAhIgYEGw/VocaixtzR2UhP1UvtznEBZAQEf2GAxVNUPI8hmcZoVOHZ1A7vQFYXH7YPDSFH4+QEBH9AsYYdp8+j4DIMDhN306IkjTSRz2YZ0TEFzSfSfQLPH4RAVGq3zZo2v991bcIk5NyieISEiKiX+D0SQKjUyugVLT/WAeFyE0eUVxCQkT0C1xeSWD06o6r6/VqGprFMyRERL/A5Zc8oqDgXEhQoFw+GprFIyRERL/A2eIRJXXiESVplDBqlaGgNRFf0LtC9AtcbWJEHZFu0ODuK6jMI14hISL6BZcUpGBIhgF6FXVgTERIiIh+QZKGhl2JDL1zxIDho6N1qLVSmUc8QkJE9Av2lTdByXMYlWOCtpPhmaOlzIMapMUfJEREwhMs7xAZQ3GmIVyIHI3A6X8BKi2M/AQAlEsUj5AQEQmP2y9AbFmeLyyPSAgAR94GPFYAQJ7fguPcZfD4SYjiDcojIhKeYA6RTq2Agm/TFK3xREiEAMDkOAOt30JCFIeQEBEJT7B+rF0yY+230s8hs4G0oVApeKQ7T8NNQhR3kBARCU9rwWubYZnfA1irpd8zRwEZI6DkOSR7zlLhaxxCMSIi4Ql6OGEFr5YKgImAPg3QpQDgoFLwSBXPI6AQ5TGU6BQSIiLh8bR4OFpVGwffWiX9TBks/dQlw5yagck6FVBA0/fxBgkRkfBMyE9GYXpS+LS9rVb6acpt3WbMAdwWwNEApA2NqY1E15AQEQlPu/IOUQQcddLvxrZClA00nGjdR8QNFKwm+h+uc1IOkVIN6FNbtxsycazGij3fHYedmujHFSRERMJzuMqCg5XNraUbjnrppyELaLvYoiELHr8I5rLA7fbE3lCiU2hoRiQ8+8qa4PAGkJeskxrnu85LO5Iywg9U6cGpNIDghs/RBKQYY28s0SHkEREJDWMsNH2vDU7fO89JP/Vp4QdzHEStNFTz28/FykSiG5AQEQmNTxAhtCwjpAvOmgU9oguFCABriRkJDhKieIKEiEhoPD4pOVGlkBIWIQrSFD3QoRBxLduEoFgRcQEJEZHQhIZlIW+oScqoVqoBTfsYEJeU1nocETdQsJpIaNoLUTA+lB4+Y9aCxpgKTslD47fFykSiG5AQEQlNsIBV19YjAjoclgHA2CEFQE2K9ETwAwpVX5tIdAMSIiKhGZKRhIVT81v7EHks0k9dSscnKLWS+Ah+wGMDkjoWLCK2UIyISGi0KgVyk3XIMmmlDcFAtS654xM4DtCapd+DokXIDgkR0b8Iios2ucPdVpcfu6sDOFptBbwUJ4oXaGhGJDT/rrPB5RNQmJaEVC0PeO3SjqDXcwEcDzT4NFD4AmBuK9qHswk5ICEiEpojZ6042+xG0jglUjk/wBigUALqpA6P1yh5eJVGiAwQ3Bb6AsQJsg7NNm7ciPHjx8NkMsFkMmHGjBn48MMP5TSJSDCCjfB1KgXgbpY2apM7nLoHALWCh08p5Rf5nZYYWEh0B1mFKC8vD7/5zW9w4MAB7N+/H1dddRVuvvlmHDt2TE6ziASitc6Mv/iMGQCO4wCtCQAguK2dHkfEFlk90/nz54c9f/bZZ7Fx40bs2bMHY8aMkckqIlFgjMHdUuKhUylalw7qJFAdhNMYAACixy4N5TrxnojYETdDZEEQ8Pbbb8PpdGLGjBkdHuP1euH1ekPPbTaa9RjIeANiaGFFrUpx8an7FpQ6IwAOAUEA/K5O40lE7JB9+v7IkSMwGAzQaDS4//778e6772L06NEdHrtu3TqYzebQIz8/P8bWEvFEMD6kVvJSwetFpu6DJGk14LUt4hOcZSNkRXYhGjFiBA4fPoy9e/figQcewKJFi3D8+PEOj129ejWsVmvoUVVVFWNriXjC45eGZaE6M0+Lh9wSA+qM+RNyMX1EAVL1asDr6EsTiW4i+9BMrVajuLgYADB58mTs27cPGzZswJ///Od2x2o0Gmg0mlibSMQpqUlqLJyaLw3PBD/gd0s7NF0LEQBAbQRQD/jII4oHZBeiCxFFMSwORBCdoVbyyE3WSU+Cxa4KFaDsxh+rloA1eUTxgaxCtHr1asybNw8FBQWw2+14/fXX8fnnn2P79u1ymkUkIsFyDY3porNgp+rtqDzjwjCXE4NzyCOKB2QVooaGBtx5552ora2F2WzG+PHjsX37dvzHf/yHnGYRCULFeSfOO33INeuQ3c34EAB4/SLqPSrk+QTARx5RPCCrEP31r3+V8/ZEgvN9vQNHqq2YMTQN2XyLZ9NBV8YL0ah4+BR6qdc1zZrFBbLPmhFEpLjblnd4uy9EWqUCPoUeAZGRRxQnkBARCUtIiNSK8BjRRZA8IoPkEflcUsN9QlZIiIiEJZjQqFW2FaLueUQBXoOWNCTA5+wjC4nuQkJEJCyesILXnnlE4Dh4OK2Ug0RCJDskRERC0rbgVcsLQKAl96wbs2ZqBQ+tSgGF1igJkd/Vl6YS3SDuEhoJojv4hNaCV53YIiRKdbeSGXmewwNzhgLfFgBNZyhgHQeQEBEJiZLn8eNp+XD7BKj8DdLG7pR2tCVYde8jj0huSIiIhETBc8gxt5R31JZKP3ssRHrpJ8WIZIeEiEh8epBDFOTL7xvhPuPA1IAPKX4SIrkhISISkga7B2eb3UhP0qCgB+UdQSwuPyxuBbycSB5RHECzZkRCUt3sxhcnG3Gk2hqRR6RR8vArdAgIJETxAAkRkZC0ZlXzPUpmDKJVKeDndS3Z1SREckNCRCQkHWdVd7yoYkdolHxrvVnAA4jixU8i+gwSIiIhCbaJ1SkCQMAnbeyhRxTgNQiIkFbyoIC1rJAQEQmJ2yd5RHqxpT2sUiMlNHYTqcyDh5fXShsol0hWSIiIhMQTaBEi1uLJ9GDGDJCGdBoVD6YK5hJRdrWckBARCUnQIwqVd/QwmbEwPQkPzinGhCGDpA1UbyYrlEcUpwiCAL/fL7cZcct1o9Lg8QtQ2+rg4fWAKhnweHp+IYUJ4PWAyxnZ+QMclUoFhULR6+tEJERnzpzBkCFDen1zoj2MMdTV1cFischtSkJQ7TcApulAQAuUlfX8AmIOYEoFHJrIzieQnJyM7OxscL1YujsiISouLsbs2bOxdOlS/OhHP4JWq43YACKcoAhlZmZCr9f36s0dELgtgOCT1ilT67p9GmMMNrcfCsGDJM4DTqntcZxpoMMYg8vlQkODVHSck5MT8bUiEqKDBw9i8+bNWLFiBR566CEsXLgQS5cuxbRp0yI2hJCGY0ERSktLk9ucuEUQGTx+AQqek1Z5VagAvb5765m1we7nwHEiNHwAvEoJ0B/UHqPTSeLf0NCAzMzMiIdpEQWrJ06ciA0bNqCmpgabNm1CbW0tLr/8cowdOxYvvvgiGhsbIzJmoBOMCen1epktiW8EUYTDG4DDGwBYS79prudfAI7jwDgeDGi9DtFjgp/X3sQ0ezVrplQqsWDBArz99tv47W9/i9OnT2PlypXIz88PrVdG9BwajnWNKPVDAw8mJSMCAN/zjzLHAQxBIaLM6kiJxue1V0K0f/9+PPjgg8jJycGLL76IlStXorS0FJ988glqampw880399pAgriQYGdGBVrEg5OSE3sKz3FgkhpJQhQUNSLmRCREL774IsaNG4eZM2eipqYGr7zyCioqKvBf//VfKCoqwhVXXIGSkhIcPHgw2vYS/YjFixfjlltu6fF5QY8oJERtvKE5c+Zg+fLlXZ5fWFiI9evXgw95RKxVjAhZiEiINm7ciJ/+9KeoqKjAtm3bcOONN4K/wDXOzMyklVwHEN0RgGjBWjwXPuQRRRYg5TjpHxb8GpAQhfHUU09h4sSJMblXRLNmn3zyCQoKCtqJD2MMVVVVKCgogFqtxqJFi6JiJEG0pTVGFPSIIhUiTooTcTwAkYRIRiLyiIYOHYpz5861297U1ISioqJeG0UkFosXL8YXX3yBDRs2tHy5OZSWlmLp0qUoKiqCTqfDiBEjsGHDhg7PX7t2LTIyMmAymXD//ffD5/N1eb9gjMjltOPO+/4ThvRByMnJwQsvvNDu2IaGBsyfPx86nQ5FRUV47bXXQvuMGiUyjVpok7Ox8S//i3k3SscNGTIE77zzTui48vJycByHt956C1dccQV0Oh2mTp2KU6dOYd++fZgyZQoMBgPmzZvXoxnjTZs2YcyYMdBoNMjJycFDDz0U2ldZWYmbb74ZBoMBJpMJt912G+rr60P7g97K3/72NxQWFsJsNuPHP/4x7HZ76/+TKOK5555DcXExNBoNCgoK8Oyzz4b2r1q1CsOHD4der8eQIUPwxBNPhGa+SkpKsHbtWnz77beh97SkpKTbr63HsAjgOI7V19e3215eXs70en0kl4wIq9XKADCr1Rqze/YlbrebHT9+nLnd7nb7vH6h04c/IHT7WF83ju0pFouFzZgxg91zzz2straW1dbWMo/Hw5588km2b98+dubMGfbqq68yvV7P3nzzzdB5ixYtYgaDgS1cuJAdPXqUvf/++ywjI4P96le/6vJ+voDAPL4Au//uJawgfxD710fvs++++47deOONzGg0sl/84hehY+fNm8cmTJjAvv76a7Z//342c+ZMptPp2O9///vQMQBYWmoKe3njf7OTJ0+yX//610yhULDjx48zxhgrKytjANjIkSPZRx99xI4fP84uvfRSNnnyZDZnzhz21VdfsYMHD7Li4mJ2//33d+v/7E9/+hPTarVs/fr17OTJk+ybb74J2SQIAps4cSK7/PLL2f79+9mePXvY5MmT2ezZs0Pnr1mzhhkMBrZgwQJ25MgRtnPnTpadnR32f/foo4+ylJQUVlJSwk6fPs2+/PJL9vLLL4f2P/PMM2zXrl2srKyM/eMf/2BZWVnst7/9LWOMMZfLxX75y1+yMWPGhN5Tl8vV4Wvp6nPb3e9oj4To4YcfZg8//DDjeZ7dd999oecPP/ww+8///E82ffp0NnPmzJ5cslcMJCF68eOTnT7ePXg27Ng/7jjV6bFv7asMO3bj56fbHRMJs2fPDhOAjli2bBn74Q9/GHq+aNEilpqaypxOZ6s9Gzcyg8HABKFrQbTb7UytVrO3Sv7MmN/DGGPs/PnzTKfThew4efIkA8C++eab0HknTpxgANoJ0f133cmYxxbaNn36dPbAAw8wxlqF6C9/+Uto/xtvvMEAsB07doS2rVu3jo0YMaJLu4Pk5uayxx9/vMN9H3/8MVMoFKyysvW9OnbsWNhrWbNmDdPr9cxma7X5kUceYdOnT2eMMWaz2ZhGowkTnovxu9/9jk2ePDn0fM2aNWzChAkXPS8aQtSjGNGhQ4eCXhSOHDkCtbq1/4tarcaECROwcuXK3rtpRL/gpZdewqZNm1BZWQm32w2fz9cu+DlhwoSwBM4ZM2bA4XCgqqoKgwcP7vTapadPw+fzYfqUSaFgdWpqKkaMGBE65sSJE1AqlZg8eXJo28iRI5GcnAwA8AsiHJ6AdN9pk8NiRDNmzMDhw4fD7jl+/PjQ71lZWQCAcePGhW0Lljt0RUNDA2pqanD11Vd3uP/EiRPIz89Hfn5+aNvo0aORnJyMEydOYOrUqQCk2T+jsbUZXE5OTuj+J06cgNfr7fQeAPDmm2/iD3/4A0pLS+FwOBAIBGAyyVPm0iMh+uyzzwAAS5YswYYNG2QzeiCy7MriTvfxF+ST3TtraKfHXph7dtdlfRPT27JlC1auXIkXXngBM2bMgNFoxO9+9zvs3bu3V9dljMHlE+DxB1o3RpDMGLyWT2gToL5IsFqlUoV+DybxXbhN7EbL2WBZRG9pe+8L73+xe3z99de4/fbbsXbtWlx77bUwm83YsmVLh3G2WBDRO7h582YSoRijVvKdPpQKvtvHqrpxbET2qdUQhNYyiV27dmHmzJl48MEHMWnSJBQXF6O0tLTded9++y3cbnfo+Z49e2AwGMK8gbaIDHB4A8jMyYNKpcLeA4dDyYzNzc04depU6NiRI0ciEAjgwIEDoW0nT54MdTZomxG8Z9/BsL7Ve/bswahRo3r2n9BNjEYjCgsLsWPHjg73jxo1ClVVVaiqqgptO378OCwWC0aPHt2tewwbNgw6na7Te+zevRuDBw/G448/jilTpmDYsGGoqKgIO+bC97Qv6bZHtGDBApSUlMBkMmHBggVdHrt169ZeG0YkFoWFhdi7dy/Ky8thMBgwbNgwvPLKK9i+fTuKiorwt7/9Dfv27Ws3q+rz+bB06VL8+te/Rnl5OdasWYOHHnqoXWpIENYyY2Yy6rH0jp/gkSeeQVreUGRmZuLxxx8PO2/EiBG47rrrcN9992Hjxo1QKpVYvnx5yFto60m+ve09TLlkEi6fOw+vvfYavvnmmz7Ng3vqqadw//33IzMzE/PmzYPdbseuXbvw85//HHPnzsW4ceNw++23Y/369QgEAnjwwQcxe/ZsTJkypVvX12q1WLVqFR599FGo1WpcdtllaGxsxLFjx7B06VIMGzYMlZWV2LJlC6ZOnYoPPvgA7777btg1CgsLUVZWhsOHDyMvLw9GoxEaTc8Ki7tLt//8mc3m0F8Qs9nc5YMYeKxcuRIKhQKjR49GRkYGrr32WixYsAALFy7E9OnTcf78eTz44IPtzrv66qsxbNgwzJo1CwsXLsRNN92Ep556qtP7tM2q/t0zT+CKmZdi/vz5mDt3Li6//PKweBAgee+5ubmYPXs2FixYgHvvvReZmZkAwj2ip1avxJZ3tmL8+PF45ZVX8MYbb3Tb+4iERYsWYf369fjTn/6EMWPG4MYbb8T3338fsuvvf/87UlJSMGvWLMydOxdDhgzBm2++2aN7PPHEE/jlL3+JJ598EqNGjcLChQtDMaSbbroJDz/8MB566CFMnDgRu3fvxhNPPBF2/g9/+ENcd911uPLKK5GRkYE33ngjOi++AzjGErfAxmazwWw2w2q19ouhosfjQVlZGYqKiqjHUyd4/AKsbj90zAUT55XWr9dG9sePMYYGuxfZZh3+77W/YsGN8wBjdvtAGtElXX1uu/sdjSgg4Ha74XK19vitqKjA+vXr8fHHH0dyOYLoNmKUyjuA1sxqAFKtGUDZ1TIRkRDdfPPNeOWVVwAAFosF06ZNwwsvvICbb74ZGzdujKqBxMCksrISBoOh3SMzNRlDctNRU1UpHRjhjFkQPqhEXHTrzTqyPfj48ssvo3KP/kTEHRp///vfAwDeeecdZGdn49ChQ/i///s/PPnkk3jggQeiaiQx8MjNzW2XxwMAdq8fXp+IQdktw7FeeEQAkJaklgLgzkZACDZaU130vIvRke1BBg0a1Ovr9zciEiKXyxVKpPr444+xYMEC8DyPSy+9tN0UIEFEglKpRHFx+9ypgCBCEBmU7pbEwV4KUShgzSkABKK29HRHthOdE5FfW1xcjG3btqGqqgrbt2/HNddcA0DKGO1J0HjdunWYOnUqjEYjMjMzccstt+DkyZORmEQMEJQKHhoFh5D8RFh5344oD82InhGRED355JNYuXIlCgsLMX36dMyYMQOA5B1NmjSp29f54osvsGzZMuzZsweffPIJ/H4/rrnmGjidtA450QXB/tI83+sZLrcvgGanD95g3h4JkSxENDT70Y9+hMsvvxy1tbWYMGFCaPvVV1+NH/zgB92+zkcffRT2vKSkBJmZmThw4ABmzZoViWlEP8flC4AX/NAA4Ho5LAOAgCiVeag5DhqAhEgmIl7pNTs7G9nZ2WHberuckNVqBSAVL3aE1+uF1+sNPbfZbL26H5FYMMZg9wSgFLxQKxi4CPpUX0hw1oyhxbMiIZKFiITI6XTiN7/5DXbs2IGGhoZ2hX5nzpzp8TVFUcTy5ctx2WWXYezYsR0es27dOqxduzYSk4l+QDD1lmMiOHBRiQ8FR3YitYuVlYiE6O6778YXX3yBO+64Azk5OVFZTmTZsmU4evQovvrqq06PWb16NVasWBF6brPZOi2OJPofbVfv4IBez5gBrR6RSB6RrEQkRB9++CE++OADXHbZZVEx4qGHHsL777+PnTt3Ii8vr9PjNBpNnxXdEb1jzpw5mDhxItavX99n9+hJr+ru2tPqEbURIsaozCPGRDTITklJ6TSO0xMYY3jooYfw7rvv4tNPP6V+1wOc7vaqjkZ5R5CgRySEhmZtFm0kYkZEQvTMM8/gySefDKs3i4Rly5bh1Vdfxeuvvw6j0Yi6ujrU1dWF9ach4p9Im+cH1zV79tlnkZubG+quuHv3bkycOBFarRZTpkzBtm3bwHEcDh0+BDAGHiKOHv835t14EwwGA7KysnDHHXeEFnToyJ7y8vIObefa/sbR8EwuIhqavfDCCygtLUVWVhYKCwvbdYrr7sKKwbq0OXPmhG3fvHkzFi9eHIlp/Q/GACHyNcUjRqHq9vBkw4YNOHXqFMaOHYunn34agOQ15+Xl4e2330ZaWhp2796Ne++9Fzk5ObjttttC5+7YsQMmkwmffPIJACnuN3/+fFx//fV4/fXXUVFREVovjTGAgwir1Yqr5t+Ku+++B79fvx5utxurVq3Cbbfdhk8//bRDezIyMjp+mTyHTKNGinM6eClHiYQo5kQkRJGsztkRCdyBJHYIfuBLGdp3XvFLQKm++HGQ+lOp1Wro9fqwlI62M5xFRUX4+uuv8dZbb4UJUVJSEv7yl7+E+p//z//8DziOw8svvwytVovRo0ejuroa99xzD9RKHslaBX7z1xJMmjAO/9+6daHrbNq0Cfn5+Th16hSGDx/eoT0dETbRwvEASIjkICIhWrNmTbTtIPoh3WmeP27cuLBFGE6ePInx48eH9bUJ5qcpeR5qXsSRoyfw2c5dMBgM7e5ZWlqK4cOHR2ZwqMwjNu1RiVYiTmi0WCx45513UFpaikceeQSpqak4ePAgsrKyqLo4mihUkncix317QXeb5yclJfXswqIIh9OJ+ddfh98+/2K73Tk5OT221ebxQxAYTBwv1bCRRxRzIhKi7777DnPnzoXZbEZ5eTnuuecepKamYuvWraisrAz1KiKiAMd1e4gkJ101zw/SUfP8CxkxYgReffVVeL3eUKrGvn37AAAefwBevw+TJozD1vc+RGFhIZTKjj/CPWn87g+ICIgMopIjIZKJiGbNVqxYgcWLF+P7778Pc6Gvv/567Ny5M2rGEYlD2+b5586dw7Bhw7B//35s374dp06dwhNPPBESlK746U9/ClEUce+99+LEiRPYvn07nn/+eQCA2y/C4/XjgbsXoam5GT/5yU+wb98+lJaWYvv27ViyZElIfC60p6tlfrhQUiNlV8tFREK0b98+3Hfffe22Dxo0CHV1db02ikg8Im2efyEmkwnvvfceDh8+jIkTJ+Lxxx/Hk08+CQBQa7TgmIBBuTnY9cXnEAQB11xzDcaNG4fly5cjOTk5tIrHhfZUVlZ2es/gah6MhEg+urcYbTgZGRns4MGDjDHGDAYDKy0tZYxJS+Xm5eVFcsmIGEhLTg9kXn31VaZSqVhZbROzN1Yx0VrDmOCP2vUtLh+rs7qlpa+tNYzZG6J27YFANJacjsgjuummm/D000/D75fyWziOQ2VlJVatWoUf/vCHUZRJYiDyyiuv4KuvvkJZWRm2bduGVatW4Ue33gq9TgMOTEpvikJWdRC+ozIPIqZEJEQvvPACHA4HMjIy4Ha7MXv2bBQXF8NoNOLZZ5+Nto3EAKOurg4/+9nPMGrUKDz88MO49dZb8dKf/gccE6R4DqeIai1YhzEiynGLKRHNmpnNZnzyySfYtWsXvv32WzgcDlxyySWYO3dutO0jBiCPPvooHn300bBtbl8ATqdT8lmi1R62hdYYESfVfDBIYhRFr4vomh4LkSiKKCkpwdatW1FeXg6O41BUVITs7GwwxqLSEoQgLkRkAI82HlEU0akU0KkUbco8xJbhGQlRrOjR0Iwxhptuugl33303qqurMW7cOIwZMwYVFRVYvHhxj9rEEkRP0Kp4GNUcVIroNERrS7AwVnpCM2dy0COPqKSkBDt37sSOHTtw5ZVXhu379NNPccstt+CVV17BnXfeGVUjBxpd5bwMVBQ8DwUnSrGhKAtRGCREPSYan9ceCdEbb7yBX/3qV+1ECACuuuoqPPbYY3jttddIiCJErVaD53nU1NQgIyMDarWahrptcXukOjA+AAieqF1WEBmcXmkG2MQJQMAPwA0I9H/fFYwx+Hw+NDY2guf5sJrBntIjIfruu+/w3HPPdbp/3rx5+MMf/hCxMQMdnudRVFSE2tpa1NTUyG1OXOELCFD47OB5DpzG3uq5RAGRMTg8AYADTIoAIPgAhQZQaS9+MgG9Xo+CgoJQMmkk9EiImpqakJWV1en+rKwsNDc3R2wMIXlFBQUFCAQC3a6VGgi889URDK37GKPy0qCdvjSq0/cBQcSre6QVin+ab4e6dj+QPhwoau/5E+EoFAoolcpee+49EiJBEDotMgwaFQgEemUQIQVPVSpVu4ZzAxVvQIDfZQG8dhh0eVDqdFG/h8Cr4AuI4JRqaEUXINgBLXlEsaJHQsQYw+LFizttYN92zTGCiBYurwBtwAYFz0Fp6H2v9I7QqhTwBUS4OS2MAOCj1YZjSY+EaNGiRRc9hgLVRLRx+gLQ+m1QKzhAm9wn99CqeNjcgBstXpC/d/3YiZ7RIyHavHlzX9lBEJ3i8gnQBqxQKXhAl9In99CppJSAkBD5XLSsUAyJ3tQDQfQRdk8A2oCtz4WI4wAv1xJ2EFtmz4iYEHGrWEJmvHagbCcQ8AIFlwKmXLkt6jMcbi80ATs0SZo+E6K5o7Nw7Zhs8DwHVKqkRQt8TkBJC3rGAhKiREQIAN9uAZzSOl5oOgNMXgIkpclrVx8xKYuHmG2ASq0GNMY+uYdK0WZwoE4C3JaWOFHfBMeJcGholoic3SeJkFoPGDKlv96ln8ptVZ9hEu1I1quRZE6PTcxGpZd++ihgHStIiBINUQSqDyAgijiqnYyjKVeDAcD5060eUn/DdV76qe8776Te5sE/j9Ri56lGySMCAD9N4ccKEqJEo+kMAm4rvq3341/nU2GFAVz6MGlf3RF5besDRJHh5JlyNNg9EHR9N/T0+AWcrLOj4ryTPCIZICFKNM6dQq3Fg0rFYOi1GkwrSgWyxkj7Gv/d7zoLOnwBlFdVouycE3wfxsCC0/cevygNeQHKJYohJESJhChCaDyFWpsHzfrBuGpkphRkTR0K8EopwOpokNvKqGL3BKD3W6BS8OCSOl6/Phpo1S15RH4BLOQR0dAsVpAQJRKOejRbLPBBCSQXYGiGtORyhdWPA3Yz7B4/0Fwms5HRxWm3Qyl6oFHygK7vYkRapSREgsjgV7TUspEQxQwSokTCWoVzDi9smhyMzE0OVTyfqLXje28qzjt9gKXz9bsSEY+1HgDA68x9uuKtSsFB2dK82kNlHjGHhCiBEJrKYXP7YdfkoDjTENpenJkEqzYXzS4fmKUSEPtP+xCvvREAwBv6blgGSB0PdC3Ds1B2NXlEMYOEKFFgDKLlLDJNWugzi5BhaM34LUhNgleTDoeggsfjAey1MhoaXQJ2KSVBaUzv83tpW8o8PGgZmgU8UroE0edQZnWi4DwHlehFYWYyCi+bFJbYp1byGJSih60hB1Z3I3S2GsCcJ6Ox0UNwnIMCgNaU2ef3unVKHlQ8Dx4MOMlJM5B+F6AxXPxkoleQR5QoOFtmwwxZHTaPH5Sig1OdIQWsbdUxNq7vmJwWwMhsI9Ky+r6WTqNUSLVmPA+oWrwiihPFBBKiBEGwN8Dm8SOg73iIMihZB4cmAzZ3AMzWT/pd+93QiU6k6NXQmrNje2+awo8pJEQJgv1cNY7V2PCP0wGwDpIWs81aeLSZUCp5CC5L//gCBXOitOaYNLIvO+fEP4/U4kBFc5syD/KIYgEJUYLgbJK8nKTUnA4blasUPO6+cjQmDB8KJc8DtsQPWDfWncXZZhfOwRyT+9ncfpyss6Pa4m4VIq8jJvce6JAQJQI+F9x2KwDAnN55rESt5AFjjvTEnvjDs6b6KlQ1u1HmSYrJ/ZI0LdnVvkBruxGvPSb3HuiQECUCzgY4vAF4lUZkpV7EOzDlgIH1C48oYJeSGTUxig/p1NIkstMrAOoWIfKREMUCEqIEwGetg9svwKlKQ5ap846Bbp+Af5wWcKCiGaKjPoYW9gGiCNEhJTNqU3Nicku9qrXejDyi2EJClADYz7UMswyZ0Ks7T/3SqnhUB4zwC4DbYU3s+Ia7GT6vDwKngim573OIAEDfMjTzBUT4VcEYEQlRLJBViHbu3In58+cjNzcXHMdh27ZtcpoTt7iapWGWJrnrIQrHcUg3G+BWmeH0BlpzjxIQn7UGPkGEW5WC5KS+qzFri1rBh+rNXGiZvvc6+l1rlXhEViFyOp2YMGECXnrpJTnNiG9EEWmwoTBNj6LBhRc9PMukhUuVJglRArcEcTSeBQAEkrKgVbVP4OwLgvVmPMfBw2ml7HUm9o9UiDhH1hKPefPmYd68eXKaEP+4m6BXAvpUE1CQf9HDs0xa1KrT4HBXAAkcJ/KcrwIAKJMHxfS+P7t0MDRKXkqRUCdJHpHXTmUefUxC1Zp5vd6wZa1tNpuM1sSIoFdjyOxW4/gskwZOVRpc1gBEe31iBgFFEdkKK1Lyk+EYMyqmtw7zvtQGSYh8CRxrSxAS6nO6bt06mM3m0CM//+IeQqLjtdSi0eGFVdG99bzMOhUC+gyIDHBbGqQVPhINZyN4MQCdToeMTBnXawvNnA2AP3gyk1BCtHr1alit1tCjqqpKbpP6HEtjNU43OLCrrntvFcdxyM/KgMmcDIABzsY+ta9PCCZjGnNjvuRzaaMDH3xXi0OVzYDGJG2kmbM+J6GGZhqNBhrNwFp502upAwDoLjJj1pZ543IAcZi08KK9LuFWgQ1YqlHe4EBAY8AwkUHBx06MrG4/TtXbwXHAJFPQI6KhWV+TUB7RgMPngtdpAQAYuyjt6BBDS+5NAs6cOc+dRaPDi8MWLWKoQQAAfUuXRqc30BqgJo+oz5HVI3I4HDh9+nToeVlZGQ4fPozU1FQUFBTIaFmc4GyEyyfAqzQi22zq0anMkAlfQIAq0QLWfjc8QS8wvaDDAt++JClU5kH1ZrFEViHav38/rrzyytDzFStWAAAWLVqEkpISmayKHwK2Orh9Apy6NKQZup/UxxjDa8e8GFppwQROCb0oSs2+EgHrWTi9fniUZqSmxH7deYNG+ko4vAEwdSo4QApWMxbzeNVAQlYhmjNnToe9dQgJR1MNGAC/Lg1GTfffKo7joDKkQuSUcLk90LubgKS+7/kcFSyVcHkF2DTZGGns+x5EF2LQSv/PfoHBqzRI63kIfsDvbl14kYg6CfJncmDiapKGKNrkjnsQdUWmSQ+nOg0ObyChEhtFSxVcvgBs2lxkGGM/MaFS8KFcIocfrXEijzXmtgwkSIjiFVFEBmfDyGwjRhcP7fHpGUYNnKp0KdZhr+sDA/uAgA+upmoIDPAk5SJZp5LFDINGKvNw+wSpOyQAeCyy2DJQSKjp+wGFuxkqTkSKUY+UQT0vc8g0afCtOg1OlwBmr0NCRDdsZ+H1+eFXGZGWliE1speBW6fkt5Z51JkBazV5RH0MCVG8EhxOJWVGFGhOS9LAo82AIDJ4mmugS4Rga3M50pI0uKxoLFxDs2QzI6zMQ5ss/SQh6lNoaBan+Kx1qGp2oUYwRhTQV/AcklKzwTgeLpczMb5ITWcAAIr0oTBq5RmWtSM0NEuA/78EhjyiOMV+vgZnm91o0Chxc4SezPCcZBjP5UGrtEoeli45ukZGE48NzNEAjuOBlEJZTamxuHGo0gKTTokr0kiIYgF5RHFKKKkvJfJ+zZcUpGDY0KFSkl68B6ybzqDR7sXXjWp8W++T1RSPX8Cpejsqm1zhwWpKNekzSIjiEb8bXnszAMCQ1st+PIYWIYv3Uo+mM7C6/TjL5cDpC8hqSjCXyOEJSELEcYAQoAZpfQgJUTziaIDLL8CrNCCth6UdF+LRpcPq9sNvjePlhUQBYlMZLC4/LNo8DE6LzfJBnWHUSPEpl09AgHFSXyKAhmd9CAlRHMIc9XD5BLhUPSvt6IitJ/04XmuH3WqJ3yry5nI4nU64OB18+mxkm2KfUd0WrYqHSiHF5eyeQGtszd0sn1H9HBKiOMTVVCtNu2vSkaLvnRCFmun74jixseEEzjt9aNIVYWimIaZtPzqC4ziYW5IprW4/oGupeXM3yWhV/4aEKA5xNVUDABTmnF5/KTNNWjjUmVKGta06GuZFF1EAO3cKTU4fmvRFKM40ym0RAMDUIkQ2jx/Qp0kbXSREfQVN38cbooA0zg5DfjKs40b2+nKZRg2OabLgtH4fn0LUXA67wwEH08CTlIvCtPgoLDXpVFDwHLwBETC0eESu8/Ia1Y8hIYo3nOfAMxE6nR66zN5nF6cbNHBqMuETGLzNZ6GJt5Yg9cegUvBIyhuD0bnJUCriw7bLi9MxZ3iGVObhbJm2dzdRO5A+goQo3nC0xHEMWVH5wKuVPHTJWRDqVXA6XdA4GwGjfOUTYfg9QONJ6FQKTJ48CzDFZkXX7qBqK4i6ZIDjpSl8rx3Q9m4mk2hPfPz5IUL4rbX4vsGO7936qPVqyk1JgkOdBZsnANjORuWaUaHhGCAGpF5Jxtisbx8RvKJ15oyGZ30CCVGcYT9Xg3MOH45YtFFrkzoqx4TiYSOQZdJKleTxAGMInD2MM40OWJJHx91wRxQZ/nmkFm98UwmPX6CZsz6GhCieYAzuZinxUJ8avZU3cpN1KC4eAZ1KAVir4qNUwVKJhpoK1DoEvFeXEnedOnmew9lmF+qsHmkKP6ll5sx5Tl7D+ikkRPGEqwlulwsip0RyepSHKuZ8aYjhscVFYp6nbA/OWtxoTBqGycV5MW+S3x2COVznHT6pHQsQ/6UyCQoJUTxhq4bTJ8ChTkemObrT2M4AhyohFQ12D9BUFtVr99yYc6j6/lsIIiAOmopROfGRO3QhqUmSEDW7fNLkAQA4G+LDo+xnkBDFEX7LWbh9AhzqTCmeE0VqrW7sbjah2uIGa+n7IxdnD36Ecw4fmvWDcdn44XHpDQGtQtTk9AH6VMmjDPiobWwfQEIURzgaKsAAiMYcJPVg1Y7uUJCaBIcuDx6/CHdjGSAKUb1+d2muLUfN6W8BcMiccE3UBTeahAkRr2jNsKbhWdQhIYoXAj74bfXgOcCQMTjql1creaRm5yPAa2GxOaSgdawRRdQdeA+CyCBkjcElI4fF3oYekNIiRBaXH4LIWodnJERRhxIa4wV7LTINGqSlpsE9bkif3KIow4gK3WCkuEqR2/Dv2HdCrDmEkXo7KtPMyJh9k2zN8buLUaOERsXDoFHC7RdgCC3jnTjLMyUKJETxglVKNFQk54VWG402xZkGHEoaisyGk3DVHId+2DUxKfewuvxICjRDeeYzcByHwVOvB4yxX8W1p3Ach3uvGNJadmJqSamwVVOpR5ShoVm8YKmQfiYX9NktDBolUgYVI8Brcb6pufWefUhpowNv7Pke33/2N2nF1JTBQO4lfX7faBFW+2bIlmJFPldcpED0J0iI4gEhgPqqUnxXbcERV0qf3mr0oBScTxoCgTGg9nCf3cfjF7DjRD3eO1SJotqPIDgaIaiSgFE3JaQnwRgDFErA2NJ6Nx47GSQwJETxgO0sbC43mgMa2Lm+zakZmmHAlVdeg8K0JKDxlJTgGEUEkeHIWSv+9nUFjlU2YGTjdgzXNGF0fgYU429tXcI5QXD5AnhzXyX+/51nIIoMMLX0EI+XUpl+AglRHCA2lcPi8sOmzcXg9L79oip4DuaMPCA5H2AiUPVN1K593uHF/+4ux79O1IPZajGt6X3MSLGjKDMZ/LgfAaY4LmztBK1SgUa7Fy6fICU2mvOkHXLMOvZjSIjiAEfNvxEQGZxJBbHr1zx4Jly+AKqPfwW4LRFdwuULoM7qCT036VQQvE4Mt+/FjcInmJbFYE5OBSbdIcWGEhCe55Bt1gEAqi1uSYg4Tqo589pltq7/QLNmcuNuhq2xGgwcDINGxqxfs02Xhz3nDTC6q+H7+v9QdOVdXcZuRJHhnNOLOqsHNRYP6qxuNLv8SNarsHhmIThnI1Q1h7FQOIgkswgFrwEyRwLDrgHU8q7K0VvyUnSoanKhqsmN8XnJUpzIViutTJszQW7z+gUkRHJz7jSaXD7YtTkYnJkWs9uadGqkTbwBnj1/RV3ZCVSLf0f+pP9ASpIajAEmrTJUevHZvxtwvNYGX0BsvQBj0PubMMhfg8A3n0LlkpL8TCpIvYWKZgFpQ2P2evqS/FQ9vi49j6pmFxhj4FKHSEJ0vpSEKEqQEMmMq/oYnF4BlpRCzMqMbSD3klHDcMI1D7Zv34ei4iscaLKiyjwVIq/EA3OGQqtSAABExuALiEjiPBiiPI9BXCMyA7Uw8Q6pk6ELUgfD9GFA7kQgpSghZ8Y6I9ukhVrJw+0T0OjwIjN1KFC+C2guk1ISFCq5TUx4SIjkxNUEznYWGUYtrIPHQqdWxPT2HMdh9JQ5sCYB5458Ao37GDLdp9GsGwx/WT20WjUQcGO6qx7TNY1IEmzgGAcwSNFFXg2kFgFpxZIIJfgQrDMUPIe8FB3ONDpR2uBE5pBcqV2sxyZ5RZm9X+RgoENCJCd130GnUqB45HgUjx8umxnmUXNgTs8FTv8LzGMB0ACuujG0P+SncTyQlAEkD5aCz8mDAWXv1l1LFEZmm8AYkGnSSN5e5iigcq/U7paEqNeQEMmF4Adqv5V+zx4nry0AkDEcSBsKrrkcsFQCPoc0va/USf2akzKkIK1KJ7elsjAi24gR2W1yvLLGSkJ07rTkGVFD/V5BQiQXdd+huuE8zCnpMKTL5w2FwSukAHM/CTL3KYZMKRfLUgVU7weGXiW3RQkN5RHJQcAH28mdqGxy4f3mfDj81PEvUXB6A9h9+py0cm7+dGljzSHA65DXsASHhEgGxPLdqKyth1dpRGrxlD6rtieizwdHarG3rAlfl56XgvTGLKlrY9kXcpuW0JAQxZrmctQe/Qx2TwDV6TNx6bBsuS0iesDMoVKu15FqK6qa3VLCJgDUfgc0npTRssQmLoTopZdeQmFhIbRaLaZPn45vvole/VNcYavFub1bUHXeiYakERg/YRpMWspBSSTyUvQYO8gMAHj/u1o08OlA/jRp54n3pEA/0WNkF6I333wTK1aswJo1a3Dw4EFMmDAB1157LRoa+lE7TlEAzh5A9Wcv43T1Odg0WTCMnYexg2imJRGZMyIDOWYtPH4Bb+2rwkHlBASSB0szod9uASr3SMtTE92GYzKvbDd9+nRMnToV//3f/w0AEEUR+fn5+PnPf47HHnusy3NtNhvMZjOsVitMpjj6Ugt+BNx2uC110DhroG46CXhsONvswlF3GlTjf4grx+TFfatUonM8fgH/PFKLivMuAMDsYcm4xPEl0HgKLl8ATmghpo2AmFwALikdnNYEhVINBc/DqFWGGq4JIoMgMih4DjyHuF3RJFK6+x2VNUrq8/lw4MABrF69OrSN53nMnTsXX3/9dbvjvV4vvF5v6LnN1v1eOtUWNz49UY+g6gblN6jD04rSMDrXBBx7F9bzdThWbWlzIAMHJmUUg6EgVYdBZqlK3u714+hZCwAGMAaeCeDElmbrAArT9Mgx6wC1HtlT5sKlHoFhWcZ+94EbaGhVCtwycRCO19qwr7wJw3NSAfUCoO47nDuwHdX19cDZ8N7WjOMR4DSYMDgVZp0G4HhUN3twutEBBunzwHEADw5ciyiNG2RGil4avtfbvSg/52xnS+mgmyEotJgzIhP5qdJ6eKWNDuw+3fmqtJcPy0BRupQJX3HeiZ2nGjs9dsbQNBRnSjlU1RY3Pv13A350SV5UKwFkFaJz585BEARkZWWFbc/KysK///3vdsevW7cOa9eujehe/oCIcw5fp/u9gZblddzN4JyN4FxdiJzHD2j9AADOGwB8rR+OYFmoyCngV5lx3liInDGXAGlDoVSoECcZQ0QU4HkOYweZMSbX1PqHJWcCbBPyYTl9FFp7GbSeRqi9TeDEAETGoGQeKHwOgJPapyg9buj8rs7v4fIDTBIi5vBAdLQXovMOHwIKHj6htSjZ6+/68962gNl3ke+Gx3/BsXYvGKI7kJJ1aFZTU4NBgwZh9+7dmDFjRmj7o48+ii+++AJ79+4NO74jjyg/P79bQzO3T0Cj3RtWixn8neM4mHUqaRrdWg2vzwur2wcOPBjHSR8yjgMHDgAHvUYJvVoJcBz8IoPNE5D2cRw4Xgmo9NDpdNCoFOT5EJJXLfiBgBsIeKWM9ZaHKAgICCIYGETGIDKpyJiJ0u96tRJBx8PhC8Dmbh97Eox5AK9AukET8lIc3gCanZ2LS2qSOrR2nssXwDl7F8ca1KEUk+D3aFCKrlstaxJiaJaeng6FQoH6+nAXtr6+HtnZ7ae1NRoNNBpNRPfSqRUoSOvGMs7mQdAAyOzmdVUA0swRmUQMFDhOqsnroC6PB9Ddaj0D2tT9XexYjbLb+Wl6tRIFad07ttvfox4i66yZWq3G5MmTsWPHjtA2URSxY8eOMA+JIIj+jewpvStWrMCiRYswZcoUTJs2DevXr4fT6cSSJUvkNo0giBghuxAtXLgQjY2NePLJJ1FXV4eJEyfio48+ahfAJgii/yJ7HlFviNs8IoIgAHT/Oyp7ZjVBEAQJEUEQskNCRBCE7MgerO4NwfBWT0o9CIKIHcHv5sVC0QktRHa7tNJmfn6+zJYQBNEVdrsdZnPnmb8JPWsmiiJOnjyJ0aNHo6qqasDNnAVLXOi1D5zXnmivmzEGu92O3Nxc8HznkaCE9oh4nsegQYMAACaTKSHemL6AXvvAe+2J9Lq78oSCULCaIAjZISEiCEJ2El6INBoN1qxZE3FVfiJDr33gvfb++roTOlhNEET/IOE9IoIgEh8SIoIgZIeEiCAI2SEhIghCdhJSiJqamnD77bfDZDIhOTkZS5cuhcPh6PKcOXPmgGtphB983H///TGyOHJ6ugru22+/jZEjR0Kr1WLcuHH45z//GSNLo09PXntJSUm791er1cbQ2uiwc+dOzJ8/H7m5ueA4Dtu2bbvoOZ9//jkuueQSaDQaFBcXo6SkpM/tjDosAbnuuuvYhAkT2J49e9iXX37JiouL2U9+8pMuz5k9eza75557WG1tbehhtVpjZHFkbNmyhanVarZp0yZ27Ngxds8997Dk5GRWX1/f4fG7du1iCoWCPffcc+z48ePs17/+NVOpVOzIkSMxtrz39PS1b968mZlMprD3t66uLsZW955//vOf7PHHH2dbt25lANi7777b5fFnzpxher2erVixgh0/fpz98Y9/ZAqFgn300UexMThKJJwQHT9+nAFg+/btC2378MMPGcdxrLq6utPzZs+ezX7xi1/EwMLoMW3aNLZs2bLQc0EQWG5uLlu3bl2Hx992223shhtuCNs2ffp0dt999/WpnX1BT1/75s2bmdlsjpF1saE7QvToo4+yMWPGhG1buHAhu/baa/vQsuiTcEOzr7/+GsnJyZgyZUpo29y5c8HzfLt10C7ktddeQ3p6OsaOHYvVq1fD5ep8YTu5Ca6CO3fu3NC2rlbBBaT/m7bHA8C1117b6fHxSiSvHQAcDgcGDx6M/Px83HzzzTh27FgszJWV/vKeJ1zRa11dHTIzw1cdUyqVSE1NRV1dXafn/fSnP8XgwYORm5uL7777DqtWrcLJkyexdevWvjY5Inq6Ci4g/d90dHxX/y/xSCSvfcSIEdi0aRPGjx8Pq9WK559/HjNnzsSxY8eQl5cXC7NlobP33Gazwe12Q6fTyWRZz4gbIXrsscfw29/+tstjTpw4EfH177333tDv48aNQ05ODq6++mqUlpZi6NChEV+XiA9mzJgRthbezJkzMWrUKPz5z3/GM888I6NlRHeIGyH65S9/icWLF3d5zJAhQ5CdnY2Ghoaw7YFAAE1NTR2uDtsZ06dPBwCcPn06LoWop6vgAkB2dnaPjo9XInntF6JSqTBp0iScPn26L0yMGzp7z00mU8J4Q0AcTd9nZGRg5MiRXT7UajVmzJgBi8WCAwcOhM799NNPIYpiSFy6w+HDhwEAOTk50X4pUSGSVXBnzJgRdjwAfPLJJwm3am40VgAWBAFHjhyJ2/c3WvSX9zzhZs0Yk6bvJ02axPbu3cu++uorNmzYsLDp+7Nnz7IRI0awvXv3MsYYO336NHv66afZ/v37WVlZGfv73//OhgwZwmbNmiXXS+gWW7ZsYRqNhpWUlLDjx4+ze++9lyUnJ4empe+44w722GOPhY7ftWsXUyqV7Pnnn2cnTpxga9asSejp+5689rVr17Lt27ez0tJSduDAAfbjH/+YabVaduzYMbleQkTY7XZ26NAhdujQIQaAvfjii+zQoUOsoqKCMcbYY489xu64447Q8cHp+0ceeYSdOHGCvfTSSzR9HyvOnz/PfvKTnzCDwcBMJhNbsmQJs9vtof1lZWUMAPvss88YY4xVVlayWbNmsdTUVKbRaFhxcTF75JFH4j6PiDHG/vjHP7KCggKmVqvZtGnT2J49e0L7Zs+ezRYtWhR2/FtvvcWGDx/O1Go1GzNmDPvggw9ibHH06MlrX758eejYrKwsdv3117ODBw/KYHXv+OyzzxiAdo/ga120aBGbPXt2u3MmTpzI1Go1GzJkCNu8eXPM7e4t1AaEIAjZiZsYEUEQAxcSIoIgZIeEiCAI2SEhIghCdkiICIKQHRIigiBkh4SIIAjZISEiEoqSkhIkJyeHnj/11FOYOHFi6PnixYtxyy23xNwuoneQEBEdsnjx4k7b6S5btgwcx4UVKfeFABQWFmL9+vVh2xYuXIhTp051es6GDRvCWqXOmTMHy5cvj6pdRPQhISI6JT8/H1u2bIHb7Q5t83g8eP3111FQUCCLTTqdrl0/qraYzeYwj4lIDEiIiE655JJLkJ+fH9Y8buvWrSgoKMCkSZN6de2OPJVbbrkl5GXNmTMHFRUVePjhh0PN8IH2Q7MLaeuZLV68GF988QU2bNgQukZZWRmKi4vx/PPPh513+PBhcBzX79uGxCskRESX3HXXXdi8eXPo+aZNm7BkyZI+v+/WrVuRl5eHp59+GrW1taitre3xNTZs2IAZM2bgnnvuCV2joKCg3WsCgM2bN2PWrFkoLi6O1ksgegAJEdElP/vZz/DVV1+hoqICFRUV2LVrF372s5/1+X1TU1OhUChgNBqRnZ0dUXM3s9kMtVoNvV4fuoZCocDixYtx8uTJ0PJEfr8fr7/+Ou66665ovwyim8RNh0YiPsnIyMANN9yAkpISMMZwww03ID09XW6zekVubi5uuOEGbNq0CdOmTcN7770Hr9eLW2+9VW7TBizkEREX5a677kJJSQn+93//N2peA8/zuLADjd/vj8q1u8Pdd98dCsRv3rwZCxcuhF6vj9n9iXBIiIiLct1118Hn88Hv9+Paa6+NyjUzMjLC4j6CIODo0aNhx6jVagiC0Kv7dHaN66+/HklJSdi4cSM++ugjGpbJDA3NiIuiUChCK6goFIpOj7NaraFe4EHS0tKQn5/f7tirrroKK1aswAcffIChQ4fixRdfhMViCTumsLAQO3fuxI9//GNoNJqIhoSFhYXYu3cvysvLYTAYkJqaCp7nQ7Gi1atXY9iwYYnX47mfQR4R0S1MJhNMJlOXx3z++eeYNGlS2GPt2rUdHnvXXXdh0aJFuPPOOzF79mwMGTIEV155ZdgxTz/9NMrLyzF06FBkZGREZPfKlSuhUCgwevRoZGRkoLKyMrRv6dKl8Pl8MZkFJLqGWsUSA5Yvv/wSV199NaqqqtotUkjEFhIiYsDh9XrR2NiIRYsWITs7G6+99prcJg14aGhGDDjeeOMNDB48GBaLBc8995zc5hAgj4ggiDiAPCKCIGSHhIggCNkhISIIQnZIiAiCkB0SIoIgZIeEiCAI2SEhIghCdkiICIKQHRIigiBk5/8BLsp91E/B/CUAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T20:34:52.148432Z", + "iopub.status.busy": "2024-03-24T20:34:52.148102Z", + "iopub.status.idle": "2024-03-24T20:34:52.291799Z", + "shell.execute_reply": "2024-03-24T20:34:52.290855Z" + }, + "papermill": { + "duration": 0.165202, + "end_time": "2024-03-24T20:34:52.293912", + "exception": false, + "start_time": "2024-03-24T20:34:52.128710", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAplElEQVR4nO3de1xUdf4/8BfXARxAEUUkcLwUaCooBGL6NXdR1HT1a1usmiC5tmm2FnmJUsj1W3QT0SJ5rImslUn2NbdvmZdIywuIgrqggIEgplzEGzcdBvj8/vDH2R0ZYDBgODOv5+PhA+dzLvM+nOE153xmzueYCSEEiIhkxtzQBRARPQiGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLloYuoKs1Njbi6tWrsLe3h5mZmaHLIaL7CCFQVVWF/v37w9y85eMrkwuvq1evwt3d3dBlEFEbLl++jIceeqjF6SYXXvb29gDu/WIcHBwMXE3n0Wg0OHDgACZPngwrKytDl0MdwFT2aWVlJdzd3aW/1ZaYXHg1nSo6ODgYfXjZ2dnBwcHBqF/opsTU9mlb3TrssCciWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyZLJfVXCWNXW1iI3N1d6XH1HjeNZBejlfApKW4XWvF5eXrCzs+vqEok6FMPLSOTm5sLX17dZ+3s65s3IyMDo0aM7vyiiTsTwMhJeXl7IyMiQHueV3ELErizEPj0Cnq49m81LJHcMLyNhZ2endTRlfuk6FEfuYOhwb/gM6G3Ayog6BzvsiUiWGF5EJEsMLyKSJYOHV3x8PFQqFWxsbBAQEID09PRW54+Li4OnpydsbW3h7u6OV155BXfv3u2iaomouzBoeCUnJyMiIgLR0dHIzMyEt7c3goODUV5ernP+HTt24LXXXkN0dDRycnKwdetWJCcn4/XXX+/iyonI0AwaXrGxsVi0aBHCw8MxbNgwJCQkwM7ODomJiTrnP378OB5//HHMnTsXKpUKkydPxpw5c9o8WiMi42Owr0rU1dUhIyMDkZGRUpu5uTmCgoKQmpqqc5mxY8fis88+Q3p6Ovz9/XHx4kXs3bsX8+fPb/F51Go11Gq19LiyshLAvYHdNBpNB21N91NfXy/9NObtNCVN+9HY96e+22ew8KqoqEBDQwNcXFy02l1cXLQuc/lPc+fORUVFBcaNGwchBOrr6/HCCy+0etoYExODtWvXNms/cOCAUV8ic7kaACyRlpaGK9mGroY60sGDBw1dQqeqra3Vaz5ZfUn18OHDePvtt/Hxxx8jICAA+fn5WLZsGdatW4c1a9boXCYyMhIRERHS46bxsSdPnmzUw0CfLb4BZJ3CmDFj4O3hZOhyqANoNBocPHgQkyZNMuphoJvOjtpisPBydnaGhYUFysrKtNrLysrQr18/ncusWbMG8+fPx5///GcAwIgRI1BTU4Pnn38eb7zxhs7bJCkUCigUimbtVlZWRv0CsLS0lH4a83aaImN/7eq7bQbrsLe2toavry9SUlKktsbGRqSkpCAwMFDnMrW1tc0CysLCAsC9e70Rkekw6GljREQEwsLC4OfnB39/f8TFxaGmpgbh4eEAgNDQULi5uSEmJgYAMGPGDMTGxmLUqFHSaeOaNWswY8YMKcSIyDQYNLxCQkJw7do1REVFobS0FD4+Pti3b5/UiV9cXKx1pLV69WqYmZlh9erVuHLlCvr06YMZM2bgrbfeMtQmEJGBmAkTO9+qrKyEo6Mjbt++bdQd9mcuXceszWnYs3gMR5UwEhqNBnv37sW0adOMus9L379Rg18eRET0IBheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEsmTQ+zYSkW61tbXIzc3Vaqu+o8bxrAL0cj4Fpa1Cavfy8oKdnV1Xl2hwDC+ibig3Nxe+vr46p7133+OMjAyMHj2684vqZhheRN2Ql5cXMjIytNrySm4hYlcWYp8eAU/XnlrzmiKGF1E3ZGdn1+xoyvzSdSiO3MHQ4d68CzrYYU9EMsXwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEssTwIiJZYngRkSwxvIhIlhheRCRLBg+v+Ph4qFQq2NjYICAgAOnp6a3Of+vWLbz44otwdXWFQqHAI488gr1793ZRtUTUXRh0SJzk5GREREQgISEBAQEBiIuLQ3BwMPLy8tC3b99m89fV1WHSpEno27cvvvrqK7i5ueHSpUvo2bNn1xdPRAZl0PCKjY3FokWLEB4eDgBISEjAd999h8TERLz22mvN5k9MTMSNGzdw/PhxWFlZAQBUKlVXlkzUaQoralCjrm9xesG1GumnpaXuP90eCksMdO7RKfV1NwYLr7q6OmRkZCAyMlJqMzc3R1BQEFJTU3Uu88033yAwMBAvvvgi/vnPf6JPnz6YO3cuVq1aBQsLC53LqNVqqNVq6XFlZSUAQKPRQKPRdOAWdS/19fXST2PeTmNRdL0Gk+KO6TXvq19ltTr94MuPQ9VbvgGm7+vVYOFVUVGBhoYGuLi4aLW7uLg0u/FAk4sXL+LHH3/EvHnzsHfvXuTn52PJkiXQaDSIjo7WuUxMTAzWrl3brP3AgQNGfdOCy9UAYIm0tDRcyTZ0NdSWpv01f0gDXGyFznk0jcANNeCkAKx09FaX3THDp/kW2J/yE9yVnVpup6qtrdVrPlkNA93Y2Ii+ffvi73//OywsLODr64srV67g/fffbzG8IiMjERERIT2urKyEu7s7Jk+eDAcHh64qvcudLb4BZJ3CmDFj4O3hZOhyqA3nrlbig6w0PDXpcTzaX/frUqPR4ODBg5g0aZLUbXL/Oj7NT8O4ceNaXIccNJ0dtcVg4eXs7AwLCwuUlZVptZeVlaFfv346l3F1dYWVlZXWKeLQoUNRWlqKuro6WFtbN1tGoVBAoVA0a7eystL5ApCT1vpILt1USz9tbFp+JzOlPpLurKkPy9LSss3XZUuv3fasozvTt3aDhZe1tTV8fX2RkpKCWbNmAbh3ZJWSkoKlS5fqXObxxx/Hjh070NjYCHPze8fNFy5cgKurq87gMmaFFTWY+MHhNudrq38EAA4tf4IBRrJj0NPGiIgIhIWFwc/PD/7+/oiLi0NNTY306WNoaCjc3NwQExMDAFi8eDE++ugjLFu2DC+99BJ++eUXvP322/jrX/9qyM0wiKYjrrgQHwzp27yDo+aOGt8eTsX0JwLRw7b5kScA5JdX4+XkM61+wkXUXRk0vEJCQnDt2jVERUWhtLQUPj4+2Ldvn9SJX1xcLB1hAYC7uzv279+PV155BSNHjoSbmxuWLVuGVatWGWoTDG5IXyWGuzk2a9doNCjtA4we0EvWpxBELTF4h/3SpUtbPE08fPhws7bAwECkpaV1clVE1N0Z/PIgIqIHwfAiIlky+GkjEQHqhrswt7mCwso8mNvo/oZpfX09rtZfRc6NHJ2XBxVWVsPc5grUDXcBNO8HNTYML6Ju4GrNJfQY+CFeb31QFQDAx/s+bnFaj4HA1Rof+MKlxXmMBcNLptp6p27rXRowvXfq7qx/jwGoKXwJG0N8MFjHV1+Ae/v02NFjeHzc4zr3aUF5NZYln0H/iQM6u9xugeElU/q+U7f2Lg2Y1jt1d6awsEHjXTcMdPDEsN6630g0Gg0KLQsx1Gmozq+/NN69jca716CwsOnscrsFhpdMtfVO3da7NGB679RkXNodXhcvXsSgQYM6oxZqh7beqdt6lwZM752ajEu7vyoxZMgQTJw4EZ999hnu3r3bGTUREbWp3eGVmZmJkSNHIiIiAv369cNf/vKXNsedJyLqaO0OLx8fH2zcuBFXr15FYmIiSkpKMG7cOAwfPhyxsbG4du1aZ9RJRKTlgb9hb2lpidmzZ2PXrl149913kZ+fj+XLl8Pd3R2hoaEoKSnpyDqJiLQ8cHidOnUKS5YsgaurK2JjY7F8+XIUFBTg4MGDuHr1KmbOnNmRdRIRaWn3p42xsbHYtm0b8vLyMG3aNGzfvh3Tpk2Thq4ZOHAgkpKSeFcfIupU7Q6vzZs347nnnsOCBQvg6uqqc56+ffti69atv7k4IqKWtDu8Dh48CA8PD61BAgFACIHLly/Dw8MD1tbWCAsL67AiiYju1+4+r8GDB6OioqJZ+40bNzBw4MAOKYqIqC3tDi8hdN9Trrq6GjY2/KY2EXUNvU8bm+59aGZmhqioKK0btjY0NODEiRPw8fHp8AKJiHTRO7xOnz4N4N6RV1ZWltatxqytreHt7Y3ly5d3fIVERDroHV6HDh0CAISHh2Pjxo1GfbdpIur+2v1p47Zt2zqjDiKidtErvGbPno2kpCQ4ODhg9uzZrc67e/fuDimMiKg1eoWXo6MjzMzMpP8TERmaXuH1n6eKPG0kou6A920kIlnS68hr1KhR0mljWzIzM39TQURE+tArvGbNmtXJZRARtY9e4RUdHd3ZdRARtQv7vIhIlvQ68nJycsKFCxfg7OyMXr16tdr/dePGjQ4rjoioJXqF14YNG2Bvby/9X9/OeyKizqJXeP3nwIILFizorFqIiPTW7j4vCwsLlJeXN2u/fv06LCwsOqQoIqK2dNhghGq1WmuYHCKizqT3qBKbNm0CcG8wwk8++QRKpVKa1tDQgJ9//hleXl4dXyERkQ56h9eGDRsA3DvySkhI0DpFtLa2hkqlQkJCQsdXSESkg97hVVhYCACYOHEidu/ejV69enVaUUREbWn3YIRNI6oSERlSu8Prueeea3V6YmJiu4uIj4/H+++/j9LSUnh7e+PDDz+Ev79/m8vt3LkTc+bMwcyZM7Fnz552Py8RyVe7w+vmzZtajzUaDbKzs3Hr1i387ne/a3cBycnJiIiIQEJCAgICAhAXF4fg4GDk5eWhb9++LS5XVFSE5cuXY/z48e1+TiKSv3aH19dff92srbGxEYsXL8bgwYPbXUBsbCwWLVqE8PBwAEBCQgK+++47JCYm4rXXXtO5TENDA+bNm4e1a9fiyJEjuHXrVrufl4jkrd3hpYu5uTkiIiLwxBNPYOXKlXovV1dXh4yMDERGRmqtKygoCKmpqS0u97e//Q19+/bFwoULceTIkVafQ61WQ61WS48rKysB3Dti1Gg0etfa3dTX10s/dW1HU1tr29jWOqjr6LMv2tqnxrI/9a29Q8ILAAoKCqRfnr4qKirQ0NAAFxcXrXYXFxfk5ubqXObo0aPYunUrzpw5o9dzxMTEYO3atc3aDxw4oHXjXLm5XA0Aljh69CguKVue7+DBg795HdT52rMvWtqnxrI/a2tr9Zqv3eHVdOfsJkIIlJSU4LvvvtO6BrIzVFVVYf78+diyZQucnZ31WiYyMlKr5srKSri7u2Py5MmyvvfkuauV+CArDePGjcOj/Ztvh0ajwcGDBzFp0iRYWVk90Dqo6+izL9rap8ayP5vOjtrS7vBqunN2E3Nzc/Tp0wfr169v85PI+zk7O8PCwgJlZWVa7WVlZejXr1+z+QsKClBUVIQZM2ZIbY2NjQAAS0tL5OXlNet3UygUUCgUzdZlZWXV4h+1HFhaWko/W9uO1rZT33VQ52vPvmhpnxrL/tS3doN+z8va2hq+vr5ISUmRhppubGxESkoKli5d2mx+Ly8vZGVlabWtXr0aVVVV2LhxI9zd3TusNiLq3jqsz+tBRUREICwsDH5+fvD390dcXBxqamqkTx9DQ0Ph5uaGmJgY2NjYYPjw4VrL9+zZEwCatRORcTN4eIWEhODatWuIiopCaWkpfHx8sG/fPqkTv7i4GObmHK2aiLQZPLwAYOnSpTpPEwHg8OHDrS6blJTU8QURUbfHQxoikqUOC69ff/0Vzz//fEetjoioVR0WXtevX8fWrVs7anVERK3iaSMRyRLDi4hkieFFRLKk91clZs+e3ep0DktDRF1J7/BydHRsc3poaOhvLoiISB96h9e2bds6sw4ionZhnxcRyZLeR176DnfzIDfgICJqL73DKykpCQMGDMCoUaMghOjMmoiI2qR3eC1evBhffPEFCgsLER4ejmeffRZOTk6dWRsRUYv07vOKj49HSUkJVq5cif/7v/+Du7s7nnnmGezfv59HYkTU5do1JI5CocCcOXMwZ84cXLp0CUlJSViyZAnq6+tx7tw5KJUyHvVfZu5oGgAA2Vdu65xec0eNU9eAfpduoodt82GwASC/vLrT6iPqbA88npe5uTnMzMwghEBDQ0NH1kR6KPj/wfPa7qxW5rLEp/kn21xXD0W3GNaNqF3a9apVq9XYvXs3EhMTcfToUUyfPh0fffQRpkyZwtFOu9jkR+/doGRwXyVsrSyaTc8ruY1Xv8rC+j+OgKdry18w7qGwxEDnHp1WJ1Fn0Tu8lixZgp07d8Ld3R3PPfccvvjiC71vP0Ydz6mHNf7k79Hi9KZ7aA7u0wPD3Vq/OoJIjvQOr4SEBHh4eGDQoEH46aef8NNPP+mcb/fu3R1WHBFRS/QOr9DQUJiZmXVmLUREemvXl1SJiLoL9rITkSwxvIhIlhheRCRLDC8ikiV+tZqoG2jrci+g7Uu+TO1yL4YXUTeg3+VegD6XfJnK5V6msZVE3Vxbl3sB+l3yZUqXezG8iLqBti73AnjJ1/3YYU9EssTwIiJZYngRkSwxvIhIlhheRCRLDC8ikiWGFxHJEsOLiGSJ4UVEstQtwis+Ph4qlQo2NjYICAhAenp6i/Nu2bIF48ePR69evdCrVy8EBQW1Oj8RGSeDh1dycjIiIiIQHR2NzMxMeHt7Izg4GOXl5TrnP3z4MObMmYNDhw4hNTUV7u7umDx5Mq5cudLFlRORIRk8vGJjY7Fo0SKEh4dj2LBhSEhIgJ2dHRITE3XO//nnn2PJkiXw8fGBl5cXPvnkEzQ2NiIlJaWLKyciQzLohdl1dXXIyMhAZGSk1GZubo6goCCkpqbqtY7a2lpoNBo4OTnpnK5Wq6FWq6XHlZWVAACNRgONRvMbqu/emi7ira+vN+rtNCWmsk/13TaDhldFRQUaGhrg4uKi1e7i4oLc3Fy91rFq1Sr0798fQUFBOqfHxMRg7dq1zdoPHDgAOzu79hctE5erAcASaWlpuJJt6GqoI5jKPq2trdVrPlkPifPOO+9g586dOHz4MGxsbHTOExkZiYiICOlxZWWl1E/m4ODQVaV2ubPFN4CsUxgzZgy8PXQflZK8mMo+bTo7aotBw8vZ2RkWFhYoKyvTai8rK0O/fv1aXfaDDz7AO++8gx9++AEjR45scT6FQgGFovmQuVZWVrCysnqwwmXA0tJS+mnM22lKTGWf6rttBu2wt7a2hq+vr1Zne1Pne2BgYIvLvffee1i3bh327dsHPz+/riiViLoZg582RkREICwsDH5+fvD390dcXBxqamoQHh4OAAgNDYWbmxtiYmIAAO+++y6ioqKwY8cOqFQqlJaWAgCUSiWUSqXBtoOIupbBwyskJATXrl1DVFQUSktL4ePjg3379kmd+MXFxTA3//cB4ubNm1FXV4c//vGPWuuJjo7Gm2++2ZWlE5EBGTy8AGDp0qVYunSpzmmHDx/WelxUVNT5BRFRt2fwL6kSET0IhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJYYXkQkSwwvIpIlhhcRyRLDi4hkieFFRLLE8CIiWWJ4EZEsMbyISJa6RXjFx8dDpVLBxsYGAQEBSE9Pb3X+Xbt2wcvLCzY2NhgxYgT27t3bRZUSUXdh8PBKTk5GREQEoqOjkZmZCW9vbwQHB6O8vFzn/MePH8ecOXOwcOFCnD59GrNmzcKsWbOQnZ3dxZUTkSEZPLxiY2OxaNEihIeHY9iwYUhISICdnR0SExN1zr9x40ZMmTIFK1aswNChQ7Fu3TqMHj0aH330URdXTkSGZGnIJ6+rq0NGRgYiIyOlNnNzcwQFBSE1NVXnMqmpqYiIiNBqCw4Oxp49e3TOr1aroVarpceVlZUAAI1GA41G8xu3oPuqr6+Xfhrzdhqr2tpa5OXlabVdKLkNdWk+ss9Yo67MUWr39PSEnZ1dV5fYafR9vRo0vCoqKtDQ0AAXFxetdhcXF+Tm5upcprS0VOf8paWlOuePiYnB2rVrm7UfOHDAqHa4Wq3Gr7/+Kj0uqwXUpRbY9UUefr5vMx966CEoFIourpDao6CgAK+++qrOafP/of14/fr1GDx4cBdU1TVqa2v1ms+g4dUVIiMjtY7UKisr4e7ujsmTJ8PBwcGAlXWs06dPIyQkpFn7ezrmPXHiBEaNGtX5RdEDq62txbhx47Taqu+osf/ISQSPfwxK23+/+RjbkVfT2VFbDBpezs7OsLCwQFlZmVZ7WVkZ+vXrp3OZfv36tWt+hUKh8yjDysoKVlZWD1h59zN8+HBkZGRIj6vvqPHdoVQ8OTFQ64UOAF5eXka17cbI0dER/v7+Wm0ajQZVt25g/NgxRr3/9N02g4aXtbU1fH19kZKSglmzZgEAGhsbkZKSgqVLl+pcJjAwECkpKXj55ZeltoMHDyIwMLALKu6+7OzsMHr0aOmxRqPBzYpyBPr7GfULnUyXwU8bIyIiEBYWBj8/P/j7+yMuLg41NTUIDw8HAISGhsLNzQ0xMTEAgGXLlmHChAlYv349nnzySezcuROnTp3C3//+d0NuBhF1MYOHV0hICK5du4aoqCiUlpbCx8cH+/btkzrli4uLYW7+7290jB07Fjt27MDq1avx+uuv4+GHH8aePXswfPhwQ20CERmAmRBCGLqIrlRZWQlHR0fcvn3bqDrs76fRaLB3715MmzaNp41GwlT2qb5/owb/kioR0YNgeBGRLDG8iEiWGF5EJEsG/7SxqzV9PqHvt3jlSqPRoLa2FpWVlUbduWtKTGWfNv1ttvVZosmFV1VVFQDA3d3dwJUQUWuqqqrg6OjY4nST+6pEY2Mjrl69Cnt7e5iZmRm6nE7TdA3n5cuXjforIabEVPapEAJVVVXo37+/1nc872dyR17m5uZ46KGHDF1Gl3FwcDDqF7opMoV92toRVxN22BORLDG8iEiWGF5GSqFQIDo6moMOGhHuU20m12FPRMaBR15EJEsMLyKSJYYXEckSw6sNCxYskIao7ihPPPGE1jDWuqhUKsTFxXXo8xIZE5MKL31Cg+TlzTffhI+Pj6HLaFF3e811t3p+C5MKLyI5qqurM3QJ3ZLJhNeCBQvw008/YePGjTAzM4OZmRkKCgqwcOFCDBw4ELa2tvD09MTGjRt1Lr927Vr06dMHDg4OeOGFF/R+QdXU1CA0NBRKpRKurq5Yv359s3nKy8sxY8YM2NraYuDAgfj888+bzWNmZobNmzdj6tSpsLW1xaBBg/DVV19J04uKimBmZoYvv/wS48ePh62tLR577DFcuHABJ0+ehJ+fH5RKJaZOnYpr167p+VsDEhMT8eijj0KhUMDV1VXrrk7FxcWYOXMmlEolHBwc8Mwzz2jdlq7pqOjTTz+FSqWCo6Mj/vSnP0kXxwP3rjV97733MGTIECgUCnh4eOCtt96Spq9atQqPPPII7OzsMGjQIKxZs0a6o3JSUhLWrl2Ls2fPSvs0KSlJ723rbA/6mmvqqnjrrbfQv39/eHp6AgCOHz8OHx8f2NjYwM/PD3v27IGZmRnOnDkjLZudnY2pU6dCqVTCxcUF8+fPR0VFRYv1FBUVddWvo+MJE3Hr1i0RGBgoFi1aJEpKSkRJSYm4e/euiIqKEidPnhQXL14Un332mbCzsxPJycnScmFhYUKpVIqQkBCRnZ0tvv32W9GnTx/x+uuv6/W8ixcvFh4eHuKHH34Q//rXv8T06dOFvb29WLZsmTTP1KlThbe3t0hNTRWnTp0SY8eOFba2tmLDhg3SPABE7969xZYtW0ReXp5YvXq1sLCwEOfPnxdCCFFYWCgACC8vL7Fv3z5x/vx5MWbMGOHr6yueeOIJcfToUZGZmSmGDBkiXnjhBb1q//jjj4WNjY2Ii4sTeXl5Ij09XaqpoaFB+Pj4iHHjxolTp06JtLQ04evrKyZMmCAtHx0dLZRKpZg9e7bIysoSP//8s+jXr5/W727lypWiV69eIikpSeTn54sjR46ILVu2SNPXrVsnjh07JgoLC8U333wjXFxcxLvvviuEEKK2tla8+uqr4tFHH5X2aW1trV7b1hV+62tu/vz5Ijs7W2RnZ4vbt28LJycn8eyzz4pz586JvXv3ikceeUQAEKdPnxZCCHHz5k3Rp08fERkZKXJyckRmZqaYNGmSmDhxYov11NfXG+JX0yFMJryEEGLChAlaoaHLiy++KJ566inpcVhYmHBychI1NTVS2+bNm4VSqRQNDQ2trquqqkpYW1uLL7/8Umq7fv26sLW1lerIy8sTAER6ero0T05OjgDQLLzuD52AgACxePFiIcS/w+uTTz6Rpn/xxRcCgEhJSZHaYmJihKenZ6t1N+nfv7944403dE47cOCAsLCwEMXFxVLbuXPntLYlOjpa2NnZicrKSmmeFStWiICAACGEEJWVlUKhUGiFVVvef/994evrKz2Ojo4W3t7eei/f1R70Nefi4iLUarXUtnnzZtG7d29x584dqW3Lli1a4bVu3ToxefJkrXVfvnxZABB5eXl61yMXJjeqxP3i4+ORmJiI4uJi3LlzB3V1dc06gL29vbVupx4YGIjq6mpcvnwZAwYMaHHdBQUFqKurQ0BAgNTm5OQknQYAQE5ODiwtLeHr6yu1eXl5oWfPns3Wd/+NdQMDA7VOGQBg5MiR0v+bbh83YsQIrbby8vIWa25SXl6Oq1ev4ve//73O6Tk5OXB3d9caF23YsGHo2bMncnJy8NhjjwG496mpvb29NI+rq6v0/Dk5OVCr1S0+BwAkJydj06ZNKCgoQHV1Nerr62U/ooI+r7kRI0bA2tpaepyXl4eRI0fCxsZGarv/jtpnz57FoUOHoFQqmz1nQUEBHnnkkY7dEAMzmT4vXXbu3Inly5dj4cKFOHDgAM6cOYPw8HBZd5D+5wibTeOV3d/W2NjY5npsbW07vJ77n7+t50hNTcW8efMwbdo0fPvttzh9+jTeeOMNWe8ffV9zPXr0aPe6q6urMWPGDJw5c0br3y+//IL/+q//6qhN6DZMKrysra3R0NAgPT527BjGjh2LJUuWYNSoURgyZAgKCgqaLXf27FncuXNHepyWlgalUtnmaKyDBw+GlZUVTpw4IbXdvHkTFy5ckB57eXmhvr4eGRkZUlteXh5u3brVbH1paWnNHg8dOrTVGh6Uvb09VCoVUlJSdE4fOnQoLl++jMuXL0tt58+fx61btzBs2DC9nuPhhx+Gra1ti89x/PhxDBgwAG+88Qb8/Pzw8MMP49KlS1rz3L9Pu5sHfc3dz9PTE1lZWVCr1VLbyZMnteYZPXo0zp07B5VKhSFDhmj9awrD7v77ag+TCi+VSoUTJ06gqKgIFRUVePjhh3Hq1Cns378fFy5cwJo1a5q9IIB7H1UvXLgQ58+fx969exEdHY2lS5e2OsojACiVSixcuBArVqzAjz/+iOzsbCxYsEBrOU9PT0yZMgV/+ctfcOLECWRkZODPf/6zzqOSXbt2ITExERcuXEB0dDTS09O1Pv3raG+++SbWr1+PTZs24ZdffkFmZiY+/PBDAEBQUBBGjBiBefPmITMzE+np6QgNDcWECRPg5+en1/ptbGywatUqrFy5Etu3b0dBQQHS0tKwdetWAPfCrbi4GDt37kRBQQE2bdqEr7/+WmsdKpUKhYWFOHPmDCoqKrT+uLuDB33N3W/u3LlobGzE888/j5ycHOzfvx8ffPABgH8fYb/44ou4ceMG5syZg5MnT6KgoAD79+9HeHi4FFj316PPUXi3ZehOt66Ul5cnxowZI2xtbQUAkZubKxYsWCAcHR1Fz549xeLFi8Vrr72m1QEcFhYmZs6cKaKiokTv3r2FUqkUixYtEnfv3tXrOauqqsSzzz4r7OzshIuLi3jvvfeadZqWlJSIJ598UigUCuHh4SG2b98uBgwY0KzDPj4+XkyaNEkoFAqhUqm0PqFq6rBv6rwVQohDhw4JAOLmzZtS27Zt24Sjo6Pev7OEhATh6ekprKyshKurq3jppZekaZcuXRJ/+MMfRI8ePYS9vb14+umnRWlpqTRdV2f6hg0bxIABA6THDQ0N4n/+53/EgAEDhJWVlfDw8BBvv/22NH3FihXS7z0kJERs2LBBq/67d++Kp556SvTs2VMAENu2bdN727rCb3nN3e/YsWNi5MiRwtraWvj6+oodO3ZI62xy4cIF8d///d+iZ8+ewtbWVnh5eYmXX35ZNDY26qynsLCwk38DnYdD4siEmZkZvv766w6/VInk6/PPP0d4eDhu377dYX2UcmLynzYSycX27dsxaNAguLm54ezZs1i1ahWeeeYZkwwugOH1mxQXF7faOX3+/Hl4eHh0YUXto+sj9Sbff/89xo8f34XVUFtKS0sRFRWF0tJSuLq64umnn9a6GsHU8LTxN6ivr2/18gqVSgVLy+77/pCfn9/iNDc3N5N9Ryd5YHgRkSyZ1FcliMh4MLyISJYYXkQkSwwvIpIlhhd1qgULFkgD31lZWcHFxQWTJk1CYmJiuy5NSUpK0jnSRmfrjHsYUMdgeFGnmzJlCkpKSlBUVITvv/8eEydOxLJlyzB9+nTU19cbujySK0Nem0TGr6Xr9FJSUgQAaSDC9evXi+HDhws7Ozvx0EMPicWLF4uqqiohxL+v0fzPf9HR0UIIIbZv3y58fX2FUqkULi4uYs6cOaKsrEx6nhs3boi5c+cKZ2dnYWNjI4YMGSISExOl6cXFxeLpp58Wjo6OolevXuIPf/iDdL1fdHR0s+c9dOhQp/yeqP145EUG8bvf/Q7e3t7YvXs3AMDc3BybNm3CuXPn8I9//AM//vgjVq5cCQAYO3Ys4uLi4ODggJKSEpSUlGD58uUAAI1Gg3Xr1uHs2bPYs2cPioqKsGDBAul51qxZg/Pnz+P7779HTk4ONm/eDGdnZ2nZ4OBg2Nvb48iRIzh27BiUSiWmTJmCuro6LF++HM8884x05FhSUoKxY8d27S+KWmbo9CTj1tKRlxBChISEiKFDh+qctmvXLtG7d2/psb6jYZw8eVIAkI7aZsyYIcLDw3XO++mnnwpPT09pxAUhhFCr1cLW1lbs37+/zfrJsHjkRQYjhJDGovrhhx/w+9//Hm5ubrC3t8f8+fNx/fp11NbWtrqOjIwMzJgxAx4eHrC3t8eECRMA3LvuFAAWL16MnTt3wsfHBytXrsTx48elZc+ePYv8/HzY29tDqVRCqVTCyckJd+/e1WuAQDIshhcZTE5ODgYOHIiioiJMnz4dI0eOxP/+7/8iIyMD8fHxAFq/Z2FNTQ2Cg4Ph4OCAzz//HCdPnpQGK2xaburUqbh06RJeeeUVaUz+plPO6upq+Pr6Nhs2+cKFC5g7d24nbz39Vt33qmEyaj/++COysrLwyiuvICMjA42NjVi/fr00yuyXX36pNb+u4Ytzc3Nx/fp1vPPOO9KQ3KdOnWr2XH369EFYWBjCwsIwfvx4rFixAh988AFGjx6N5ORk9O3bt8WbehjTsMnGhkde1OnUajVKS0tx5coVZGZm4u2338bMmTMxffp0hIaGYsiQIdBoNPjwww9x8eJFfPrpp0hISNBah0qlQnV1NVJSUlBRUYHa2lp4eHjA2tpaWu6bb77BunXrtJaLiorCP//5T+Tn5+PcuXP49ttvpXH/582bB2dnZ8ycORNHjhxBYWEhDh8+jL/+9a/49ddfpef917/+hby8PFRUVEg3vKVuwNCdbmTcwsLCpK8ZWFpaij59+oigoCCRmJiodd/L2NhY4erqKmxtbUVwcLDYvn17syGsX3jhBdG7d2+tr0rs2LFDqFQqoVAoRGBgoPjmm2+a3ctw6NChwtbWVjg5OYmZM2eKixcvSussKSkRoaGhwtnZWSgUCjFo0CCxaNEicfv2bSGEEOXl5WLSpElCqVTyqxLdDIfEISJZ4mkjEckSw4uIZInhRUSyxPAiIllieBGRLDG8iEiWGF5EJEsMLyKSJYYXEckSw4uIZInhRUSyxPAiIln6fzr1xfrKTyTWAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T20:34:52.334082Z", + "iopub.status.busy": "2024-03-24T20:34:52.333134Z", + "iopub.status.idle": "2024-03-24T20:34:52.562840Z", + "shell.execute_reply": "2024-03-24T20:34:52.561936Z" + }, + "papermill": { + "duration": 0.25185, + "end_time": "2024-03-24T20:34:52.564924", + "exception": false, + "start_time": "2024-03-24T20:34:52.313074", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAToAAAEmCAYAAADyVly8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABmJklEQVR4nO2dd3xT5ff439ndA0oHUChL9pJRwYGjCoqKij/BLSp+FMGBA5Aloh9QEFFEUBRwoyIKXweK/YDK1jJkFJBZVgeU7pH1/P5IkyZN0ialbdr0eb9eeZHc+9x7z+1NDuc85zznKIQQAolEIvFjlL4WQCKRSGobqegkEonfIxWdRCLxe6Sik0gkfo9UdBKJxO+Rik4ikfg9UtFJJBK/Ryo6iUTi96h9LUBdYzabOXPmDKGhoSgUCl+LI5FIKiCEID8/n+bNm6NU1owt1ugU3ZkzZ4iPj/e1GBKJpApOnjxJy5Yta+RcjU7RhYaGApY/YlhYmI+lkUgkFcnLyyM+Pt72W60JGp2is7qrYWFhUtFJJPWYmpxaksEIiUTi90hFJ5FI/B6p6CQSid/T6OboPEEIgdFoxGQy+VoUiaTBolKpUKvV9SKNSyq6Cuj1es6ePUtRUZGvRZFIGjxBQUHExcWh1Wp9KodUdHaYzWaOHTuGSqWiefPmaLXaevG/kUTS0BBCoNfrycrK4tixY7Rq0xaFQkmARuUTeaSis0Ov12M2m4mPjycoKMjX4kgkDZrAwEA0Gg0nTpzgveSD/HogmwX39OaSmJrLj/MUnwcjFi5cSEJCAgEBASQmJrJ9+/ZKx+fk5PDkk08SFxeHTqfjkksu4aeffqpRmWpq2YlE0thRKpWUGEys2nGKgxn5nMz2zZSQTy26r776ivHjx7N48WISExOZP38+gwcP5uDBg0RHRzuN1+v1XH/99URHR7Ny5UpatGjBiRMniIiIqHvhJRJJlRhMZi4U6QF4aGAC13WO8YkcPlV08+bNY/To0YwaNQqAxYsX8+OPP7J06VImTpzoNH7p0qVkZ2ezefNmNBoNAAkJCXUpskQi8RAhBBl5JZjM0CYqmIk3dvKZLD7z0fR6PSkpKSQlJZULo1SSlJTEli1bXB6zZs0aBgwYwJNPPklMTAzdunXjv//9b6VpIKWlpeTl5Tm8JBYeeughbrvttho959VXX80zzzxT6ZiEhATmz59fo9eV+IbKnve5Aj2FpUYUCpgytIvPAhHgQ0V37tw5TCYTMTGOpmxMTAzp6ekujzl69CgrV67EZDLx008/MXXqVN58801effVVt9eZNWsW4eHhtpe/Vi7xRMFIGhYvv/wyvXr18rUY1aJYbyQ9rwSA8EANCVHBPpWnQc26m81moqOj+eCDD+jTpw8jRoxg8uTJLF682O0xkyZNIjc31/Y6efJkHUoskdR/DAZDjZ7PZBakZRcjhCBEpyZE5/vkDp8puqioKFQqFRkZGQ7bMzIyiI2NdXlMXFwcl1xyCSpVuQncuXNn0tPT0ev1Lo/R6XS2SiXVqVgihKBIb/TJSwjhkYwPPfQQv//+O2+//TYKhQKFQsGRI0d45JFHaNOmDYGBgXTs2JG3337b5fEzZsygWbNmhIWF8fjjj7v9W1aksLCQBx54gJCQEOLi4njzzTedxmRmZnLLLbcQGBhImzZt+Pzzz53GKBQKFi1axI033khgYCBt27Zl5cqVtv3Hjx9HoVDw9ddfc+WVVxIYGEi/fv04dOgQf/31F3379iUkJIQbb7yRrKwsj2QHy5xv165d0el0xMXFMXbsWNu+tLQ0hg0bRkhICGFhYdx1110O31WrtfXpp5+SkJBAeHg4I0eOJD8/3zbGbDbzxhtv0L59e3Q6Ha1ateK1116z7Z8wYQKXXHIJQUFBtG3blqlTp9qUzvLly5kxYwa7d++2PdPly5dXeU8HDhzgiiuuICAggC5duvDbb7+hUCj4/vvvHf6WX331FYMGDSIgIIDPP/+c8+fPc/fdd9OiRQuCgoLo3r07X375pcO5PXneAGdziik1mtColMSEBVQpc13gM1Wr1Wrp06cPycnJtnkis9lMcnKywxfOnssvv5wvvvgCs9lsSwE5dOhQrWZeFxtMdJn2S62cuyr2vzKYIG3Vj+jtt9/m0KFDdOvWjVdeeQWAyMhIWrZsyTfffEPTpk3ZvHkzjz32GHFxcdx11122Y5OTkwkICGDDhg0cP36cUaNG0bRpU4cfpDteeOEFfv/9d1avXk10dDQvvfQSO3bscHC3HnroIc6cOcP69evRaDQ89dRTZGZmOp1r6tSpzJ49m7fffptPP/2UkSNHsmfPHjp37mwbM336dObPn0+rVq14+OGHueeeewgNDeXtt98mKCiIu+66i2nTprFo0aIqZV+0aBHjx49n9uzZ3HjjjeTm5rJp0ybA8j20Krnff/8do9HIk08+yYgRI9iwYYPtHEeOHOH777/nhx9+4MKFC9x1113Mnj3b9rebNGkSS5Ys4a233uKKK67g7NmzHDhwwHZ8aGgoy5cvp3nz5uzZs4fRo0cTGhrKiy++yIgRI9i7dy9r167lt99+AyA8PLzSezKZTNx22220atWKbdu2kZ+fz3PPPedy7MSJE3nzzTfp3bs3AQEBlJSU0KdPHyZMmEBYWBg//vgj999/P+3ataN///6AZ887p0hPdlmUNT4yCDXGKp9FnSB8yIoVK4ROpxPLly8X+/fvF4899piIiIgQ6enpQggh7r//fjFx4kTb+LS0NBEaGirGjh0rDh48KH744QcRHR0tXn31VY+vmZubKwCRm5vrtK+4uFjs379fFBcX27YVlhpE6wk/+ORVWGrw+L4GDRoknn766UrHPPnkk2L48OG2zw8++KBo0qSJKCwstG1btGiRCAkJESaTqdJz5efnC61WK77++mvbtvPnz4vAwECbHAcPHhSA2L59u21MamqqAMRbb71l2waIxx9/3OH8iYmJ4oknnhBCCHHs2DEBiA8//NC2/8svvxSASE5Otm2bNWuW6NixY6VyW2nevLmYPHmyy32//vqrUKlUIi0tzbZt3759Dvcyffp0ERQUJPLy8mxjXnjhBZGYmCiEECIvL0/odDqxZMkSj+QRQog5c+aIPn362D5Pnz5d9OzZ0+Pjf/75Z6FWq8XZs2dt29atWycA8d133wkhyv+W8+fPr/J8Q4cOFc8995wQwrPnXWowir2nc8TukxfE2ZwiIYTr31RVVPYbrS4+dZ5HjBhBVlYW06ZNIz09nV69erF27VpbgCItLc0heTc+Pp5ffvmFZ599lh49etCiRQuefvppJkyYUGsyBmpU7H9lcK2dv6prXwwLFy5k6dKlpKWlUVxcjF6vd5rc7tmzp8MqkAEDBlBQUMDJkydp3bq123MfOXIEvV5PYmKibVuTJk3o2LGj7XNqaipqtZo+ffrYtnXq1Mll3uOAAQOcPu/atcthW48ePWzvrd+R7t27O2xzZS1WJDMzkzNnznDddde53J+amkp8fLxD4KpLly5ERESQmppKv379AEv02L4KblxcnO36qamplJaWur0GWPJI33nnHY4cOUJBQQFGo/GiisEePHiQ+Ph4h6kfqzVWkb59+zp8NplM/Pe//+Xrr7/m9OnT6PV6SktLbd+Nqp63EIKT2cWYzIIgrZroeuKyWvH5LOHYsWPduqr2boKVAQMGsHXr1lqWqhyFQuGR+1jfWLFiBc8//zxvvvkmAwYMIDQ0lDlz5rBt2zZfi1ZtrLmTUF59tuI2s9lc5XkCAwNrXJ6K16/qGlu2bOHee+9lxowZDB48mPDwcFasWOF23qumCQ52jILOmTOHt99+m/nz59O9e3eCg4N55plnPJ6vzcwvpVBvRKVQEN8kEGU9WyPeoKKuEvdotVqHfMJNmzYxcOBAxowZQ+/evWnfvj1HjhxxOm737t0UFxfbPm/dupWQkJAq03DatWuHRqNxUJwXLlzg0KFDts+dOnXCaDSSkpJi23bw4EFycnKczlfxP6+tW7c6zM/VJKGhoSQkJJCcnOxyf+fOnTl58qRDhH7//v3k5OTQpUsXj67RoUMHAgMD3V5j8+bNtG7dmsmTJ9O3b186dOjAiRMnHMZUfKZV0bFjR06ePOkQNPnrr788OnbTpk0MGzaM++67j549e9K2bVuHZ1nZ8zaYzGSWpZK0iAxEp/Zdvpw7Gp6pInFJQkIC27Zt4/jx44SEhNChQwc++eQTfvnlF9q0acOnn37KX3/9RZs2bRyO0+v1PPLII0yZMoXjx48zffp0xo4dW+V635CQEB555BFeeOEFmjZtSnR0NJMnT3Y4rmPHjgwZMoT//Oc/LFq0CLVazTPPPOPS2vnmm2/o27cvV1xxBZ9//jnbt2/no48+qpk/jgtefvllHn/8caKjo7nxxhvJz89n06ZNjBs3jqSkJLp37869997L/PnzMRqNjBkzhkGDBjm5fO4ICAhgwoQJvPjii2i1Wi6//HKysrLYt28fjzzyCB06dCAtLY0VK1bQr18/fvzxR7777juHcyQkJHDs2DF27dpFy5YtCQ0NRafTub3m9ddfT7t27XjwwQd54403yM/PZ8qUKUDV/Rc6dOjAypUr2bx5M5GRkcybN4+MjAybYq/seeeXGBFAZJCWiCDflmNyh7To/ITnn38elUpFly5daNasGYMHD+aOO+5gxIgRJCYmcv78ecaMGeN03HXXXUeHDh246qqrGDFiBLfeeisvv/yyR9ecM2cOV155JbfccgtJSUlcccUVDvNxAMuWLaN58+YMGjSIO+64g8cee8zlOuYZM2awYsUKevTowSeffMKXX37psfVUHR588EHmz5/Pe++9R9euXbn55pv5999/AYtSWL16NZGRkVx11VUkJSXRtm1bvvrqK6+uMXXqVJ577jmmTZtG586dGTFihG0O79Zbb+XZZ59l7Nix9OrVi82bNzN16lSH44cPH86QIUO45ppraNasmVO6R0VUKhXff/89BQUF9OvXj0cffZTJkycDFsVbGVOmTOHSSy9l8ODBXH311cTGxjqtmqn4vC+//HK69uiFWQi0aiXNIypx1w/9Aj+9AIaSSuWoLRRCeJis5Sfk5eURHh5Obm6u08RvSUkJx44do02bNlV+MSQ1h0Kh4Lvvvqvx5WgSi0t6xRVXcPjwYdq1a1ej584u1HPqQhEKFLSLDnY5l11SUsKxI//S5rdHCbhwAG6cA4mPVXreyn6j1UW6rhKJH/Hdd9/Zpi4OHz7M008/zeWXX17jSq7EYOJMjmVuNyZc5z5gJ8xQeB4MhdBqIPR9uEbl8BTpukpckpaWRkhIiNtXWlqar0WslMpk//PPP30tXrX4/PPP3d5T165dAcjPz+fJJ5+kU6dOPPTQQ/Tr14/Vq1fXqBxmITiZXYS5bIlXsxD384YUZIGpFLRhMHwJqHxjW0nX1Q7pupZjNBo5fvy42/0JCQmo1fXXITh8+LDbfS1atKixFJO6JD8/32nJpBWNRlNp3mNNcianmHMFpaiVCjrEhKJRubGXSvIoyTzMsdNZtAkzE9D1Ro/OL11XSZ2hVqtp3769r8WoNg1ZdneEhoY6JCj7gvwSA+cKSgFoGRnkXsmZDJBTli6jC4V2l9aRhK6Rik4ikXiEwWTmZLZlXq5piI6wQI3rgULAhRNgNoIqAOqBdyTn6CQSSZUIITh1oRij2UyARkVcZUu8CjJBnw8KJYS1gHqwSkJadBKJpErOFejJLzGgVCho1SQIpdKN8tIXQv5Zy/uwFqCqJFBRh0iLTiKRVIp9teC48AD3JdHNRrhwHBAQEAFBTetKxCqRik4ikbjFvlpwWICGJsFulngJATknwaQHlRYi4uuFy2pFKjqJU7Ma+4q0ksbN2dzyasEtIwPdr5ktOg8lOYACIhNAWb9mxeqXNJJ6wdmzZ4mMjPS1GBIfk1ukJ7vQWi04ELW7VBJDCeSetrwPjQOtbxvhuEIqOokT7np21CVCCEwmU71OSvZn9EYzp8qWeEWH6ggJcJNKYjaXzcuZQRsKIc4FG+oD0nWtCiEskSRfvLxYtJKfn8+9995LcHAwcXFxvPXWW9VugeiqmcqqVau45pprCAoKomfPnk69dzdu3GhrXBMfH89TTz1FYWGhbf+nn35K3759CQ0NJTY2lnvuucehGvCGDRtQKBT8/PPP9OnTB51Ox8aNG72WXXLxiLIlXh5VC847DcZii6sa2bpezcvZI/+7rApDEfy3uW+u/dIZj92A8ePHs2nTJtasWUNMTAzTpk1zalxyMUyePJm5c+fSoUMHJk+ezN13383hw4dRq9UcOXKEIUOG8Oqrr7J06VKysrJslaOXLVsGWFrqzZw5k44dO5KZmcn48eN56KGH+OmnnxyuM3HiRObOnUvbtm2l++wjPK4WXJwDRecs7yNag8qN1VcPkIrOD8jPz+fjjz/miy++sPUosNaBqymef/55hg4dClhqx3Xt2pXDhw/TqVMnZs2axb333muzHjt06MA777zDoEGDWLRoEQEBATz8cHnVirZt2/LOO+/Qr18/CgoKCAkJse175ZVXuP7662tMbol3FJYabdWCm1dWLdioh5yywg7B0RBQM2tSawup6KpCE2SxrHx1bQ84evQoBoPBoRFKeHi4Q6Oai8W+MU1cXBxgaTLTqVMndu/ezT///OPQs1UIgdls5tixY3Tu3JmUlBRefvlldu/ezYULF2y9FdLS0hwKbHpawVdS8xjNZk5mF9mqBUe6qxYsBOQcB2GyfEfD4upSzGohFV1VKBT1MopU17hqTGNVVgUFBfznP//hqaeecjquVatWFBYWMnjwYAYPHsznn39Os2bNSEtLY/DgwU7NVyo2bZHUDUIITl8oRm8yV10tOD/dMoesUFpSSRT1f6pfKjo/oG3btmg0Gv766y9atWoFQG5uLocOHeKqq66q9etfeuml7N+/323FkD179nD+/Hlmz55ta7rz999/17pcEs+5UGQgt9iAAssSL5W7JV6l+VCQbnkfHg/q+rHEqyrqvyqWVEloaCgPPvggL7zwAuvXr7c1YFEqlVU2RakJJkyYwObNmxk7diy7du3i33//ZfXq1bY2lq1atUKr1bJgwQKOHj3KmjVrmDlzZq3LJfGMUk+rBZuMlqokAIFNIKhJHUl48dQLRbdw4UISEhIICAggMTGR7du3ux27fPlyFAqFw6uxF8kEmDdvHgMGDODmm2+2NS7p3LlznfxtevTowe+//86hQ4e48sor6d27N9OmTbMFQ5o1a8by5cv55ptv6NKlC7Nnz2bu3Lm1LpekasxCkOZJtWAhLPXlzAbLQv3wlnUr6MUifMyKFSuEVqsVS5cuFfv27ROjR48WERERIiMjw+X4ZcuWibCwMHH27FnbKz093ePr5ebmCkDk5uY67SsuLhb79+8XxcXF1b6f+kJBQYEIDw8XH374oa9FkdRjzlwoErtPXhD7TucIvdHkfmB+phCndwhxeqcQpYUen786v6nKfqPVxecW3bx58xg9ejSjRo2iS5cuLF68mKCgIJYuXer2GIVCQWxsrO0VExNThxLXT3bu3MmXX37JkSNH2LFjB/feey8Aw4YN87FkkvpKfomBLE+qBeuLLInBYCm9pPUsG6A+4VNFp9frSUlJISkpybZNqVSSlJTklHlvT0FBAa1btyY+Pp5hw4axb98+t2NLS0vJy8tzePkrc+fOpWfPniQlJVFYWMiff/5JampqpY1iJI0Tj6sFm03lpZd04RAcVWcy1iQ+jbqeO3cOk8nkZJHFxMRw4MABl8d07NiRpUuX0qNHD3Jzc5k7dy4DBw5k3759tGzpPG8wa9YsZsyYUSvy1yd69+5NSkqK0/bi4mJ27dpV9wJJ6i3Cm2rBuacsXbyUGohoVW+XeFVFg0svGTBgAAMGDLB9HjhwIJ07d+b99993GcmbNGkS48ePt33Oy8uzpTg0BgIDA/2yUYyk+pz3tFpwUTYUZ1veRyb4rFVhTeBTyaOiolCpVE4t3DIyMjyuoKHRaOjdu7fb9nY6nQ6dzrtcH9G4OkBKGhHFeiNnPakWbCyF3JOW9yGxoKveNEd9+S35dI5Oq9XSp08fkpOTbdvMZjPJyckOVltlmEwm9uzZY1uWdDFYs/+Lioou+lwSSX3D82rBZaWXhNmyKii0+mW7rL8l+5U1vsDntuj48eN58MEH6du3L/3792f+/PkUFhYyatQoAB544AFatGjBrFmzAMui78suu4z27duTk5PDnDlzOHHiBI8++uhFy6JSqYiIiLCVDwoKCqqThFuJpC5Izy2mpMSAWqkkKlBLaWmp64H5GVBcCKggLBbcjasEIQRFRUVkZmYSERGBSuXGcqwjfK7oRowYQVZWFtOmTSM9PZ1evXqxdu1aW4AiLS0NpbLc8Lxw4QKjR48mPT2dyMhI+vTpw+bNmx0Whl8MVpfZvlaaRNLQKdabOF+oRwFEhWg5WehG8RhKoLDsux/cDApPX9R1IyIi6kUhV4WoL050HZGXl0d4eDi5ubmEhbkvLWMymTAYDHUomURSO2TmlfDoJ39TWGrk7v6tePTKtq4HFp6DFfdYAhDd/h9cPeGirqvRaKplyXn6G/UGn1t09RWVSuVzc1siuVhMZsHz3+3k0LlSesZH8J9rO7lODDab4esnIWs3xHSD6yaAxn+WVvp8ZYREIqk9Fq4/zPZj2YTo1Lwzspf71Q+b5sPRDZb6cncu9SslB1LRSSR+S8qJbN5O/heAmbd1pXVTN7X+Tv4F/3vV8v7GN6BZzRVsrS9IRSeR+CG5xQae+nIXJrPg9t4tuL23m2ojxTnw7cOWasHdhkPv++pUzrpCKjqJxM8QQjD5uz2czimmVZMgXhnW1d1A+OEZS++HiNZw81sNdolXVUhFJ5H4GStTTvHDP2dRKxW8c3dvQt31ZN3xCez7ztKq8M5lEBBet4LWIVLRSSR+xNGsAqavsVTzefb6S+gVH+F6YOYB+LksfeTaqdCyT90I6COkopNI/AS90czTK3ZRpDcxoG1THh/UzvVAQzGsfNjSeLrdtTDQuamRvyEVnURSDSat+od7P9yK3mj2tSg25v56kD2nc4kM0vDWiF7uG9z8OgUy91lWPty2GJT+rwb8/w4lkmoghOBQRj4ms/PCob2nc/ly+0k2HT7P3jO5PpDOmT8OZfHBH0cBeH14D2LD3eTBpf4f/PWh5f3t70No46jOLRWdROKCDQezuOGtPxj7xQ6nUkOfb0uzvT+SWVDXojlxrqCU8V/vBuD+y1pzQ1c3a0tzTsLqJy3vBz4F7a+rIwl9z0UpuoKCgkZTplzSuDhcpsB+3pvOT3vSbdvzSwys3lW+0P1IVmGdy2aPEIIXvtnNuYJSLokJYfLQzq4Hmozw7aNQkgst+lgCEI0IrxXdsWPHGDp0KMHBwYSHhxMZGUlkZCQRERFERkbWhowSSZ1TbDDZ3k9fs5ecIj0A3+86Q5G+fN+RLN9adMs3H2f9wSy0aiXv3N3bfSHN31+Hk1tBFwbDPwK1m1p0forXi/rvu+8+hBAsXbqUmJgYWa9N4pfYK7pzBXrm/nqQmcO68flWSwPnpM7R/Jaa6VNFt/9MHrN+svRWmTK0M51i3VT6OPYn/DHH8v7mt6BJmzqSsP7gtaLbvXs3KSkpdOzof+vhJBIrxWVWW7cWYew9ncf6A1kcGVjIgfR8dGolzw/uyG+pmaSdL8JgMrtfLF+L8o37cgd6k5mkzjHcf1lr1wMLz8Oq0YCwLO/qfmedyllf8Prp9OvXj5MnT9aGLBJJvcGq6Aa0bQrA6ZxiTl2wlAWPCw+gY0woQVoVRrOl031d88oP+zmSVUhMmI437uzh2rMSwhJ8yD8LUZdYFuw3Ury26D788EMef/xxTp8+Tbdu3Zxqwffo0aPGhJNIfIXVdY0LD6RZqI6s/FL2nLKkkgRoVCgUCto2C2bv6TyOZBbQrlnd9chdu/csX25PQ6GAt+7q5b73w7b34dDPoNJZSi9p3VQvaQR4reiysrI4cuSIracDgEKhQAiBQqHAZDJVcrRE0jCwBhwCtSouiQkhK7+U3WWKLkhrmfBv1yzEoujqMPJ6JqeYCd/uAeDxQe0Y2N5NQ+mzu2FdWWT1hlchtnsdSVg/8VrRPfzww/Tu3Zsvv/xSBiMkfktJmUUXpFXRITqUTYfP88+pHMCi/ACbFXe0jgISJrPg2a92kVtsoGfLcMZff4nrgaUFliVeJj10HAr9R9eJfPUZrxXdiRMnWLNmjWyKLPFrrK5rgEbFJTGhAGTmW7phBZalcLRtZnEF6yry+t76w2w7lk2wVsU7d/d2HwD56QU4fxjCWsCwd/229JI3eB2MuPbaa9m9e3dtyCKR1DlHsgq46o31fL7thMN2azAiUKOiY6zj/Fug1mIfWC26I1mFtd6oOeXEBebbqgV3c18t+J+vYfcXoFDCHUsgqEmtytVQ8Nqiu+WWW3j22WfZs2cP3bt3dwpG3HrrrTUmnERS27yT/C9p2UVM/m4v9yaWp2hYLbpArYr20aEOxwRqLPZBm6hgFApLNd/zhXqiQnS1ImNeiYGnV+zEZBbc1qs5d1zqplrw+SPww7OW94MmQMLltSJPQ8RrRff4448DlkbSFZHBCElDI0BdvpLAGlADR4suPFBDbFgA6XklAASVWXQBGhUtIwM5mV3MkcyCWlF0lmrBezl1wVIteOZt3VwPNOrh20dAXwCtL4erXqhxWRoyXruuZrPZ7UsqOUlDIy6ivMpHTlF5H197iw6gQ0y5+2q/zMoWkDhXO5HXb3ec5v92n0GlVPD2yF7uqwUnz4AzOyEw0uKyKmWrTnu8UnQGgwG1Ws3evXtrVIiFCxeSkJBAQEAAiYmJbN++3aPjVqxYgUKh4LbbbqtReSSNBwXlE/X2y77sLTqAjjHl7mugnaJrG1U2T1cLVUyOnStk2mrLb2389ZfQu5WbteT/roMt71reD1sI4S1qXJaGjleKTqPR0KpVqxq13L766ivGjx/P9OnT2bFjBz179mTw4MFkZmZWetzx48d5/vnnufLKK2tMFknjo8RY/l221p4zmszoTZaCmtacuUvsFJ11G0C76NqJvOqNZp76cidFehOXtW3ivlpwfjp8Z5lOov9j0GlojcrhL3jtuk6ePJmXXnqJ7OzsGhFg3rx5jB49mlGjRtGlSxcWL15MUFAQS5cudXuMyWTi3nvvZcaMGbRt27ZG5JA0TkoN5RWCzWWR0xK7qsFWN9XBddU6u641nTT8Zlm14IggDfNH9HZdLdhshu/+A0XnIKY7XD+zRmXwJ7wORrz77rscPnyY5s2b07p1a4KDHcPcO3bs8Phcer2elJQUJk2aZNumVCpJSkpiy5Ytbo975ZVXiI6O5pFHHuHPP/+s9BqlpaWUlpbaPsuaeRJ7Sl1YdEV6I2BJP9OpLbZABzeuq1XRnbpQRInB5L5Mkhf8+W8W75dVC36jsmrBm+bD0Q2gCbIs8dK4GSfxXtHV5HzYuXPnMJlMxMQ4lnOOiYnhwIEDLo/ZuHEjH330Ebt27fLoGrNmzWLGjBkXK6rETyk1urDo9GVua9maVoAQnZoWEYGczil2cF2jQrSEBqjJLzFy4nwRHWMdU1G85bxdteD7Lmvlvlrwye3wv1ct7298A5q5WSUhAaqh6KZPn14bcnhEfn4+999/P0uWLCEqys0avwpMmjSJ8ePH2z7n5eURHx9fWyJKGhj2iq5sWs4p4mrllp7N+eqvNLq3KO9/qlAoaNcshF0ncziSVXBRik4IwQsr/yErv5QO0SFMGdrF9cDiHFj5CAgTdLvTUn5JUileKzorKSkppKamAtC1a1d69+7t9TmioqJQqVRkZGQ4bM/IyCA21vl/siNHjnD8+HFuueUW2zaz2fLtVKvVHDx4kHbtHCdtdTodOl3tJHJKGj6lBveua0U3dOKNnXhxcEeUFebLbIruIiOvH28+zv8OZKJVK1lwj5tqwULA/z0NuWkQmWAppCmXeFWJ14ouMzOTkSNHsmHDBiIiIgDIycnhmmuuYcWKFTRr1szjc2m1Wvr06UNycrLNJTabzSQnJzN27Fin8Z06dWLPnj0O26ZMmUJ+fj5vv/22tNQkXuPKdbVZdC4UTUUlBzUTeU09m8d/f7ZM10y+qZJqwTs+hv3fg1INw5dCgJtxEge8jrqOGzeO/Px89u3bR3Z2NtnZ2ezdu5e8vDyeesr7Rrjjx49nyZIlfPzxx6SmpvLEE09QWFhoKwP1wAMP2IIVAQEBdOvWzeEVERFBaGgo3bp1Q6ttXHXwJRePq2CEfeUST7Dm0lU3adhSLXgneqOZpM7RPDDATbXgzAPw80TL++umQcs+1bpeY8Rri27t2rX89ttvdO5c3m2oS5cuLFy4kBtuuMFrAUaMGEFWVhbTpk0jPT2dXr16sXbtWluAIi0tDWUjaLAr8Q0Oc3RWi64sGOFpBLW91aLLLHBYRuYpM3/cz+HMAqJDdbxxZ0/XxxuKYeUoMBZDu+tgwDivrtHY8VrRmc1mp4X8YEkmts6XecvYsWNduqoAGzZsqPTY5cuXV+uaEglUyKOrMEdXMRjhjlZNglEpFRTqTWTklbpPB3HB2r3pfLGtrFrwiEqqBf8yGTL3Q3A03L4Y5H/+XlGtMk1PP/00Z86csW07ffo0zz77LNdd518NcYv0Ro7X0hpGSf3A3nU1VtN11aqVtG4SBHg3T3c2t5iJq/4B4D9XteNyd9WC96+Bvz+yvL99MYREe3wNiQWvFd27775LXl4eCQkJtGvXjnbt2tGmTRvy8vJYsGBBbcjoE9YfzKTfq7/x3Dey9p4/sPnwOS77bzLrDzouLXQIRpgdgxHeJP9ai3B6Wm3YZBY8s2IXOUUGelRWLTgnDdaUeTuXPw3t/cuYqCu8dl3j4+PZsWMHv/32my2pt3PnziQlJdW4cL6ka1wYxQYTKScucDSrgLZ12PxEUrMIIbjnw20AJKdmcE3HcouoxOA8R1ekdx91dUe7ZiFlfV498wAWbbCrFjyyN1q1C5vDZIRvR0NJLrToA9dO9VgeiSPVyqNTKBRcf/31XH/99TUtT70hOiyAQZc0Y/3BLFbtOM3zg2Uf24aKffHfg+n5DvtcRV0rSy9xR/ma16otupQTF3jrN0u14FeGdSMhyk214N9nw8mtoAuD4R+Byk2JJkmVVEvRJScnk5ycTGZmplMAorLF+A2N4X1asv5gFt/uOMWz11/iemG1pN5jttN0BpNjyXPXS8C8m6MDu1y6KpKG7asFD+vVnDsudVNS6dgf8Mdcy/tb5kOTNh7LInHG6zm6GTNmcMMNN5CcnMy5c+e4cOGCw8ufSOocQ1iAmrO5JWw5ct7X4kiqidlOtxlM5YpNCIHexRIwq+sa4IWis+bSncktsUVtKyKEYEpZteD4JoG8els316kkhedh1WOAgN73Q7fhHsshcY3XFt3ixYtZvnw5999/f23IU68I0Ki4tVdzPtuaxsqUk1zRwbP1tZL6haNFV67Y7K05uDjXNTJYS5NgLdmFeo5mFdLNbj2slVU7TrPGVi24t+tqwULA6jGQfxaiLoEbX/dYBol7vLbo9Ho9AwcOrA1Z6iV39rEsK1u7L538EkMVoyX1EeFg0ZV/qKjobK6rl+klVtpV0v6wWG9i+pp9gKVa8KXuqgVvWwyH1oJKB3cuA62b+TuJV3it6B599FG++OKL2pClXtKzZTjtmgVTYjDz056zvhZHUg1Mbi06x0rZ5Yv6vU8vgcqLcJ7NLaag1EiwVuW+WvDZ3bBumuX94Ncg1k0jHInXeO26lpSU8MEHH/Dbb7/Ro0cPp1US8+bNqzHh6gMKhYI7+8Tz+toDrEw5xYh+rXwtksRL7F1Xo71FZ3Bt0VXHdYXKI6+5xRZvICJI6zqoVVoA34wCkx463Qz9HvXq2pLK8VrR/fPPP/Tq1QvAqUmOt2v8Ggq3927BnF8O8NfxCxw/V+g+HUBSLxF2+syjOTpb1NW7n0d50rCzRZdXYglQhAe6SRH56QXIPgJhLeDWBbL0Ug3jtaJbv359bchRr4kND+CKDs3441AWq3acYvwNMqeuIWFv0ekrcV0rLgEL1Ho3s2NrfZhVgNksHEo6WS26sEAXP7ndX8HuL0ChhOEfQlATr64rqRq5MthD7uxj6Y7+7Y7TtqVCkoaBW9e1YjDiIufoWkYGolUpKTWaOZ1T7LAvr0zROVl054/Aj2UVsAdNgNaNJ9BXl0hF5yE3dIkhNEDN6Zxith6VOXUNCZMbi67EYHI5rrpzdGqVkoQo14v7bRadfUqJUQ8rHwZ9AbS+Aq56wavrSTxHKjoPCdCouKVncwBW7jjlY2kk3mCfXmIyV23RlaeXeL9wyNbQusI8XV6JC4sueQac3QWBkXDHB6C8+A5iEtdIRecFwy+1uK8/70mnoNR19ruk/mHvutpTMepqMgsMJrMt185biw7Kl4JVrGKSZ5ujK1N0/66DLe9a3g97D8LdLAWT1AheK7o//vgDo9H5R240Gvnjjz9qRKj6yqWtImgbFUyxwSRz6uoh6w9msnD9YUQFxeZuStUpj06Uu60AAV4GI8B9iklesV3UNT8dvnvcsqP/f6DTTV5fR+IdXj/Ja665huzsbKftubm5XHPNNTUiVH1FoVAwvCwosTJFuq/1iRKDiXFf7GTOLwdJPetYocRd8MiV62pd0K9SKtCqLkbRObqu5XN0Sss61qJzENMdrn/F62tIvMfrJ+muJv758+cJDvb//LI7Lm2BQgHbj2WTdr7I1+JIykhOzbRNJxRWWFRf0XO1Kj6nPDohHGrRVScv1JpLl5VfapuXg/I5uh7Hl8Gx30ETBP9vGWg8L7suqT4ez7becccdgMWqeeihhxx6pZpMJv75559GsQY2LjyQK9pH8ee/52zlmyS+Z/Wu07b39knB4Bh1BUvkNUCpcujpCpY5uupUF7YnNEBDdKiOzPxSjmYV0is+ArBYdJcqDtF2z9uWgTfNgagO1bqGxHs8tujCw8MJDw9HCEFoaKjtc3h4OLGxsTz22GN89tlntSlrvaE8p+6UzKmrB+QWGdhwMMv22WiqOEfnrOjAtetaXM0F/fbY3Fe72nTmohze0b6LQpig253Q695qn1/iPR5bdMuWLQMgISGB559/vlG4qe64oUssoTo1py4Us/14Npe1beprkRo1P+8965AfZ6xQDLZicMJag86V61pcjTLqFWkXHcyWo+dtAQlhNjPJtIiWynMYw1ujvvktucSrjvF6jm769OmNWsmBpQ3e0B5xgAxK1AdW7zrj8LliFeGKRne5onPhulaj6GZFKkZeS7cv4yblNgxChemOjyAgrNrnllQPrxVdRkYG999/P82bN0etVqNSqRxe1WHhwoUkJCQQEBBAYmIi27dvdzt21apV9O3bl4iICIKDg+nVqxeffvppta57MVjd15/2nKVQ5tT5jPTcErYes6xUaRkZCHjguloVnYs8uvJVEdVPMW1rH3nNTEW37iUA3jSPRNuqb7XPK6k+Xqd+P/TQQ6SlpTF16lTi4uIuumLJV199xfjx41m8eDGJiYnMnz+fwYMHc/DgQaKjnftXNmnShMmTJ9OpUye0Wi0//PADo0aNIjo6msGDB1+ULN7Qp3UkCU2DOH6+iJ/3ptsUn6Ru+eGfMwgBfVtHotMoOXWh2Ml1rdhX3d0cnUnYz9FVq50KUF6AM/38BcQ3L6IwlfC7qQffam9jonRZfYLXT3Pjxo38+eeftlJNF8u8efMYPXo0o0aNAiyl2n/88UeWLl3KxIkTncZfffXVDp+ffvppPv74YzZu3Finik6hUDD80pa8ue4Q36ackorOR1jd1mG9mvNbqqVnq7Pr6s6ic3RdzeaamaNrHh5IgEbJRPEJiqwD6AOieC7nCcIitNU+p+Ti8No+j4+Pd5rcrS56vZ6UlBSHnrBKpZKkpCS2bNlS5fFCCJKTkzl48CBXXXWVyzGlpaXk5eU5vGqKO/q0RKGALUfPczJb5tTVNUeyCthzOheVUsFN3ePQqCzWktHkuqCmldIKwQhrhNVkrl7z6ooolQruD9vNfepkAP7p/wbnCHdfi05S63it6ObPn8/EiRM5fvz4RV/83LlzmEwmYmJiHLbHxMSQnp7u9rjc3FxCQkLQarUMHTqUBQsWuO0xO2vWLIdUmPj4+IuW20qLiEAGtrNEXFftOF3FaElNs6bMmruqQxRNQ3SolZavs8HsXTDCqujMdlHXi0kvISeNZ4sXALCz1UMcC+sPVKhcIqlTvFZ0I0aMYMOGDbRr147Q0FCaNGni8KoLQkND2bVrF3/99RevvfYa48ePZ8OGDS7HTpo0idzcXNvr5MmTNSqLfU5dTVm6Es/YdTIHgOs6W/6jVJdZdAY3TW+sGEwVLTrLDI5DMKK6is5khG9HE2QuYKe5PV+HPmirLhwmLTqf4fUc3fz582vs4lFRUahUKjIyMhy2Z2RkEBsb6/Y4pVJJ+/btAejVqxepqanMmjXLaf4OQKfTOaziqGkGd40lWLuXtOwi/jp+gf5tZHXYusIadAgNsHyNNWVrUz3OozNUcF3tLLpqu66/z4aTWzGoQxhXOJbYcyU0i7CWaKp+gENycXj9l3/wwQdr7OJarZY+ffqQnJzMbbfdBoDZbCY5OZmxY8d6fB6z2UxpaWmNyeUNQVo1Q3vE8fXfp1iZclIqujrEmkZibTajLvu3YjDC5Dbq6mi9mc0CfZmSrFYw4ujv8MdcANIHzebUj00ozCqw9XiVrqvvqFay0JEjR5gyZQp33303mZmWSNfPP//Mvn37vD7X+PHjWbJkCR9//DGpqak88cQTFBYW2qKwDzzwAJMmTbKNnzVrFuvWrePo0aOkpqby5ptv8umnn3LfffdV51ZqBGvv1x//Oeu2S7uk5rEW0bQqOLXVovM06lr2b7C961rdObrCc5aqJAi49AGiEu8B4EKRgePnLZVMZDDCd3it6H7//Xe6d+/Otm3bWLVqFQUFluzv3bt3M336dK8FGDFiBHPnzmXatGn06tWLXbt2sXbtWluAIi0tjbNny2u/FRYWMmbMGLp27crll1/Ot99+y2effcajj/quPVy/hEhaNQmiUG/il33ugyiSiyOnSM+7//uXUxcsEW5rMxtVWRBCa426Vsyjq0LRBdq7rtUpoy4EfD8GCtIhqiMMeZ1ArYoWEZYEZutcopyj8x1eK7qJEyfy6quvsm7dOrTa8ryga6+9lq1bt1ZLiLFjx3LixAlKS0vZtm0biYmJtn0bNmxg+fLlts+vvvoq//77L8XFxWRnZ7N582ZGjBhRrevWFNacOpBLwmqTF1b+w9xfD3H/R5aVM+4suoqua8UYUanJMY+uPL2kmkvAti6Cf38Blc5Seklr6RthLdmUU+SmMY6kzvBa0e3Zs4fbb7/daXt0dDTnzp2rEaEaIndcaimFvfnIeZvFIalZ1u23BK2OnbO4guUWnVXReZZHV9Gis1d0RVbl56lFd2YXrJtmeT/4NYjpattlXfNqRc7R+Q6vFV1ERISDK2ll586dtGjhZ3XvD/0CaZ5ZqfFNghjQtilCwHcyp65OMJW5qFYFp1Fao66e5tFZAw/qsnHlFYY9Si8pzbd08TIboNPN0M9x+qRdtKOikxad7/Ba0Y0cOZIJEyaQnp6OQqHAbDazadMmnn/+eR544IHakNE37FkJX9wFKx+BIufS8a4YLnPq6hRr0MGaKGzLo6to0ZndWXQuXFdvVkb89AJkH4GwlnDrAqfSS+2iHKv8uGxeLakTvFZ0//3vf+nUqRPx8fEUFBTQpUsXrrrqKgYOHMiUKVNqQ0bfcMlgaNIW8k7BmnHOEz0uuLFbLEFaFcfPF5Fy4kIdCNm4qei6ajyNuppMZR2/yrp9uVgCVmXUdfdXsPtLUChh+BIIck4rkhZd/cFrRafValmyZAlHjhzhhx9+4LPPPuPAgQN8+umn1S7TVC/RhcKdS0GpgQM/wF8fVnlIsE7NTd1lnbraoGKzaXARjLDm0TlFXR2P0xvNDrXoXC0BqzTqev4I/Dje8n7QRGjtuoVAdKiOEF25FRcq5+h8RrWLbrVq1YqbbrqJu+66iw4d/LT2ffPecP0My/tfJkP63ioPsUZff/znrO1HI7k4cosNdJ621vZZW2EFhKoaeXT2tehcua5u5+iMpbByFOgLoPUVcNXzbuVWKBS2kk2hOrVNTknd49Gkwfjx45k5cybBwcGMHz++0rHz5s2rEcHqDZeNgaMb4N9fLRPPj60HrfsKy4ltmtAyMpBTF4r5dX86w3r5WYDGB/xxKMth5iCgrCimzaKzBiPc5NE5LQEzmW2BCLVSYXN5Swwm2zndztH9NgPO7obAJnDHB6Cs3Itp2yyE3adyZQ6dj/FI0e3cuRODwWB7746LLcJZL1Eo4LZFsOhyOHcQfp4Aw951O1yptOTUvZ38LytTTklFVwvYLDdzhWCE0nUeXcUlYKV2rqtOrbRZWvZtEl3O0R36FbYutLy/7T0Ir/rZWi06qeh8i0eKbv369S7fNxqCoyz/e38yDHZ+Cu2ugW7D3Q63KrqNh89xJqeY5mUZ8pLqUfH/T+t8nclUYY7OXdTVletaZtHpNCqboisoqzJib+XZyDsL3z9ueZ/4OHS80SPZretcrWXeJb6h+oXxGxttB8GVz1ne/98zcOG426GtmgbRv00TS07dTplTV9MUG0wIIWxBh/KoqzVhuPI5OoOpfI5Op1aiKtOkBaVuAhFmE3z3GBSdh9jucP0rHss66JJmLHmgL6/d1s3jYyQ1j0cWnbV5tSesWrWq2sLUe66eBMf/hJPbLPl1D68FlWuX5M4+Ldl+LJtvU04x5up2/unW+wghLO5nxTm6cte14hyd4/H2UdcAjQql1aIrtUzPOAUiNr4Fx/4ATTDcuQzUnpf9UigUXN8lpuqBklrFI4vOvkJvWFgYycnJ/P3337b9KSkpJCcnEx4eXmuC1gtUahj+IQSEw+m/4X+vuh16U/c4AjUqjp4rZEdaTq2Is3rXacZ/vcuWAOuvKHD+T6LUYHafR2cWZOaX2BShqwbWNtfVzqIrMTgu8gcgbRus/6/l/U1zIMpPMwz8HI8U3bJly2yvmJgY7rrrLo4dO8aqVatYtWoVR48eZeTIkURFRdW2vL4nopUlCx5g03w48j+Xw0J0am7sbikeWls5dQvXH2bVjtO26hj+yPoDmfy813nJYaHeaLPUrJac1XXdfTKHxP8mM+X7PUDleXT2wQgrNte1OAe+fRSECbr/P+h1Tw3dlaSu8XqObunSpTz//PMOycEqlYrx48ezdOnSGhWu3tJlGPSx1Mtj1X+gINPlsDvLcup++OeMy4TXi8VqydXGuesDZrNgzOc7+OEfZ0VXYNdL1ymPziwQAvaezrOdxx77PDqdutx1tRKoVVn83TXjIDcNItvA0HnOURFJg8FrRWc0Gjlw4IDT9gMHDmCu2EDTnxkyC5p1hsJM+O5x5+ahwGVtm9IiIpD8EiO/7s9wcZKLw1Rm0lSck/IXzHb14SqSX1Ku6DS2Rf2Oiii32GA7jz2lDlHXctfVSqBGBSnLIHUNKNVw50cQEHZxNyPxKV4rulGjRvHII48wb948Nm7cyMaNG3nzzTd59NFHbVWBGwWaQEvtMXUAHEmGLc65dZacOkuuVW24r1bd6q+KzlTJ+uLCSiw6KzlFesCF62oy26xgnVqJssKvoL1Ig7VlVa2TXoYWfaohvaQ+4XU5hblz5xIbG8ubb75pK9cUFxfHCy+8wHPPPVfjAtZrojtbLLsfnoXkGZBwudOP4o5LW/LO/w6z8d8s0nNLiA0PqLHLWyfb9Sb3CqGhcr6glM1Hzrvdb++6VqxeYiWvxIjJLCrPo1OrHCy6AEp5/NxrYCyB9klw2ZMXfS8S3+O1RadUKnnxxRc5ffo0OTk55OTkcPr0aV588UX/WtTvKX1GWebszEbLErESxwbZCVHB9EuIxFwLOXU219UPo663LNjIuC/dr8LJLzHY3ls9Vk1F06xsnKsuYO6CEVPVn9FcfxxCYuC2xTiZe5IGyUU9xbCwMMLCGvnchUIBt7wD4a0sScQ/POuUuGXt/boy5WSN1qmzWnT+6LqeyS2pdH++3SoGa45iRYsOLGXMXbmutmCERmkLRgxRbudedTJmFHD7+xDS7GJvQ1JPqJaiW7lyJXfddReXXXYZl156qcOrURIYYZmwVqhg70rY9bnD7pu6xxGgUXIkq7BGU0H8WdFVRWHZKgZ7a0zjStEVG2x/JysVXVe1UkELsnhd8wEAW+Putyzzk/gNXiu6d955h1GjRhETE8POnTvp378/TZs25ejRo9x4o2fr//yS+P5wzUuW9z+9AFmHbLtCAzQM6WrJqft2R80FJaxpE6V+6LpWhXUVg9pO0alduJm5xQY3c3R2wQhh4m3tQsIVRewwt+fvNk/UouQSX+C1onvvvff44IMPWLBgAVqtlhdffJF169bx1FNPkZubWxsyNhyueBbaXAWGIst8naHc/bL2fl2zq+Zy6srTS/wvGOGOsABL/MwajLCPtLp2XfVOS8CMdnXndGolsTveoq/yEHkikKcMY9HpPF/iJWkYeK3o0tLSGDjQUlE1MDCQ/Px8AO6//36+/PLLmpWuoaFUwe0fQFBTyNhT3h0KGNCuKc3DA8grMfJbas3k1DVG1zUy2NJi036OzopTxRFcW3T2x7ctSKHpTktq0CTDaE6JaM8a40gaFF4rutjYWLKzLc1iWrVqZevleuzYsWpPtC9cuJCEhAQCAgJITExk+/btbscuWbKEK6+8ksjISCIjI0lKSqp0fJ0TFmeJ1gFsfx8O/ARY5pJuL8up+7aGcurMfp4w7IqIIIuis+bRqRxcV2eLLtdFMAIsiq4JeVx/YCoKBF8ar+FH82WAl82rJQ0CrxXdtddey5o1awBL8vCzzz7L9ddfz4gRI1z2e62Kr776ivHjxzN9+nR27NhBz549GTx4MJmZrpdVbdiwgbvvvpv169ezZcsW4uPjueGGGzh9uh6VQ7rkhvL8q9VjINcim7XM+u+HssjMqzyq6AnleXSNR9FFBlmqxbi06NTOX+ccNxZdYbGeuZrFBOvPoY/swAxjeQc7adH5H14rug8++IDJkycD8OSTT7J06VI6d+7MK6+8wqJFi7wWYN68eYwePZpRo0bRpUsXFi9eTFBQkNt1s59//jljxoyhV69edOrUiQ8//BCz2UxycrLX165VkqZDXE8ovgCrHgOzibbNQujTumZy6oQQNkvFYGw8c3RNyiw66xydym5ezlUeXU6RwWmtK8C1ud9yrWoXRqWO80Pep4TyeTlp0fkfXik6o9HIq6++Snp6um3byJEjeeeddxg3bhxardari+v1elJSUkhKSioXSKkkKSmJLVu2eHSOoqIiDAYDTZo4t5sDKC0tJS8vz+FVJ6h1ltpl2hA4sRH+mAvY59RdXO9X+5SJxuS6Os/RVR6MsMzROW7rpjjKwyUfA7Cv+wRM0Z0d9ktF5394pejUajVvvPEGRqOx6sEecO7cOUwmEzExjoUJY2JiHJRpZUyYMIHmzZs7KEt7Zs2a5VBPLz4+/qLl9pim7SxVLwB+nw3HNzG0Rxw6tZJ/MwvYc7r6UWr7daCNStGVua7W/g7qquboivUOrmswxSzQLECDkbWmfpxtf49zmSbpuvodXruu1113Hb///nttyOI1s2fPZsWKFXz33XcEBLheQzpp0iRyc3Ntr5MnT9atkD1HQM+7QZhh1WjCzPkM7nrxdersi6U0pjk6azDC2t/BXkkpFAonZZdT5LgE7BXNMtooMzgtmjLBMBqdVuVcvUQqOr/D60X9N954IxMnTmTPnj306dOH4GDH1n+33nqrx+eKiopCpVKRkeGYbpGRkUFsbGylx86dO5fZs2fz22+/0aNHD7fjdDqd7/OibpoLJ7dD9hFYPZY7L53Pmt1nWL3rDJOHdkan9v6H5WjRNZ45uogyi85YoYy6FbVKYdsH1mCE5f3tyj8ZrtqISSh4Wv8kuYSUVS9xU3hT4jd4rejGjBkDuO7fqlAoMJk8T4bVarX06dOH5ORkbrvtNgBbYGHs2LFuj3vjjTd47bXX+OWXX+jbt693N+ALdCFw51L4MAkO/sgVba8mNqwd6XklJKdmclP3OK9P6TBH14hWRlRsQ6iqEIDQKJWUUP73yC02YBKCBMVZXtVYAlzzjcP5W3QCnKuXgLTo/BGvXVez2ez25Y2SszJ+/HiWLFnCxx9/TGpqKk888QSFhYW22nYPPPAAkyZNso1//fXXmTp1KkuXLiUhIYH09HTS09MpKCjw+tp1SvNetu5Ryl+n8FjHQqD6OXXmRhiMaB4e4NRYuqKrWtHC0xvNlBYXsUCzgGBFKVtMXVhous22X6dWOkRuQVp0/ojPa9CMGDGCuXPnMm3aNHr16sWuXbtYu3atLUCRlpZmq3sHsGjRIvR6PXfeeSdxcXG219y5c311C55z2RPQYTCYSrnv5AwCKWHDoSwy873PqbN3z/xtjq6yaHRFRVcxkFCx+CbAFWnv0V15nGwRwjOGMZjtvvYBLioMV7yGpOHjsetaXFxMcnIyN998M2CZ5C8tLbXtV6lUzJw5021QoDLGjh3r1lXdsGGDw+fjx497ff56g0Jh6fC+6HK0OYd5J2IFo3MeYvXOM4y+qq1XpzL7cdTV6GopQxkVra2KFl3FcurXKHdy7YVvAHjB8B8ycExD0qlVThVQXC0lkzRsPH6iH3/8Me+//77t87vvvsvmzZvZuXMnO3fu5LPPPqtWwnCjIzgKhi8BFFxf8is3K7dUK6fOMY/Of4IRWfml/JvhehpCoVA4K7oKSsn+czQXmKuxLMdbahxCstm5JLpOo0RpZ9FJt9U/8VjRff755zz22GMO27744gvWr1/P+vXrmTNnDl9//XWNC+iXtLkKrnoegFmaDynKPMy+M94lMvtjwrAQgmvmbuCmd/50O6ZioMDdHJ0SM/M1C2mqyOegog2zjXe7PF9Fi04GIvwTjxXd4cOH6d69u+1zQEAASruIV//+/dm/f3/NSufPDJoI8ZcRqihmgeZdVv193KvD7V1Xf2lgXaQ3OfSCcEWAuvI5OusysCdUaxio2k+h0PG0cRx6NC7Pp1MrsT+FtOj8E48VXU5OjsOcXFZWFgkJCbbPZrPZYb+kClRqGL4EgyaMXsojxO980yuFZWrAwQiDycySP46yv4IVa9/C0B0B2gquakVFp1ZwqeIQz6pXAjDNMIoDBvc5mTq1EoVCYVN2gVqvM64kDQCPFV3Lli3Zu3ev2/3//PMPLVu2rBGhGg0RrVAOs9RCG8Vqdm9Y5fGhDTkY8fHm47z2U6qTi2rf8MYdWpWjBVbRogunkHe076JWmFljvoJvzVe6P1eZkrM/T6BGBiL8EY+f6k033cS0adMoKXFOhSguLmbGjBkMHTq0RoVrDKi6DWNHtKW81SVbnocC1+WpKmJ0SBhuWMEId/OReR5YdBUDEg4WnRCMK3iHlopzHDfH8E7AE4Dz+lcrOruyTtaAhJyj8088VnQvvfQS2dnZdOzYkTlz5rB69WpWr17NG2+8QceOHblw4QIvvfRSbcrqt4QNe4MD5njCTRfQrxztuJDVDf4YjMjzwKIDxzw3h5URKcu4rHQTeqFinGEcAaHhlZ7HfulduUUnXVd/xGNFFxMTw+bNm+ncuTMTJ07k9ttv5/bbb2fSpEl06dKFjRs3OlUhkXhG+xbRvNtkEsVCi/b4BtiyoMpj/HFRf1VzdNYsEHtFZ+v8lbEf1lpW0LxhHMke0ZYmwZWvcba36FTSovNrvJqQaNOmDWvXriUrK4utW7eydetWsrKyWLt2LW3bepfwKnEkMfFyXjHeb/mQ/AqcSql0vL+UaRoy/w/be0/m6MBRGamUCtAXwcpRYCxhT2A/PjJZutFFBVdeH1FnNx+nlHN0fk21nmqTJk3o378//fv3d1vwUuIdt/Rszrck8YMpEcxG+PZhKHGfW+cvCcMH0vNt7z2JugLOc3S/TIKsAxASw/LoCYiyr3WTqhSd2nmuT6aX+Cfyv696QkSQlqQuMbxkeJQcbSxcOA4/PItTr74y7KOuJrNwatLcEKnKoit3Xcu/tt1y10PKckABd3xAibapbV+TkKoUnQuLTqaX+CVS0dUj7uzTkjyCecY4DqFQwd6VsOtzl2MrKraG4L7mFht4/pvdbDp8zuV+Ty066xxdS0UWt5yYbdl4xbPQ9mqH6iVNq7ToXMzRSYvOL5GKrh5xVYdmRIXo2FDUhsNdn7Zs/OkFyDroNLYyRVdYanTZEMbXvPnrQVamnCIz33ViuTeuqxojb2veJcBUAC37wTWWiL99D4nIoMoVnWP01mrRyZ+EPyKfaj1CrVJyR1nv17mFN0KbQWAogpUPg8Exf9FZ0Vk+n80tpuv0X7j3w211I7QXnLpQXOl+b4IRz6i/pY/yX0pUITD8I1BZlnhp7Cw6nUZFiM69K+roulrPLV1Xf0QqunqGtfdr8sFzZA9+F4KiIGMvrJvqMM4kXFt0q3edAWDL0fN1IK13/O9A5cnQVSUMK8qSf7uW7GKMytJbOLnDFIhsbRtj77qqlQrCA12vcQWLIrQiXVf/Riq6ekbH2FC6twjHaBasPmKC2y1lhtj+ARz40TauomtqXSfrohFWvSCnSF/lmKpcV7MQUHiOu0+/ilIh+MJ4DcdjrncYY++6KhVVKDpXwQip6PwSqejqIfa9X+lwPQwoK0q6+knItTS+djdHp1TUT03nSbm9vOLKXVeT0QTfPU6o4RyHzC14xfiA01pXrZ3yUqsUtmY6rnAVjKjYk0LiH0hFVw+5tWdzNCoF+87kkXo2D66bDnG9oPgCrBoNZpNDegmUz9Ep6qmi84Sq5uj+n/H/4PA6jEod4wzjKEHnXI/O7rNSUZWiK1dqXZqHoVUp6RATUk3pJfUZqejqIZHBWq7rZFlO923KKVBrLV3EtCFwYhP8MYeK2SRW19VFs/oGgRCi0lp03RRHGWf+DIDN7Z/joGgFVN4zwtUcnf14+5URb93Vi7+mJNEyMqj6NyGpt0hFV0+xuq/f7zptcUubtoOb37Ls/P11wjO3O4y3rnet2KO0oVCoN+EuIyaYYhZoFqBRmKDzrRxPuMu2r7KeESqlgvBAxxQTrZ0irDhHV9l8nqRhIxVdPWVQx2ZEhWg5V6Dn94NZlo097oKe94Awc+nfLxJB+fIp6xxdQ3VdK3NbZ2qW0UaZwWkRBbe+Q4BdCkhlPSNUSmfX1X4OrzqNwyUNE6no6ikalZJhvSw5dWt2nynfcdMcaNqeoJJ03tB8AFjMIKuiy8j1vnVifcBdxPUO5R/codqIUSh5Sv8kBEY6REadSqmrKlp0mgr7XVt0Ev9GPul6zJUdogDYf9Zucb8uBO5cikmp4QZVCg+ofgXKFd3Xf5+sczk9oaqgqyuLro3iLDM1ywCYbxxOiugIOK5oqCwYoVIqiKig6OyVm05WKmk0yCddj+kYGwrAsXOFlBpN5TvierKr43gAJqu/oLPiBPqyKsNZBZ717Ticmc9/f0rlvIfja5uKycJaDCzQLCBYUcoWUxfeMw2z7avMonNwXV3k0TmsnJCua6PB54pu4cKFJCQkEBAQQGJiItu3b3c7dt++fQwfPpyEhAQUCgXz58+vO0F9QGxYAKE6NSaz4Ni5Qod9qa3u4TdTb3QKA+9q3sFcaumF6ml72Ovf+oMP/jjKhG/31LTYXiOEcHJdJ6hX0E15nGwRwjOGMZjtvqr261HtE4TB2XWtaLXZz9EFSIuu0eDTJ/3VV18xfvx4pk+fzo4dO+jZsyeDBw8mM9P1UqGioiLatm3L7NmziY1139nJX1AoFFxSZtUdOJvvsM+MpfN8uoiknfIsXXa/5tW5rQrxn1M5NSCpa06cL+S9DYerzI8zmYXDmGuVO3hE/TMAzxseJwPHmoeuFuNbsVd8KqXCSRHKYETjxKeKbt68eYwePZpRo0bRpUsXFi9eTFBQEEuXLnU5vl+/fsyZM4eRI0ei01VeJttfuLRVBADLNx9H2JlrRpPgAmE8axiDWShISFsFe1b6SErX3PT2n7yx9iAzf6i8369JCPKKLRZdDNnM1ViWvS01DuF/5kudxlc6R1dhrataVTFYIYMRjRGfPWm9Xk9KSgpJSUnlwiiVJCUlsWXLlhq7TmlpKXl5eQ6vhsToq9oSpFWx62QOP+1Jt223rozYYu7Ku9b5q/97hnhFhi/EdEmh3jKvuO1YNjvTLrgdZ7XolJiZr3mPJooC9poTmG282+V4hwrDlSgypVLhkDcH5Q2uQSq6xoTPnvS5c+cwmUxODXViYmJIT093c5T3zJo1i/DwcNsrPj6+xs5dF0SHBvDYVZZ+HG/8csC2AsJ+revbxuFkhPcCfT4LNO+ixrO6blB1NLQmOHG+iEc+/tvtfouiMzJGtZoBqv0UCssSLz2uE3gdS6k7foXtDTyLRVfhK64oTxrWyQX8jQa//y9t0qRJ5Obm2l4nT9bP9IvKGH1lW5qF6jhxvojPt50AHMs0mVDxW5fXMOvC6aU8wvPqbwDnCif1FbMZorJ38Iz6WwCmGkZxTMS5He/UHMcO+4RppVLh5NoiyufppEXXePDZk46KikKlUpGR4ehqZWRk1GigQafTERYW5vBqaATr1DybdAkA7yT/S16JwUmJXdDEcOzy1wF4XP1/XKn8x2nhf33FVJTNfadnolaYWWW6glXmqyodr6tQocQe++otaqXCwZW10ik2lBCdmhaRgRcpuaSh4DNFp9Vq6dOnD8nJybZtZrOZ5ORkBgwY4Cux6i139W1J++gQLhQZWLThiPOifpPgSNNr+NRomfOcp3kPU37VUwBZ+aUcr5C6UrcIgn55lqamTI6ZY5hqGFXlEQqFwpYaUtGis/+oVDgHIwSCz0cnsmnCtYQFyLWtjQWf2u7jx49nyZIlfPzxx6SmpvLEE09QWFjIqFGWL/sDDzzApEmTbOP1ej27du1i165d6PV6Tp8+za5duzh8+LCvbqHOUKuUTBzSCYClG49x6kKRw36Dycy5Aj2vGu8j1RxPM0Ue6tVPOHa6dsPVczf4zM29V5VMwL8/YkDNOMM4CvHMyrLO01V0TRUV5ug0SuevuE6tIryS8k0S/8Onim7EiBHMnTuXadOm0atXL3bt2sXatWttAYq0tDTOnj1rG3/mzBl69+5N7969OXv2LHPnzqV37948+uijvrqFOuW6ztH0b9OEUqOZb1JOOez78M+jnC8opRQt4wzjKBZaVMc2wOZ3PDp3xdLsdcElipNMVX8KwCL1fewVnjdBb9ssBLVSQVy4o2K0n6NTKRVo1I6K8JErZKP1xojPO4GMHTuWsWPHuty3YcMGh88JCQkOuWSNDYVCwUs3dea2hZuc9hlMgvOFlnLlh0VLZhgfYLbmQ/jfTEi4Alr2rWtxKyWAUt7VvEOAwkBx62v56MQQLGnQnvHxw/3JKdLTLNQxn9JerSkUjgnDL9/ShSHd/D/RXOKMDDs1MHrFR3BzD9cRSft1ritM16DvOAzMRksXsZLcSs9b1/9/TFN/wiXK02SKCNKveYu8Uu/60obo1C6LZFYsJW+/JCwuQgYfGitS0TVAnruho8vtju0EFRTd8CaEt4KcE/DDs5VqM1EnGXUWblJu5R71esxCwTOGMeQow2tM0VYsx2fvyjaUdBtJzSMVXQMkoanrct9HMwscPht1YXDnR6BQwd5vYednbs9ZVxZdS0WWxaUG3jPdymZzN3LKmuI45bxVg06x7tOHfDEPKakfSEXXAHFXRTi/Qs+FvadzIb4/XDvFsuHnFyHrYG2L5xY1Rt7RLCBMUUSKuQPzjcOB8laIoQEXP2XcLFTHHy9cw99Tkpz2VeycJmk8SEXXQLm6Y7Mqxzy07C/Lm8ufgbZXg6HIMl9ncK5CfLHJxbtP5rB807FK3cNn1Su5VHmYPBHE04axGMtiYTlFFosutIby2lo1DSIqxLnoQ0NJoJbUPFLRNVAW3nMpSx/qy03dPYgiKpVw+/uIoCjI2AvrpjoNuVgdMGzhJl7+v/38sOesy/0DlXt5QvV/AEwwjOaUKFfUF2yKrnaTAGLCAmr1/JL6i1R0DZRgnZprO8UQqPFMOWw4o+ShnEcsH7Z/wPVKx0X2NWXr/JuR77StKbnM17yHUiH4wngtP5sTHfbn1qDr6oqPH+7PlKGdGdC2aa2cX1L/kYqugeNiKacD1rzDZ77axe/mnnxgHArAHM37xHHeaVx1SDmR7XafAjNzNYuJVuRw0NySV4z3O42xWnS1tSRr0CXNePTKtg22Q5rk4pGKroFTVaZ/UVlNuOCyFoFzjCOgeW8iFIXM1y5EhWW/vZpLO19EXhVVga2YzILhi9zXD3xY9TPXqHZTIjSMM4yjBOe5M2vUtabm6CSSikhF18CxNtBxh1Vhhegsis6AGoZ/RIEIIFF5gHHq74DyOboT5wu5as56er+yzqPrbzt23u2+7oqjTFCvAGCm8X4OCde1ALMLLYnOtT1HJ2m8SEXnBwzuGuN2n7VEeZCuvIbb+nOhTDY8DMA41XckKlJtJt22oxY31NNUjNSzznNyAJTksUCzAK3CxE+m/nxuus7tOfaetlR9DnOh6K7rFF3p/UkkniAVnR9w32Wt3e6raNEBjFr2F6vNV7DSdBUqhWC+diEUl1lmdtNYnszbVewHobAcCD8+R4Iyg1MiiomGRx1P7IaKrmtUiJaPHupH66bBVR4rkVSGVHR+QGXdrPLK5r+sc3T2TDM8xBFzHHGKbAJ/fhqEcFgr+n//uE4VqZLdX8KerzEKJU/px5JHiEeHqZQKxl9/idP25uEyLURycUhF5wfYV9zVVCg0+VuqpXVksM5Z0RURwDjDOEqFGu3htbB9iYPdteGg67aTlRFRfAJ+fB6At4x3skM4Ky5XtG0WzM0943jqug68Prw7wVoVC++xdAC7J7E1I/vF8969zh3BJBJPkLO/foB9+z9LCkW5y/nl9jS+3J7GdZ2iXR67XyQwy3gPL2s+Qfw6hYgr7aK4VXiuGXmOKyy0GBh6aAYYCiHhShYduNXje1j37CBbteAR/VpxZ59422etWsns4T08PpdEUhFp0fkB9hadu5mw5APurbPlpsGsM12KwlRK/7+fJxDnJWKuSPxvssPnieoviSk8CEFN4Y4lmD38er0yrKtTSfSKnyWSi0EqOj9Ap7HrZVqtpFgFLxoeI11EElpwjJfVn1g3e8y1yh08rF4LwNMlj3HSGO7RcXtnDOaBAQleyiuReIdUdH6AfTCiugvXLxDGM4YnESgYod7ArcrNrNpx2qNjY8hmrmYxAB8Zb2R1UXce+zSl0mP+M6gtx2cPdYgGSyS1hVR0fkCAnUVnsGsPNn9EL6/Os9XchW3xlvy61zQf0UqRQXZZeXZ7So0mpn6/FwAlZuZr3qOJooC95gReN44E4JCLNa/2jBnU3ivZJJKLQSo6P0Brt+DVPs/3tt4t+OmpK/nfc4M8Pte9/17NdnNHQhXFvKNZQHFJsdOYT7ec4NOtlkbaT6q+Z4BqP4VCxzjDOPRYcuHcJRxf3yWGw6/dKLtwSeoUqej8AHUlK/u7NA+jbTPP8tgATKh4Rv8kOSKYXsqj/PDWGNbudewPezbXEqzoqzjAM+pvAZhieJhjwnUvCyv/vHwDSx7oW6m8EkltIL9xEifOEMUEw2MA/Ef9A198sZQZ/7ePtPNFCCH4aOMxwingbe1CVArBt6Yr+M58pdvzLXmgL8dm3SQbRkt8hpwJbsQcnz2UT7ccZ+rqfU77fjH34xPj9TygXsebmkXctKk1v6VmcF2nGEDwumYJLRTnOWaOYZphlMvzLxvVj2s6us7fk0jqknph0S1cuJCEhAQCAgJITExk+/btlY7/5ptv6NSpEwEBAXTv3p2ffvqpjiT1P+4fkMDx2UNd7nvNeC+p5niaKfJ4U7OIU9mFLN98nPtUvzFE9Rd6oWKcYRyFOLcR3D75OqnkJPUGnyu6r776ivHjxzN9+nR27NhBz549GTx4MJmZrhNcN2/ezN13380jjzzCzp07ue2227jtttvYu3dvHUvu/5SiZazhKYqFlqtUe3hM9SPd1KeYqrZ0E3vdeDd7hWM9vEOv3sjx2UOJDpXrUyX1B58runnz5jF69GhGjRpFly5dWLx4MUFBQSxdutTl+LfffpshQ4bwwgsv0LlzZ2bOnMmll17Ku+++W8eS+yeXtorg0Ks32j4fES142fggAJM0X/KD+kV0CgP/M/XiI1P5uO2Tr+P47KFo1T7/SkkkTvj0W6nX60lJSSEpqbw1nVKpJCkpiS1bXFet3bJli8N4gMGDB7sdX1paSl5ensNL4h6VUuGkrL4yXc0PpstsnzNEBM8bHgcUfDdmoLTgJPUenyq6c+fOYTKZiIlxLKwYExNDenq6y2PS09O9Gj9r1izCw8Ntr/h411VuGzrTb+lCp9hQfn32KrrEhTFlaGeH/S/f0gWAkf3iCdSoeP/+Pg77701sBcCzZWWSHr68DQAzb+sGKJhkeJTj5hiMQskzhid54qZEjs8eSu9WkbV8ZxJJDSB8yOnTpwUgNm/e7LD9hRdeEP3793d5jEajEV988YXDtoULF4ro6GiX40tKSkRubq7tdfLkSQGI3NzcmrmJBkSpwSSEEMJoMjvtM5vNIrdY7/D51IUiIYQQ+SUGsenfLPHcxxvExm3b60ZYSaMlNze3xn+jPk0viYqKQqVSkZGR4bA9IyOD2FjX/UpjY2O9Gq/T6dDpnBuyNEasLqmryiAKhcIhz02hUNAiwhJNDdGpGdg+ioHtPV9hIZHUJ3zqumq1Wvr06UNycnm5H7PZTHJyMgMGDHB5zIABAxzGA6xbt87teIlEIvF5wvD48eN58MEH6du3L/3792f+/PkUFhYyapQlCfWBBx6gRYsWzJo1C4Cnn36aQYMG8eabbzJ06FBWrFjB33//zQcffODL25BIJPUYnyu6ESNGkJWVxbRp00hPT6dXr16sXbvWFnBIS0tDqSw3PAcOHMgXX3zBlClTeOmll+jQoQPff/893bp189UtSCSSeo5CiIto0d4AycvLIzw8nNzcXMLCwnwtjkQiqUBt/EZldqdEIvF7pKKTSCR+j1R0EonE7/F5MKKusU5JyqVgEkn9xPrbrMnwQaNTdPn5ll4G/roUTCLxF/Lz8wkP96ybXFU0uqir2WzmzJkzhIaGljV7rn3y8vKIj4/n5MmTDTrSK++jfuEP9+HqHoQQ5Ofn07x5c4fUsouh0Vl0SqWSli1b+uTaYWFhDfYLaY+8j/qFP9xHxXuoKUvOigxGSCQSv0cqOolE4vdIRVcH6HQ6pk+f3uCrqMj7qF/4w33U1T00umCERCJpfEiLTiKR+D1S0UkkEr9HKjqJROL3SEUnkUj8Hqnoaojs7GzuvfdewsLCiIiI4JFHHqGgoKDS8ePGjaNjx44EBgbSqlUrnnrqKXJzcx3GKRQKp9eKFStqTO6FCxeSkJBAQEAAiYmJbN++vdLx33zzDZ06dSIgIIDu3bvz008/OewXQjBt2jTi4uIIDAwkKSmJf//9t8bkdYU397BkyRKuvPJKIiMjiYyMJCkpyWn8Qw895PQ3HzJkSK3eA3h3H8uXL3eSMSDAseWkL54FeHcfV199tcvv+NChQ21jauR51FibnUbOkCFDRM+ePcXWrVvFn3/+Kdq3by/uvvtut+P37Nkj7rjjDrFmzRpx+PBhkZycLDp06CCGDx/uMA4Qy5YtE2fPnrW9iouLa0TmFStWCK1WK5YuXSr27dsnRo8eLSIiIkRGRobL8Zs2bRIqlUq88cYbYv/+/WLKlClCo9GIPXv22MbMnj1bhIeHi++//17s3r1b3HrrraJNmzY1JvPF3sM999wjFi5cKHbu3ClSU1PFQw89JMLDw8WpU6dsYx588EExZMgQh795dnZ2rchf3ftYtmyZCAsLc5AxPT3dYUxdP4vq3Mf58+cd7mHv3r1CpVKJZcuW2cbUxPOQiq4G2L9/vwDEX3/9Zdv2888/C4VCIU6fPu3xeb7++muh1WqFwWCwbQPEd999V5Pi2ujfv7948sknbZ9NJpNo3ry5mDVrlsvxd911lxg6dKjDtsTERPGf//xHCGFpkRgbGyvmzJlj25+TkyN0Op348ssva+EOvL+HihiNRhEaGio+/vhj27YHH3xQDBs2rKZFrRRv72PZsmUiPDzc7fl88SyEuPjn8dZbb4nQ0FBRUFBg21YTz0O6rjXAli1biIiIoG/fvrZtSUlJKJVKtm3b5vF5rKWj1WrHJchPPvkkUVFR9O/fn6VLl9ZI+Rq9Xk9KSgpJSUm2bUqlkqSkJLZs2eLymC1btjiMBxg8eLBt/LFjx0hPT3cYEx4eTmJiottz1vU9VKSoqAiDwUCTJk0ctm/YsIHo6Gg6duzIE088wfnz52tUdnuqex8FBQW0bt2a+Ph4hg0bxr59+2z76vpZXMx92PPRRx8xcuRIgoODHbZf7POQiq4GSE9PJzo62mGbWq2mSZMmpKene3SOc+fOMXPmTB577DGH7a+88gpff/0169atY/jw4YwZM4YFCxZctMznzp3DZDLZmhBZiYmJcStzenp6peOt/3pzzouhOvdQkQkTJtC8eXOHH+eQIUP45JNPSE5O5vXXX+f333/nxhtvxGQy1aj8VqpzHx07dmTp0qWsXr2azz77DLPZzMCBAzl16hRQ988CLv55bN++nb179/Loo486bK+J59Hoqpd4w8SJE3n99dcrHZOamnrR18nLy2Po0KF06dKFl19+2WHf1KlTbe979+5NYWEhc+bM4amnnrro6zZ2Zs+ezYoVK9iwYYPDRP7IkSNt77t3706PHj1o164dGzZs4LrrrvOFqE4MGDDAoZfxwIED6dy5M++//z4zZ870oWTV56OPPqJ79+7079/fYXtNPA9p0VXCc889R2pqaqWvtm3bEhsbS2ZmpsOxRqOR7OxsYmNjK71Gfn4+Q4YMITQ0lO+++w6NRlPp+MTERE6dOkVpaelF3VtUVBQqlYqMjAyH7RkZGW5ljo2NrXS89V9vznkxVOcerMydO5fZs2fz66+/0qNHj0rHtm3blqioKA4fPnzRMrviYu7DikajoXfv3jYZ6/pZwMXdR2FhIStWrOCRRx6p8jrVeR5S0VVCs2bN6NSpU6UvrVbLgAEDyMnJISUlxXbs//73P8xmM4mJiW7Pn5eXxw033IBWq2XNmjVO6QGu2LVrF5GRkRe9CFqr1dKnTx+Sk5Nt28xmM8nJyQ6Wgj0DBgxwGA+wbt062/g2bdoQGxvrMCYvL49t27a5PWdd3wPAG2+8wcyZM1m7dq3DvKo7Tp06xfnz54mLi6sRuStS3fuwx2QysWfPHpuMdf0s4OLu45tvvqG0tJT77ruvyutU63lcVChDYmPIkCGid+/eYtu2bWLjxo2iQ4cODuklp06dEh07dhTbtm0TQgiRm5srEhMTRffu3cXhw4cdQudGo1EIIcSaNWvEkiVLxJ49e8S///4r3nvvPREUFCSmTZtWIzKvWLFC6HQ6sXz5crF//37x2GOPiYiICFuawv333y8mTpxoG79p0yahVqvF3LlzRWpqqpg+fbrL9JKIiAixevVq8c8//4hhw4bVenqJN/cwe/ZsodVqxcqVKx3+5vn5+UIIIfLz88Xzzz8vtmzZIo4dOyZ+++03cemll4oOHTqIkpKSWrmH6tzHjBkzxC+//CKOHDkiUlJSxMiRI0VAQIDYt2+fw73W5bOozn1YueKKK8SIESOcttfU85CKroY4f/68uPvuu0VISIgICwsTo0aNsv14hBDi2LFjAhDr168XQgixfv16Abh8HTt2TAhhSVHp1auXCAkJEcHBwaJnz55i8eLFwmQy1ZjcCxYsEK1atRJarVb0799fbN261bZv0KBB4sEHH3QY//XXX4tLLrlEaLVa0bVrV/Hjjz867DebzWLq1KkiJiZG6HQ6cd1114mDBw/WmLwXew+tW7d2+TefPn26EEKIoqIiccMNN4hmzZoJjUYjWrduLUaPHu2Uo+br+3jmmWdsY2NiYsRNN90kduzY4XA+XzwLb+9DCCEOHDggAPHrr786naumnocs0ySRSPweOUcnkUj8HqnoJBKJ3yMVnUQi8XukopNIJH6PVHQSicTvkYpOIpH4PVLRSSQSv0cqOolE4vdIRSepF7gql11XJcwl/o8s0ySpNwwZMoRly5Y5bHNXvMBgMDhVetHr9Wi1Wq+vW93jJA0HadFJ6g06nY7Y2FiHV2RkJGBpErRo0SJuvfVWgoODee2113j55Zfp1asXH374IW3atLFVf0lLS2PYsGGEhIQQFhbGXXfd5VA6yN1xEv9FKjpJg+Hll1/m9ttvZ8+ePTz88MMAHD58mG+//ZZVq1axa9cuzGYzw4YNIzs7m99//51169Zx9OhRRowY4XCuisdJ/BvpukrqDT/88AMhISEO21566SVeeuklAO655x5GjRrlsF+v1/PJJ5/QrFkzwFIfb8+ePRw7doz4+HgAPvnkE7p27cpff/1Fv379XB4n8W+kopPUG6655hoWLVrksM2+aY2rIpmtW7d2UFapqanEx8fblBxAly5diIiIIDU11aboKh4n8W+kopPUG4KDg2nfvn2l+z3Z5um1JI0HOUcn8Ss6d+7MyZMnOXnypG3b/v37ycnJoUuXLj6UTOJLpEUnqTeUlpY6tcVTq9VERUV5fI6kpCS6d+/Ovffey/z58zEajYwZM4ZBgwZ51B9C4p9Ii05Sb1i7di1xcXEOryuuuMKrcygUClavXk1kZCRXXXUVSUlJtG3blq+++qqWpJY0BGQpdYlE4vdIi04ikfg9UtFJJBK/Ryo6iUTi90hFJ5FI/B6p6CQSid8jFZ1EIvF7pKKTSCR+j1R0EonE75GKTiKR+D1S0UkkEr9HKjqJROL3SEUnkUj8nv8Po+cCrVGO874AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.019265, + "end_time": "2024-03-24T20:34:52.603032", + "exception": false, + "start_time": "2024-03-24T20:34:52.583767", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4805.649015, + "end_time": "2024-03-24T20:34:55.343581", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/tab_ddpm_concat/0/mlu-eval.ipynb", + "output_path": "eval/treatment/tab_ddpm_concat/0/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/treatment/tab_ddpm_concat/0", + "path_prefix": "../../../../", + "random_seed": 0, + "single_model": "tab_ddpm_concat" + }, + "start_time": "2024-03-24T19:14:49.694566", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/tab_ddpm_concat/model.pt b/treatment/tab_ddpm_concat/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..c8759acb05f56adb4dee7b17334bf2f213078228 --- /dev/null +++ b/treatment/tab_ddpm_concat/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b7564d23ddd2da7b3fb62d10de4c6320ce9bceaaec1940f9f238d907c248b97 +size 73318657 diff --git a/treatment/tab_ddpm_concat/params.json b/treatment/tab_ddpm_concat/params.json new file mode 100644 index 0000000000000000000000000000000000000000..5d1ce0c9d81197e46bbf76394825dcd1f643787f --- /dev/null +++ b/treatment/tab_ddpm_concat/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 1, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "torch", "grad_clip": 0.75, "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.1, "loss_balancer_beta": 0.7999999999999999, "loss_balancer_r": 0.96, "tf_pma_low_exp_2": 2, "dataset_size_exp_2": 11, "batch_size_exp_2": 2, "epochs": 100, "lr_mul": 0.04, "n_warmup_steps": 140, "Optim": "diffgrad", "fixed_role_model": "tab_ddpm_concat", "mse_mag_target": 0.1, "g_loss_mul": 0.1, "d_model_exp_2": 9, "attn_activation": "leakyhardtanh", "tf_d_inner_exp_2": 9, "tf_n_layers_enc": 4, "tf_n_head_exp_2": 6, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds_exp_2": 5, "ada_d_hid_exp_2": 10, "ada_n_layers": 7, "ada_activation": "selu", "ada_activation_final": "leakyhardsigmoid", "head_d_hid_exp_2": 8, "head_n_layers": 8, "head_n_head_exp_2": 6, "head_activation": "leakyhardsigmoid", "head_activation_final": "leakyhardsigmoid", "mse_mag": true, "mse_mag_multiply": true, "models": ["tab_ddpm_concat"], "max_seconds": 3600} \ No newline at end of file diff --git a/treatment/tvae/eval.csv b/treatment/tvae/eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..8d08cf3b21ff1b2fcc8220762ef2e9054901a893 --- /dev/null +++ b/treatment/tvae/eval.csv @@ -0,0 +1,2 @@ +,avg_g_cos_loss,avg_g_mag_loss,avg_loss,grad_duration,grad_mae,grad_mape,grad_rmse,mean_pred_loss,pred_duration,pred_mae,pred_mape,pred_rmse,pred_std,std_loss,total_duration +tvae,0.0,0.239999996821087,0.004334433722427396,5.680645942687988,0.06174943223595619,1.489418625831604,0.09651356935501099,0.00020048483565915376,7.251855134963989,0.039978448301553726,256361.25,0.06583641469478607,0.2199508100748062,2.2841795725980774e-05,12.932501077651978 diff --git a/treatment/tvae/history.csv b/treatment/tvae/history.csv new file mode 100644 index 0000000000000000000000000000000000000000..adc8a63b101cfe097d91f0986b5910f6064687a8 --- /dev/null +++ b/treatment/tvae/history.csv @@ -0,0 +1,5 @@ +,avg_role_model_loss_train,avg_role_model_std_loss_train,avg_role_model_mean_pred_loss_train,avg_role_model_g_mag_loss_train,avg_role_model_g_cos_loss_train,avg_non_role_model_g_mag_loss_train,avg_non_role_model_g_cos_loss_train,avg_non_role_model_embed_loss_train,avg_loss_train,n_size_train,n_batch_train,duration_train,duration_batch_train,duration_size_train,avg_pred_std_train,avg_role_model_loss_test,avg_role_model_std_loss_test,avg_role_model_mean_pred_loss_test,avg_role_model_g_mag_loss_test,avg_role_model_g_cos_loss_test,avg_non_role_model_g_mag_loss_test,avg_non_role_model_g_cos_loss_test,avg_non_role_model_embed_loss_test,avg_loss_test,n_size_test,n_batch_test,duration_test,duration_batch_test,duration_size_test,avg_pred_std_test +0,0.03363065153919415,4.061211425165913,0.003072351892546645,0.0959731253138226,0.0,0.0,0.0,0.0,0.04791451765126276,900,450,580.8831059932709,1.290851346651713,0.6454256733258565,0.12362771289730214,0.010208010552554698,1.7609046706683165,0.0012737641513725704,0.0,0.0,0.0,0.0,0.0,0.010208010552554698,450,225,203.08173871040344,0.9025855053795708,0.4512927526897854,0.09662675946161875 +1,0.010233858526780142,0.7669666670105895,0.0007310967529919379,0.152462607157989,0.0,0.0,0.0,0.0,0.010440422089125236,900,450,579.9043016433716,1.288676225874159,0.6443381129370795,0.20330373977095204,0.010850908685869168,1.70096435137354,0.0007107436713112964,0.0,0.0,0.0,0.0,0.0,0.010850908685869168,450,225,196.19050812721252,0.8719578138987223,0.43597890694936114,0.09977313449461185 +2,0.008459381934637867,1.3255586893503324,0.0004864757848670276,0.12002694543341123,0.0,0.0,0.0,0.0,0.008653573781455684,900,450,573.4812700748444,1.274402822388543,0.6372014111942715,0.19907333407094685,0.00886493952675291,1.6276623941000343,0.0003420339755327389,0.0,0.0,0.0,0.0,0.0,0.00886493952675291,450,225,198.55576705932617,0.8824700758192274,0.4412350379096137,0.10561421838104858 +3,0.005182997622466448,0.5467010652780068,0.00016614465866880747,0.1050477642213486,0.0,0.0,0.0,0.0,0.0052792941385990215,900,450,575.0789859294891,1.2779533020655314,0.6389766510327657,0.21151418934407248,0.006986802332486501,2.1908077004363427,0.00018339116417594552,0.0,0.0,0.0,0.0,0.0,0.006986802332486501,450,225,197.8352234363556,0.8792676597171359,0.43963382985856797,0.09777659590274804 diff --git a/treatment/tvae/mlu-eval-load.ipynb b/treatment/tvae/mlu-eval-load.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..5939ad62638b996516fce997026a385e5bfc5286 --- /dev/null +++ b/treatment/tvae/mlu-eval-load.ipynb @@ -0,0 +1,1982 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:21.204576Z", + "iopub.status.busy": "2024-03-26T11:11:21.203640Z", + "iopub.status.idle": "2024-03-26T11:11:21.247501Z", + "shell.execute_reply": "2024-03-26T11:11:21.246180Z" + }, + "papermill": { + "duration": 0.061151, + "end_time": "2024-03-26T11:11:21.250127", + "exception": false, + "start_time": "2024-03-26T11:11:21.188976", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:21.277170Z", + "iopub.status.busy": "2024-03-26T11:11:21.276590Z", + "iopub.status.idle": "2024-03-26T11:11:21.286210Z", + "shell.execute_reply": "2024-03-26T11:11:21.285191Z" + }, + "papermill": { + "duration": 0.025784, + "end_time": "2024-03-26T11:11:21.288536", + "exception": false, + "start_time": "2024-03-26T11:11:21.262752", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:21.315211Z", + "iopub.status.busy": "2024-03-26T11:11:21.314331Z", + "iopub.status.idle": "2024-03-26T11:11:21.319222Z", + "shell.execute_reply": "2024-03-26T11:11:21.318209Z" + }, + "papermill": { + "duration": 0.020123, + "end_time": "2024-03-26T11:11:21.321494", + "exception": false, + "start_time": "2024-03-26T11:11:21.301371", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:21.346210Z", + "iopub.status.busy": "2024-03-26T11:11:21.345663Z", + "iopub.status.idle": "2024-03-26T11:11:21.350619Z", + "shell.execute_reply": "2024-03-26T11:11:21.349573Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.019915, + "end_time": "2024-03-26T11:11:21.352898", + "exception": false, + "start_time": "2024-03-26T11:11:21.332983", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:21.379395Z", + "iopub.status.busy": "2024-03-26T11:11:21.378781Z", + "iopub.status.idle": "2024-03-26T11:11:21.385080Z", + "shell.execute_reply": "2024-03-26T11:11:21.384183Z" + }, + "papermill": { + "duration": 0.021745, + "end_time": "2024-03-26T11:11:21.387284", + "exception": false, + "start_time": "2024-03-26T11:11:21.365539", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "19566fb6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:21.411621Z", + "iopub.status.busy": "2024-03-26T11:11:21.411072Z", + "iopub.status.idle": "2024-03-26T11:11:21.416660Z", + "shell.execute_reply": "2024-03-26T11:11:21.415732Z" + }, + "papermill": { + "duration": 0.020763, + "end_time": "2024-03-26T11:11:21.419001", + "exception": false, + "start_time": "2024-03-26T11:11:21.398238", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"tvae\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 1\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/tvae/1\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011406, + "end_time": "2024-03-26T11:11:21.441744", + "exception": false, + "start_time": "2024-03-26T11:11:21.430338", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:21.467564Z", + "iopub.status.busy": "2024-03-26T11:11:21.467205Z", + "iopub.status.idle": "2024-03-26T11:11:21.478570Z", + "shell.execute_reply": "2024-03-26T11:11:21.477595Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.026594, + "end_time": "2024-03-26T11:11:21.480855", + "exception": false, + "start_time": "2024-03-26T11:11:21.454261", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/tvae/1\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:21.509801Z", + "iopub.status.busy": "2024-03-26T11:11:21.508738Z", + "iopub.status.idle": "2024-03-26T11:11:23.838086Z", + "shell.execute_reply": "2024-03-26T11:11:23.837005Z" + }, + "papermill": { + "duration": 2.346397, + "end_time": "2024-03-26T11:11:23.840269", + "exception": false, + "start_time": "2024-03-26T11:11:21.493872", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:23.865086Z", + "iopub.status.busy": "2024-03-26T11:11:23.864569Z", + "iopub.status.idle": "2024-03-26T11:11:23.883003Z", + "shell.execute_reply": "2024-03-26T11:11:23.881737Z" + }, + "papermill": { + "duration": 0.033617, + "end_time": "2024-03-26T11:11:23.885417", + "exception": false, + "start_time": "2024-03-26T11:11:23.851800", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:23.910407Z", + "iopub.status.busy": "2024-03-26T11:11:23.910066Z", + "iopub.status.idle": "2024-03-26T11:11:23.919166Z", + "shell.execute_reply": "2024-03-26T11:11:23.918067Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.023974, + "end_time": "2024-03-26T11:11:23.921363", + "exception": false, + "start_time": "2024-03-26T11:11:23.897389", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:23.947347Z", + "iopub.status.busy": "2024-03-26T11:11:23.947044Z", + "iopub.status.idle": "2024-03-26T11:11:24.058104Z", + "shell.execute_reply": "2024-03-26T11:11:24.056937Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.127513, + "end_time": "2024-03-26T11:11:24.060802", + "exception": false, + "start_time": "2024-03-26T11:11:23.933289", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:24.085724Z", + "iopub.status.busy": "2024-03-26T11:11:24.085383Z", + "iopub.status.idle": "2024-03-26T11:11:29.418209Z", + "shell.execute_reply": "2024-03-26T11:11:29.417185Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 5.348314, + "end_time": "2024-03-26T11:11:29.420799", + "exception": false, + "start_time": "2024-03-26T11:11:24.072485", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-26 11:11:26.605320: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-26 11:11:26.605381: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-26 11:11:26.607182: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:29.449372Z", + "iopub.status.busy": "2024-03-26T11:11:29.447953Z", + "iopub.status.idle": "2024-03-26T11:11:29.455792Z", + "shell.execute_reply": "2024-03-26T11:11:29.454962Z" + }, + "papermill": { + "duration": 0.024758, + "end_time": "2024-03-26T11:11:29.458456", + "exception": false, + "start_time": "2024-03-26T11:11:29.433698", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:29.485653Z", + "iopub.status.busy": "2024-03-26T11:11:29.484807Z", + "iopub.status.idle": "2024-03-26T11:11:55.808522Z", + "shell.execute_reply": "2024-03-26T11:11:55.807064Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 26.3409, + "end_time": "2024-03-26T11:11:55.811986", + "exception": false, + "start_time": "2024-03-26T11:11:29.471086", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-26T11:11:55.847611Z", + "iopub.status.busy": "2024-03-26T11:11:55.847191Z", + "iopub.status.idle": "2024-03-26T11:11:55.855362Z", + "shell.execute_reply": "2024-03-26T11:11:55.854307Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.030365, + "end_time": "2024-03-26T11:11:55.857738", + "exception": false, + "start_time": "2024-03-26T11:11:55.827373", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 95,\n", + " 'realtabformer': (69, 281, Embedding(281, 768), True),\n", + " 'lct_gan': 75,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:55.887063Z", + "iopub.status.busy": "2024-03-26T11:11:55.886108Z", + "iopub.status.idle": "2024-03-26T11:11:55.961492Z", + "shell.execute_reply": "2024-03-26T11:11:55.960348Z" + }, + "papermill": { + "duration": 0.092656, + "end_time": "2024-03-26T11:11:55.964066", + "exception": false, + "start_time": "2024-03-26T11:11:55.871410", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/tvae/all inf False\n", + "../../../../ml-utility-loss/aug_test/treatment 0\n", + "Caching in ../../../../treatment/_cache_bs_test/tvae/all inf False\n", + "../../../../ml-utility-loss/bs_test/treatment 0\n", + "Caching in ../../../../treatment/_cache_synth_test/tvae/all inf False\n", + "../../../../ml-utility-loss/synthetics/treatment 600\n", + "600\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:55.995445Z", + "iopub.status.busy": "2024-03-26T11:11:55.995042Z", + "iopub.status.idle": "2024-03-26T11:11:56.351968Z", + "shell.execute_reply": "2024-03-26T11:11:56.351003Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.374884, + "end_time": "2024-03-26T11:11:56.354152", + "exception": false, + "start_time": "2024-03-26T11:11:55.979268", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Body': 'twin_encoder',\n", + " 'loss_balancer_meta': True,\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'tf_pma_low': 16,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.75,\n", + " 'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'bias_weight_decay': 0.05,\n", + " 'loss_balancer_beta': 0.75,\n", + " 'loss_balancer_r': 0.96,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 2,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.06,\n", + " 'n_warmup_steps': 80,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'tvae',\n", + " 'g_loss_mul': 0.2,\n", + " 'd_model': 512,\n", + " 'attn_activation': torch.nn.modules.activation.SELU,\n", + " 'tf_d_inner': 256,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 2048,\n", + " 'ada_n_layers': 6,\n", + " 'ada_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 7,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tvae'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.5, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "# params = {\n", + "# **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "# }\n", + "params = getattr(PARAMS, dataset_name).BEST_DICT[gp][gp_multiply][single_model]\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).DEFAULTS,\n", + " **params,\n", + "}\n", + "if isinstance(params, (list, tuple)):\n", + " params = params[param_index]\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-26T11:11:56.380003Z", + "iopub.status.busy": "2024-03-26T11:11:56.379630Z", + "iopub.status.idle": "2024-03-26T11:11:57.090034Z", + "shell.execute_reply": "2024-03-26T11:11:57.088984Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.725565, + "end_time": "2024-03-26T11:11:57.092191", + "exception": false, + "start_time": "2024-03-26T11:11:56.366626", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n", + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['tvae'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:57.121474Z", + "iopub.status.busy": "2024-03-26T11:11:57.121080Z", + "iopub.status.idle": "2024-03-26T11:11:57.125954Z", + "shell.execute_reply": "2024-03-26T11:11:57.124857Z" + }, + "papermill": { + "duration": 0.021848, + "end_time": "2024-03-26T11:11:57.128241", + "exception": false, + "start_time": "2024-03-26T11:11:57.106393", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:57.154680Z", + "iopub.status.busy": "2024-03-26T11:11:57.154394Z", + "iopub.status.idle": "2024-03-26T11:11:57.161826Z", + "shell.execute_reply": "2024-03-26T11:11:57.160890Z" + }, + "papermill": { + "duration": 0.022972, + "end_time": "2024-03-26T11:11:57.163968", + "exception": false, + "start_time": "2024-03-26T11:11:57.140996", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "29729665" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:57.190916Z", + "iopub.status.busy": "2024-03-26T11:11:57.190592Z", + "iopub.status.idle": "2024-03-26T11:11:57.301732Z", + "shell.execute_reply": "2024-03-26T11:11:57.300619Z" + }, + "papermill": { + "duration": 0.128142, + "end_time": "2024-03-26T11:11:57.304509", + "exception": false, + "start_time": "2024-03-26T11:11:57.176367", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 95] --\n", + "├─Adapter: 1-1 [2, 2648, 95] --\n", + "│ └─Sequential: 2-1 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 2048] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 2048] 196,608\n", + "│ │ │ └─LeakyHardtanh: 4-2 [2, 2648, 2048] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 2048] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 2048] 4,196,352\n", + "│ │ │ └─LeakyHardtanh: 4-4 [2, 2648, 2048] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 2048] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 2048] 4,196,352\n", + "│ │ │ └─LeakyHardtanh: 4-6 [2, 2648, 2048] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 2048] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 2048] 4,196,352\n", + "│ │ │ └─LeakyHardtanh: 4-8 [2, 2648, 2048] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 2048] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 2048] 4,196,352\n", + "│ │ │ └─LeakyHardtanh: 4-10 [2, 2648, 2048] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 512] 1,049,088\n", + "│ │ │ └─LeakyHardsigmoid: 4-12 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 95] (recursive)\n", + "│ └─Sequential: 2-2 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-7 [2, 661, 2048] (recursive)\n", + "│ │ │ └─Linear: 4-13 [2, 661, 2048] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-14 [2, 661, 2048] --\n", + "│ │ └─FeedForward: 3-8 [2, 661, 2048] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 2048] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-16 [2, 661, 2048] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 2048] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 2048] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-18 [2, 661, 2048] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 2048] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 2048] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-20 [2, 661, 2048] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 2048] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 2048] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-22 [2, 661, 2048] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-24 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 8192] --\n", + "│ └─Encoder: 2-3 [2, 16, 512] --\n", + "│ │ └─ModuleList: 3-14 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-25 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-6 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 2648, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 256] 131,328\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 256] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 131,584\n", + "│ │ │ └─EncoderLayer: 4-26 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-18 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 2648, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 256] 131,328\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 256] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 131,584\n", + "│ │ │ └─EncoderLayer: 4-27 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-30 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 2648, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 256] 131,328\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 256] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 131,584\n", + "│ │ │ └─EncoderLayer: 4-28 [2, 16, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-42 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 2648, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 256] 131,328\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 256] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 131,584\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 512] 8,192\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 16, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 16, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-54 [2, 16, 512] --\n", + "│ └─Encoder: 2-4 [2, 16, 512] (recursive)\n", + "│ │ └─ModuleList: 3-14 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-60 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 661, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 661, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-31 [2, 661, 256] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-72 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 661, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 661, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 661, 256] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-40 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-84 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 32, 661, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 32, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-42 [2, 661, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-43 [2, 661, 256] --\n", + "│ │ │ │ │ └─Linear: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-45 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-46 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 32, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-96 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 32, 661, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 32, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-48 [2, 661, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-49 [2, 661, 256] --\n", + "│ │ │ │ │ └─Linear: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-18 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-51 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-52 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 32, 16, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 32, 16, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-108 [2, 16, 512] --\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-15 [2, 128] --\n", + "│ │ │ └─Linear: 4-33 [2, 128] 1,048,704\n", + "│ │ │ └─LeakyHardsigmoid: 4-34 [2, 128] --\n", + "│ │ └─FeedForward: 3-16 [2, 128] --\n", + "│ │ │ └─Linear: 4-35 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-36 [2, 128] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 1] --\n", + "│ │ │ └─Linear: 4-45 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 29,729,665\n", + "Trainable params: 29,729,665\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 116.36\n", + "========================================================================================================================\n", + "Input size (MB): 2.51\n", + "Forward/backward pass size (MB): 1231.11\n", + "Params size (MB): 118.92\n", + "Estimated Total Size (MB): 1352.55\n", + "========================================================================================================================" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = test_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "90576617", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:57.337664Z", + "iopub.status.busy": "2024-03-26T11:11:57.337165Z", + "iopub.status.idle": "2024-03-26T11:11:57.456906Z", + "shell.execute_reply": "2024-03-26T11:11:57.455773Z" + }, + "papermill": { + "duration": 0.139202, + "end_time": "2024-03-26T11:11:57.459488", + "exception": false, + "start_time": "2024-03-26T11:11:57.320286", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torch\n", + "model.load_state_dict(torch.load(\"model.pt\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:11:57.493782Z", + "iopub.status.busy": "2024-03-26T11:11:57.492877Z", + "iopub.status.idle": "2024-03-26T11:17:18.488500Z", + "shell.execute_reply": "2024-03-26T11:17:18.487481Z" + }, + "papermill": { + "duration": 321.015893, + "end_time": "2024-03-26T11:17:18.491497", + "exception": false, + "start_time": "2024-03-26T11:11:57.475604", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:17:18.525943Z", + "iopub.status.busy": "2024-03-26T11:17:18.525457Z", + "iopub.status.idle": "2024-03-26T11:17:18.551388Z", + "shell.execute_reply": "2024-03-26T11:17:18.550375Z" + }, + "papermill": { + "duration": 0.045615, + "end_time": "2024-03-26T11:17:18.553726", + "exception": false, + "start_time": "2024-03-26T11:17:18.508111", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tvae0.00.240.0043345.6806460.0617491.4894190.0965140.00027.2518550.039978256361.250.0658360.2199510.00002312.932501
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "tvae 0.0 0.24 0.004334 5.680646 0.061749 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "tvae 1.489419 0.096514 0.0002 7.251855 0.039978 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "tvae 256361.25 0.065836 0.219951 0.000023 12.932501 " + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:17:18.584279Z", + "iopub.status.busy": "2024-03-26T11:17:18.583917Z", + "iopub.status.idle": "2024-03-26T11:17:19.017708Z", + "shell.execute_reply": "2024-03-26T11:17:19.016530Z" + }, + "papermill": { + "duration": 0.451632, + "end_time": "2024-03-26T11:17:19.020161", + "exception": false, + "start_time": "2024-03-26T11:17:18.568529", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:17:19.056001Z", + "iopub.status.busy": "2024-03-26T11:17:19.055593Z", + "iopub.status.idle": "2024-03-26T11:22:45.566345Z", + "shell.execute_reply": "2024-03-26T11:22:45.565287Z" + }, + "papermill": { + "duration": 326.532202, + "end_time": "2024-03-26T11:22:45.569403", + "exception": false, + "start_time": "2024-03-26T11:17:19.037201", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/tvae/all inf False\n", + "Caching in ../../../../treatment/_cache_bs_test/tvae/all inf False\n", + "Caching in ../../../../treatment/_cache_synth_test/tvae/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:22:45.601812Z", + "iopub.status.busy": "2024-03-26T11:22:45.600920Z", + "iopub.status.idle": "2024-03-26T11:22:45.623788Z", + "shell.execute_reply": "2024-03-26T11:22:45.622973Z" + }, + "papermill": { + "duration": 0.041149, + "end_time": "2024-03-26T11:22:45.626043", + "exception": false, + "start_time": "2024-03-26T11:22:45.584894", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:22:45.656657Z", + "iopub.status.busy": "2024-03-26T11:22:45.656354Z", + "iopub.status.idle": "2024-03-26T11:22:45.662239Z", + "shell.execute_reply": "2024-03-26T11:22:45.661397Z" + }, + "papermill": { + "duration": 0.023676, + "end_time": "2024-03-26T11:22:45.664486", + "exception": false, + "start_time": "2024-03-26T11:22:45.640810", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tvae': 0.46125430537475043}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:22:45.695427Z", + "iopub.status.busy": "2024-03-26T11:22:45.694655Z", + "iopub.status.idle": "2024-03-26T11:22:46.142681Z", + "shell.execute_reply": "2024-03-26T11:22:46.141646Z" + }, + "papermill": { + "duration": 0.466057, + "end_time": "2024-03-26T11:22:46.144896", + "exception": false, + "start_time": "2024-03-26T11:22:45.678839", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABAjklEQVR4nO3deXxU9b34/9eZM/tkZrJvkEBYBAFZXLC4FFBbF74q1y7YRUGL2lut9XK9t9LvQ61aH9p71Wqt9fbbW6H+rNraor1Xr1JvK+JSEVkUBAExJCELCdkmmX05vz9OMhAhkEwmOZPJ+/l4zCOZkzNz3jOZvPPZP4qmaRpCCGEgk9EBCCGEJCIhhOEkEQkhDCeJSAhhOElEQgjDSSISQhhOEpEQwnCSiIQQhpNEJIQwnCQiIYThJBGJtHr33Xf58Y9/TEdHh9GhiFFEEpFIq3fffZd77rlHEpEYFElEQgjDSSISafPjH/+Yf/mXfwGgqqoKRVFQFIWcnBwWL158zPmJRIJx48bx1a9+NXnsoYce4pxzzqGgoACHw8EZZ5zBH//4x+Ne75lnnuGMM87A4XCQn5/P1VdfTV1d3fC8ODGsFFkGRKTLRx99xIMPPshzzz3Hz372MwoLCwHYv38/9957L/X19ZSWlibP37hxIwsXLuSFF15IJqOKigquuOIKZsyYQSQS4fnnn+f999/n5ZdfZsmSJcnH3n///dx55518/etfZ+HChbS0tPD444+Tk5PDtm3byM3NHdHXLoZIEyKN/v3f/10DtOrq6uSxPXv2aID2+OOP9zn3e9/7npaTk6MFAoHksaO/1zRNi0Qi2qxZs7QLLrggeezAgQOaqqra/fff3+fcHTt2aGaz+ZjjIvNJ1UwMu1NOOYW5c+fy+9//PnksHo/zxz/+kcsvvxyHw5E8fvT37e3tdHZ2cv7557N169bk8XXr1pFIJPj617/O4cOHk7fS0lKmTp3KG2+8MTIvTKSN2egAxNiwbNkyfvSjH1FfX8+4cePYsGEDzc3NLFu2rM95L7/8Mj/5yU/Yvn074XA4eVxRlOT3+/btQ9M0pk6detxrWSyW4XkRYthIIhIjYtmyZaxevZoXXniB2267jT/84Q94vV4uueSS5DlvvfUWV1xxBV/84hf55S9/SVlZGRaLhTVr1vDss88mz0skEiiKwquvvoqqqsdcKycnZ0Rek0gfSUQirY4uuRytqqqK+fPn8/vf/55bbrmFdevWsXTpUmw2W/KcP/3pT9jtdtavX9/n+Jo1a/o81+TJk9E0jaqqKk455ZTheSFiREkbkUgrl8sFcNwBjcuWLeO9997jqaee4vDhw8dUy1RVRVEU4vF48tiBAwd46aWX+px31VVXoaoq99xzD9rnOn01TaO1tTU9L0aMGOm+F2m1efNm5s+fz2WXXcbVV1+NxWLh8ssvx+VycfDgQSorK8nJycFisdDU1NSnPedvf/sbF154Ieeffz7f/OY3aW5u5oknnqC0tJSPPvqoT9J58MEHWb16Neeccw5Lly7F7XZTXV3Niy++yI033sjtt99uxMsXqTKwx05kqfvuu08bN26cZjKZjunKP/fcczVAW7ly5XEf+5vf/EabOnWqZrPZtOnTp2tr1qzR7r77bu14H9U//elP2nnnnae5XC7N5XJp06dP126++WZtz549w/XSxDCREpEQwnDSRiSEMJwkIiGE4SQRCSEMJ4lICGE4SURCCMNJIhJCGG5UT/FIJBI0NDTgdrv7nVoghDCOpml0dXVRXl6OydR/uWdUJ6KGhgYqKiqMDkMIcRJ1dXWMHz++35+P6kTkdrsB/UV6PB6DoxFCfJ7P56OioiL5t9qfUZ2IeqtjHo9HEpEQGexkTSfSWC2EMJwkIiGE4SQRCSEMN6rbiAZC0zRisVifxbbEwKiqitlslqERYthldSKKRCI0NjYSCASMDmXUcjqdlJWVYbVajQ5FZLGsTUSJRILq6mpUVaW8vByr1Sr/2QdB0zQikQgtLS1UV1czderUEw5IE2IosjYRRSIREokEFRUVOJ1Oo8MZlRwOBxaLhZqaGiKRCHa73eiQRsz+lm621XZQ7LZx7pRCVJP8ExtOWZuIesl/8aEZi+9fa3eYlz9sJKFp1LUFMCkK500tNDqsrDb2PmVCnESO3cz8qnzcdv3/9LbadkJR6ewYTpKIhPgcm1llweQCvnNeFYU5VmIJjb2HuowOK6tJIhrjJk6cyKOPPmp0GBlJURSml+lThz5r8RscTXaTRCTEUXY1+NjT1JWsik0o0Ds6DrYHiMUTRoaW1bK+sXosiEQiMs4nTd77rJXOYJSl88ZRVeiiKMfGFyYVMD7PgUmGfwybMVkiisQS/d4+/1/vROdGB3BuKhYtWsQtt9zCLbfcgtfrpbCwkDvvvDO50+nEiRO57777uPbaa/F4PNx4440AvP3225x//vk4HA4qKiq49dZb8fuPVCmam5u5/PLLcTgcVFVV8bvf/S6l+LKVLxSlMxhFUaA8Vx+qoCgKCyYXUJHvxCRd+MNmTJaInnjj035/VlXoYum8ccn7/2/jfqLx4+9BOT7PwdfOPLIw21PvVBOM9O1d+acvnZJSjL/97W/5zne+w/vvv88HH3zAjTfeSGVlJTfccAMADz30EHfddRd33303APv37+eSSy7hJz/5CU899RQtLS3JZLZmzRoAVqxYQUNDA2+88QYWi4Vbb72V5ubmlOLLRs2+EABFbhs2s2pwNGPLmExEo0FFRQU/+9nPUBSFadOmsWPHDn72s58lE9EFF1zAP//zPyfPX7lyJd/61re47bbbAJg6dSo///nPWbhwIU8++SS1tbW8+uqrvP/++5x11lkA/OY3v+HUU08d8deWqZq7wgAU5dj6HI/GE9S0+ukIRDlzYr4RoWW9jElEDz74IKtXr+YHP/jBsPfi3Lx4Sr8/+3zp+8YvTu733M83GVx/btVQwurjC1/4Qp8pKQsWLODhhx9OTt4988wz+5z/4Ycf8tFHH/WpbmmalpzqsnfvXsxmM2eccUby59OnTyc3NzdtMY92Lb2JyH1sIvrvDxsBmD0+F6t5TLZoDKuMSESbN2/mV7/6FbNnzx6R6w3mgzRc5w6Vy+Xqc7+7u5ubbrqJW2+99ZhzKysr2bt370iFNmod7o4AUPi5EpHTasZpVQlE4rQHIpR4xs5Ul5FieGrv7u7mW9/6Fr/+9a/Jy8szOpyMsWnTpj7333vvPaZOnYqqHr/t4vTTT2fXrl1MmTLlmJvVamX69OnEYjG2bNmSfMyePXvo6OgYzpcxaoSicXzBKHBsiQggz6X3Srb5IyMa11hheCK6+eabWbJkCRdddNFJzw2Hw/h8vj63bFVbW8uqVavYs2cPzz33HI8//jg/+MEP+j3/hz/8Ie+++y633HIL27dvZ9++ffz5z3/mlltuAWDatGlccskl3HTTTWzatIktW7awcuVKHA7HSL2kjGYzm7j+vCr+Yd447JZjk32BJKJhZWjV7Pnnn2fr1q1s3rx5QOc/8MAD3HPPPcMcVWa49tprCQaDzJ8/H1VV+cEPfpDspj+e2bNn8+abb/J//+//5fzzz0fTNCZPnsyyZcuS56xZs4aVK1eycOFCSkpK+MlPfsKdd945Ei8n4ymKgtdhweuwHDnYUQu7X4Z4hDL3Aj4iTxLRMFG03sEpI6yuro4zzzyT119/Pdk2tGjRIubOndtvY3U4HCYcDifv925V0tnZecwuHqFQiOrqaqqqqkbd8hUnex9G0mh+H4ckGoRN/wFRvUu/IxTnOe3LOHOLWX7ORGNjG0V8Ph9er/e4f6NHM6xEtGXLFpqbmzn99NOTx+LxOBs3buQXv/gF4XD4mPYQm82GzXZs/V2Iodpa2044mmBaqZt8lxUObtaTkLMALHYcsVrGtW6n2rqYeEKT9YnSzLBEdOGFF7Jjx44+x6677jqmT5/OD3/4w34bZYUYDh/Xd3K4O0KZ106+0wJNPZ/NieeB3YO182nO9bZwzlmlxwzxEENnWCJyu93MmjWrzzGXy0VBQcExx8eaDRs2GB3CmKJpGh0Bvccs12mBzoMQ8oHZCoVTwWRGceRTQDuE6sA7w+CIs4/hvWZCGK0rHCOW0DApCh67BQ73jLkqPAVUiz5ytahnqk57tXGBZrGMGNDYS0oCwggdfr005HWY9YmtHTX6D/InHTkpdwLd+96hdf8nRPMXMqU4x4BIs5eUiMSY1x7Qu+TzXFa9t6y7ZyJwbuWRk7zj6QzFaWhqpPpggwFRZjdJRGLM6wz2loh62oc0Te8ts7mPnGS2YfaWAhBrqzUizKwmiUiMeV2hGABuuwV8PaUd77hjzrPm6ccS3YdGLLaxIqPaiIQwwsUzSzh3SoE+afmTniTjLj3mPGe+nohUfzPReAKLKv/H00XeSTHmmVUTuU4rTosKXfpyH7jLjjnPmluGalJwRQ7T3VOdE+khiUiIXmEfRAKgmMBVfMyPFVcRFrOKORHG39VmQIDZSxKRGNO6QlHWf9zE5gNt0NWkH3QVgnqcVgvVrDdiA8EOWWI3ncZWItI0iEVG/jaIecVPP/00BQUFfSb3AixdupRrrrkm3e/ImNfuj7KrwceuBh/4W/SDOSX9nj95wgRmj/cy0REaoQjHhrHVWB2PwlsPj/x1z/9nfbrAAHzta1/j1ltv5b/+67/42te+Bui7b7zyyiv85S9/Gc4oxyRfSG/r8TjM4D+sH3T1v8+9J78EuvZDuH0kwhszxlaJaBRwOBx885vfTO68AfDMM89QWVnJokWLjAssSyW77m0WCPQkImf/iai3akagdZgjG1vGVolIteilEyOuOwg33HADZ511FvX19YwbN461a9eyYsWKPovpi/ToLRG5bSZo6SnluAr6Pb9b9dDZESTeXUflvJGIcGwYW4lIUQZcRTLSvHnzmDNnDk8//TRf/vKX+fjjj3nllVeMDisr9ZaI8kwBSMT0Bml7br/nB8251LYFsKhBKqNBsMhSu+kwthLRKLJy5UoeffRR6uvrueiii6ioqDj5g8Sg9S6Y70l06AecBcfuE3UUd46LiOqCuJ9Y92HMefJ7SQdpI8pQ3/zmNzl48CC//vWvuf76640OJytpmoY/rJeIcuKd+sETtQ+hL7Ifs+pLngZ80k6ULpKIMpTX6+UrX/kKOTk5LF261OhwspKiKHx30WSuP7cKZ7SnfcjZf/tQ72MURy4AQZ8MakwXqZplsPr6er71rW/JOt3DyKKa8DpNEOrQDzhPvqW0qScRRf3ShZ8ukogyUHt7Oxs2bGDDhg388pe/NDqcsaE3ETlOvsmn6tKTlSSi9JFElIHmzZtHe3s7P/3pT5k2bZrR4WSt2tYAu5t8jHebmBkJ6AcHkIgsOXloQCIgiShdJBFloAMHDhgdwphwqCvErgYf1twQMwGsTjCfvBp8alUllmYvFksMEgkwSVPrUMk7KMas3h4zD136gROMHzqa15uH027DoqDP2BdDlvWJyKCNbLNGNr9//nAcgBzNrx8YQLUM0McZ2b36971tS2JIsjYRWSz6tIpAIGBwJKNb7/vX+35mk+QYokRPiWiAiSgUjfNpl5maNj+EOocrvDEla9uIVFUlNzeX5mZ93Rin0ylztQZB0zQCgQDNzc3k5uZm5c67/oieiBzxnurVABORpsHOVijpDjE+0EH2vTMjL2sTEUBpqb7ucG8yEoOXm5ubfB+zydGjqh2x3hJR7oAea7eYiJr1HT7C/g6cwxHgGJPViUhRFMrKyiguLiYalTWGB8tisWRlSQggEk8QjWsoWgxL3A8KAy4RKYqC6nBDB0T8nZKI0iCrE1EvVVWz9g9KpMZmVrnlgikE2w9h3oG+KoNl4ClFdeiN1dGAtBGlw5hIREIcj0U1YaGnx8yee8JZ98c81qUnonjQpzcaSfvjkGRtr5kQA9Lb6zXA9qFeNmdPiSgSglj4JGeLk5ESkRiTPm3uYn+LnxmBJioAbN5BPd7hsBMz2YjGExDuAot9WOIcKyQRiTGpoUOf3jFeOQwqYPcM6vGnjfOiTK3AHm7VR1fnFA1PoGOEVM3EmNTbde/qHVVtH1yJyG23kOMpwGwyyTSPNJBEJMYkf0Sf3uFM9CQi2+BKRPpj9LFEhLvSFNXYJYlIjEn+cAxFi2GLB/UDg6yaRWIJPm7TqG71o4WkRDRUkojEmNQdjmGN+bGaTfrOHYMYQ9RrW1OMps4QsaAkoqGSRCTGnGg8QSSWwBbvxqIqeo/ZIMcBWc0mtJ6qmQxqHDpJRGLMCfQs/+HS/KgmZdDVsl6mnsfFJBENmXTfizHH67Tw/QumENnfgFKvpNZQDVgcPYkoEoZYZFRs3pmppEQkxiSzajrSY5ZiicjmcBBXLPqgxkh3GqMbewxNRE8++SSzZ8/G4/Hg8XhYsGABr776qpEhibGkt7drkGOIejmtZqKq88joapEyQxPR+PHjefDBB9myZQsffPABF1xwAVdeeSUff/yxkWGJLLezvpPXdjbR0tqiH0ixaua0qkRUB9GEBhF/GiMcewxtI7r88sv73L///vt58sknee+995g5c6ZBUYlsd7A9yO6GTmYF28FjTblqNnu8F06diKNjj1TNhihjGqvj8TgvvPACfr+fBQsWHPeccDhMOHxkprPPJ+M3xOAFIjEsiSBWJaF326dYInLbLeDJBZ9JEtEQGd5YvWPHDnJycrDZbHz3u9/lxRdfZMaMGcc994EHHsDr9SZvFRUVIxytyAb+cAxrrFsfzGjNAdMQFs2z5ehfw5KIhsLwRDRt2jS2b9/Opk2b+Md//EeWL1/Orl27jnvu6tWr6ezsTN7q6upGOFqRDfyROLZ4N1bVlHK1DPTdPD5qjum7eUgb0ZAYXjWzWq1MmTIFgDPOOIPNmzfz2GOP8atf/eqYc202GzbbyXfiFKI/8YRGMBInN9aNRTWlXC3rfa4PGiPM6AhREe4y/r/6KJZx710ikejTDiREOiW3EEp0Y1aVlLvuARwWlYjqQgNiIem+HwpDS0SrV6/m0ksvpbKykq6uLp599lk2bNjA+vXrjQxLZLHe6R0eJYhC6tM7AEwmBdWutxHFQgGs8Zg+gVYMmqHvWnNzM9deey2NjY14vV5mz57N+vXr+dKXvmRkWCKLlXrtfP+CKcQ3b4Qgg14i9vNsdicJRSUa1yDSNfBtq0Ufhiai3/zmN0ZeXoxRZtWEOdbTyzWEEhGAw2omqjp6pnn4JRGlKOPaiIQYdrEIRHsXRBtaichpNRNRXT3TPKQLP1VSoRVjypaadjoON3JmMIrXnQPmofXCOq0qAdVJLO6TLvwhkEQkxpS6tgBt9Y2cRnzI1TKA0yfkoUQm4mzp0NuIREqkaibGlO5wrGdlRtOQG6oBvA4LHm+evpuHlIhSJolIjCmBSAxbrKtnVPXQExEAVpf+VdqIUiaJSIwZiYRGoGd6h8U8tOkdvfzhGNuaYtS1B2Ti6xBIIhJjhj8SQ9PAHvdjMaU+6/5okViC9xvCNHaGJBENgSQiMWYEejZVzCGAogxtVHUvh1Wf5hFPaMTDAUgkhvycY5EkIjFmBCJx0BK4tIB+IA1tRDazCc3sQEMhFo9DVBqsUyGJSIwZVYUubj2vlFNLe9YgsuYM+TkVRcFp6x1dLUvGpkoSkRhT1EhXT9e9e9CbKvbHYVWPWkRf2olSIYlIjC3hnuWF09BQ3UtfRN8p2woNgYysFmPGu/sPo9Z+xoxwFHe6xhABDouZsMlBTKpmKUupRPTZZ5+lOw4hht2BwwEam5v1tpw09Jj1WjCpgMWzJ1HisUsiSlFKiWjKlCksXryYZ555hlAolO6YhBgW+qjqnrWq01g18zoteDy5qCZF5pulKKVEtHXrVmbPns2qVasoLS3lpptu4v333093bEKkTSKh4Q/Hsca7sajpGUPUR+9uHlIiSklKiWju3Lk89thjNDQ08NRTT9HY2Mh5553HrFmzeOSRR2hpaUl3nEIMSTAaJ5FIYE/u3pGbtuf2haJsaYxwsD0giShFQ+o1M5vNXHXVVbzwwgv89Kc/5dNPP+X222+noqIiuQSsEJnAH4mhJsLYlJg+qtrmTttzByNxNtWHOeQL671mmpa25x4rhpSIPvjgA773ve9RVlbGI488wu23387+/ft5/fXXaWho4Morr0xXnEIMiT8cP1IasjpBtaTtuR1WlahJ777X4lGIyS40g5VS9/0jjzzCmjVr2LNnD5dddhlPP/00l112GSaTnteqqqpYu3YtEydOTGesQqQsFI1jjfXOuk9f1z2A06KSMJmJmSzEExrmiB8s9rReI9ullIiefPJJrr/+elasWEFZWdlxzykuLpbF8UXGOLXMwynz8tD25aS1xwz0xfitZlPP2tUa5kg3uArSeo1sl1Iiev3116msrEyWgHppmkZdXR2VlZVYrVaWL1+eliCFSAc10gmm9JeIAFxWlajJQTTuxyEN1oOWUhvR5MmTOXz48DHH29raqKqqGnJQQgyLUM/0jmFIRE6r+ch8M0lEg5ZSItL66RXo7u7Gbpe6scg8b3zSzM7P6ghEYmmvmkHvukSOIxstikEZVNVs1apVgL70wV133YXT6Uz+LB6Ps2nTJubOnZvWAIVIh/0t3aith5lQZhmWEtH5Uwsx2SfhOtggJaIUDCoRbdu2DdBLRDt27MBqtSZ/ZrVamTNnDrfffnt6IxRiiDRNIxQKY0kEsaq29I+qBnKdVvDmQYMiiSgFg0pEb7zxBgDXXXcdjz32GB5P+n+hQqRbMBpHjerVJYvVDuZhaj7o3c1DlgIZtJR6zdasWZPuOIQYNt1hfbKrRVUwObxpWxDtaB2BCPsaIozrDFJukUQ0WANORFdddRVr167F4/Fw1VVXnfDcdevWDTkwIdIlENa3EErrXmaf0xWK8d7BMOf7wpR7g5CI68vRigEZcCLyer36HJ2e74UYLfQSURdWc3qX/zia06oSM9mJ9G7iEfEPS1tUthpwIjq6OiZVMzGahGO9y38MX4nIaTWDohDETkLTMEkiGpSU2oiCwSCapiW772tqanjxxReZMWMGX/7yl9MaoBBDdcaEfOa1uki0O4ctOdgtJkyKQlR1EEvEsUrP2aCkNKDxyiuv5Omnnwago6OD+fPn8/DDD3PllVfy5JNPpjVAIdLBFPZhHqbpHaCPrXNYTUctoi+DGgcj5RUazz//fAD++Mc/UlpaSk1NDU8//TQ///nP0xqgEEOWSEC4JzEMUxsRgMNq7plvJtM8BiulRBQIBHC79YWl/vKXv3DVVVdhMpn4whe+QE1NTVoDFGKo1m/7lE8PdRKOa2nZVLE/TouanIEviWhwUl48/6WXXqKuro7169cn24Wam5tlkKPIKJqmUd94iJbuCJrNo8++HyaLpxdz8bzJ5LusMqhxkFL6rdx1113cfvvtTJw4kbPPPpsFCxYAeulo3rx5aQ1QiKEIROKYY10ogNWVO6zXyndZ8XhzURVFdnwdpJR6zb761a9y3nnn0djYyJw5c5LHL7zwQv7hH/4hbcEJMVT+cAx7rOvIqOrhlpzmIVWzwUh5p9fS0lJKS0v7HJs/f/6QAxIinbrDMWzxrmFZIvbzWrvDfNoQYYIvRKnJrC+iPwzTSbJRSlUzv9/PnXfeyTnnnMOUKVOYNGlSn9tAPfDAA5x11lm43W6Ki4tZunQpe/bsSSUkIY7LH47ro6rTvIXQ8XQEo7xXG6KlKwyJmCyiPwgplYhWrlzJm2++yTXXXENZWVly6sdgvfnmm9x8882cddZZxGIxfvSjH/HlL3+ZXbt24XK5UnpOIY7WHY5hj/mwOkzgyB3Wazmt+iL6Qa3nzyrSLYvoD1BKiejVV1/llVde4dxzzx3SxV977bU+99euXUtxcTFbtmzhi1/84pCeWwiAaCyGLd6NxWwf9hKR06L/OQWxo6GhRLrBVTis18wWKSWivLw88vPz0x0LnZ2dAP0+dzgcJhw+Utz1+Xxpj0Fkly9W2kjU56Epalo3VTweh1WfbR9SHMQTIX1bITEgKbUR3Xfffdx1110EAoG0BZJIJLjttts499xzmTVr1nHPeeCBB/B6vclbRUVF2q4vslSoA5OioDrzhr3h2Go+sq1QTAY1DkpKJaKHH36Y/fv3U1JSwsSJE7FY+u6auXXr1kE/580338zOnTt5++23+z1n9erVyXWzQS8RSTISJxTs0L8Oc/tQL4dFJarq0zzsMqhxwFJKREuXLk1rELfccgsvv/wyGzduZPz48f2eZ7PZsNlsab22yF6JhMZbH+1jXGcXE0rdpG+T6f45rWrPtkKaDGochJQS0d13352Wi2uaxve//31efPFFNmzYIHuiibTqjsToaj9MazDCZFfeiFzzSzNKsBRNxlW9T6pmg5DyxJuOjg7+8z//k9WrV9PW1gboVbL6+voBP8fNN9/MM888w7PPPovb7aapqYmmpiaCwWCqYQmR1BWKYYv5sKgmFMfIJKKCHNuRaR5SNRuwlEpEH330ERdddBFer5cDBw5www03kJ+fz7p166itrU2uVXQyvWsXLVq0qM/xNWvWsGLFilRCEyKpO6RP77BZh38wYx/Wnt45KRENWEololWrVrFixQr27dvXZ2fXyy67jI0bNw74eTRNO+5NkpBIB3+gG3MipK9VPUKN1c1dId6rD9HcFYJozyL64qRSSkSbN2/mpptuOub4uHHjaGpqGnJQQqRD0Kc3GZhtLjCPTCdHmz/C32sCtPhj+gGpng1ISonIZrMddzDh3r17KSoqGnJQQqRDpEtPRKor/YNv++O09C6i35P4pHo2ICkloiuuuIJ7772XaDQK6Ov11tbW8sMf/pCvfOUraQ1QiFSZIj4UwDLM6xAdrXd0tR+HfkAS0YCklIgefvhhuru7KSoqIhgMsnDhQqZMmYLb7eb+++9Pd4xCpGRhhYX5VfkUF5ee/OQ0cfYkogB2NE2TqtkApdRr5vV6ef3113nnnXf48MMP6e7u5vTTT+eiiy5Kd3xCpC7YjklRwJk7Ypd0WFQUBSImB9GED6sMahyQQSeiRCLB2rVrWbduHQcOHEBRFKqqqigtLUXTtJSXBBEi7YJ6GxHOghG7pMmk4LCoREz6tkKyv9nADKpqpmkaV1xxBStXrqS+vp7TTjuNmTNnUlNTw4oVK2SZWJExmtq72flZLTVtfnCMXGM1gMtmJqo6iMQSUjUboEGViNauXcvGjRv561//yuLFi/v87G9/+xtLly7l6aef5tprr01rkEIMlq+9ha5gFEyWI+tIj5BLZpViHXcKOft2S2P1AA2qRPTcc8/xox/96JgkBHDBBRdwxx138Lvf/S5twQmRqrDvMACKM3/E140uzLHh8eTq7VNSIhqQQSWijz76iEsuuaTfn1966aV8+OGHQw5KiKGKdumJSHWNXPtQH0fv5qFpxsQwigwqEbW1tVFSUtLvz0tKSmhvbx9yUEIMVdTfCoDFPfKJqKUrzLt1IZp8IX2KRyw04jGMNoNKRPF4HLO5/2YlVVWJxWJDDkqIoUr49R4zm3vk14zuCETYVOPjUO8iEtJOdFKDaqzunZDa3+JkR68nLYSREoFWTIAzt3jEr+206X9WAa1nQni4SxbRP4lBJaLly5ef9BzpMRNGi4aDOBJBIkBO7sjPfcyx6n9W3ZodjSCKlIhOalCJaM2aNcMVhxBpY4l0cnplHgmLE5Nj5PfHc9qO3s0jILt5DEDKKzQKkbECevuQyTmyAxl7WVQTNoupZxF9mW82EJKIRPbpndoxQsvDHo/LaiaiOntGV0uJ6GRSmvQqRCbbW32AaEMn7jwHlQbF4LSqhFR9vpmUiE5OEpHIOsGOQwRCMVSLcSWii04tQS2fSs4nH0qJaAAkEYnskkig+fVR1U5v/4Nvh1ueywpaz+6yUiI6KWkjEllFC7YRicZIKGbcuQaP3bHl6F+jIYjLQN8TkUQkskqo8xDxhEbQkovbMRJ7ux5fuz/Cuwe6qffpyykTPnaNd3GEJCKRVQLthwDQnIWYVeM+3t3hGJsOtFMf7Gn9kER0QpKIRFYJd+qJyJRj7G4yrp5pHj7NqR8IdxkYTeaTRCSyihJoxWY2YfGM/Byzo7l6RlcHFCexRAJCUiI6Eek1E9kjkaDMEqCsMg9t7nRDQ7GZVWwWE2E1h0jsEGYpEZ2QlIhE9gi26+v/qBaUkdzrvh9um5mI2UUknpA2opOQRCSyh79Z/+oqHPHlYY8nx27uKRFJIjoZSUQia0Q6Gtha2847TSrxhPHLs+bYLERUl56IpI3ohKSNSGSNYFsj4ViCQ1ouqsn4EtHZk/KZP96F+4O/QSwMsQiYrUaHlZGkRCSyg6YR7mgAwOIZuS2mT8Rjt+D15GCy9KxoKtWzfkkiEtkh0k040AUo2PMyIxEl2Tz611CnsXFkMElEIjt0HSIcjRO0eMl1j/yqjMcTisZ599PD7Gztaa+SLvx+SSIS2aH7EMFoAr+lkFwD55gdTVFgU3Ub+30mfVCjVM36JYlIZAWt+xDBaBy/tUBfgiMDHBnU2NNzJiWifkkiElkh1tmIy6pCTgl5zsxIRNAzqFHN0Qc1Shd+vyQRjTKhaJzW7r77x9V3BPs5e4yIBLBEfMws93L1BWdlRNd9rxy7mZDZ0zOWqMPocDKWJKJRRNM0XvmokXVb6+kM6uvcvLWvhT9srmNn/Rjukelq1L86C8DiMDaWz8mxWQib3YR7BzUmEkaHlJEMTUQbN27k8ssvp7y8HEVReOmll4wMJ+PtqO+kti1AOBbXF2VH37oG4M29LXSFokaGZxxfPZqmgafc6EiO4bbru3mE44AmDdb9MTQR+f1+5syZwxNPPGFkGKNCJJbgvc9aATh3SiGFOfogufkT8ynz2onEErxf3WZkiMbxNbCzwcefqzOvmup1WEAx0aX0LBsrY4mOy9ApHpdeeimXXnqpkSGMGjvqO/GH43gdFmaPz9WL+R21mKwuzptSxAtbGvi4wcf8qnzc9szovh4RmobWWU8wEqNJy8dhUY2OqI+qQhfXnTsRz95KaD/Q0040weCoMs+ommsWDocJh4801Pp8Y6OYq2kaHx3sAOCsifmojdvg0//Vl7wAxnvKmOBeQE0XfFjXyXlTDV40fiQFWgmHgkQxE7YVZMwYol52i4rdouqbPbYfgGCH0SFlpFHVWP3AAw/g9XqTt4qKCqNDGhG1bQE6AlGsZhPTTbWwd72ehHKK9UmUvkbODfwNRYuxq7GTRAbMPB8xvnoC0Tjd1iLycuyYMqjHrA+7V/8qVbPjGlWJaPXq1XR2diZvdXV1Roc0Iura9HaP04oULPv/oh+smA9nXg9nXAcWB0W0M9W/FYD2QMSoUEdeRy3+cIwuWwlFPe1mmWZ7XQdvH4zSHY5JF34/RlXVzGazYbNl5odtOJ03tZBZ4zxY97+mLyXhKYdJi/U5BM58mP5/MO14gcU5tdjOuAzVNYbeo55E5HOVM9mTma+7+nA3Le0qpyRi5EjV7LhGVYloLMvVunC27tLvTLkQTEf96gqnQP4knGYFteYtYwI0QrADQj66Iwm6bCUUe+xGR3RcXoc+ligUTejbT8fH6DCLEzA0EXV3d7N9+3a2b98OQHV1Ndu3b6e2ttbIsDJKrGe8EPVbQNOgYAp4xx974qSF+tfmT9CCHYSi8ZEL0igdtSQ0DWdhBcV57oytmnnsFmImG8FET4+etBMdw9Cq2QcffMDixYuT91etWgXA8uXLWbt2rUFRZY5QNM5v3q6m0mPisu4PUQHGn3n8k92lkFtJe8OnvLf+f4hVLeLyOZk3wC+tOmoxKQozTp3FjEmVRkfTL30skYJPyQGiEGjT19UWSYYmokWLFukjYsVx1bYFiMQSmJp3o9pj+oc3b2L/D6iYj7X5M9zNO9meM49IrBSrOUtr35oGHTX697mZm4QAPD1DCjrwAK0QHKMDT08gSz+l2eGzFj8AUzioHyibc+LdKfIn43Tn4jLF8PgPUNsWGIEoDRJohZCPQAxiOeOMjuaEvMlE5NYX9Q+0GhxR5pFElKE0TaOm1Y8lHqCUw/rBomknfpDJhFJ6GrkuK0Xde6k+7B/+QI3Suh8NjffaXPzyrVpausInf4xB7BYVp1UlbMklHIvrVTPRhySiDNXkCxGIxCmJ1uG2qXobUO+guBMpOY08pwVv6CAHm1qyt+rbtp9QNMEhy3gUIM+ZWSOqP+8bZ1fyjUVzcVrNUjU7DklEGaq3NDNVqcekKFA0wC2UXQW4iydgNoGj/ZOMLimkLBaGzoN0haJ02MdT4rFjVjP7o+yxW1BdBfqdSACimTU512iZ/dsbw6oP+1HjIcq1Q/qBk1XLjqKWzsLrsJAfrOazbKyetR+ARJy2hIuQJZfy3Mxag6hfZivY3Pr3Uj3rY1SNrB4rNE1jeqmHNv8+8sxmvbfMmT/wJyiaRqHbhi3YiSMnC8cTtXwCQJ1Jb6Quz83MgYxH6whE2FTdRlmbymwXeoO1N7Mb2UeSJKIMpCgKZ0zIA18HHDYNvFrWy+amoHwyBR11EK4BiocjTGPEY9D6KeFYnAOmChSFUVEiSmiwq8FHNGTjNJeGIu1EfUjVLFPFItBWrX8/iGpZUm/yatmdvpgyQXs1xCK0x+10W0so8dj1ZTYynNdhwaQodJm8+vrV0oXfhySiDBOLJ9jV4CN4aC8kYvo6Nq6iwT9R0TTiGnQ0VbOrOotWKWjWE6t7/Cy+OK2IOeNzjY1ngFSTgtdhJmjJ0+ec+Q8bHVJGkUSUYRo6Qqz/uIl3/v4OGppeGjrRIMb+2Nx020rY3djFro8+0AfSjXaxCBzeC4CrYhZnTMhnRrnH4KAGLs9lxW8pIBiNQ7Bdfz0CkESUcapb/ShajAk0oqCkVi3r4ak8Dauq4On6lPr2LOgubtmtz1x35oNn9DX0FubYiKkOfHGbPkUlIKWiXpKIMsyBw35ygwfJtwN2D7jLUn4upXg6XqcNd/gQNQ2N6QvSKI0fAXDAMpmPG32jboWB3g0P2pSegandzQZGk1kkEWWQzkCUNn+EglCNPj+pMMVqWS+bG3fJRP25a3eM7lHW/lboPAiKwjvdZfzl40McaB1dY6QKc6woCvgt+Xq1299idEgZQxJRBtGrZXEmaPWYTSYoOmXIz5lXNQeTAra2PbQHRvGCXPVbAOhyTaA5bEE1KVQVugwOanDyXVa+t2gKF545S692S4koSRJRBjlw2I8n1EihLQFWJ3iOswDaIFlLTsXtsJITaaG2vj4NURogGoSmDwHYZzkVgMp8JzZz5nfbH01RFH1ZFlfPuK7uQ3pbkZBElCniCY2D7QHyg9XkOnuqZaY0/HpsObiKJwEQOPjx0J/PCA3bIB5DyylmW5feSza9zG1wUEPgKgSTWZ8zF2w3OpqMICOrM4RqUrjunAn4N7TisKppqZb1Kp4yl8JYI073KKwKxCJwcDMAhzyz8XXEsFlMTC7KMTiw1NS1Bfj7/lZO7bRzmrsbfA2Dm76TpaRElEFcoSaKbXEUsx1y07cbqKNsBi6bFaX70OibbFn/gT5b3ZHH1rDegzitxI0lw2fb90c1KdR3BKmLefUGa1+D0SFlhNH528xWLXv0r4VTwZTG9g+rM7nErHZoFFXPokGofQ8AbeK5xDQTigKnjR/AukwZqthtQzUpHDYVEo4moEsSEUjVLCM0dYbYuLeZ81u2U+YAik5N+zUiBdOo/eQjmhs3clbFOVhGQ0Nv9Vt6O4qrEKV4JleUmugMRpNLr45GZtVEkdtGe6SY7nAMe3ezPpFXHdt/ilIiygCfNnfja6qmu6sdzDbIr0r7NSylp9IeASXYTn3N3rQ/f9r5GqFB37mWKRclG+5HcxLqVeq1E1bddMbM+tbh3U1Gh2Q4SUQG0zSNfc1dFAQ+I99lhcJT0lst66GYbTjHzwLg8N7NaX/+tIrHYO+retd2yQx2hwvoDI7iMVCfU+a1g6LQpPR043fIPn6SiAzW6o/Q4Q9TFKrW/9sPdu2hQSg+5WwAYk0fEwll8A4f1W9C1yGwOGgtPY/Xdx3it+8eoCOQHZNEy7z6+kl1WhGxRALaawyOyHiSiAy291AXnnAThdYoZqtjWKplvQrLJoKrEC0Rp2HP+8N2nSFp3Q91emyJUy7l9U/9xBMaEwqc5DqtBgeXHl6HhRKPndxxU4klNH3qSjxmdFiGkkRkIE3T2N3YRaF/nz4hsmjasFTLeikmE86q+QB07n0XEolhu1ZKupth10v69+NOZ0ugiMbOEFaziQumZ9Eqk8A35ldwyVkzsTs9+rpTvlE66j1NJBEZqL4jSLc/QEmomjyXFUpnD/s1J874AjHVQcDXhq9ux7Bfb8CCHbDjBX0AY24ldfnn8M6n+jIZC08pwm0f/Y3UR1MURZ/QnNczXqztM2MDMpgkIgOZTSZm2xopdir6VjPeoc8tOxmv24lj4nwq8hzYGzdnxlynQBts/x2EfOAsoGXCEl7e2YymwYxyDzNH0eJng9XpnEA0noDD+4wOxVCSiAxU6rWz2NNIVZELSk8b2pIfg3DmORcyvigXa7AFDu0ckWv2q6MOtv1/ySTE3G/wYVOYUDROmdfOBdOL9dJDFnptZxO/3WOmJRDT17Aew8vHSiIykq8ROg+iKCqUzBq561qdUHmO/n31RmOWLNU0qNsMHz6nT+HIKYa53wSbm8XTizl7Uj5L540btVM5BqLYYyNuslKb6Gn/OjwKxncNk+z9LWcwTdPYfKAN/2f69AWKp+urMY5kDOPOoCVmY09NPfH9G0b02vgP61WxT/8XEnFihdPYnLeEmNkJ6POxzplcOCp25xiKKcX6xN39jCMST8ChjzOjqmwASUQGqGkNsGl3DTu3v0dC02D8/BGPIa6Y+Wt8Hm3+CC2fvKN3mw+3YDt88gps/k/oqENTLRwoOJffdszl7Wof7302yibkDpHHbqE8106ro4oWf1xP0GO092xsT3AxgKZpvPdZK+N82ynJsWDKrQBP6utSp8qsmpg+cx4HOqsxt+2mYMeLWM64Btwl6b1QLKL3CDV9pH/VNBKaRr25gneUuTQesgEx3HYz4/Myf6PEdJtZ7qWhI8Tu+DjKOYTSsH1EOi0yjSSiEXagNUBbawunBz6hvMIDVecbFsvscV52Viyk7bN2ag61MWX772DWVcmZ+imJR6GrSf/P3lHXs0/9kcF6+2LFvK+dSnO8EACr2cSZE/I4fUJeVrcH9WdaqZuN+1qotk6lI3CQvObdMGkR2EbnekupkkQ0guIJjY17W6js2ESZ24K1cNLQ/uiHyGRSWDyjnHVdX0JpWY+33UfRh89D2RyoOPvEC3bFY3pVK3BYr1Ikv7aBpg+U1DSN7nAMV24RpuJpUDaX6uoIzQ0+cmxm5lbmcto4b9a3BZ2IRTUxs9zL1gNxGsJ55CX8ULcJplxodGgjShLRCNpa207i8D5Kw9WUl+TD5MVGh8S4XAdnTR3HJpYQ6XyH89QGchu2Q8P2nl1mC8Hi0JNLPArhLgj7IOI/bsNqNJ6gLWahIVFAdTSPZnM5V546m7JcvSF6bmWISUU5VBW6UE3Z2S0/WKdX5lKUY2O65TLY+YK+6sD4M8E+etddGixJRCPkcHeYD/bWMaPtLSbku7BUzgd3qdFhATB/Yj6Hu8PsMy1k8vgIudHd0LZfL/GcaE1lsxWchXSpuezttlEXdlEfdRFRXaAqYAG7RaUrHKe3FazYbad4FC83PRzcdgszyi2guSG3Qq/S7nsdZn1lxMaWGU0S0QjJs5u4SPs7CXuMwpKJMNG4tqHPM5kULp1VRkVeJ9PHe0GZBdEQDbX7ycGP0xTBpKiENRPdmp3WmB1PXgHlhQWgKHR1BNm4uU5/MjMUuW1UFbqYWOiizGPHJCWfgVEUwpMuwvfW/6OIfXrJaNwZRkc1IiQRjYR4DHX3n5lqbUcrz0eZdZVemsggqklhTkVu8n5EsfKnA1ZiiePP8Zodj1NepCeYEo+duZW5jM91MC7PgdMqH6tURGIJnt0ZxBk+lYXxnZTu+1+w50LBZKNDG3YZ0U3xxBNPMHHiROx2O2effTbvv5+hS1QMUlNniPVb9hLb9ow+l8hkRpn1Fb3dJcN1haIUum2YjyrNKArk2MxU5juT2yeDnsQWTytmaolbktAQWM0mTi3z0OCezQeBUg62daPt+CM0fpj1Ax0N/9T8/ve/Z9WqVfzHf/wHZ599No8++igXX3wxe/bsobh4dC790O6PsG1/PW1736PM9yEHPWYmluTDzKXDut5QOhXk2PjG/EoSCY1IPIGmgc1skmrWMDu7Kp9ILMFWFkHrG3TU11IR+DOecZ+gVJ0PnnKjQxwWimbwhuhnn302Z511Fr/4xS8ASCQSVFRU8P3vf5877rjjhI/1+Xx4vV46OzvxeAyaoZ1IkIgG2d/Qgq/1EG0tB0m01eAON6FoCYpyrFROmoZ15uWyf5UYsJ31nby55xBFbVsZ37kFl9XE7HFeFFchFEwiZC/B4i5EteeAxZWezTiHwUD/Rg0tEUUiEbZs2cLq1auTx0wmExdddBF///vfjzk/HA4TDoeT930+38Au1FYN+/8KmsbmmnYSiUSyqKvnYQ008NjNzEjuIKqx5UAb0XhC338qma/1c912lVllbohHULQEHQfaiSU0cgAFyHVaKBk/hbxTzoPiU8dM74dIj1njvEwocLLpMy+76iZzamQnitqiz9IPtLKrtp1wLIFqUjApgGImYTKDyYzTbmPWuNzkZ257XQfhWM8ieMnPof7VYTUz+6jtmT482EkoEk/et5pNzKvMPX6Qp309bXMkDU1Ehw8fJh6PU1LSd1pBSUkJn3zyyTHnP/DAA9xzzz2Dv1AsDN0tAGjdbSQSxy8EapoZQkdWLdTCPrT48c+NKyrE9AZnBYV8bw4RSy6u/BKKyydgL5kmJSAxJG67hYtmlBCaWog/PBtsmj6sovMgHY07UDUf1niQOBoQ7bkBWhCCRz63CX87idjxV+PUrCp0H1l9QevqIBE9kog0swm6+9m4QEvfCp+GtxENxurVq1m1alXyvs/no6Ki4uQPzK2AOVcDMH5iOPlfQf+iJFfLs5pVcFp7fqBQNT1C8tep6OcpgKKYUFUT2K1gsYPZzuRhXOJVjG12i3pk9HnJTCiZyaIpXyYSTxCKRImFA2jxKIl4DC0exUwcXL29shrjJoeIH/3P96jSvdmkgPtIx0PZpDCxo/75qmrfn/dhdaXtNRqaiAoLC1FVlUOHDvU5fujQIUpLjx3sZ7PZsNn6eVNOxOpKNhKXD6KQUpC9CwOKUc5kUrCbehKUy37Cc0tyB/68JQYV4g1t4bJarZxxxhn89a9/TR5LJBL89a9/ZcGCBQZGJoQYSYZXzVatWsXy5cs588wzmT9/Po8++ih+v5/rrrvO6NCEECPE8ES0bNkyWlpauOuuu2hqamLu3Lm89tprxzRgCyGyl+HjiIYiI8YRCSH6NdC/0cwcBSWEGFMkEQkhDCeJSAhhOMMbq4eit3lrwFM9hBAjqvdv82RN0aM6EXV1dQEMbHS1EMIwXV1deL39L307qnvNEokEDQ0NuN3urN2W+ER6p7jU1dVJr+EAyPs1eEN9zzRNo6uri/LyckwnWCFgVJeITCYT48ePvT2gPs/j8cgf1iDI+zV4Q3nPTlQS6iWN1UIIw0kiEkIYThLRKGaz2bj77rtTW5FgDJL3a/BG6j0b1Y3VQojsICUiIYThJBEJIQwniUgIYThJREIIw0kiymCD3QH3hRdeYPr06djtdk477TT+53/+Z4QizRyDec/Wrl2rb4hw1M1uP/H6z9lk48aNXH755ZSXl6MoCi+99NJJH7NhwwZOP/10bDYbU6ZMYe3atWmJRRJRhurdAffuu+9m69atzJkzh4svvpjm5ubjnv/uu+/yjW98g+985zts27aNpUuXsnTpUnbu3DnCkRtnsO8Z6COGGxsbk7eampoRjNhYfr+fOXPm8MQTTwzo/OrqapYsWcLixYvZvn07t912GytXrmT9+vVDD0YTGWn+/PnazTffnLwfj8e18vJy7YEHHjju+V//+te1JUuW9Dl29tlnazfddNOwxplJBvuerVmzRvN6vSMUXWYDtBdffPGE5/zrv/6rNnPmzD7Hli1bpl188cVDvr6UiDJQ7w64F110UfLYiXbABfj73//e53yAiy++uN/zs00q7xlAd3c3EyZMoKKigiuvvJKPP/54JMIdlYbzMyaJKAOdaAfcpqam4z6mqalpUOdnm1Tes2nTpvHUU0/x5z//mWeeeYZEIsE555zDwYMHRyLkUae/z5jP5yMYDA7puUf17HshhmLBggV99s8755xzOPXUU/nVr37FfffdZ2BkY4+UiDLQYHfABSgtLR3U+dkmlffs8ywWC/PmzePTTz8djhBHvf4+Yx6PB4fDMaTnlkSUgVLZAXfBggV9zgd4/fXXx8yOuenYNTgej7Njxw7KysqGK8xRbVg/Y0Nu7hbD4vnnn9dsNpu2du1abdeuXdqNN96o5ebmak1NTZqmado111yj3XHHHcnz33nnHc1sNmsPPfSQtnv3bu3uu+/WLBaLtmPHDqNewogb7Ht2zz33aOvXr9f279+vbdmyRbv66qs1u92uffzxx0a9hBHV1dWlbdu2Tdu2bZsGaI888oi2bds2raamRtM0Tbvjjju0a665Jnn+Z599pjmdTu1f/uVftN27d2tPPPGEpqqq9tprrw05FklEGezxxx/XKisrNavVqs2fP1977733kj9buHChtnz58j7n/+EPf9BOOeUUzWq1ajNnztReeeWVEY7YeIN5z2677bbkuSUlJdpll12mbd261YCojfHGG29owDG33vdo+fLl2sKFC495zNy5czWr1apNmjRJW7NmTVpikWVAhBCGkzYiIYThJBEJIQwniUgIYThJREIIw0kiEkIYThKREMJwkoiEEIaTRCSEMJwkIjGqrF27ltzc3OT9H//4x8ydOzd5f8WKFSxdunTE4xJDI4lIHNeKFStQFIXvfve7x/zs5ptvRlEUVqxY0ef8dCeAiRMn8uijj/Y5tmzZMvbu3dvvYx577LE+6ygvWrSI2267La1xifSTRCT6VVFRwfPPP99n0atQKMSzzz5LZWWlITE5HA6Ki4v7/bnX6+1TYhKjgyQi0a/TTz+diooK1q1blzy2bt06KisrmTdv3pCe+3gllaVLlyZLWYsWLaKmpoZ/+qd/Su6wAcdWzT7v6JLZihUrePPNN3nssceSz1FdXc2UKVN46KGH+jxu+/btKIoiaxEZRBKROKHrr7+eNWvWJO8/9dRTXHfddcN+3XXr1jF+/Hjuvffe5A4bg/XYY4+xYMECbrjhhuRzVFZWHvOaANasWcMXv/hFpkyZkq6XIAZBEpE4oW9/+9u8/fbb1NTUUFNTwzvvvMO3v/3tYb9ufn4+qqridrspLS1NaaVJr9eL1WrF6XQmn0NVVVasWMGePXuSe55Fo1GeffZZrr/++nS/DDFAsma1OKGioiKWLFnC2rVr0TSNJUuWUFhYaHRYQ1JeXs6SJUt46qmnmD9/Pv/93/9NOBzma1/7mtGhjVlSIhIndf3117N27Vp++9vfpq3UYDKZ+PxSWNFoNC3PPRArV65MNsSvWbOGZcuW4XQ6R+z6oi9JROKkLrnkEiKRCNFolIsvvjgtz1lUVNSn3Scejx+zK63VaiUejw/pOv09x2WXXYbL5eLJJ5/ktddek2qZwaRqJk5KVVV2796d/L4/nZ2dbN++vc+xgoICKioqjjn3ggsuYNWqVbzyyitMnjyZRx55hI6Ojj7nTJw4kY0bN3L11Vdjs9lSqhJOnDiRTZs2ceDAAXJycsjPz8dkMiXbilavXs3UqVPHzCYDmUpKRGJAPB4PHo/nhOds2LCBefPm9bndc889xz33+uuvZ/ny5Vx77bUsXLiQSZMmsXjx4j7n3HvvvRw4cIDJkydTVFSUUty33347qqoyY8YMioqKqK2tTf7sO9/5DpFIZER6AcWJyZrVYsx66623uPDCC6mrqztmB1MxsiQRiTEnHA7T0tLC8uXLKS0t5Xe/+53RIY15UjUTY85zzz3HhAkT6Ojo4N/+7d+MDkcgJSIhRAaQEpEQwnCSiIQQhpNEJIQwnCQiIYThJBEJIQwniUgIYThJREIIw0kiEkIY7v8HfcWXO0Q6VNgAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:22:46.176597Z", + "iopub.status.busy": "2024-03-26T11:22:46.175805Z", + "iopub.status.idle": "2024-03-26T11:22:46.528425Z", + "shell.execute_reply": "2024-03-26T11:22:46.527476Z" + }, + "papermill": { + "duration": 0.370709, + "end_time": "2024-03-26T11:22:46.530583", + "exception": false, + "start_time": "2024-03-26T11:22:46.159874", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/CElEQVR4nO3de3xU9Z34/9c5Z+6Tmcn9RhISLnK/qlBQK6j1ulbqbmtba8FLa6t2l+XnqnQfq6utS7tbXWlrbb/tCnUrtbX18uh2K2utSotKEQiCIGDIDZKQkNtk7pdzfn+cZCCQQDKZ5Ewmn+fjkUfIzJmZd4bknc/1/ZE0TdMQBEEwkGx0AIIgCCIRCYJgOJGIBEEwnEhEgiAYTiQiQRAMJxKRIAiGE4lIEATDiUQkCILhTEYHMBKqqtLU1ITL5UKSJKPDEQThDJqm0dPTQ2lpKbI8eLtnXCeipqYmysvLjQ5DEITzaGxspKysbND7x3UicrlcgP5Nut1ug6MRBOFMXq+X8vLyxO/qYMZ1IurrjrndbpGIBCGNnW/oRAxWC4JgOJGIBEEwnEhEgiAYblyPEQ2FpmnEYjHi8bjRoWQ0RVEwmUxiGYWQlIxORJFIhObmZgKBgNGhTAgOh4OSkhIsFovRoQjjTMYmIlVVqa2tRVEUSktLsVgs4q/1KNE0jUgkQltbG7W1tUyfPv2ci9cE4UwZm4gikQiqqlJeXo7D4TA6nIxnt9sxm83U19cTiUSw2WxGhzQiNW0+9jR0Ueiycsm0fBRZ/BEbTRmbiPqIv8xjJ1Pe63ZfmP/Z24yqaTR2BJAliUun5xsdVkbLjJ8cQUihLJuJJVW5uGz63+k9DZ2EomKyYzSJRCQIZ7CaFJZNzePOS6vIz7IQUzUOn+gxOqyMJhKRIAxCkiRmluhbh462+Q2OJrOJRJSGVqxYwdq1a40OY0I60OTlUEtPois2OU+f6DjWGSAWV40MLaNl/GC1IAzHe0fb6Q5GWbVoElX5TgqyrHxiSh5lOXZksfxj1EzIFlEkpg76ceZfvXNdGx3CtcO1Zs0a3n77bTZu3IgkSUiSRFlZGc8880y/6/bs2YMsy9TX1wPw5JNPMm/ePJxOJ+Xl5dxzzz34fL5+j/nLX/7CZZddht1up7y8nL//+7/H7xddjj7eUJTuYBRJgtJsffmBJEksm5pHea4DWUzhj5oJ2SJ6+s2PB72vKt/JqkWTEl//v201ROMDn8pdlmPnsxedKsz27PZagpH+syv/+KkLhhXbxo0bOXz4MHPnzuWxxx4D4IknnmDLli18/etfT1z3/PPPc8kllzB58mRAnzr//ve/T1VVFUePHuWee+7hgQce4Ec/+hEANTU1XHvttXz729/m2Wefpa2tjfvuu4/77ruPTZs2DSvGTNXqDQFQ4LJiNSkGRzOxTMgWUTrzeDxYLBYcDgfFxcUUFxdz6623sn37dhoaGgB91fgLL7zArbfemnjc2rVrWblyJZWVlVxxxRV8+9vf5te//nXi/g0bNnDrrbeydu1apk+fzvLly/n+97/Pc889RygUGvPvMx219oQBKMiy9rs9Glf5uLWH9+s6jAhrQkibFtF3vvMd1q9fzz/8wz/w1FNPjepr3bty2qD3ndn6/uonpw567ZlDBndcUjWSsAa1cOFCZs2axZYtW3jooYd4++23aW1t5bOf/Wzimj/+8Y9s2LCBjz76CK/XSywWIxQKEQgEcDgc7N27lw8++IDnn38+8RhN0xJbYWbNmjUqsY8nbX2JyHV2Ivrd3mYA5pdlYzGJv9+plhbv6M6dO/nJT37C/Pnzx+T1LCZ50A+TIg/5WvMQrk2VW2+9lS1btgCwZcsWrr32WvLy8gCoq6vjb/7mb5g/fz6//e1v2bVrF08//TSgb3UB8Pl83H333VRXVyc+9u7dy5EjR5g6dfBkO5Gc9OnvVf4ZLSKHxYTDonfVOgORMY9rIjA8Efl8Pm699VZ++tOfkpOTY3Q4acFisZxVtuSLX/wi+/fvZ9euXfzmN7/p1y3btWsXqqryxBNP8IlPfIILLriApqamfo9fvHgxBw4cYNq0aWd9iN3yEIrG8QajwNktIoAcp/4edfhFIhoNhieie++9lxtuuIGrrrrqvNeGw2G8Xm+/j0xUWVnJjh07qKur4+TJk6iqSmVlJcuXL+fOO+8kHo/z6U9/OnH9tGnTiEaj/OAHP+Do0aP893//Nz/+8Y/7PeeDDz7IO++8w3333Ud1dTVHjhzh1Vdf5b777hvrby8tWU0yd1xaxWcWTcJmPnugOk8kolFlaCJ64YUX2L17Nxs2bBjS9Rs2bMDj8SQ+MvUoofvvvx9FUZg9ezYFBQWJQepbb72VvXv38pnPfAa73Z64fsGCBTz55JN897vfZe7cuTz//PNnvafz58/n7bff5vDhw1x22WUsWrSIhx9+mNLS0jH93tKVJEl47GYq852nbuxqgHd/BH95ipJIHSAS0WiRNE0beG56lDU2NnLRRRfx+uuvJ8aGVqxYwcKFCwcdrA6Hw4TD4cTXfUeVdHd3n3WKRygUora2lqqqqnFfkmK8yKj3PBqEHT+GqD6j2BWK80vtahzZhaxeXmlsbOOI1+vF4/EM+Dt6OsNmzXbt2kVrayuLFy9O3BaPx9m2bRs//OEPCYfDKEr/JrLVasVqPbv/Lggjtbuhk3BUZUaxi1ynBY7t1JOQIw/MNuyxBia1V1NrWUlc1UR9ohQzLBFdeeWV7Nu3r99tt99+OzNnzuTBBx88KwkJwmj68Hg3J30RSjw2ch1maOn92ay8FGxuLN3PcYmnjeUXF5+1xEMYOcMSkcvlYu7cuf1uczqd5OXlnXW7IIwmTdPoCugzZtkOM3Qfg5AXTBbInw6yCcmeSx6dEGoEz2yDI848hs+aCYLResIxYqqGLEm4bWY4eVi/I/8CUMz6ytWC3q06nbXGBZrB0mZlNcBbb71ldAjCBNTl11tDHrtJ39japW8kJnfKqYuyJ+M7sp32mo+I5l7OtMIsAyLNXKJFJEx4faulc5wWfbbM16rfkV1x6iJPGd2hOE0tzdQeaxrgWYSREIlImPC6g30tot7xIU3TZ8usrlMXmayYPMUAxDoajAgzo4lEJEx4PaEYAC6bGby9rR3PpLOus+Tot6m+E2MW20SRVmNEgmCEa+YUccm0PH2T8ke9ScZVfNZ1jlw9ESn+VqJx9axNz0LyxDuZhtKtZnW6xZNqJkUm22HBYVagRy/3gavkrOss2SUosoQzchJfb3dOSA2RiDJUX/kPYRjCXogEQJLBWXjW3ZKzALNJwaSG8feIImmpJBJRmhmoZnVNTQ133nknVVVV2O12ZsyYwcaNG8963KpVq3j88ccpLS1lxowZALzzzjssXLgQm83GRRddxCuvvIIkSVRXVyceu3//fq677jqysrIoKiritttu4+TJk4PGU1dXN1Zvx6jrCUXZ+mELO+s6oKdFv9GZD8oAoxaKSR/EBoJdrWMYZeabWGNEmgZxA5rUfYvihmCgmtU5OTmUlZXx4osvkpeXxzvvvMNXv/pVSkpK+NznPpd47BtvvIHb7eb1118H9A2HN954I9dffz1btmyhvr7+rC5WV1cXV1xxBXfddRf/+Z//STAY5MEHH+Rzn/scf/rTnwaMp6CgIAVvSnro9Ec50OQl12nh4tI2/casokGvnzp5MkpWCItdlNdNpYmViOJR+PMTY/+6l/1/+naBITizZnWfRx99NPHvqqoq3n33XX7961/3S0ROp5Of/exniUJnP/7xj5EkiZ/+9KfYbDZmz57N8ePH+cpXvpJ4zA9/+EMWLVrEv/3bvyVue/bZZykvL+fw4cNccMEFA8aTKbwh/Q+T224Cv94KxDn4Offu3CLoqYFw51iEN2FMrEQ0jj399NM8++yzNDQ0EAwGiUQiLFy4sN818+bN61dt8dChQ8yfP79fSY4lS5b0e8zevXt58803yco6e6VwTU0NF1wwvFNIxpvE1L3VfCoROQZPRH1dMwLtoxzZxDKxEpFi1lsnRrzuCLzwwgvcf//9PPHEEyxbtgyXy8V//Md/sGPHjn7XOZ3OQZ5hcD6fjxtvvJHvfve7Z91XUnL2zFGm6WsRuawytPW2cpx5g17vU9x0dwWJ+xqpWDQWEU4MEysRSdKQu0hGOrNm9fbt21m+fDn33HNP4raamprzPs+MGTP4xS9+QTgcTtRx2rlzZ79rFi9ezG9/+1sqKysxmQb+cRiohnam6GsR5cgBUGP6gLQte9Drg6ZsGjoCmJUgFdEgmO2DXisMnZg1S0Nn1qyePn0677//Plu3buXw4cP8y7/8y1kJZSBf/OIXUVWVr371qxw8eJCtW7fyve99D9BLo4JeM7yjo4MvfOEL7Ny5k5qaGrZu3crtt9+eSD4D1dDOFH0F891ql36DI++cEwuuLCcRxUk0rhHznRyDCCcGkYjS0Jk1q6+55hpuvvlmbrnlFpYuXUp7e3u/1tFg3G43v/vd76iurmbhwoX88z//Mw8//DBAYtyotLSU7du3E4/Hufrqq5k3bx5r164lOzsbWZYHjKevhvZ4p2ka/rDeIsqKd+s3nmt8CL3IfsyilzwNeMU4UaoYVrM6Fc5VDzej6ien0PPPP8/tt99Od3d3vwL8qTAe3/NoXCUQjuOq/V/k1gNQ9UmovOScj/nTy/+FveMgU5ZcT9G8K8Yo0vEp7WtWC2PjueeeY8qUKUyaNIm9e/cm1gilOgmNV2ZFxuOQIdSl3+DIPe9jZHs2AFG/mMJPFZGIMlxLSwsPP/wwLS0tlJSU8NnPfpbHH3/c6LDST18isp//kE/FqScrkYhSRySiDPfAAw/wwAMPGB1GWmpoD3CwxUuZS2ZOJKDfOIREZM7KQQPUgEhEqSISkTBhnegJcaDJiyU7xBwAiwNM5z+ualZVBeZWD2ZzDFQVZDHnM1LiHRQmrL4ZMzc9+g3nWD90Oo8nB4fNillC37EvjFjGJ6JxPCk47oy399of1tdJZWl+/YYhdMsAfZ2RzaP/u29sSRiRjE1EZrO+rSIQCBgcycTR9173vffpLrGGSO1tEQ0xEYWicT7uMVHf4YdQ92iFN6Fk7BiRoihkZ2fT2qrXjXE4HInVxEJqaZpGIBCgtbWV7OzscXNKrz+iJyJ7vLd7NcREpGmwvx2KfCHKAl2Mj+82vWVsIgISZSv6kpEwurKzs8dNqZDTV1XbY30touwhPdZmloma9BM+wv4uHKMR4AST0YlIkiRKSkooLCwkGhU1hkeT2WweNy0hgEhcJRrXkLQY5rgfJIbcIpIkCcXugi6I+LtFIkqBjE5EfRRFGVe/JMLos5oU7rtiGsHOE5j2oVdlMA89pSh2fbA6GhBjRKkwIRKRIAzErMiY6Z0xs2UPuZwvgNmpJ6J40KsPGonxxxHJ2FkzQRiSvlmvIY4P9bE6eltEkRDEwikOauIRLSJhQvq4tYeaNj+zAy2UA1g9w3q83W4jJluJxlUI94B5fFQbSFciEQkTUlOXvr2jTDoJCmAbvETFQOZN8iBNL8cWbtdXV2dlzskmRhBdM2FC6pu6d/atqrYNr0XkspnJcudhkmWxzSMFRCISJiR/RN/e4VB7E5F1eC0i/TH6WiLCPSmKauISiUiYkPzhGJIWwxoP6jcMs2sWial82KFR2+5HC4kW0UiJRCRMSL5wDEvMj8Uk6yd3DGMNUZ89LTFaukPEgiIRjZRIRMKEE42rRGIq1rgPsyLpM2bDXAdkMclovV0zsahx5EQiEiacQG/5D6fmR5GlYXfL+si9j4uJRDRiYvpemHA8DjPfuGIakZompONScgPVgNnem4giYYhFxsXhnelKtIiECcmkyKdmzJJsEVntduKSWV/UGPGlMLqJx9BE9MwzzzB//nzcbjdut5tly5bxhz/8wciQhImkb7ZrmGuI+jgsJqKK49TqaiFphiaisrIyvvOd77Br1y7ef/99rrjiCm666SY+/PBDI8MSMtz+4928tr+FtvY2/YYku2YOi0JEsRNVNYj4UxjhxGPoGNGNN97Y7+vHH3+cZ555hvfee485c+YYFJWQ6Y51BjnY1M3cYCe4LUl3zeaXeWBWJfauQ6JrNkJpM1gdj8d58cUX8fv9LFu2bMBrwuEw4fCpnc5er1i/IQxfIBLDrAaxSKo+bZ9ki8hlM4M7G7yySEQjZPhg9b59+8jKysJqtfK1r32Nl19+mdmzZw947YYNG/B4PImP8vLyMY5WyAT+cAxLzKcvZrRkgTyConnWLP1zWCSikTA8Ec2YMYPq6mp27NjB17/+dVavXs2BAwcGvHb9+vV0d3cnPhobG8c4WiET+CNxrHEfFkVOulsG+mkeH7TG9NM8xBjRiBjeNbNYLEybNg2ACy+8kJ07d7Jx40Z+8pOfnHWt1WrFaj3/SZyCMJi4qhGMxMmO+TArctLdsr7ner85wuyuEOXhHuP/qo9jaffeqarabxxIEFIpcYSQ6sOkSElP3QPYzQoRxYkGxEJi+n4kDG0RrV+/nuuuu46Kigp6enrYsmULb731Flu3bjUyLCGD9W3vcEtBJJLf3gEgyxKKTR8jioUCWOIxfQOtMGyGvmutra18+ctfprm5GY/Hw/z589m6dSuf+tSnjAxLyGDFHhvfuGIa8Z3bIMiwS8SeyWpzoEoK0bgGkZ6hH1st9GNoIvqv//ovI19emKBMiowp1jvLNYIWEYDdYiKq2Hu3efhFIkpS2o0RCcKoi0Ug2lcQbWQtIofFRERx9m7zEFP4yRIdWmFC2VXfSdfJZi4KRvG4ssA0sllYh0UhoDiIxb1iCn8ERCISJpTGjgAdx5uZR3zE3TKAxZNzkCKVONq69DEiISmiayZMKL5wrLcyozzigWoAj92M25Ojn+YhWkRJE4lImFACkRjWWE/vquqRJyIALE79sxgjSppIRMKEoaoagd7tHWbTyLZ39PGHY+xpidHYGRAbX0dAJCJhwvBHYmga2OJ+zHLyu+5PF4mp/LUpTHN3SCSiERCJSJgwAr2HKmYRQJJGtqq6j92ib/OIqxrxcABUdcTPORGJRCRMGIFIHDQVpxbQb0jBGJHVJKOZ7GhIxOJxiIoB62SIRCRMGFX5Tv7+0mJmFffWILJkjfg5JUnCYe1bXS1KxiZLJCJhQlEiPb1T965hH6o4GLtFOa2IvhgnSoZIRMLEEu4tL5yCgeo+ehF9hzhWaATEymphwnin5iRKw1Fmh6O4UrWGCLCbTYRlOzHRNUtaUi2io0ePpjoOQRh1dScDNLe26mM5KZgx67NsSh4r50+hyG0TiShJSSWiadOmsXLlSn7xi18QCoVSHZMgjAp9VXVvreoUds08DjNudzaKLIn9ZklKKhHt3r2b+fPns27dOoqLi7n77rv561//murYBCFlVFXDH45jifswK6lZQ9RP32keokWUlKQS0cKFC9m4cSNNTU08++yzNDc3c+mllzJ37lyefPJJ2traUh2nIIxIMBpHVVVsidM7slP23N5QlF3NEY51BkQiStKIZs1MJhM333wzL774It/97nf5+OOPuf/++ykvL0+UgBWEdOCPxFDUMFYppq+qtrpS9tzBSJwdx8Oc8Ib1WTNNS9lzTxQjSkTvv/8+99xzDyUlJTz55JPcf//91NTU8Prrr9PU1MRNN92UqjgFYUT84fip1pDFAYo5Zc9ttyhEZX36XotHISZOoRmupKbvn3zySTZt2sShQ4e4/vrree6557j++uuRZT2vVVVVsXnzZiorK1MZqyAkLRSNY4n17bpP3dQ9gMOsoMomYrKZuKphivjBbEvpa2S6pBLRM888wx133MGaNWsoKSkZ8JrCwkJRHF9IG7NK3FywKAftSFZKZ8xAL8ZvMcm9tas1TBEfOPNS+hqZLqlE9Prrr1NRUZFoAfXRNI3GxkYqKiqwWCysXr06JUEKQiookW6QU98iAnBaFKKynWjcj10MWA9bUmNEU6dO5eTJk2fd3tHRQVVV1YiDEoRREerd3jEKichhMZ3abyYS0bAllYi0QWYFfD4fNpvoGwvp582PWtl/tJFAJJbyrhn01SWynzpoURiWYXXN1q1bB+ilDx5++GEcDkfivng8zo4dO1i4cGFKAxSEVKhp86G0n2RyiXlUWkSXTc9Htk3BeaxJtIiSMKxEtGfPHkBvEe3btw+LxZK4z2KxsGDBAu6///7URigII6RpGqFQGLMaxKJYU7+qGsh2WMCTA02SSERJGFYievPNNwG4/fbb2bhxI2536v9DBSHVgtE4SlTvLpktNjCN0vBB32keohTIsCU1a7Zp06ZUxyEIo8YX1je7mhUJ2e5JWUG003UFIhxpijCpO0ipWSSi4RpyIrr55pvZvHkzbrebm2+++ZzXvvTSSyMOTBBSJRDWjxBK6VlmZ+gJxXjvWJjLvGFKPUFQ43o5WmFIhpyIPB6Pvken99+CMF7oLaIeLKbUlv84ncOiEJNtRPoO8Yj4R2UsKlMNORGd3h0TXTNhPAnH+sp/jF6LyGExgSQRxIaqacgiEQ1LUmNEwWAQTdMS0/f19fW8/PLLzJ49m6uvvjqlAQrCSF04OZdF7U7UTseoJQebWUaWJKKKnZgaxyJmzoYlqQWNN910E8899xwAXV1dLFmyhCeeeIKbbrqJZ555JqUBCkIqyGEvplHa3gH62jq7RT6tiL5Y1DgcSVdovOyyywD4zW9+Q3FxMfX19Tz33HN8//vfT2mAgjBiqgrh3sQwSmNEAHaLqXe/mdjmMVxJJaJAIIDLpReW+r//+z9uvvlmZFnmE5/4BPX19SkNUBBGauuej/n4RDfhuJaSQxUH4zAriR34IhENT9LF81955RUaGxvZunVrYlyotbVVLHIU0oqmaRxvPkGbL4Jmdeu770fJypmFXLNoKrlOi1jUOExJ/a88/PDD3H///VRWVrJ06VKWLVsG6K2jRYsWpTRAQRiJQCSOKdaDBFic2aP6WrlOC25PNookiRNfhympWbO/+7u/49JLL6W5uZkFCxYkbr/yyiv5zGc+k7LgBGGk/OEYtljPqVXVoy2xzUN0zYYj6XZqcXExixYt6lccbcmSJcycOXPIz7FhwwYuvvhiXC4XhYWFrFq1ikOHDiUbkiCcxReOYY33jEqJ2DO1+8LsaIrQ4g3piUgU0R+ypFpEfr+f73znO7zxxhu0traiqmq/+4d6Euzbb7/Nvffey8UXX0wsFuOb3/wmV199NQcOHMDpdCYTmiD04w/H9VXVKT5CaCBdwSjvNYRw9IQpdtv0IvqidvWQJJWI7rrrLt5++21uu+02SkpKEls/huu1117r9/XmzZspLCxk165dfPKTn0zqOQXhdL5wDFvMi8Uugz17VF/LYdGL6Ae13l+riE8koiFKKhH94Q9/4Pe//z2XXHJJSoPp7u4GIDc3N6XPK0xc0VgMa9yH2WQb9RaRw6z/OgWxoaEhRXzgzB/V18wUSSWinJyclCcLVVVZu3Ytl1xyCXPnzh3wmnA4TDh86swor9eb0hiEzPPJCivq8Rw0SUnpoYoDsVv03fYhyU5cDenHCglDktRg9be+9S0efvhhAoFAygK599572b9/Py+88MKg12zYsAGPx5P4KC8vT9nrCxkq1IUsSSiOnFGpQ3Q6i+nUsUIxsahxWJJqET3xxBPU1NRQVFREZWUlZnP/UzN37949rOe77777+J//+R+2bdtGWVnZoNetX78+UTcb9BaRSEbCOQW79M+jPD7Ux25WiCr6Ng+bWNQ4ZEklolWrVqXkxTVN4xvf+AYvv/wyb7311nmPIrJarVit1pS8tpD5VFXjzx8cYVJ3D5OLXaTukOnBOSxK77FCmljUOAxJJaJHHnkkJS9+7733smXLFl599VVcLhctLS2AXnjNbren5DWEicsXidHTeZL2YISpzpwxec1PzS7CXDAVZ+0R0TUbhqQXNHZ1dfGzn/2M9evX09HRAehdsuPHjw/5OZ555hm6u7tZsWIFJSUliY9f/epXyYYlCAk9oRjWmBezIiPZxyYR5WVZT23zEF2zIUuqRfTBBx9w1VVX4fF4qKur4ytf+Qq5ubm89NJLNDQ0JGoVnc9gBzUKQir4Qvr2Dqtl9Bcz9mPpnZ0TLaIhS6pFtG7dOtasWcORI0f6nex6/fXXs23btpQFJwgj4Q/4MKkhvVb1GA1Wt/aEeO94iNaeEER7i+gL55VUItq5cyd33333WbdPmjQpMc4jCEYLevUhA5PVCaaxmeTo8Ed4tz5Amz+m3yC6Z0OSVCKyWq0DLiY8fPgwBQUFIw5KEFIh0qMnIsU5div1Hea+Ivq9iU90z4YkqUT06U9/mscee4xoNAro9XobGhp48MEH+du//duUBigIyZIjXiTAPMp1iE7Xt7raT++sr0hEQ5JUInriiSfw+XwUFBQQDAa5/PLLmTZtGi6Xi8cffzzVMQpCUi4vN7OkKpfCwuIxe01HbyIKYNMnY0TXbEiSmjXzeDy8/vrrbN++nb179+Lz+Vi8eDFXXXVVquMThOQFO5ElCRzZY/aSdrOCJEFEthNVvVjEosYhGXYiUlWVzZs389JLL1FXV4ckSVRVVVFcXIymaUmXBBGElAvqY0Q48sbsJWVZwm5WiMj6sULifLOhGVbXTNM0Pv3pT3PXXXdx/Phx5s2bx5w5c6ivr2fNmjWiTKyQNlo6few/2kB9hx/sY1tWxmk1EVXsRGKq6JoN0bBaRJs3b2bbtm288cYbrFy5st99f/rTn1i1ahXPPfccX/7yl1MapCAMl7ezjZ5gFGTzqTrSY+TaucVYJl1A1pGDYrB6iIbVIvrlL3/JN7/5zbOSEMAVV1zBQw89xPPPP5+y4AQhWWHvSQAkR+6ol/84U36WFbc7Wx+fEi2iIRlWIvrggw+49tprB73/uuuuY+/evSMOShBGKtqjJyLFOXbjQ/2cfpqH2Mp0XsNKRB0dHRQVFQ16f1FREZ2dnSMOShBGKupvB8DsGvtE1NYT5p3GkH6ahxqHWGjMYxhvhpWI4vE4JtPgw0qKohCLxUYclCCMlOrXZ8ysrrGvGd0ViLCj3suJYO8NYpzovIY1WK1pGmvWrBm0ONnp9aQFwUhqoB0ZcGQXjvlrO6z6r1VA690QHu4RRfTPY1iJaPXq1ee9RsyYCUaLhoPY1SARICt77Pc+Zln0XyufZkMjiCRaROc1rES0adOm0YpDEFLGHOlmcUUOqtmBbB/7gzod1tNP8wiI0zyGIOkKjYKQtgL6+JDsMOZ8PLMiYzXLvUX0xX6zoRCJSMg8fVs7xqg87ECcFhMRxdG7ulq0iM4nqU2vgpDODtfWEW3qxpVjp8KgGBwWhZCi7zcTLaLzE4lIyDjBrhMEQjEUs3EtoqtmFaGUTifro72iRTQEIhEJmUVV0fz6qmqHZ/DFt6Mtx2kBrfd0WdEiOi8xRiRkFC3YQSQaQ5VMuLINXrtjzdI/R0MQFwt9z0UkIiGjhLpPEFc1guZsXPaxONt1YJ3+CO/U+Tju1cspEz67xrtwikhEQkYJdJ4AQHPkY1KM+/H2hWPsqOvkeLB39EMkonMSiUjIKOFuPRHJWcaeJuPs3ebh1Rz6DeEeA6NJfyIRCRlFCrRjNcmY3WO/x+x0zt7V1QHJQUxVISRaROciZs2EzKGqlJgDlFTkoC2caWgoVpOC1SwTVrKIxE5gEi2icxItIiFzBDv1+j+KGWksz7ofhMtqImJyEomrYozoPEQiEjKHv1X/7Mwf8/KwA8mymXpbRCIRnY9IRELGiHQ1sbuhk+0tCnHV+PKsWVYzEcWpJyIxRnROYoxIyBjBjmbCMZUTWjaKbHyLaOmUXJaUOXG9/yeIhSEWAZPF6LDSkmgRCZlB0wh3NQFgdo/dEdPn4raZ8bizkM29FU1F92xQIhEJmSHiIxzoASRsOemRiBKsbv1zqNvYONKYSERCZug5QTgaJ2j2kO0a+6qMAwlF47zz8Un2t/eOV4kp/EGJRCRkBt8JglEVvzmfbAP3mJ1OkmBHbQc1Xllf1Ci6ZoMSiUjICJrvBMFoHL8lTy/BkQZOLWrsnTkTLaJBiUQkZIRYdzNOiwJZReQ40iMRQe+iRiVLX9QopvAHJRLROBOKxmn39T8/7nhXcJCrJ4hIAHPEy5xSD5+/4uK0mLrvk2UzETK5e9cSdRkdTtoSiWgc0TSN33/QzEu7j9Md1Ovc/PlIG7/e2cj+4xN4RqanWf/syAOz3dhYzpBlNRM2uQj3LWpUVaNDSkuGJqJt27Zx4403UlpaiiRJvPLKK0aGk/b2He+moSNAOBbXi7KjH10D8PbhNnpCUSPDM473OJqmgbvU6EjO4rLpp3mE44AmBqwHY2gi8vv9LFiwgKefftrIMMaFSEzlvaPtAFwyLZ/8LH2R3JLKXEo8NiIxlb/WdhgZonG8Texv8vJqbfp1Uz12M0gyPVJv2VixlmhAhm7xuO6667juuuuMDGHc2He8G384jsduZn5Ztt7M72pAtji5dFoBL+5q4sMmL0uqcnHZ0mP6ekxoGlr3cYKRGC1aLnazYnRE/VTlO7n9kkrchyugs653nGiywVGln3G11ywcDhMOnxqo9XonRjNX0zQ+ONYFwMWVuSjNe+DjP+olL4AydwmTXcuo74G9jd1cOt3govFjKdBOOBQkiomwNS9t1hD1sZkVbGZFP+yxsw6CXUaHlJbG1WD1hg0b8Hg8iY/y8nKjQxoTDR0BugJRLCaZmXIDHN6qJ6GsQn0TpbeZSwJ/QtJiHGjuRk2DnedjxnucQDSOz1JATpYNOY1mzPqxefTPoms2oHGViNavX093d3fio7Gx0eiQxkRjhz7uMa9Awlzzf/qN5UvgojvgwtvBbKeATqb7dwPQGYgYFerY62rAH47RYy2ioHfcLN1UN3bxl2NRfOGYmMIfxLjqmlmtVqzW9PxhG02XTs9n7iQ3lprX9FIS7lKYslLfQ+DIhZl/g7zvRVZmNWC98HoU5wR6j3oTkddZylR3en7ftSd9tHUqXKDGyBJdswGNqxbRRJat9eBoP6B/Me1KkE/7r8ufBrlTcJgklPo/GxOgEYJdEPLii6j0WIsodNuMjmhAHru+ligUVfXjp+MTdJnFORiaiHw+H9XV1VRXVwNQW1tLdXU1DQ0NRoaVVmK964U4vgs0DfKmgafs7AunXK5/bv0ILdhFKBofuyCN0tWAqmk48sspzHGlbdfMbTMTk60E1d4ZPTFOdBZDu2bvv/8+K1euTHy9bt06AFavXs3mzZsNiip9hKJx/usvtVS4Za737UUBKLto4ItdxZBdQWfTx7y39X+JVa3gxgXpt8AvpboakCWJ2bPmMntKhdHRDEpfSyThlbKAKAQ69LraQoKhiWjFihX6ilhhQA0dASIxFbn1IIotpv/w5lQO/oDyJVhaj+Jq3U911iIisWIspgztfWsadNXr/85O3yQE4O5dUtCFG2iH4ARdeHoOGfpTmhmOtvkBmMYx/YaSBec+nSJ3Kg5XNk45httfR0NHYAyiNEigHUJeAjGIZU0yOppz8iQSkUsv6h9oNzii9CMSUZrSNI36dj/meIBiTuo3Fsw494NkGal4HtlOCwW+w9Se9I9+oEZpr0FD470OJz/6cwNtPeHzP8YgNrOCw6IQNmcTjsX1rpnQj0hEaarFGyIQiVMUbcRlVfQxoL5FcedSNI8chxlP6BjHWtoyt+vbUUMoqnLCXIYE5DjSa0X1mb6wtIIvrFiIw2ISXbMBiESUpvpaM9Ol48iSBAVDPELZmYercDImGeydH6V1SyFpsTB0H6MnFKXLVkaR24ZJSe8fZbfNjOLM07+IBCCaXptzjZbe/3sTWO1JP0o8RKl2Qr/hfN2y0yjFc/HYzeQGazmaid2zzjpQ43SoTkLmbEqz06sG0aBMFrC69H+L7lk/42pl9UShaRozi910+I+QYzLps2WO3KE/QcEM8l1WrMFu7FkZuJ6o7SMAGmV9kLo0Oz0XMp6uKxBhR20HJR0K853oA9ae9B5kH0siEaUhSZK4cHIOeLvgpDz0blkfq4u80qnkdTVCuB4oHI0wjRGPQfvHhGNx6uRyJIlx0SJSNTjQ5CUasjLPqSGJcaJ+RNcsXcUi0FGr/3sY3bKEvuTVdjB1MaWDzlqIReiM2/BZiihy2/QyG2nOYzcjSxI9skevXy2m8PsRiSjNxOIqB5q8BE8cBjWm17FxFgz/iQpmENegq6WWA7UZVKWgVU+srrK5fHJGAQvKso2NZ4gUWcJjNxE05+h7zvwnjQ4prYhElGaaukJs/bCF7e9uR0PTW0PnWsQ4GKsLn7WIg809HPjgfX0h3XgXi8DJwwA4y+dy4eRcZpe6DQ5q6HKcFvzmPILROAQ79e9HAEQiSju17X4kLcZkmpGQkuuW9XJXzMOiSLh7PuZ4ZwZMF7cd1HeuO3LBPf4GevOzrMQUO964Vd+iEhCtoj4iEaWZupN+soPHyLUBNje4SpJ+LqlwJh6HFVf4BPVNzakL0ijNHwBQZ57Kh83ecVdhoO/Agw6pd2Gqr9XAaNKLSERppDsQpcMfIS9Ur+9Pyk+yW9bH6sJVVKk/d8O+8b3K2t8O3cdAktjuK+H/PjxBXfv4WiOVn2VBksBvztW73f42o0NKGyIRpRG9WxZnsnYckyxDwQUjfs6cqgXIElg7DtEZGMcFuY7vAqDHOZnWsBlFlqjKdxoc1PDkOi3cs2IaV140V+92ixZRgkhEaaTupB93qJl8qwoWB7gHKIA2TJaiWbjsFrIibTQcP56CKA0QDULLXgCOmGcBUJHrwGpK/2n700mSpJdlcfau6/Kd0MeKBJGI0kVc1TjWGSA3WEu2o7dbJqfgv8eahbNwCgCBYx+O/PmM0LQH4jG0rEL29OizZDNLXAYHNQLOfJBN+p65YKfR0aQFsbI6TSiyxO3LJ+N/qx27RUlJt6xP4bSF5MeacbjGYVcgFoFjOwE44Z6PtyuG1SwztSDL4MCS09gR4N2admZ125jn8oG3aXjbdzKUaBGlEWeohUJrHMlkg+zUnQZqL5mN02pB8p0Yf5stj7+v71a357A7rM8gzihyYU7z3faDUWSJ411BGmMefcDa22R0SGlhfP5vZqq2Q/rn/Okgp3D8w+JIlJjVToyj7lk0CA3vAaBVXkJMk5EkmFc2hLpMaarQZUWRJU7K+YSjKvSIRASia5YWWrpDbDvcymVt1ZTYgYJZKX+NSN4MGj76gNbmbVxcvhzzeBjorf2zPo7izEcqnMOni2W6g9FE6dXxyKTIFLisdEYK8YVj2Hyt+kZeZWL/KooWURr4uNWHt6UWX08nmKyQW5Xy1zAXz6IzAlKwk+P1h1P+/CnnbYYm/eRapl2VGLgfz0moT7HHRlhx0R0z6UeH+1qMDslwIhEZTNM0jrT2kBc4Sq7TAvkXpLZb1ksyWXGUzQXg5OGdKX/+lIrH4PAf9KntotkcDOfRHRzHa6DOUOKxgSTRIvVO43eJc/xEIjJYuz9Clz9MQahW/2s/3NpDw1B4wVIAYi0fEgml8QkftW9Dzwkw22kvvpTXD5zg5+/U0RXIjE2iJR69flKjVkBMVaGz3uCIjCcSkcEOn+jBHW4h3xLFZLGPSresT35JJTjz0dQ4TYf+OmqvMyLtNdCox6ZecB2vf+wnrmpMznOQ7bAYHFxqeOxmitw2sidNJ6Zq+taVeMzosAwlEpGBNE3jYHMP+f4j+obIghmj0i3rI8kyjqolAHQffgdUddReKym+Vjjwiv7vSYvZFSiguTuExSRzxcwMqjIJfGFJOddePAebw63XnfKO01XvKSISkYGOdwXx+QMUhWrJcVqgeP6ov2bl7E8QU+wEvB14G/eN+usNWbAL9r2oL2DMrqAxdznbP9bLZFx+QQEu2/gfpD6dJEn6huac3vViHUeNDchgIhEZyCTLzLc2U+iQ9KNmPCPfW3Y+HpcDe+USynPs2Jp3psdep0AHVD8PIS848mibfAP/s78VTYPZpW7mjKPiZ8PV7ZhMNK7CySNGh2IokYgMVOyxsdLdTFWBE4rnjazkxzBctPxKygqysQTb4MT+MXnNQXU1wp7/TiQhFn6BvS1hQtE4JR4bV8ws1FsPGei1/S38/JCJtkBMr2E9gcvHikRkJG8zdB9DkhQomjt2r2txQMVy/d+124wpWapp0LgT9v5S38KRVQgLvwhWFytnFrJ0Si6rFk0at1s5hqLQbSUuW2hQe8e/To6D9V2jJHP/l9OYpmnsrOvAf1TfvkDhTL0a41jGMOlC2mJWDtUfJ17z1pi+Nv6Telfs4z+CGieWP4OdOTcQMzkAfT/W8qn54+J0jpGYVqhv3K1hEpG4Cic+TI+usgFEIjJAfXuAHQfr2V/9HqqmQdmSMY8hLpl4I76IDn+Eto+269Pmoy3YCR/9Hnb+DLoa0RQzdXmX8POuhfyl1st7R8fZhtwRctvMlGbbaLdX0eaP6wl6gs6eTewNLgbQNI33jrYzyVtNUZYZObsc3MnXpU6WSZGZOWcRdd21mDoOkrfvZcwX3gauotS+UCyizwi1fKB/1jRUTeO4qZzt0kKaT1iBGC6bibKc9D8oMdXmlHpo6gpxMD6JUk4gNVWPyaRFuhGJaIzVtQfoaG9jceAjSsvdUHWZYbHMn+Rhf/nldBztpP5EB9Oqn4e5Nyd26iclHoWeFv0ve1dj7zn1pxbrHYkV8ldtFq3xfAAsJpmLJueweHJORo8HDWZGsYttR9qotUynK3CMnNaDMGUFWMdnvaVkiUQ0huKqxrbDbVR07aDEZcaSP2Vkv/QjJMsSK2eX8lLPp5DatuLp9FKw9wUoWQDlS89dsCse07tagZN6lyLxuQM0faGkpmn4wjGc2QXIhTOgZCG1tRFam7xkWU0srMhm3iRPxo8FnYtZkZlT6mF3XZymcA45qh8ad8C0K40ObUyJRDSGdjd0op48QnG4ltKiXJi60uiQmJRt5+Lpk9jBDUS6t3Op0kR2UzU0VfeeMpsPZrueXOJRCPdA2AsR/4ADq9G4SkfMTJOaR200h1ZTKTfNmk9Jtj4QvbAixJSCLKrynShyZk7LD9fiimwKsqzMNF8P+1/Uqw6UXQS28Vt3abhEIhojJ31h3j/cyOyOPzM514m5Ygm4io0OC4Allbmc9IU5Il/O1LII2dGD0FGjt3jOVVPZZAFHPj1KNod9VhrDTo5HnUQUJygSmMFmVugJx+kbBSt02Sgcx+WmR4PLZmZ2qRk0F2SX613aI6/D3L8ds7VlRhOJaIzk2GSu0t5FtcXIL6qESuPGhs4kyxLXzS2hPKebmWUekOZCNERTQw1Z+HHIEWRJIazJ+DQb7TEb7pw8SvPzQJLo6QqybWej/mQmKHBZqcp3UpnvpMRtQxYtn6GRJMJTrsL75/9HAUf0ltGkC42OakyIRDQW4jGUg68y3dKJVpqLNPdmvTWRRhRZYkF5duLriGTht3UWYurAe7zmx+OUFugJpshtY2FFNmXZdibl2HFYxI9VMiIxlS37gzjCs7g8vp/iI38EWzbkTTU6tFGXFtMUTz/9NJWVldhsNpYuXcpf/5qmJSqGqaU7xNZdh4nt+YW+l0g2Ic39W33cJc31hKLku6yYTmvNSBJkWU1U5DoSxyeDnsRWzihkepFLJKERsJhkZpW4aXLN5/1AMcc6fGj7fgPNezN+oaPhPzW/+tWvWLduHT/+8Y9ZunQpTz31FNdccw2HDh2isHB8ln7o9EfYU3OcjsPvUeLdyzG3icqiXJizalTrDaVSXpaVLyypQFU1InEVTQOrSRbdrFG2tCqXSExlNyug/U26jjdQHngV96SPkKouA3ep0SGOCkkz+ED0pUuXcvHFF/PDH/4QAFVVKS8v5xvf+AYPPfTQOR/r9XrxeDx0d3fjdhu0Q1tVUaNBapra8LafoKPtGGpHPa5wC5KmUpBloWLKDCxzbhTnVwlDtv94N28fOkFBx27KunfhtMjMn+RBcuZD3hRCtiLMrnwUWxaYnak5jHMUDPV31NAWUSQSYdeuXaxfvz5xmyzLXHXVVbz77rtnXR8OhwmHw4mvvV7v0F6ooxZq3gBNY2d9J6qqJpq6eh7WQAO3zcTsxAmiGrvqOojGVf38qUS+1q912RTmlrggHkHSVLrqOompGlmABGQ7zBSVTSPngkuhcNaEmf0QUmPuJA+T8xzsOOrhQONUZkX2Iylt+i79QDsHGjoJx1QUWUKWAMmEKptANuGwWZk7KTvxM1fd2EU41lsEL/FzqH+2W0zMP+14pr3HuglF4omvLSaZRRXZAwc573Mp2yNpaCI6efIk8XicoqL+2wqKior46KOPzrp+w4YNPProo8N/oVgYfG0AaL4OVHXgRqCmmSB0qmqhFvaixQe+Ni4pENMHnCUkcj1ZRMzZOHOLKCydjK1ohmgBCSPispm5anYRoen5+MPzwarpyyq6j9HVvA9F82KJB4mjAdHeD0ALQvDUz63q70SNDVyNU7Mo4DtVfUHr6UKNnkpEmkkG3yAHF2ipq/Bp+BjRcKxfv55169YlvvZ6vZSXl5//gdnlsODzAJRVhhN/FfRPUqJansWkgMPSe4dE1cwIif9OSb9OAiRJRlFksFnAbAOTjamjWOJVmNhsZuXU6vOiOVA0hxXTriYSVwlFosTCAbR4FDUeQ4tHMREHZ9+srMakqSHip//xPa11b5IlcJ2aeCiZEiZ22h9fRel/fz8WZ8q+R0MTUX5+PoqicOLEiX63nzhxguLisxf7Wa1WrNZB3pRzsTgTg8Slw2ik5GVuYUBhnJNlCZvcm6CctnNeW5Q99OctMqgRb+gIl8Vi4cILL+SNN95I3KaqKm+88QbLli0zMDJBEMaS4V2zdevWsXr1ai666CKWLFnCU089hd/v5/bbbzc6NEEQxojhieiWW26hra2Nhx9+mJaWFhYuXMhrr7121gC2IAiZy/B1RCORFuuIBEEY1FB/R9NzFZQgCBOKSESCIBhOJCJBEAxn+GD1SPQNbw15q4cgCGOq73fzfEPR4zoR9fT0AAxtdbUgCIbp6enB4xm89O24njVTVZWmpiZcLlfGHkt8Ln1bXBobG8Ws4RCI92v4RvqeaZpGT08PpaWlyOeoEDCuW0SyLFNWNvHOgDqT2+0Wv1jDIN6v4RvJe3aullAfMVgtCILhRCISBMFwIhGNY1arlUceeSS5igQTkHi/hm+s3rNxPVgtCEJmEC0iQRAMJxKRIAiGE4lIEATDiUQkCILhRCJKY8M9AffFF19k5syZ2Gw25s2bx//+7/+OUaTpYzjv2ebNm/UDEU77sNnOXf85k2zbto0bb7yR0tJSJEnilVdeOe9j3nrrLRYvXozVamXatGls3rw5JbGIRJSm+k7AfeSRR9i9ezcLFizgmmuuobW1dcDr33nnHb7whS9w5513smfPHlatWsWqVavYv3//GEdunOG+Z6CvGG5ubk581NfXj2HExvL7/SxYsICnn356SNfX1tZyww03sHLlSqqrq1m7di133XUXW7duHXkwmpCWlixZot17772Jr+PxuFZaWqpt2LBhwOs/97nPaTfccEO/25YuXardfffdoxpnOhnue7Zp0ybN4/GMUXTpDdBefvnlc17zwAMPaHPmzOl32y233KJdc801I3590SJKQ30n4F511VWJ2851Ai7Au+++2+96gGuuuWbQ6zNNMu8ZgM/nY/LkyZSXl3PTTTfx4YcfjkW449Jo/oyJRJSGznUCbktLy4CPaWlpGdb1mSaZ92zGjBk8++yzvPrqq/ziF79AVVWWL1/OsWPHxiLkcWewnzGv10swGBzRc4/r3feCMBLLli3rd37e8uXLmTVrFj/5yU/41re+ZWBkE49oEaWh4Z6AC1BcXDys6zNNMu/ZmcxmM4sWLeLjjz8ejRDHvcF+xtxuN3a7fUTPLRJRGkrmBNxly5b1ux7g9ddfnzAn5qbi1OB4PM6+ffsoKSkZrTDHtVH9GRvxcLcwKl544QXNarVqmzdv1g4cOKB99atf1bKzs7WWlhZN0zTttttu0x566KHE9du3b9dMJpP2ve99Tzt48KD2yCOPaGazWdu3b59R38KYG+579uijj2pbt27VampqtF27dmmf//znNZvNpn344YdGfQtjqqenR9uzZ4+2Z88eDdCefPJJbc+ePVp9fb2maZr20EMPabfddlvi+qNHj2oOh0P7p3/6J+3gwYPa008/rSmKor322msjjkUkojT2gx/8QKuoqNAsFou2ZMkS7b333kvcd/nll2urV6/ud/2vf/1r7YILLtAsFos2Z84c7fe///0YR2y84bxna9euTVxbVFSkXX/99dru3bsNiNoYb775pgac9dH3Hq1evVq7/PLLz3rMwoULNYvFok2ZMkXbtGlTSmIRZUAEQTCcGCMSBMFwIhEJgmA4kYgEQTCcSESCIBhOJCJBEAwnEpEgCIYTiUgQBMOJRCSMK5s3byY7Ozvx9b/+67+ycOHCxNdr1qxh1apVYx6XMDIiEQkDWrNmDZIk8bWvfe2s++69914kSWLNmjX9rk91AqisrOSpp57qd9stt9zC4cOHB33Mxo0b+5UvXbFiBWvXrk1pXELqiUQkDKq8vJwXXnihX62ZUCjEli1bqKioMCQmu91OYWHhoPd7PJ5+LSZhfBCJSBjU4sWLKS8v56WXXkrc9tJLL1FRUcGiRYtG9NwDtVRWrVqVaGWtWLGC+vp6/vEf/zFR2B7O7pqd6fSW2Zo1a3j77bfZuHFj4jlqa2uZNm0a3/ve9/o9rrq6GkmSRAkQg4hEJJzTHXfcwaZNmxJfP/vss9x+++2j/rovvfQSZWVlPPbYY4nC9sO1ceNGli1bxle+8pXEc1RUVJz1PQFs2rSJT37yk0ybNi1V34IwDCIRCef0pS99ib/85S/U19dTX1/P9u3b+dKXvjTqr5ubm4uiKLhcLoqLi5Mq8ObxeLBYLDgcjsRzKIrCmjVrOHToUOKooWg0ypYtW7jjjjtS/W0IQyRKxQrnVFBQwA033MDmzZvRNI0bbriB/Px8o8MakdLSUm644QaeffZZlixZwu9+9zvC4TCf/exnjQ5twhItIuG87rjjDjZv3szPf/7zlLUaZFnmzAo00Wg0Jc89FHfddVdiIH7Tpk3ccsstOByOMXt9oT+RiITzuvbaa4lEIkSjUa655pqUPGdBQUG/cZ94PH7WYZAWi4V4PD6i1xnsOa6//nqcTifPPPMMr732muiWGUx0zYTzUhSFgwcPJv49mO7ubqqrq/vdlpeXR3l5+VnXXnHFFaxbt47f//73TJ06lSeffJKurq5+11RWVrJt2zY+//nPY7Vak+oSVlZWsmPHDurq6sjKyiI3NxdZlhNjRevXr2f69OkTprZ3uhItImFI3G43brf7nNe89dZbLFq0qN/Ho48+OuC1d9xxB6tXr+bLX/4yl19+OVOmTGHlypX9rnnssceoq6tj6tSpFBQUJBX3/fffj6IozJ49m4KCAhoaGhL33XnnnUQikTGZBRTOTZSKFSasP//5z1x55ZU0NjaedXCgMLZEIhImnHA4TFtbG6tXr6a4uJjnn3/e6JAmPNE1EyacX/7yl0yePJmuri7+/d//3ehwBESLSBCENCBaRIIgGE4kIkEQDCcSkSAIhhOJSBAEw4lEJAiC4UQiEgTBcCIRCYJgOJGIBEEwnEhEgiAY7v8HI+HGvByWQjQAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:22:46.563220Z", + "iopub.status.busy": "2024-03-26T11:22:46.562550Z", + "iopub.status.idle": "2024-03-26T11:22:46.823754Z", + "shell.execute_reply": "2024-03-26T11:22:46.822807Z" + }, + "papermill": { + "duration": 0.280379, + "end_time": "2024-03-26T11:22:46.826245", + "exception": false, + "start_time": "2024-03-26T11:22:46.545866", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2Q0lEQVR4nO3de1wU9f4/8BfssstNvCGgiK5KeTleUBTUSrxwKcI0s2NqgdhPTyplrnihUxBZcVIDvJMmqZmXUrIeRsqGl0ghDdMDpigq4gUQvICA7i7L5/cH353jugssBszO7vv5ePCQmfnMznuc2Tczn8/M52PFGGMghBCBseY7AEIIeRKUvAghgkTJixAiSJS8CCGCRMmLECJIlLwIIYJEyYsQIkiUvAghgiTmO4DWVltbi5s3b6JNmzawsrLiOxxCyGMYY7h//z66dOkCa+v6r68sLnndvHkTHh4efIdBCGnEtWvX0LVr13qXW1zyatOmDYC6/xgnJyeeo2k5arUaaWlpCAwMhI2NDd/hkGZgKce0oqICHh4e3He1PhaXvLS3ik5OTmafvOzt7eHk5GTWJ7olsbRj2li1jklU2K9btw4ymQy2trbw9fXFiRMn6i07evRoWFlZ6f28+OKLrRgxIYRvvCev3bt3Qy6XIyYmBqdOncKgQYMQFBSEW7duGSyfkpKCoqIi7ic3NxcikQivvvpqK0dOCOET78krPj4es2bNQnh4OPr164ekpCTY29sjOTnZYPkOHTrAzc2N+1EoFLC3t6fkRYiF4bXOS6VSITs7G1FRUdw8a2tr+Pv7IzMz06jP2Lx5M1577TU4ODgYXK5UKqFUKrnpiooKAHX1B2q1+m9Eb9q0+2bO+2hpLOWYGrt/vCavsrIyaDQauLq66sx3dXXF+fPnG13/xIkTyM3NxebNm+stExcXh9jYWL35aWlpsLe3b3rQAqNQKPgOgTQzcz+m1dXVRpUTdGvj5s2bMWDAAPj4+NRbJioqCnK5nJvWNsMGBgaafWujQqFAQECARbRMWQJLOabau6PG8Jq8nJ2dIRKJUFJSojO/pKQEbm5uDa5bVVWFXbt24aOPPmqwnFQqhVQq1ZtvY2Nj1ieAlqXsp7mprq7Wu/uofKDE8ZxLaO/sAke7/53Tffr0Mau7CGPPV16Tl0Qigbe3N9LT0zFx4kQAda/vpKenIyIiosF1v/vuOyiVSrz++uutECkhrev8+fPw9vY2uGz5Y9PZ2dkYMmRIywdlYni/bZTL5QgLC8PQoUPh4+ODxMREVFVVITw8HAAQGhoKd3d3xMXF6ay3efNmTJw4ER07duQjbEJaVJ8+fZCdna0zL6/oHuTf5SD+1QHo3bmdTllLxHvymjJlCkpLSxEdHY3i4mJ4eXnhwIEDXCV+YWGh3suZeXl5+O2335CWlsZHyIS0OHt7e72rKeurtyHNeIC+/QfBqzv90eY9eQFAREREvbeJR44c0ZvXu3dv0IhthFg23h9SJYSQJ0HJixAiSCZx20j+vseb1v/XrP6HTrM6YH5N68QyUfIyE/U1rT/erA5YbtM6MS+UvMzE403r9TWra8sSInSUvMzE403r1KxOzB1V2BNCBImSFyFEkCh5EUIEiZIXIUSQKHkRQgSJkhchRJAoeRFCBImSFyFEkCh5EUIEiZIXIUSQKHkRQgSJkhchRJAoeRFCBIn35LVu3TrIZDLY2trC19cXJ06caLD8vXv3MG/ePHTu3BlSqRRPP/00UlNTWylaQoip4LVLnN27d0MulyMpKQm+vr5ITExEUFAQ8vLy4OLioldepVIhICAALi4u2LNnD9zd3XH16lW0a9eu9YMnhPCK1+QVHx+PWbNmcWM0JiUl4aeffkJycjKWLl2qVz45ORl37tzB8ePHuVF1ZTJZa4ZMCDERvCUvlUqF7OxsREVFcfOsra3h7++PzMxMg+v8+OOPGDFiBObNm4cffvgBnTp1wrRp07BkyRKIRKLWCp2QFnGlrApVypp6l18qreL+FYsNf3UdpGL0cHZokfhMDW/Jq6ysDBqNhhtcVsvV1VVnIIlHXb58GYcOHcL06dORmpqK/Px8zJ07F2q1GjExMQbXUSqVUCqV3HRFRQUAQK1WQ61WN9PemJ6amhruX3PeT3NRcLsKAYnHjCq7cE9Og8sV7z4DWUfhJjBjz1dBdQNdW1sLFxcXbNy4ESKRCN7e3rhx4wZWrFhRb/KKi4tDbGys3vy0tDSzHkHnWiUAiJGVlYUbuXxHQxqjPV5veGrgamd4QGV1LXBHCXSQAjYGmtpKHljh63wRDqYfhYdji4bboqqrq40qx1vycnZ2hkgkQklJic78kpISuLm5GVync+fOsLGx0blF7Nu3L4qLi6FSqSCRSPTWiYqKglwu56YrKirg4eGBwMBAODk5NdPe8KPgdhWqlBqDy5TF5UDOObh4DkB3t7b1foaDVCTov9Lm4uzNCqzMycIrAc/gH10Mn5dqtRoKhQIBAQFcne/jn/F1fhaeffbZej9DCLR3R43hLXlJJBJ4e3sjPT0dEydOBFB3ZZWeno6IiAiD6zzzzDPYsWMHamtrYW1d96fnwoUL6Ny5s8HEBQBSqRRSqVRvvo2NjcETQCiulBl3m7H4+3ONljkcOdpi6klMlbYOSywWN3pe1nfuNuUzTJmxsfN62yiXyxEWFoahQ4fCx8cHiYmJqKqq4lofQ0ND4e7ujri4OADAnDlzsHbtWsyfPx9vv/02Ll68iE8//RTvvPMOn7vBC23FbuIUL3i66N8jVD1QYv+RTISMHgEHO/3kDQD5tyrx7u7TDVYSE2KqeE1eU6ZMQWlpKaKjo1FcXAwvLy8cOHCAq8QvLCzkrrAAwMPDAwcPHsSCBQswcOBAuLu7Y/78+ViyZAlfu8A7TxdH9HfXvy1Uq9Uo7gQM6d5e0H+FCakP7xX2ERER9d4mHjlyRG/eiBEjkJWV1cJREUJMHe+vBxFCyJOg5EUIESRKXoQQQeK9zosQAig1D2FtewNXKvJgbWv4CdOamhrcrLmJc3fOGXw96EpFJaxtb0CpeQig/mf7zAUlL4Fq7GRv7EQHLO9kN2U3q67CoccavNdwj1AAgPUH1te7zKEHcLPKC95wrbeMuaDkJVDGnuwNneiAZZ3spqyLQ3dUXXkbq6Z4oZeB5/aAuj9Ix347hmeefcbgH6RLtyoxf/dpdBnTvaXDNQmUvASqsZO9sRMdsLyT3ZRJRbaofeiOHk690a+j4atgtVqNK+Ir6Nuhr8Fn92oflqP2YSmkItuWDtckUPISqMZO9sZOdMDyTnZiXqi1kRAiSJS8CCGCRMmLECJIlLwIIYJEyYsQIkiUvAghgkTJixAiSJS8CCGCRMmLECJIlLwIIYJEyYsQIkiUvAghgmQSyWvdunWQyWSwtbWFr68vTpyov5+XLVu2wMrKSufH1pZeLCbE0vCevHbv3g25XI6YmBicOnUKgwYNQlBQEG7dulXvOk5OTigqKuJ+rl692ooRE0JMAe/JKz4+HrNmzUJ4eDj69euHpKQk2NvbIzk5ud51rKys4Obmxv1ox3kkhFgOXvvzUqlUyM7ORlRUFDfP2toa/v7+yMzMrHe9yspKdO/eHbW1tRgyZAg+/fRT/OMf/zBYVqlUQqlUctMVFRUA6vq7UqvVzbQnra+mpob719B+aOc1tI+NfQZpPcYci8aOqbkcT2Nj5zV5lZWVQaPR6F05ubq64vz58wbX6d27N5KTkzFw4ECUl5dj5cqVGDlyJM6ePYuuXbvqlY+Li0NsbKze/LS0NNjb2zfPjvDgWiUAiPHbb7/hquFegwEACoXib38GaXlNORb1HVNzOZ7V1dVGlRNcT6ojRozAiBEjuOmRI0eib9+++OKLL7Bs2TK98lFRUZDL5dx0RUUFPDw8EBgYCCcnp1aJuSWcvVmBlTlZePbZZ/GPLvr7oVaroVAoEBAQUG9Pqo19Bmk9xhyLxo6puRxP7d1RY3hNXs7OzhCJRCgpKdGZX1JSAjc3N6M+w8bGBoMHD0Z+fr7B5VKpFFKp1OB69X2phUDbL71YLG5wPxraT2M/g7S8phyL+o6puRxPY2PntcJeIpHA29sb6enp3Lza2lqkp6frXF01RKPRICcnB507d26pMAkhJoj320a5XI6wsDAMHToUPj4+SExMRFVVFcLDwwEAoaGhcHd3R1xcHADgo48+wvDhw+Hp6Yl79+5hxYoVuHr1Kv7f//t/fO4GIaSV8Z68pkyZgtLSUkRHR6O4uBheXl44cOAAV4lfWFgIa+v/XSDevXsXs2bNQnFxMdq3bw9vb28cP34c/fr142sXePFArQEA5N4oN7i86oESf5QCblfvwsFO/7YZAPJvVbZYfIS0NN6TFwBEREQgIiLC4LIjR47oTCckJCAhIaEVojJtl/4v8SxNyWmglBhf559s9LMcpCZxGhDSJHTWClTgP+oaNHq5OMLORqS3PK+oHAv35ODzyQPQu7PhQUyBusTVw9mhxeIkpKVQ8hKoDg4SvObTrd7l2gcWe3VyQH/3+pMXIULF++tBhBDyJOjKixAT0FgDDNB4I4ylNcBQ8iLEBBjXAAMY0whjKQ0wlrGXhJi4xhpgAOMaYSypAYaSFyEmoLEGGIAaYR5HFfaEEEGi5EUIEaQmJ6/Lly+3RByEENIkTU5enp6eGDNmDLZv346HDx+2REyEENKoJievU6dOYeDAgZDL5XBzc8O//vWvBkf7IYSQltDk5OXl5YVVq1bh5s2bSE5ORlFREZ599ln0798f8fHxKC0tbYk4CSFExxNX2IvFYkyaNAnfffcdPvvsM+Tn5yMyMhIeHh4IDQ1FUVFRc8ZJCCE6njh5/fHHH5g7dy46d+6M+Ph4REZG4tKlS1AoFLh58yYmTJjQnHESQoiOJj+kGh8fj6+++gp5eXkIDg7Gtm3bEBwczHUY2KNHD2zZsgUymay5YyWEEE6Tk9eGDRswc+ZMzJgxo95+411cXLB58+a/HRwhhNSnyclLoVCgW7duOl0zAwBjDNeuXUO3bt0gkUgQFhbWbEESQsjjmlzn1atXL5SVlenNv3PnDnr06NEsQRFCSGOanLwYYwbnV1ZWwtbW9omCWLduHWQyGWxtbeHr62v0c2O7du2ClZUVJk6c+ETbJYQIl9G3jdpRp62srBAdHQ17e3tumUajwe+//w4vL68mB7B7927I5XIkJSXB19cXiYmJCAoKQl5eHlxcXOpdr6CgAJGRkXjuueeavE1CiPAZnbz+/PNPAHVXXjk5OZBIJNwyiUSCQYMGITIysskBxMfHY9asWdw4jUlJSfjpp5+QnJyMpUuXGlxHo9Fg+vTpiI2NRUZGBu7du9fk7RJChM3o5HX48GEAQHh4OFatWgUnJ6e/vXGVSoXs7GxERUVx86ytreHv74/MzMx61/voo4/g4uKCN998ExkZGQ1uQ6lUQqlUctMVFRUAALVaDbVa/Tf3wHRp+36qqakx6/20JJZyTI3dtya3Nn711VdNDqY+ZWVl0Gg03ACzWq6urjh//rzBdX777Tds3rwZp0+fNmobcXFxiI2N1Zuflpamc+trbq5VAoAYWVlZuJHLdzSkOVjKMa2urjaqnFHJa9KkSdiyZQucnJwwadKkBsumpKQYteEncf/+fbzxxhvYtGkTnJ2djVonKiqKq68D6q68PDw8EBgY2CxXj6bqTOEdIOcPDB8+HIO6deA7HNIMLOWYau+OGmNU8mrbti2srKy435uLs7MzRCIRSkpKdOaXlJTAzc1Nr/ylS5dQUFCA8ePHc/Nqa2sB1L1rmZeXh169eumsI5VKIZXqj7RiY2MDGxub5tgNkyQWi7l/zXk/LYmlHFNj982o5PXorWJz3jZKJBJ4e3sjPT2de9yhtrYW6enpiIiI0Cvfp08f5OTojq7y/vvv4/79+1i1ahU8PDyaLTZCiGnjfQAOuVyOsLAwDB06FD4+PkhMTERVVRXX+hgaGgp3d3fExcXB1tYW/fv311m/Xbt2AKA3nxBi3oxKXoMHD+ZuGxtz6tSpJgUwZcoUlJaWIjo6GsXFxfDy8sKBAwe4SvzCwkK9V5EIIcSo5NXST7BHREQYvE0EgCNHjjS47pYtW5o/IEKIyTMqecXExLR0HIQQ0iR0P0YIESSjrrw6dOiACxcuwNnZGe3bt2+w/uvOnTvNFhwhhNTHqOSVkJCANm3acL8bW3lPCCEtxajk9WjHgjNmzGipWAghxGhNrvMSiUS4deuW3vzbt29DJBI1S1CEENKYZuuMUKlU6nSTQwghLcnoJ+xXr14NoK4zwi+//BKOjo7cMo1Gg19//RV9+vRp/ggJIcQAo5NXQkICgLorr6SkJJ1bRIlEAplMhqSkpOaPkBBCDDA6eV25cgUAMGbMGKSkpKB9+/YtFhQhhDSmyS9ma3tUJYQQPjU5ec2cObPB5cnJyU8cDCGEGKvJyevu3bs602q1Grm5ubh37x7Gjh3bbIERQkhDmpy8vv/+e715tbW1mDNnjl4vpoQQ0lKa5cVsa2tryOVyrkWSEEJaWrP1KnHp0iVuaCZCCGlpTb5tfHQkHqDuua+ioiL89NNPOu9AEkJIS2py8tKOnK1lbW2NTp064fPPP2+0JZK0DpVKhW82b8Adxe/4RvIn+r2/mF7dImaHnvMyE9XV1Th//jxWrVqFb775BhqNBgCw8tR+JHz6IaZPn4758+cDqBuFyZwH3CWWwSR6Ul23bh1kMhlsbW3h6+uLEydO1Fs2JSUFQ4cORbt27eDg4AAvLy98/fXXrRitaTp//jy8vb2xbds2LnFpaTQabNu2Dd7e3vD29q53NHJChIT35LV7927I5XLExMTg1KlTGDRoEIKCggx2uwPU9er673//G5mZmfjvf/+L8PBwhIeH4+DBg60cuWnp2bMnRCIRHB0d9QbsdXNzg6OjI0QiETIzM+kFemIWeE9e8fHxmDVrFsLDw9GvXz8kJSXB3t6+3if1R48ejZdffhl9+/ZFr169MH/+fAwcOBC//fZbK0duWrZs2QKNRoPKykqUl5frLCsvL0dlZSU0Gg2ysrLolpGYBV6Tl0qlQnZ2Nvz9/bl51tbW8Pf3R2ZmZqPrM8aQnp6OvLw8jBo1qiVDNXkXL17kfh83bhwyMjKwc+dOZGRkYNy4cQbLESJkzTZi9vXr1/HRRx9h48aNRq9TVlYGjUbDDTCr5erq2mC9THl5Odzd3aFUKiESibB+/XoEBAQYLKtUKqFUKrnpiooKAHWvNanVaqNjNXXaeq5evXphz5490Gg0uH37NoYMGYI9e/agX79+uHz5MjQajVnttyXRPkdZU1Nj1sfQ2H1rtuR1+/ZtbN68uUnJ60m1adMGp0+fRmVlJdLT0yGXy9GzZ0+MHj1ar2xcXBxiY2P15qelpZnV7VNZWRkAoLi4GPv374dYXHdoFQoFampqUFJSwpVLTU3lLU7y5K5VAoAYWVlZuJHLdzQtp7q62qhyzZa8noSzszNEIhH3xdIqKSnRq3R+lLW1NTw9PQEAXl5eOHfuHOLi4gwmr6ioKJ0HaysqKuDh4YHAwEA4OTk1z46YgN9//x0AUFVVhblz5+KDDz6Ag4MDqqqqsGzZMlRVVQEAnn76aQQHB/MZKnlCZwrvADl/YPjw4RjUrQPf4bQY7d1RY3hNXhKJBN7e3khPT8fEiRMB1L3knZ6ejoiICKM/p7a2VufW8FFSqRRSqVRvvo2NDWxsbJ4oblM0btw4xMXFwd3dHcXFxXj77be5ZWKxGO7u7rhx4wbGjRtnVvttSbRX02Kx2KyPobH7xmvyAupeNwoLC8PQoUPh4+ODxMREVFVVITw8HAAQGhoKd3d3xMXFAai7DRw6dCh69eoFpVKJ1NRUfP3119iwYQOfu8G70aNHo1OnTrhx4waCg4MhkUhw6dIl9OrVCyqVCqmpqXBxcTF4dUqIEBmdvCZNmtTg8nv37j1RAFOmTEFpaSmio6NRXFwMLy8vHDhwgKvELywshLX1/xpFtbdF169fh52dHfr06YPt27djypQpT7R9cyESiZCUlIRXXnkFP//8MzfKU05ODjdI8IYNG2h4OmI2jE5ebdu2bXR5aGjoEwURERFR723ikSNHdKY//vhjfPzxx0+0HUtgZWUFqVSKhw8fcvNsbW11pgkxB0Ynr6+++qol4yB/k0ajwcKFCxESEoK9e/fi6NGj+Pnnn/HCCy/Az88Pr7zyCiIjIzFhwgS6+iJmgfcn7EnzyMjIQEFBAd577z3Y2NjAz88Po0aNgp+fH2xsbBAVFYUrV64gIyOD71AJaRZGX3kZ290NDcDBj6KiIgBA//79DS7XzteWI0TojE5eW7ZsQffu3TF48GCuMpiYjs6dOwMAcnNzMWzYMBw9ehS//vorHBwcMGbMGOTm5uqUI6ZN28XRo/KK7kFZnI9zuXaovd2Om2+pXRxZMSMz0bx587Bz5050794d4eHheP3119Ghg/AelKuoqEDbtm1RXl5uVg+pajQaeHp6wtnZGWVlZSgoKOCWyWQyODs74/bt27h48SLVeQnAqVOn4O3tbVTZ7OxsDBkypIUjaj3GfkeNTl5A3XuCKSkpSE5OxvHjx/Hiiy/izTffRGBgINccb+rMNXkBwOLFi7FixQq4urriww8/5FoZP/zwQ5SUlGDRokVYvnw532ESIxi68qp8oMRPhzPx4pgRcLT734PX5nbl1SLJ61FXr17Fli1bsG3bNtTU1ODs2bNwdHR84oBbi7kmr0evvEpLS3H16lVuGV15mQe1Wo3U1FQEBweb9RP2xn5Hn7i10draGlZWVmCM6fXcSVqftrVxzZo1uHTpEhQKBeRyORQKBfLz87F69WpqbSRmpUnJS6lUYufOnQgICMDTTz+NnJwcrF27FoWFhYK46jJn1NpILI3RrY1z587Frl274OHhgZkzZ2Lnzp1wdnZuydhIE2hbEdeuXYsvvviCq7CPj4+HTCbD7NmzdcoRInRG13lZW1ujW7duGDx4cIOV8ykpKc0WXEsw5zqvzp07o7S0FCEhIViyZAmuX7+Orl274rPPPsP+/fvh4uKCmzdvUp2XAD148AByuRxZWVkYPnw44uPjYWdnx3dYLcLY76jRV16hoaGCaVG0VI8eH+3fJHomT/gmTpyIH374gZs+ffo0kpKSMGHCBOzbt4+/wPjGLEx5eTkDwMrLy/kOpVkdPnyYAWBxcXFMJpMxANxPjx492KeffsoAsMOHD/MdKmmCCRMm6BzLx38mTJjAd4jNztjvKL3baCa0FfERERHIy8vDypUrERwcjJUrV+L8+fNcrx1UYS8cDx484K64Hr9F1E7/8MMPePDgQavHZgooeZmJRyvse/fujcjISKSmpiIyMhK9e/fG2rVrdcoR0/do9+UNjQj1aDmL0kpXgibDXG8ba2pqWKdOnRgAZmdnp3NroZ12cXFhNTU1fIdKjOTt7c0AMG9vb6bRaJhKpWL79u1jKpWKaTQaNmTIEG65OaHbRgukUqkAQK/jQe10ff38E9PE/q+x5amnntLpTRjQHYSGWWijDCUvM3HkyBFupGxbW1udZdrp8vJyvZ5pienSDsa8Z88ePHjwgOsp5OjRo3jw4AH3WNKjgzZbEkpeZuLQoUMA6oY2c3Fx0Vnm4uKCp556SqccMX2BgYEA6gaZtbe3R0BAAOLj4xEQEAB7e3tuEFptOUtDyctMFBYWAgAuXLiAgQMH6lTuDhw4EBcvXtQpR0zf6NGjjRo7wlJHhDKJ5LVu3TrIZDLY2trC19cXJ06cqLfspk2b8Nxzz6F9+/Zo3749/P39GyxvKbp27QoAaN++PVJSUuDr6ws7Ozv4+voiJSUF7du31ylHhEEikQDQrwrQPiphaExSS8F78tq9ezfkcjliYmJw6tQpDBo0CEFBQbh165bB8keOHMHUqVNx+PBhZGZmcqNf37hxo5UjNy3a90zv3r2Ll19+GVlZWXjw4AGysrLw8ssv4+7duzrliOnLyMhAaWkppk+fzt0iaqnVakybNg23bt2y3J5CWqXtswE+Pj5s3rx53LRGo2FdunRhcXFxRq1fU1PD2rRpw7Zu3WpUeXN9VGL79u16j0Zof+zt7bnft2/fzneoxEg7duxgAJiVlRULCQlhq1evZhEREWz16tUsJCSEWVlZMQBsx44dfIfarIz9jvI6YrZKpUJ2djaioqK4edbW1vD390dmZqZRn1FdXQ21Wl1vl9RKpVLnEYGKigoAdX+51Gr134jetGgH6QX0m84fnXZ1dTWr/TZn2nN65MiR2LNnDzQaDRQKBQICAjB79myMHTsWx48fR4cOHczqmBq7L7wmr7KyMmg0Gp0vHlD3BXu8C9z6LFmyBF26dKm3uTguLg6xsbF689PS0syq61yNRgMXFxc4OTmhvLxc51kvR0dHuLu74/79+6ioqEBqaiqPkRJjnTlzBgBw584dpKamcs96KRQK1NbWclUBv//+u1kNKlxdXW1UOV6T19/1n//8B7t27cKRI0f0KjS1oqKidF6fqKio4OrJzKlLHKCuSf21115DcHAwxo0bh8uXL6Nnz55IT09Hamoqdu3ahfHjx/MdJjGS9i7h3Llz+PLLL+Hv788d019++QXnzp0DAPTo0QPBwcF8htqstPvdqFa5ia2HUqlkIpGIff/99zrzQ0ND2UsvvdTguitWrGBt27ZlJ0+ebNI2zbXOS2vv3r0Ge5XYu3cv36GRJtL2FDJ9+nQmFot1jqlYLGbTpk0zy55CjP2OPvEAHM3F19cXPj4+WLNmDQCgtrYW3bp1Q0REBJYuXWpwneXLl+OTTz7BwYMHMXz48CZtz1w7I3yURqPB4cOH8fPPP+OFF17AmDFjqANCAXq0g8kXX3wRgYGBuHjxIp566imkpaXhp59+MssOJpu9M8KWIpfLERYWhqFDh8LHxweJiYmoqqpCeHg4gLpOEN3d3REXFwcA+OyzzxAdHY0dO3ZAJpOhuLgYQF29DvWjX0ckEsHPzw9VVVXw8/MzqxPb0mg7mLSyssLgwYPh4uICd3d3KBQKniMzAa1yHdiINWvWsG7dujGJRMJ8fHxYVlYWt8zPz4+FhYVx0927dzfYKVtMTIxR2zL320atR3sgIML0aAeTj5/3MpnMbDuYFMSjEloRERFcZ3mPe/xF4kdHgibEnGk7jiwuLsb169d1ll27dg0lJSU65SyNSSQvQog+bceRq1atgqurK2JjYyGVSqFUKhETE4NVq1bplLM0vL8eRJqfRqPR6T6FBgUWJl9fXwB17zcWFhZi5syZaN++PWbOnInCwkLuvUdtOUtDycvMpKSkwNPTU6f7FE9PT5Mfko7o++KLLwDUPXE+efJknfdVJ0+ezD2Jri1naSh5mZGUlBRMnjwZAwYM0OkSZ8CAAZg8eTIlMIG5dOkSgLqeVHJycjBq1ChMnToVo0aNQm5uLjZu3KhTztJQ8jITGo0GCxcuREhICPbt26fTJc6+ffsQEhKCyMhIuoUUkF69egGoezc1Pz8fCoUCcrkcCoUCFy9eRG1trU45i9M6jZ+mw1wfldA2q2dmZjLG9B+VOH78uFk2q5szpVLJxGIxc3V1ZWq1WueYqtVq5urqysRiMVMqlXyH2qxoAA4Lo20u79+/v8Hl2vmW2qwuRBKJBAsWLEBJSQm6du2KL7/8Enfu3MGXX36Jrl27oqSkBAsWLOAq7i0NPSphJrTN5bm5uQZfmcrNzdUpR4Rh+fLlAICEhATMnTuXmy8Wi7Fo0SJuuSXi/d3G1mau7zZqNBp4enpiwIAB2LdvHzQaDVJTUxEcHAyRSISJEyciNzcXFy9epNeFBEilUmHNmjU4dOgQxo4di7fffttsr7gE824jaR4ikQiff/45Jk+ejIkTJ2LRokVcs/qKFSuwf/9+7NmzhxKXQEkkErzzzjvw9PREcHAwbGxs+A6Jd5S8zMikSZOwZ88eLFy4EKNGjeLm9+jRA3v27MGkSZN4jI6Q5kW3jWaIusQxP3TbqI+Sl5lSq9VcnRfdYgjb4sWLkZCQoDOCkFgsxoIFC8yywp7qvAgxA4sXL8aKFSsMvpi9YsUKADDLBGYMes6LEBOlUqmQkJAAV1dXXL9+XefF7OvXr8PV1RUJCQlQqVR8h8oLSl6EmKj169ejpqYGH3/8McRi3ZsksViMjz76CDU1NVi/fj1PEfKLkhchJkr7wnVISIjB5dr59GI2IcSkaF+43r9/v8Hl2vmW+mI2JS9CTNTcuXMhFovx/vvv67Q0AnVjdEZHR0MsFuu8NmRJeE9e69atg0wmg62tLXx9fXHixIl6y549exavvPIKZDIZrKyskJiY2HqBEtLK6MXshvH6qMTu3bshl8uRlJQEX19fJCYmIigoCHl5eXBxcdErX11djZ49e+LVV1/FggULeIiYkNZFL2Y3oMU752mAj48PmzdvHjet0WhYly5dWFxcXKPrdu/enSUkJDR5m+ban9fjaOgz81JdXc3eeust5uXlxd566y1WXV3Nd0gtxuSHPlOpVMjOzkZUVBQ3z9raGv7+/sjMzGy27SiVSiiVSm66oqICQN0T6No+wM2Rdt/MeR8txffff48lS5Zww/6dPn0aBw4cwGeffYaXX36Z3+BagLHnLG/Jq6ysDBqNBq6urjrzXV1dcf78+WbbTlxcHGJjY/Xmp6Wlwd7evtm2Y6poZGVhy8zMxPLlyzF06FDMmTMH3bp1Q2FhIfbs2YPXXnsNixcvxogRI/gOs1lVV1cbVc7sXw+KioqCXC7npisqKuDh4YHAwECzf7dRoVAgICCA3m0UKI1Gg3fffRfBwcHYu3cvNBoNFAoFIiIiMH/+fLzyyiv49ttv8eGHH5rVi/fau6PG8Ja8nJ2dIRKJuFF/tUpKSuDm5tZs25FKpZBKpXrzbWxsLOJLbSn7aY6OHTuGgoIC7Ny5E1KplLud0h7Tf//73xg5ciSysrIwevRofoNtRsaer7w9KiGRSODt7Y309HRuXm1tLdLT083uMpiQJ0HjEjSM1+e85HI5Nm3ahK1bt+LcuXOYM2cOqqqqEB4eDgAIDQ3VqdBXqVQ4ffo0Tp8+DZVKhRs3buD06dPIz8/naxcIaTGPjktgiMWPS9BKrZ/1WrNmDevWrRuTSCTMx8eHZWVlccv8/PxYWFgYN33lyhUGQO/Hz8/P6O3RoxJEKGpqaphMJmPjx49nGo1G55hqNBo2fvx41qNHD1ZTU8N3qM3K2O8odUZopqgzQvOgHQU9JCQEixYtwo0bN+Du7q4zLoG5de9NnRESYgZoXIL6UfIixMRNmjQJEyZMoHEJHkPJixABEIlE8PPzQ1VVFfz8/Cw+cQEm0KsEIYQ8CUpehBBBouRFCBEkSl6EEEGi5EUIESRKXoQQQaLkRQgRJEpehBBBouRFCBEkSl6ECIBGo8HRo0fx66+/4ujRo9BoNHyHxDtKXoSYuJSUFHh6eiIgIADx8fEICAiAp6cnUlJS+A6NV5S8CDFh2i5xBgwYgIyMDOzcuRMZGRkYMGAAJk+ebNEJjJIXISZKo9Fg4cKFCAkJwb59++Dr6ws7Ozv4+vpi3759CAkJQWRkpMXeQlLyIsREZWRkoKCgAO+99x5qamqwevVqbNy4EatXr0ZNTQ2ioqJw5coVZGRk8B0qL6hLHEJMlHZgjV27duG5555DTU0NACA1NRVLly7FvHnzdMpZGkpehJgo7cAaq1atgqurK2JjYyGVSqFUKhETE4NVq1bplLM4rdKjfiPWrl3LunfvzqRSKfPx8WG///57g+W//fZb1rt3byaVSln//v3ZTz/9ZPS2LGEAjmHDhukMUDJs2DC+QyJPoLq6mgFgEomEjR07VueYjh07lkkkEgaAVVdX8x1qszL2O8p78tq1axeTSCQsOTmZnT17ls2aNYu1a9eOlZSUGCx/7NgxJhKJ2PLly9lff/3F3n//fWZjY8NycnKM2p65Jy8YGF1J+0OEJSEhocHjqf1JSEjgO9RmZex3lPcK+/j4eMyaNQvh4eHo168fkpKSYG9vj+TkZIPlV61aheeffx6LFi1C3759sWzZMgwZMgRr165t5chNj5WV1d9aTkzLpUuXmrWcueE1ealUKmRnZ8Pf35+bZ21tDX9/f2RmZhpcJzMzU6c8AAQFBdVb3lL4+Phwv0+dOhUqlQr79u2DSqXC1KlTDZYjpu3AgQPc77Nnz4ZCoYBcLodCocDs2bMNlrMkvFbYl5WVQaPRwNXVVWe+q6srzp8/b3Cd4uJig+WLi4sNllcqlVAqldx0RUUFgLpxDdVq9d8J36ScPHmS+33r1q3cvqnVamzduhU7d+7kypnTfpuzR0eCT0xMBGMMVVVVGDlyJJ555hls3LiRK2dOx9TYfTH71sa4uDjExsbqzU9LS4O9vT0PEbW81NRU7neFQtHgciIMXbp0wdSpUzFs2DBERkZyf4y0zOmYVldXG1WO1+Tl7OwMkUiEkpISnfklJSVwc3MzuI6bm1uTykdFRUEul3PTFRUV8PDwQGBgoNmOmB0cHAy1Wg2FQoGAgAC9EbODg4N5iow8qcrKSmzYsAEbNmwAAIjFul9dczqm2rujxvCavCQSCby9vZGeno6JEycCAGpra5Geno6IiAiD64wYMQLp6el49913uXkKhQIjRowwWF4qlUIqlerNt7Gx0ftSC9mwYcO4W8ewsDBs3boVQN1+hoWF6ZQzp/02Z/7+/vjll18AAOHh4ejduzcOHTqEsWPHIi8vD5s2beLKmdMxNXpfWqfxs367du1iUqmUbdmyhf31119s9uzZrF27dqy4uJgxxtgbb7zBli5dypU/duwYE4vFbOXKlezcuXMsJiaGHpX4PzCiWZ0IiyUeU2O/o7zXeU2ZMgWlpaWIjo5GcXExvLy8cODAAa5SvrCwENbW/2sUHTlyJHbs2IH3338f7733Hp566ins27cP/fv352sXTAZjrMHHIRhjrRgNaQ50TOtnxSxs7ysqKtC2bVuUl5ebbZ2Xj4+PTuvjsGHDcOLECR4jIn9XQEAAdwsJ1N0qGmqMMQfGfkd5f0iVNL8TJ07oPOdFiUv4FAqFzjE118TVFJS8CCGCRMmLECJIlLwIIYLEe2tja9O2Txj7IJxQqdVqVFdXo6KiwqyeAbJklnJMtd/NxtoSLS553b9/HwDg4eHBcySEkIbcv38fbdu2rXe5xT0qUVtbi5s3b6JNmzZm3UWM9jWoa9eume0jIZbGUo4pYwz3799Hly5ddJ7xfJzFXXlZW1uja9eufIfRapycnMz6RLdElnBMG7ri0qIKe0KIIFHyIoQIEiUvMyWVShETE2OwRw0iTHRMdVlchT0hxDzQlRchRJAoeRFCBImSFyFEkCh5EUIEiZKXAIwePVqnz34iHKZ27Ewtnr+DkhchJk6lUvEdgmlquW70SXMICwvTG3DB3d2drV+/XqfcqVOnmJWVFSsoKGCMMfb555+z/v37M3t7e9a1a1c2Z84cdv/+fZ11MjIy2LPPPstsbW1Z165d2dtvv80qKytbbd/MnaFjl5+fz2bOnMlkMhmztbVlTz/9NEtMTNRbb8KECezjjz9mnTt3ZjKZjDFWN/jMoEGDmFQqZd7e3uz7779nANiff/7JrZuTk8Oef/555uDgwFxcXNjrr7/OSktL643nypUrrfXf0ewoeZm4e/fusREjRrBZs2axoqIiVlRUxCIjI9mzzz6rU27hwoU68xISEtihQ4fYlStXWHp6OuvduzebM2cOtzw/P585ODiwhIQEduHCBXbs2DE2ePBgNmPGjFbbN3Nn6Ng9fPiQRUdHs5MnT7LLly+z7du3M3t7e7Z7925uvbCwMObo6MjeeOMNlpuby3Jzc1l5eTnr0KEDe/3119nZs2dZamoqe/rpp3WS1927d1mnTp1YVFQUO3fuHDt16hQLCAhgY8aMqTeempoaPv5rmgUlLwHw8/Nj8+fP56b//PNPZmVlxa5evcoYY0yj0TB3d3e2YcOGej/ju+++Yx07duSm33zzTTZ79mydMhkZGcza2po9ePCgeXfAgj1+7AyZN28ee+WVV7jpsLAw5urqypRKJTdvw4YNrGPHjjrHZtOmTTrJa9myZSwwMFDns69du8YAsLy8PKPjEQqq8xIgLy8v9O3bFzt27AAAHD16FLdu3cKrr77Klfnll18wbtw4uLu7o02bNnjjjTdw+/Ztbij1M2fOYMuWLXB0dOR+goKCUFtbiytXrvCyX5Zi3bp18Pb2RqdOneDo6IiNGzeisLBQp8yAAQMgkUi46by8PAwcOBC2trbcPB8fH511zpw5g8OHD+sc0z59+gAALl261IJ7xA9KXgI1ffp0Lnnt2LEDzz//PDp27AgAKCgoQEhICAYOHIi9e/ciOzsb69atA/C/yt/Kykr861//wunTp7mfM2fO4OLFi+jVqxc/O2UBdu3ahcjISLz55ptIS0vD6dOnER4erlcp7+Dg0OTPrqysxPjx43WO6enTp3Hx4kWMGjWquXbBZFhcf15CJJFIoNFodOZNmzYN77//PrKzs7Fnzx4kJSVxy7Kzs1FbW4vPP/+c68zt22+/1Vl/yJAh+Ouvv+Dp6dnyO2DBHj92x44dw8iRIzF37lxunjFXRb1798b27duhVCq5F7MfHZsTqDume/fuhUwmg1hs+Ktt6FwSKrryEgCZTIbff/8dBQUFKCsrQ21tLWQyGUaOHIk333wTGo0GL730Elfe09MTarUaa9asweXLl/H111/rJDcAWLJkCY4fP46IiAjur/MPP/yAiIiI1t49s/b4sXvqqafwxx9/4ODBg7hw4QI++OADvSRkyLRp01BbW4vZs2fj3LlzOHjwIFauXAkAXI/A8+bNw507dzB16lScPHkSly5dwsGDBxEeHs4lLEPnkmDxXelGGpeXl8eGDx/O7OzsdJq3169fzwCw0NBQvXXi4+NZ586dmZ2dHQsKCmLbtm1jANjdu3e5MidOnGABAQHM0dGROTg4sIEDB7JPPvmklfbKMjx+7M6fP89mzJjB2rZty9q1a8fmzJnDli5dygYNGsSto31U4nHHjh1jAwcOZBKJhHl7e7MdO3Zwn6l14cIF9vLLL7N27doxOzs71qdPH/buu++y2tpag/EI+VEJ6hKHEIH65ptvEB4ejvLyctjZ2fEdTqujOi9CBGLbtm3o2bMn3N3dcebMGSxZsgT//Oc/LTJxAZS8CBGM4uJiREdHo7i4GJ07d8arr76KTz75hO+weEO3jYQQQaLWRkKIIFHyIoQIEiUvQoggUfIihAgSJS9CiCBR8iItasaMGbCysoKVlRVsbGzg6uqKgIAAJCcnN+nVlC1btqBdu3YtF2g9ZsyYgYkTJ7b6dknjKHmRFvf888+jqKgIBQUF+PnnnzFmzBjMnz8fISEhqKmp4Ts8IlT8vp1EzF197+mlp6czAGzTpk2MsYa7rT58+LBe98UxMTGMMca2bdvGvL29maOjI3N1dWVTp05lJSUl3Hbu3LnDpk2bxpydnZmtrS3z9PRkycnJ3PLCwkL26quvsrZt27L27duzl156iXvfLyYmRm+7hw8fbpH/J9J0dOVFeDF27FgMGjQIKSkpAABra2usXr0aZ8+exdatW3Ho0CEsXrwYADBy5EgkJibCyckJRUVFKCoqQmRkJABArVZj2bJlOHPmDPbt24eCggLMmDGD284HH3yAv/76Cz///DPOnTuHDRs2wNnZmVs3KCgIbdq0QUZGBo4dOwZHR0c8//zzUKlUiIyMxD//+U/uyrGoqAgjR45s3f8oUj++sycxb/VdeTHG2JQpU1jfvn0NLnu82+qvvvqKtW3bttHtnTx5kgHgrtrGjx/PwsPDDZb9+uuvWe/evbkeFxhjTKlUMjs7O3bw4MFG4yf8oisvwhvGGNcXVWPdVtcnOzsb48ePR7du3dCmTRv4+fkBANet8pw5c7Br1y54eXlh8eLFOH78OLfumTNnkJ+fjzZt2nDdJnfo0AEPHz40y26TzQ0lL8Kbc+fOoUePHkZ1W21IVVUVgoKC4OTkhG+++QYnT57E999/r7PeCy+8gKtXr2LBggW4efMmxo0bx91yVlZWwtvbW6/b5AsXLmDatGktvPfk76JeJQgvDh06hJycHCxYsMCobqsNdV98/vx53L59G//5z3/g4eEBAPjjjz/0ttWpUyeEhYUhLCwMzz33HBYtWoSVK1diyJAh2L17N1xcXODk5GQwTnPqNtnc0JUXaXFKpRLFxcW4ceMGTp06hU8//RQTJkxASEgIQkNDjeq2WiaTobKyEunp6SgrK0N1dTW6desGiUTCrffjjz9i2bJlOutFR0fjhx9+QH5+Ps6ePYv9+/ejb9++AOoGMXF2dsaECROQkZGBK1eu4MiRI3jnnXdw/fp1brv//e9/kZeXh7KyMqjV6tb5TyON47vSjZi3R0dpFovFrFOnTszf358lJyczjUbDlTOm2+q33nqLdezYUedRiR07djCZTMakUikbMWIE+/HHH/XGMuzbty+zs7NjHTp0YBMmTGCXL1/mPrOoqIiFhoYyZ2dnJpVKWc+ePdmsWbNYeXk5Y4yxW7ducV1lgx6VMCnUnxchRJDotpEQIkiUvAghgkTJixAiSJS8CCGCRMmLECJIlLwIIYJEyYsQIkiUvAghgkTJixAiSJS8CCGCRMmLECJIlLwIIYL0/wE893mpOrjOaAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-26T11:22:46.860128Z", + "iopub.status.busy": "2024-03-26T11:22:46.859661Z", + "iopub.status.idle": "2024-03-26T11:22:47.145077Z", + "shell.execute_reply": "2024-03-26T11:22:47.144122Z" + }, + "papermill": { + "duration": 0.305149, + "end_time": "2024-03-26T11:22:47.147632", + "exception": false, + "start_time": "2024-03-26T11:22:46.842483", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABSCUlEQVR4nO2deXxMZ/fAvzOTPbIiEYRYY6eWKC1aoqhSXX710ldRpapKpa2iSlVb2qJokZamK6VV1FulRe17CaW2EsSSRYSskklm7u+Pm5nMyDaTzJo8389nPu7cee6958qdM+c55zznKCRJkhAIBAInQ2lvAQQCgaA8COUlEAicEqG8BAKBUyKUl0AgcEqE8hIIBE6JUF4CgcApEcpLIBA4JUJ5CQQCp8TF3gLYGq1Wy40bN/Dx8UGhUNhbHIFAcA+SJJGRkUHt2rVRKku2r6qc8rpx4wahoaH2FkMgEJTB1atXqVu3bomfVznl5ePjA8j/Mb6+vnaWRiAQ3Et6ejqhoaH672pJVDnlpZsq+vr6CuUlEDgwZbl1hMNeIBA4JUJ5CQQCp0QoL4FA4JRUOZ+XKUiSRH5+PhqNxt6iCCyESqXCxcVFpMdUIoTyuge1Wk1CQgLZ2dn2FkVgYby8vAgJCcHNzc3eoggsgFBeBmi1Wi5duoRKpaJ27dq4ubmJX+pKgCRJqNVqbt68yaVLl2jSpEmpyY8Cy3NXrcHDVWnR75NQXgao1Wq0Wi2hoaF4eXnZWxyBBfH09MTV1ZUrV66gVqvx8PCwt0hVind/Pc2p62m8M7AFHeoHWuScQnkVg/hVrpyIv6t9uHY7m5/+ukq+VkJrwY4Z4q8pEAisytKdF8nXSjzQuDqdwixjdYFQXgKBwIrorC6Aib2aWvTcQnkJKg0jRoxg0KBB9hZDYMDSnRfJ08hWV0QDy1ldIJRXpeGhhx7i1VdftbcYAoGe63fuWs3qAqG8BA5GXl6evUUQWIilOy6Qp5Ho2sjyVhcI5VUmkiSRrc63y8vUZuYjRoxg165dLFq0CIVCgUKhoG7duixbtsxoXGxsLEqlkitXrgCwYMECWrdujbe3N6GhoYwbN47MzEyjY/bu3Uu3bt3w9PQkNDSUCRMmkJWVZZJcCQkJ9O/fH09PTxo0aMCqVasICwtj4cKF+jEKhYJly5YxcOBAvL29ef/999FoNIwaNYoGDRrg6elJeHg4ixYtMjq3RqMhKioKf39/qlevzuTJk03+/xJYn+t37vKj3upqYpVriFSJMribp6HFjN/tcu3T7/bBy63sP9GiRYs4f/48rVq14t133wVg/vz5rFq1ipdeekk/buXKlTzwwAPUr18fkFMHFi9eTIMGDYiLi2PcuHFMnjyZpUuXAnDx4kX69u3Le++9R0xMDDdv3mT8+PGMHz+er776qky5nnvuOVJSUti5cyeurq5ERUWRnJxcZNw777zD3LlzWbhwIS4uLmi1WurWrctPP/1E9erV2b9/P2PGjCEkJIRnnnlGf39ff/01MTExNG/enPnz57N+/Xp69uxZ9n+swOoYWl2dG1a3yjWE8qoE+Pn54ebmhpeXF7Vq1QLg2WefZf78+cTHx1OvXj20Wi2rV69m+vTp+uMMfWRhYWG89957jB07Vq+85syZw7PPPqsf16RJExYvXkyPHj1YtmxZqYmeZ8+eZdu2bRw5coSOHTsCsGLFCpo0KforPHToUEaOHGm0b9asWfrtBg0acODAAX788Ue98lq4cCFTp07lySefBCA6Oprff7fPj4zAmBs2sLpAKK8y8XRVcfrdPna7dnlp164dzZs3Z9WqVUyZMoVdu3aRnJzM//3f/+nHbNu2jTlz5nD27FnS09PJz88nJyeH7OxsvLy8OHHiBH///TcrV67UHyNJkn4ZVfPmzUu8/rlz53BxcaF9+/b6fY0bNyYgIKDIWJ1yM2TJkiXExMQQHx/P3bt3UavVtGvXDoC0tDQSEhLo3LmzfryLiwsdO3YUU0cHYOlO2erq0tB6VhcI5VUmCoXCpKmbI/Lss8/qldeqVavo27cv1avLD9Ply5d57LHHeOmll3j//fcJDAxk7969jBo1CrVajZeXF5mZmbz44otMmDChyLnr1atnMTm9vb2N3q9evZrXX3+d+fPn06VLF3x8fPj44485dOiQxa4psA437txlzZECqyvSelYXCOVVaXBzcytSwmfo0KFMnz6do0ePsnbtWqKjo/WfHT16FK1Wy/z58/XLZn788Uej49u3b8/p06dp3Lix2fKEh4eTn59PbGwsHTp0AODChQvcvn27zGP37dtH165dGTdunH7fxYsX9dt+fn6EhIRw6NAhunfvDkB+fj5Hjx41svQEtsfQ6rrfilYXiGhjpSEsLIxDhw5x+fJlUlJS0Gq1hIWF0bVrV0aNGoVGo2HgwIH68Y0bNyYvL49PP/2UuLg4vvvuOyPlBvDmm2+yf/9+xo8fz/Hjx/n333/55ZdfGD9+fJnyNGvWjMjISMaMGcPhw4eJjY1lzJgxeHp6lllZoEmTJvz111/8/vvvnD9/nrfffpsjR44YjZk4cSJz585lw4YNnD17lnHjxnHnzh3T/8MEFufGnbv8eOQaYH2rC4TyqjS8/vrrqFQqWrRoQc2aNYmPjwfkqeOJEyd44okn8PT01I9v27YtCxYs4MMPP6RVq1asXLmSOXPmGJ2zTZs27Nq1i/Pnz9OtWzfuu+8+ZsyYQe3atU2S6dtvvyU4OJju3bvzxBNPMHr0aHx8fMqs6PDiiy/y5JNPMnjwYDp37sytW7eMrDCA1157jWHDhjF8+HD91PKJJ54wSS6BdVi28yJqjZb7GwZa3eoCUEhVzMOZnp6On58faWlpRboH5eTkcOnSJRo0aCBKpliBa9euERoayrZt2+jVq5fNry/+vtbjxp27PPTxTtQaLT+Mvp8ujcqvvEr7jhoifF4Cq/Hnn3+SmZlJ69atSUhIYPLkyYSFhen9VILKg6HVVRHFZQ5CeQnKxZ49e+jXr1+Jn2dmZpKXl8e0adOIi4vDx8eHrl27snLlSlxdXW0oqcDaJKQZRBitsIaxJITyEpSLjh07cvz48VLH9OnThz597JMjJ7AdOqurcwPbWV0glJegnHh6epYrhUJQuUhIu8vqw7LV9Wqk7awuENFGgUBQAaLtZHWBUF4CgaCcJKbl8MNh22TTF4dQXgKBoFws23kBtUZLRINAutggr+tehPISCARmY2h1vRrZxC79TYXyEggEZhO966JdrS4QyksAxVY33bBhg93kETg2iWk5rDosLz97tZd9rC4QqRKCYkhISCi27pZAAAVWV76WiDDbRxgNEcpLUARdNVZ7IkkSGo0GFxfxiDoSSekGVpedfF067D5tXLJkCWFhYXh4eNC5c2cOHz5c6viFCxcSHh6ubwgxadIkcnJyrCegJIE6yz4vM9bMZ2Rk8Oyzz+Lt7U1ISAiffPJJuduhGU4bL1++jEKhYN26dTz88MN4eXnRtm1bDhw4YHRMWY06vvvuOzp27IiPjw+1atVi6NChRvXsd+7ciUKhYPPmzXTo0AF3d3f27t1rtuwC67Jsp2x1dQoLsKvVBXa2vNasWUNUVBTR0dF07tyZhQsX0qdPH86dO0dQUFCR8bqqoDExMXTt2pXz588zYsQIFAoFCxYssI6QednwgWklYCzOtBvg5l32OCAqKop9+/axceNGgoODmTFjBseOHdOXTq4ob731FvPmzaNJkya89dZbDBkyhAsXLuDi4mJSo468vDxmz55NeHg4ycnJREVFMWLECH777Tej60yZMoV58+bRsGFDMXV1MIytrqZ2tbrAzsprwYIFjB49Wt98ITo6mk2bNhETE8OUKVOKjN+/fz8PPPAAQ4cOBWRH85AhQ6p8eeCMjAy++eYbVq1apS8189VXX5lcd8sUXn/9dfr37w/IzTFatmzJhQsXaNasmUmNOp5//nn9uRo2bMjixYvp1KkTmZmZVKtWTf/Zu+++S+/evS0mt8ByGFpdXe1sdYEdlZdarebo0aNMnTpVv0+pVBIZGVlkSqKja9eufP/99xw+fJiIiAji4uL47bffGDZsWInXyc3NJTc3V/8+PT3dPEFdvWQLyB64epk0LC4ujry8PCIiIvT7/Pz8CA8Pt5gobdq00W+HhIQAkJycTLNmzUxq1HH06FHeeecdTpw4we3bt9FqtQDEx8fTokUL/XHFNeMQ2J/k9Bx+cCCrC+yovFJSUtBoNAQHBxvtDw4O5uzZs8UeM3ToUFJSUnjwwQeRJIn8/HzGjh3LtGnTSrzOnDlzjNpomY1CYfLUrTJjWMZG9+DqFFBZjTqysrL0FSZWrlypr/Tap08f1Gq10fh7m3EIHINluy6Sm6+lY33HsLrAARz25rBz504++OADli5dyrFjx1i3bh2bNm1i9uzZJR4zdepU0tLS9K+rV6/aUGLb0LBhQ1xdXY3qvKelpXH+/HmbXN+wUce9Lzc3N86ePcutW7eYO3cu3bp1o1mzZsU2nxU4JsnpOaw65FhWF9jR8qpRowYqlYqkpCSj/UlJSSWG6t9++22GDRvGCy+8AEDr1q3JyspizJgxvPXWW/ouOIa4u7vj7u5u+RtwIHx8fBg+fDhvvPEGgYGBBAUFMXPmTJRKpU0etDfffJP777+f8ePH88ILL+Dt7c3p06fZunUrn332GfXq1cPNzY1PP/2UsWPHcurUqVJ/cASOhaHV9UBjx7C6wI6Wl5ubGx06dGD79u36fVqtlu3bt9OlS5dij8nOzi6ioFQquTFrFSvFX4QFCxbQpUsXHnvsMSIjI3nggQdo3ry5TWq1l9Woo2bNmnz99df89NNPtGjRgrlz5zJv3jyryyWoOI5qdQEg2ZHVq1dL7u7u0tdffy2dPn1aGjNmjOTv7y8lJiZKkiRJw4YNk6ZMmaIfP3PmTMnHx0f64YcfpLi4OOmPP/6QGjVqJD3zzDMmXzMtLU0CpLS0tCKf3b17Vzp9+rR09+7dit+cncnMzJT8/PykFStW2FsUh6Ey/X1txayN/0j13/xVenLpPkmr1drkmqV9Rw2xa6rE4MGDuXnzJjNmzCAxMZF27dqxZcsWvRM/Pj7eyNKaPn06CoWC6dOnc/36dWrWrMmAAQN4//337XULDkNsbCxnz54lIiKCtLQ03n33XQAef/xxO0smcFaS03NYeegKYP9s+uIQrc8McObWWLGxsbzwwgucO3dOPyVfsGABd+7cKbNRRlXBmf++9uDd/50mZt8lOtQPYO3YLjZTXqL1WRXjvvvu4+jRo0X23717t8xGGQLBvSRnOLbVBUJ5VXpEowxBefh8Vxy5+Vra1/PnwcY17C1OsThVnpetqGIz6SqD+LuaRnJGDt8f1FldDhZhNEAoLwN0WeTZ2dl2lkRgDXR/V9H0tnQMra5uTRzT6gIxbTRCpVLh7++vz/728vJy2F8dgelIkkR2djbJycn4+/vrcwMFRTH0dU10YKsLhPIqgi67XyxfqXz4+/s7RKFFR+aLXXHk5Gm5r54/3R3Y6gKhvIqgUCgICQkhKCiIvLw8e4sjsBCurq7C4iqDmxm5fH/I8X1dOoTyKgGVSlUlH3aNVkKldOyHVmAdvth90WmsLhAOe4EBX+27RMuZWzh65ba9RRHYmJsZuXxXEGGcaMeOQOYglJdAz4GLt8jJ0/K/E3YqviiwGzqrq12oPz2a1rS3OCYhlJdAj1ojFxc8dCnVzpIIbImh1eWo2fTFIZRXaZxaB3G77C2FzVDny8rrbGI6aXdFsKKqsHxPnNNZXSCUV8mcWANrR8K60ZB5097S2IS8AstLkuDoFWF9VQVSMnP59sBlACY6kdUFQnmVTPMBULMZZCbBhrFQUK+9MqOzvEBMHasKX+yWra62of485ERWFwjlVTJuXvD0V+DiARe2wYHP7C2R1ck1UF6HhfKq9BhaXc7k69IhlFdpBLeAvnPl7e2z4FrRkjOVCd20EeDktTSy1fl2lEZgbZY7sdUFQnmVTYcR0GIQaPNlH1hOmr0lshpqA+WVr5U4Hn/HfsIIrIpsdRVEGJ0kr+tehPIqC4UCBiwC/3pw5wr8b6Ls0a6E6HxejWrKvROF36vysnx3HHfzNLSt68dD4c5ndUEFlVdmZibp6elGr0qJpz88FQNKF/hnPRz71t4SWYU8jayUuzWRH2bh96qc3DK0upxgDWNJmK28Ll26RP/+/fH29sbPz4+AgAACAgLw9/cnICDAGjI6BqGdoOfb8vbmNyH5jH3lsQI6y0tXOfNY/G2jCKSgcvDFHtnqauPEVheUY2H2f//7XyRJIiYmhuDgYKfV2uWi6wS4tAsu/gk/jYQxO8DV095SWQydomoW4kOgtxupWWpOXk+jQ/1K/KNUxbiVmcu3+50vm744zFZeJ06c4OjRo4SHh1tDHsdGqYQnPodlD8DNM7BlKgxYaG+pLIIkSXqHvbuLik5hAfz+TxKHL6UK5VWJMLS6Hg4Psrc4FcLsaWOnTp24evWqNWRxDqoFwZOfy9tHv5J9YJUAnb8LwM1FSUQDua374Uu37CWSwMLcyszluwOVw+qCclheK1asYOzYsVy/fp1WrVoVqQfepk0biwnnsDTqCQ9Ogr2fwMaJULs9BNS3t1QVwjBNwk2lpHODQAD+unxb1PiqJCzfc4lsdeWwuqAcyuvmzZtcvHiRkSNH6vcpFAokSUKhUKDRaCwqoMPy8FtweS9cOwI/j4KRm0HlvI0d8gwc824uSpqH+FLN3YWM3HzOJqbTsrafHaUTVJRbhmsYnTSv617MnjY+//zz3HfffRw4cIC4uDguXbpk9G+VQeUKT30J7n6yAtvxvr0lqhA6y0ulVOhfOl+XSJlwfnRWV+s6fvRs5vxWF5TD8rpy5QobN24UjUxBnioOXAw/DZenkGHdoHEve0tVLnSRRldV4S9yRINAdp2/yeFLqYx8oIG9RBNUkNQstVOvYSwJsy2vnj17cuLECWvI4py0HAQdn5e3178IGUl2Fae86CwvN1XhI6Hzex2+lCoatjoxy/fEVTqrC8pheQ0YMIBJkyZx8uRJWrduXcRhP3DgQIsJ5zT0+QDiD0LyaVmB/XednFbhROgsLzeXwqYjrev64e6i5FaWmriULBrVrGYv8QTlJDVLzTf7LwOVx9elw2zlNXbsWADefffdIp9VKYe9Ia6ecvmcLx6CuB2wf5EcjXQi9MrLYNro7qKiXag/hy6lcvhSqlBeTsiKAqurVR1fejWvPFYXlGPaqNVqS3xVScWlI6gZPPqRvL19Nlw9bF95zERXDsfNxfiRMJw6CpwLQ6vr1V7Ou4axJMxSXnl5ebi4uHDq1ClryePc3DcMWj0FkgbWjoK7d+wtkckUThuNH4nCZFWhvJyNFXviyKqkVheYqbxcXV2pV69e1bawSkOhgMc+gYAwSIuH/01wmvI5uRpdtNH4kWhf3x8XpYLrd+5y7Xa2PUQTlIPbRr6uymd1QTmmjW+99RbTpk0jNVX8EheLh19h+ZzTv8DRr+0tkUnklWB5ebm50LKOnKB65LL4mzsLK/bKVlfL2r5EVkKrC8qhvD777DN2795N7dq1CQ8Pp3379kYvc1myZAlhYWF4eHjQuXNnDh8u3Vd0584dXn75ZUJCQnB3d6dp06b89ttvZl/XqtTtAL1myttbpkDSafvKYwLFpUroEH4v5+J2lpqv910GKl+E0RCzo42DBg2y2MXXrFlDVFQU0dHRdO7cmYULF9KnTx/OnTtHUFDRXwu1Wk3v3r0JCgpi7dq11KlThytXruDv728xmSxGl/Fy+ZwL2+Ty0aN3yE09HJSSfF4AEWGBfLE7TlRWdRJ0VleLEF96twi2tzhWw2zlNXPmTItdfMGCBYwePVq/TjI6OppNmzYRExPDlClTioyPiYkhNTWV/fv36/PLwsLCLCaPRVEqYVA0RD8AN8/KFtjAxfaWqkTySrG8OoUFolBA3M0sbmbkUtPH3dbiCczg56PXAZjQq3GltbqgAmWgjx49yvfff8/3339PbGys2cer1WqOHj1KZGRkoTBKJZGRkRw4cKDYYzZu3EiXLl14+eWXCQ4OplWrVnzwwQelBhByc3PtV6q6Wk148gtAAce+gVM/2+7aZlKa5eXn5Up4sA8Afwm/l0OTnpNHYnoOAA8UVMStrJitvJKTk+nZsyedOnViwoQJTJgwgQ4dOtCrVy9u3jS9s3RKSgoajYbgYGOzNjg4mMTExGKPiYuLY+3atWg0Gn777Tfefvtt5s+fz3vvvVfidebMmYOfn5/+FRoaarKMFqHhQ9DtNXn7f69C6iXbXt9EcvOLjzbqiCjwe4mpo2NzMTkTgGBfd3w8nLfKiSmYrbxeeeUVMjIy+Oeff0hNTSU1NZVTp06Rnp7OhAkTrCGjHq1WS1BQEF988QUdOnRg8ODBvPXWW0RHR5d4zNSpU0lLS9O/7FJI8aGpEHo/5KbL5XPy1baXoQx0xQiLs7ygUHkJp71jc/FmFkCVWA1htvLasmULS5cupXnz5vp9LVq0YMmSJWzevNnk89SoUQOVSkVSkvFC5qSkJGrVqlXsMSEhITRt2hSVqnD9XfPmzUlMTEStLl4huLu74+vra/SyOSoXeGqFnEZx/Sj8Odv2MpRBadNGkJ32AGcS00m7m2czuQTmcaHA8mocJJRXEbRabZHF2CAnsGq1pneacXNzo0OHDmzfvt3o3Nu3b6dLly7FHvPAAw9w4cIFo+ucP3+ekJAQ3NzczLgLO+AfCo8vkbf3L4Z/t9lXnntQF/gNi3PYAwT5ehBW3QtJgmNXbttSNIEZXLwpKy9heRVDz549mThxIjdu3NDvu379OpMmTaJXL/NqWUVFRbF8+XK++eYbzpw5w0svvURWVpY++vjcc88xdepU/fiXXnqJ1NRUJk6cyPnz59m0aRMffPABL7/8srm3YR+aD4BOo+Xt9S9CRvG+PXtQ1rQRhN/LGXBY5ZV4Cs6ZPjMzhXIlqaanpxMWFkajRo1o1KgRDRo0ID09nU8//dSscw0ePJh58+YxY8YM2rVrx/Hjx9myZYveiR8fH09CQoJ+fGhoKL///jtHjhyhTZs2TJgwgYkTJxabVuGwPPIeBLeC7BRYNwbMsFatSWFVidKUl2jK4cio87VcuSUv4XKoaaM6S851/OE/cOw7i53W7Dyv0NBQjh07xrZt2zh79iwg+50MUx7MYfz48YwfP77Yz3bu3FlkX5cuXTh48GC5ruUQuHoUlM/pISex7vukMBppR8qKNkJhpv3f19K4q9bg6aYqcazA9sSnZqHRSni7qQj2daBcvM2TIeU8+IRAeD+LndZs5QVy3a7evXvTu3dviwlSpajZFB79GH55Gf58H+o/CPU621WkkkriGFI3wJNavh4kpucQG3+brpU8j8jZuJBcEGkMquY4yakn10Ls94ACnlwO3pZ7ZsqlvLZv38727dtJTk4u4qSPiYmxiGCVnnbPQtxOOPmTnD4xdg942q+5a1nRRpB/tCIaBLLxxA0OX04VysvB0Pm7GjuKvys1Ts5tBOj+BjToZtHTm+3zmjVrFo888gjbt28nJSWF27dvG70EJqJQQP8FENAA0q7CxlfsWj6nuEqqxSHyvRwXXYJqI0fwd+WrYe3zoM6Ael2gx5sWv4TZlld0dDRff/01w4YNs7gwVQ4PX3g6Br58BM78D/76Ejq9YBdRTJk2QqHf61j8bdT52jLHC2xHYaTR286SANtnwY1Y8PCXcxxV5ZrklYrZT55araZr164WF6TKUqc99J4lb2+ZJoeU7YDaROXVOKgaAV6u5ORpOXk9zRaiCUxAkiTHya4//wcc+EzeHrQU/Opa5TJmK68XXniBVatWWUOWqsv946BJH9DkyiFldZbNRTAl2giy36tTQba9KE7oOCSl55KZm49KqaB+dTtaXukJsEFu0kPEi9Csv9UuZbYtl5OTwxdffMG2bdto06ZNkWz7BQsWWEy4KoNCIf9CRT8oh5Q3Ty7MxrcRpZXEuZeIBoH8cTqJw5dSGdujkbVFE5jA7vNyUYT6gV72m8prNbBuNGTfglqtoXfRDmOWxGzl9ffff9OuXTuAIo04HCY864x415BDyd8MkEPLDR6CNv9ns8ubEm3U0bkgWfXI5VQ0WgmVUvzd7cWllCze33SabWeSgcKAil3YswAu7wFXbzmX0dXDqpczW3nt2LHDGnIIQA4l95gMuz6EXyfJ/rDqtrFsTMmw19E8xAdvNxUZOfmcTUynZW0/a4snuIe0u3l89ue/fL3/MnkaCRelgmFd6hPVu6l9BLqyH3Z+IG/3nw81mlj9kiJU5Gh0nwz1usohZhuWzzE12gjgolLSQef3EikTNkWjlVh56Ao95+1k+Z5L5GkkHg6vyZZXuzNzQEv71PDKToWfXwBJC23+A+2G2OSyQnk5GioXeGq5HGK+ESuHnG2AOdNGMGjKIZz2NiM5PYf+i/fw1vpT3MpS06imN1+P7MRXIyPst5ZRkuCX8ZB+HQIbQf95Nru0UF6OiF9d2YEPcsj5/B9Wv6S6hL6NJWGYrCrdk1x7KzNX1PyyAl/uvcTZxAz8PF15Z0ALtrzanYfC7dzW7PByOLcJVG5yzqK7j80uLZSXo9KsvxxqBjn0nJ5Q+vgKYq7l1aauH24uSlIy1cSlFKZ25ORp6LVgF499ugeN1jka7joLZxIzAJjSrxkjHmhg8g+N1Uj4G/54S97u/S7UbmfTy5t997t37yY/P7/I/vz8fHbv3m0RoQQF9H5XDjln35JD0FrrdSovrW9jcbi7qGgX6g8YLxVKzVJzJzuPq6l3RRKrhTlfoLyaBtvOuimR3Ex5+Y9GDU37QuexNhfBbOX18MMPF9stOy0tjYcfftgiQgkK0JXPcfWWQ9B7rJdDZ67lBYV+L0OnvS7ZFWDvv6Y3ZBGUTlp2YVegpsEOsHZx82S49S/41IbHl8q5ijbGbOUlSVKx+Vy3bt3C29sB1lRVNmo0kUPPIIeir+y3+CU0WgndDM9UywuKr6yam19oHe75N8UyAgo4nyxbXXX8Pe3fFejEGji+EhRKObjkXd0uYpic5/Xkk08CciLqiBEjcHcvLHam0Wj4+++/xZpHa9FuiFw+5+/Vckh67F7wslwyotrAWjLH8mpfLwCVUsH1O3e5djubugFe5OYVnutY/G2y1fl4uVl+UW5V43ySbspoZ6vr1kXYFCVv93gTwh60mygmP6m6voeSJOHj42PUC7FWrVqMGTOG77//3pqyVm36z5ND0enX5dC0BcvnGCovc5zA3u4utKotd2PSrXM0nDbmaSRR795COIS/K1+39jZTLqDZ/Q37yYIZltdXX30FQFhYGK+//rqYItoad5+C8jm95dD04eXQeYxFTq1z1gO4llHP614iGgRy4loahy/d5on76hpNGwH2/pvCw/YO51cCziU5gPLa9g4knADPQLkTvNK+ZcDN9nnNnDlTKC57Ubsd9C7o+fjHW3Ko2gIYlsMxd33qvU05DKeNICsvQcWQJIlzBZZXeC07Ka9zm+FgQe7hoGXgV8c+chhgtvJKSkpi2LBh1K5dGxcXF1QqldFLYGU6vwhN+8kh6rUj5ZB1BTFnXeO9dKwvl66+eDOLlMxc/bSxaXA1FArZYkguiJIJykdKpprb2XkoFHbqCpR2HTaMk7fvHwfhfW0vQzGY7UkdMWIE8fHxvP3224SEhIhKErZGoZDL5UQ/CLcuyCFrXTZ+OTFnXeO9BHi7ER7sw7mkDI5cStVPG4N9PXB3UXHyehr7LqbwxH3WKUhXFfi3YMpYP9ALD1cbGwi6Mjd3UyGkLUS+Y9vrl4LZymvv3r3s2bNHXxZHYAe8q8sh6m8GyCHrBj2g7eByn64ilhfIfq9zSRkcvpyq98m4u6h4oLEfJ6+nMWnNCQK83Oy/lMVJsau/a/fHcGUfuFWTcw5dHKelmtlPa2hoaJG1bAI7EPZgYVODTVFyCLuc6KuoupTPijZc55ibJ1te7q5KujUp7C404qsj5ZavqqNLk7C5v+vyPrk8E8jNYmxUnslUzFZeCxcuZMqUKVy+fNkK4gjMovsbcshanSn7v/Jzy3Uac6qoFodOeZ1OSCclUy7h4+6ipEN9+7Vyq0ycs0eahGGZm7ZDK2TZWwuzn9bBgwezc+dOGjVqhI+PD4GBgUYvgQ1RquSQtWegHMLe9k65TlO4NKh8/pRgXw/qV/dCkmDfRTm66O6isr1/phIiSRL/JslBGZspL0mSHfQZN6B6Y7lBsgNits9r4cKFVhBDUG786sih6x8Gy6HsBt3Nbqluas/G0ogIC+TKrWxOXL0DyJYXwPz/a8trP53gvnr+5T53VSYhLYeM3HxclAoa1LBRitKhz+H85oIyN1+BuwOspSwGs5XX8OHDrSGHoCKE95VD2AeXyr+YY/ealYdTkWijjogGgfx09Jp+jaS7q3yuAG95HV6+RvhJy4POWd+wprdtGmvcOA5b35a3H3kfQtpY/5rlpFz/GxcvXmT69OkMGTKE5GS58P/mzZv5559/LCqcwAwi35FD2XdTzS6fY2rPxtK4t/GDe8EU1EUpnzNf1PYqF7plQU1sMWXMzSgscxPeHyJGW/+aFcDsp3XXrl20bt2aQ4cOsW7dOjIz5fn4iRMnmDlzpsUFFJiIi7ts4rtVk0Pbu033U5jas7E06gV6EexbGEbXTRtdCqai+RptsccJSkdneYXbQnlteh1SL4JvXXj8M7uUuTEHs5/WKVOm8N5777F161bc3Nz0+3v27MnBgwctKpzATKo3gsc+kbd3fQiX95p0WEWjjSBXG9EtFQL0znqdQhSWV/mwmbP++A9y1RKFEp5aYdGqJdbC7Kf15MmTPPHEE0X2BwUFkZIi1rHZnTbPyKFtSQs/j5ZD3mVQnkKExWE4ddRbXgU9HfOE5WU2Gq3Ev8k2yPFK+Rc2vSZvPzQN6nex3rUsiNlPq7+/PwkJReupx8bGUqeO/RdrCpBD29Uby6HuDePKLJ9T0Qx7HRFhRZWX3vISDnuzuZqaTU6eFncXJfUCvaxzkbwcOUcwLwvCukG3KOtcxwqY/bT+5z//4c033yQxMRGFQoFWq2Xfvn28/vrrPPfcc9aQUWAu7gVLOVRucsj70OelDrdEtBGgSVA1/L3k6KJ7wbRR7/PSCsvLXHT+rsZB1azXlXzrDEg8CV7V5Y7tdi5zYw5mP60ffPABzZo1IzQ0lMzMTFq0aEH37t3p2rUr06dPL5cQS5YsISwsDA8PDzp37szhw4dNOm716tUoFAoGDRpUrutWakLayKFukEPfN46XONRS00alUsGANrVxc1HSrGCaUzhtFJaXufxrbWf92d/gcMEP26Bl4BtinetYCbOfVjc3N5YvX87Fixf59ddf+f777zl79izfffdduUrirFmzhqioKGbOnMmxY8do27Ytffr00adglMTly5d5/fXX6datm9nXrDJEjJZD3hq1HALPzSh2WK6ZPRtL493HW3J8Rm+9g1mfKiF8XmZzLP4OYKU0ibRr8EtBmZsu46FpH8tfw8qU+2mtV68ejz76KM888wxNmjQptwALFixg9OjRjBw5khYtWhAdHY2XlxcxMTElHqPRaHj22WeZNWsWDRs2LPe1Kz0KhRzy9q0rh8A3vV7ssLx82SqyRBKkQqEwqllfOG0Ulpc5nElI58+z8g94z2YWrsahyZeDOXdvQ0g76OWcKU4mZdhHRUUxe/ZsvL29iYoq3aG3YIHp7bnUajVHjx5l6tSp+n1KpZLIyEgOHDhQ4nHvvvsuQUFBjBo1ij179pR6jdzcXHJzCxcsp6enmyxfpcArUA59f/2oHApv+JDc0MMAtUZOaK2ow744RKpE+Vi47TwA/duEWD7SuPsjiN8PbgWlxV3cyj7GATFJecXGxpKXl6ffLglzCxOmpKSg0WgIDg422h8cHMzZs2eLPWbv3r18+eWXHD9+3KRrzJkzh1mzZpklV6Wjfhc5BL7jPTkkXrej3FKtAEv5vIpD5/PSaKUS2+YJjDl1PY3f/0lCoYBXe5V/VlMsl3bDro/k7QELHa7MjTmYpLx27NhR7LatycjIYNiwYSxfvpwaNWqUfQAwdepUI2sxPT2d0NBQa4nouHSLgku75Oa1a0fCqG1yU1sKnenWsLxcDM6Zp5FwK2fNsKqEzuoa2La2Zf1dWSmwbgwgwX3/hdZPW+7cdsCuDfVq1KiBSqUiKSnJaH9SUhK1atUqMv7ixYtcvnyZAQMG6PdpC0LwLi4unDt3jkaNjH9J3N3djXpMVlmUKjkUHv2AHBrfOgMelX+BbWF5gZwu4VZ+N2uV4MTVO2w7k4xSARMsaXVJEmx4CTISoEZT6PeR5c5tJ0xSXrqGs6awbt06k8e6ubnRoUMHtm/frk930Gq1bN++nfHjxxcZ36xZM06ePGm0b/r06WRkZLBo0aKqaVGZg28IDIqGVf8nh8gb9oBm/S2ytrEkXAzK7Ih0ibLRWV2D7qtDo5oWLEVzcCn8+weodGtgnb8DmEnKy8/PT78tSRLr16/Hz8+Pjh07AnD06FHu3LljlpLTERUVxfDhw+nYsSMREREsXLiQrKwsRo4cCcBzzz1HnTp1mDNnDh4eHrRq1croeH9/f4Ai+wUl0PQROTR+4DP45WUIaWuxJNXicFUWnlOkS5TOsfjb7Dh3E5VSwYSeFrS6rh+DrQURxb4fQK3K8V0xSXnpGs4CvPnmmzzzzDNER0fr87o0Gg3jxo3D19fXbAEGDx7MzZs3mTFjBomJibRr144tW7bonfjx8fEolWKqYVF6zZQXbScch59Hk58nJxdbQ3kplQqUCtBKstPeEK1W4nxyBk2DfFBaK4PcCTh9I52NJ25wME7ufflU+zqEWarwYE66nOOnzYPmA6DjKMuc1wFQSGZ206hZsyZ79+4lPDzcaP+5c+fo2rUrt27dsqiAliY9PR0/Pz/S0tLKpWwrDbcuwuc9QJ3Bj95DmXzrMaL/256+rSyfZd10+mbU+Vr2TelJHX9P/f4FW8+zePu/jO3RiCn9mln8us5C2JRN+m0XpYIdrz9EqCXWMkqSXNvt5E/gFwpj94Cn4/cVMPU7avZPbX5+frFpDGfPntU7zwVOQPVGcqgceDrrB7oo/7FapU4vN9lCT7+bZ7R/8fZ/AYjeVf7OR87OvbbD/3UMtYziAji+SlZcCpWc6+cEissczI42jhw5klGjRnHx4kUiIiIAOHToEHPnztX7qQROQuunIW4Hytjv+cR1KZfzHgeCyzzMXBrU8CY2/g7fHbyCVivxVv/m+Hi46j+32qJjJyAhzbib+PiejS1z4pvn4beCFRUPT4N691vmvA6E2cpr3rx51KpVi/nz5+tL44SEhPDGG2/w2muvWVxAgZXp9xFXju+kPtdwPzgZWm2weAXNpkE+xMbfYdWheED2rb37eKHTOMin6qaynLqept9uW9fPaFpdbvRlbrLlhsQPTqr4OR0Qs+cJSqWSyZMnc/36de7cucOdO3e4fv06kydPLtfCbIGdcfNmhttr5EquBFzfKYfULUyTYOOQ/5HLt43eV2nldaNwudqq0Rayjv6YDkmnwKuG3BrPicrcmEOFnBy+vr5V2+ldSTijrcfs/P/Kb7bOlEPrFuTeEsZnEtLJzS9sEFLTx8Oi13N0JEnixNU73FVr+KfA8npnQAu83S2QM37mf3Bkubz9xOfgUzTZu7JQrv+ttWvX8uOPPxIfH49arTb67Ngxyz74Auuj1mj5XhPJlKaJVLu0WQ6tv7gbPCzzw1Rc/fWktMLF8r4edl3oYXM2HL/OpDUn6N60pr47UKs6fmUcZQJ34uXcPYCuE6BJZMXP6cCYbXktXryYkSNHEhwcTGxsLBEREVSvXp24uDj69TOv2anAMcjL1wIKbkfOl0Pqty/Br5PKLB9tKsG+7ni4Gj9qsVcLp45aC13HWfh8VxwAu8/fJDE9B4UCmodU8IdCkw8/vwA5aVCnA/R82wKSOjZmK6+lS5fyxRdf8Omnn+Lm5sbkyZPZunUrEyZMIC0trewTCBwOXd9Gl2qB8NSXcmj91Fo4vtIi51coFEWWHm05lajfrmqrhlKzjGcrDWt4V3zKuHMOXD0E7r7y39BJy9yYg9nKKz4+nq5duwLg6elJRoZs9g4bNowffvjBstIJrI5WK+nXHLqqlFCvsxxaB/jtDTnkbgHuTYfYee5moQxVyPLSaCWSMwqnzAoFvPxwBdMj4nbCnvny9oCFENigYudzEsxWXrVq1SI1VW6nVa9ePX2vxkuXLhVJuBM4PnkGicX6JNUHJ8kh9rzsgpB7TglHm47ynvSLu3mFDnttFSpU+PPRa0bvvxzekSfb1y3/CTNvFpa5aT8cWj1VMQGdCLOVV8+ePdm4cSMgJ6xOmjSJ3r17M3jw4GL7OQocG105HDCo56VUySF2rxpyyP2P8jVWMeRe5WVIVbK8/vf3DQB6NQti/5Se9GxWgaRgrVYuc5OZBDWbQd+5FpLSOTB7ov3FF1/olwG9/PLLVK9enf379zNw4EBefPFFiwsosC7FKi+QQ+xPfA4rn5JD7w17yAt7y0lp1XaqSrGJtOw8DlyU1/6+1b85tSuakHpwCVzYCi4ecjlnNyv1dnRQzLK88vPzee+990hMLHS2/uc//2Hx4sW88soruLlVfidhZUPn73JRKopWdmgSKYfcQQ7B34kv93WE5QV/nksiXyvRJKgaDStaq+v6Udj2jrzddw4Et6ywfM6GWcrLxcWFjz76iPz8fGvJI7AxZVZR7fm2HHrPSZND8Zry/e0NlVerOsZpAVVFef1+Sq4Y3LdVBRNHc9IKytzkQ4vHoUPVXFNsts+rV69e7Nq1yxqyCOyArnNQiVVUXdzk0Lu7rxyK3zmnXNcxLMnWvp5xdYN763ydvJbGn2eNS4M7O5m5+ew6L0dY+7SsgPKSJPjfq3D7MvjVgwGLLb4W1Vkw2+fVr18/pkyZwsmTJ+nQoQPe3sZF0wYOHGgx4QTWR21Kz8bABnIIfu3zcki+QTe5hZoZqAy+YBENAtl8KpGbBSkD9xpeAz7bC8C2qB40DrJgKWQ7svpwPHfzNDSs6U3L2hVISI39Dv5ZJ+fiPR0Dnv4Wk9HZMFt5jRsnd9ktrj+jQqFAo9EU2S9wXHQJqmV2Dmr1FMTtgmPfyKH5sfugWk2Tr2PoT/P1cGXvmw+z+3wKo7/9i1v3JG3quJySVSmUlzpfy4o9lwAY061h+du/JZ+F3ybL273ehtBOFpLQOTF72qjVakt8CcXlfJjVOajvXDkkn5kEG8bKoXoTMfR5KRTg7qKiWUEz1TMJ6Ty5dB+x8cbVJjSVxBf2y/HrJKbnEOTjzhPt65TvJHl35Zy7/LvQ8GHoOtGyQjohojh8FSfPVMsL5FD801/JofkL2+QmHiZiOG1UIG/X8ffUV1k9Fn+Hp6ONu6RXhqRnrVbi893yWsbnH2yAu0s5y9P8Pg2ST4N3UEGZG/HVNXnaePfuXbZv385jjz0GyM1cc3MLlzmoVCpmz56Nh0fVKm/i7JjdszG4hRya/3USbJ8FYQ/I0cgyMJw26jaVSgVNg304fvUOUNRx74z5X5IkMeXnkwT7uhP1SDg7ziVzITkTHw8Xnu1cr3wn/WcD/BUjbz/5OVQLspi8zozJ6vubb77h888/17//7LPP2L9/P7GxscTGxvL999+zbNkyqwgpsB6FPRvN8MN0GCmH6LX5shM/p+wF+UYpZAbbuqmjjm2nC6OMzjhtPBZ/hzV/XWXxnxcA2HBczqh/pmOoUelrk7l9BTYW5No9OAka9bSUqE6Pycpr5cqVjBkzxmjfqlWr2LFjBzt27ODjjz/mxx9/tLiAAutSrp6NCoUcoverJ4fs//dqmeVzVEaWV+H2vbW+Xvj2L/22M04br9+5q9/OydPw5xlZGT/WphxdmTR5cm5dbhrU7QQPv2UpMSsFJj+xFy5coHXr1vr3Hh4eRv0UIyIiOH36tGWlE1idwmmjmb4YT385VK9QyaH72O9KHW7ksDfYf6/lZcjE1ceLLGR2dFIMKkYs2XGBLLWGOv6etK3rb/7JdnwA1w6Du5/c/UdVDsutEmOy8rpz546Rj+vmzZuEhYXp32u1WqPPBc5BYapEOcL3oZ3kkD3IIfzkoi3xdBhOGw1TBcJLUV4Ar/10Qm8dOgM3Mwu/A0t3yi3d3uzXzPymuhf/hL2fyNsDF0FAmIUkrDyYrLzq1q3LqVOnSvz877//pm7dCpT2ENiFck0bDek6UQ7d5xeE8vPuFjtMVYzDHqB6tbKbb8z741z5ZLMDhpaXRivRq1kQA8ydMmYmw7oXAUn2L7YU1VqKw+Qn9tFHH2XGjBnk5BSt7XT37l1mzZpF//79LSqcwProp42mpEoUh1Iph+69g+RQ/u/Tih92T56XIQsHtyv1Et/sv1w+2exAioHlVc3dhfeeaGVeUqpWC+tfhKxkCCqI7AqKxeQndtq0aaSmphIeHs7HH3/ML7/8wi+//MJHH31EeHg4t2/fZtq04h9cgeNSGG2sQN5QtSA5hA9ySP+fDUWGGCsv4y9zaGDppWE8XB2/dZdGK3H4Uirxqdn6fVP6NSPEz8yyN/sXy1NGF0/Zp+hqgT6OlRST87yCg4PZv38/L730ElOmTNFHghQKBb1792bp0qUEB1u+27LAulR42qijUU85lL/3Ezm0X/s+CKiv/9hw2nivHdKydumdczzKm9hpQ77cG8cHvxX6/Hq3CGZohJl5XVePwJ+z5e1+H0JQcwtKWPkw64lt0KABW7Zs4ebNmxw8eJCDBw9y8+ZNtmzZQsOGDa0lo8CKmJ2kWhoPvyWH9HPT4OdRcqi/AEUJDnuQLavp/Uv+onq6Ob7y0nUD1/HhU23Mc9LfvQM/F5S5afkktH/OsgJWQsr1xAYGBhIREUFERASBgYGWlklgQyrs8zJE5VpQPscPrh2BHe8XflSCw17HC91K/vFzt4RitTL+XoWFON1clAR4mZHWIEnwv4lysUf/+nIFjypa5sYcHP+pEFgVi00bdQTUl0P7AHsXyv4bil/baCrOYHkFehcqrxA/D/Oc9Ee/htMbQOkirx31sEAD2iqAUF5VHJNL4phDyycKqntKcsg/M9noy2yuUaHzed1Va8jJc8zKJf4Gllawrxnre5NOw5Yp8navmVC37HWiAhmhvKo4+mijpadmfefIof6sZFj/Ii6KwkRTs5WXq5LcfA3NZ2yh/eytDrlsKNBg2tiqjACEHnW2vDY0Pwca9YIu460kXeXEIZTXkiVLCAsLw8PDg86dO3P48OESxy5fvpxu3boREBBAQEAAkZGRpY4XlI6uAYdFLS+QQ/xPx8gh/4t/0iftJ/1H5k4bPVxVXL8tJ79mqzV6mR0Jwy7Yo7qZ2PT196lw8wxUC5Y7NYkyN2Zh9/+tNWvWEBUVxcyZMzl27Bht27alT58+JCcnFzt+586dDBkyhB07dnDgwAFCQ0N55JFHuH79uo0lrxyo8+VpmMV8XoYENZdD/sDAW19yn+JfwPzvqLuL0ihPzNEadmw/k8S6WPn5G9ujEXVMaWl2ap3s60IhKy4zqtIKZOyuvBYsWMDo0aMZOXIkLVq0IDo6Gi8vL2JiYoodv3LlSsaNG0e7du1o1qwZK1asQKvVsn37dhtLXjmwaLSxONo/By2fRIWGxa6f4UtWmZbXW48WTZswnGo6kvLKys1n1DeFlTB6tzAh1/H2ZTm6CNAtCho9bB3hKjl2VV5qtZqjR48SGRmp36dUKomMjOTAgQOlHFlIdnY2eXl5ImWjnOinjdZKR1AoYMBCbrmGEKq8yQeuK1BQuvIJ8jVe76iRjDP07y1aaE9WHrqi3170n3Z0qB9Qymjk3Le1z0NuOoR2hoemWlnCyotdlVdKSgoajaZIZn5wcLBRY9vSePPNN6ldu7aRAjQkNzeX9PR0o5egEIsmqZaEhx/f1Z1JnqTiMdUh/M+sLPOQmBEd9dsardbY8nKgIhO//yPX65o9qBWPtzOhPv2fs+WGsR6izE1Fsfu0sSLMnTuX1atXs379+hLLT8+ZMwc/Pz/9KzQ01MZSOja5GgusbTSB614t+Ch/MAA19r4jpwiUgEKhoGezYBb9px0Av51MNMpgd5QKq9fv3OXoFblpSGRzE0ozX9gG+wpy4AZ+Bv7lLAstAOysvGrUqIFKpSIpybjBaFJSErVqld6Yc968ecydO5c//viDNm3alDhu6tSppKWl6V9Xr161iOyVhTxbWF7IGfYrNI+yU9MWpSZHLp+jzi71mHah/vptXW0scJxp4wNz/9Rvl7kAOyOxoMwN0OkFaCH6m1YUuyovNzc3OnToYORs1znfu3TpUuJxH330EbNnz2bLli107NixxHEA7u7u+Pr6Gr0EhVglSbUYFAoFEkpeyxtLvlcQ3DxbmJx579iCf+sFehX7uT0d9olpOWw5lcCSHRdMP0irlXtdZqdAcCt45P2yjxGUidlNZy1NVFQUw4cPp2PHjkRERLBw4UKysrIYOXIkAM899xx16tRhzhy5rtGHH37IjBkzWLVqFWFhYXrfWLVq1ahWzfkblNqaQp+XddfS6XTjLfxI6b2YWr8MkRvYNnwIWj1pNFbn31IoFDxxXx3WxxqnwdjT8uq7aDd3svOM9q0dW/IPLQD7PoFLu8DVq6DMjeiwZQns7vMaPHgw8+bNY8aMGbRr147jx4+zZcsWvRM/Pj6ehIQE/fhly5ahVqt5+umnCQkJ0b/mzZtnr1twagr7Nlp3/aBhtFBdr4ecIgByysDty0ZjDVMpFjzTtsi5vj1wpcg+W3Gv4moaXI2OYaVEuq8ehj8LLK1+H0HNcCtKV7Wwu+UFMH78eMaPL35pxM6dO43eX7582foCVSFsEm2kmEqqD02Fy3vh6iE5deD534s9TqFQMO6hRkY+r+hdF5nSr5lV5TWVmBGdSv7w7m1YOwokDbR6Gu77r+0EqwLY3fIS2Bd1efo2lgPVvXVwVK5yqoCHn5w6oCvCR9G1j890dNwIcd2A4v1ySJJclDEtXm6e8dgnosyNhRHKq4qjtnRJnBIw1F36In3+9eSUAYB9i+iuPAEUrbQaVsO72HPuv5jC7/+Ylg9oDdrULWUB9l8xcGYjKF0LytyIQJGlEcqrCiNJku2UV0lloFsMlFMHgAWuy6jJbZMMFEmSGLr8EC9+d5SEtOI7FlkCdb6WzScTuJ2lLhIo+PHFEhz1iadgS0HmfOQ7UKe91eSrygjlVYXJ10r6RtfWTpVQlVbP65H3IbgVNRTpfOK6FKSiKfSrx9xv9L7je9v028np1usX+sm287y08hj//fIQ288U5iP+Mal78Y1B1FmyD0+TC00egfvHWU22qo5QXlUYw2autnTYK+/VXq4e8HQM2ZI7D6r+oeG5FUWOv79hdfZMLlzAfMugBE2+FVMn1h2TO3b/cyOdTScLo95Ng0tolrv5TUg5B9VqwaBlosyNFRH/s1UYnbMerG95lTht1FEznJn5wwFo/M8iiD9UZEjtEkrNJKfnsCH2ulWqrCYZWHW/HL8BwH86lRBAOLkWYr8DFPDUcvCuYXF5BIUI5VWF0SkvhaKYaKCFUZXSt1HHT5oebNB0RSlp5O5Dd28bn0Op4IthRcskv7TyGK+uOc6CrectKrNhA1lDXunVpOjO1Dj436vydvc3oEF3i8oiKIpQXlUYw6VBZjWMKAeGurHkSymYnvc82d71IO0qbHwF7lkK9EjLkte8WjrymHCnaHd4oGixwXy17OdSZ0C9LtDjTYvKISgeobyqMLZKUAUTpo0FZOLFyS6fyCkGZ/4npxzcw5fDi1/PqrKwAs7MzS+yb8ur3YoO3D4LbsSCh39BmRuHyP2u9AjlVYWx1aJsMLa2ijjs7yEjsLWcYgByykHiKaPPezUvvlqppae+WcUor2a17snX+ncrHCjIVRu0FPzqWlQGQckI5VWFycu3chVVAwzXK5pkIN0/Tk410OQWTMmyyjzEksorJ0/DkcupRvtWPHePxZeeAOsLytxEjIFm/S12fUHZCOVVhVFrrNh84x4MFVZZNewVCuQUg0HL5JSDlHNyCoIBsW/3LnKcJZXXsC8P8fnuOKN9kYb16bUaWD8Gsm9BcGvoPRuBbRHKqwqj79loi2mj4XYZl9MrOu8acsoBCjkF4eRa/ZgAgw7VOlwsqLyOXDaOdBapTb93AVzaDa7e8H9fiTI3dkAoryqM1Xo2FoNRVQlzDmzQXU49ADkVIbXQGjowtafRUGtGTH8YbZDhf+UA7JDry9F/HtQoJnVCYHWE8qrC2DLaaI7Dvsi0ssebcgqCOqOgw7ScXX9v6eXjV+8UGyGsKEufbV/4f5SdCj+/IJe5aTMY2g6x+PUEpiGUVxXG6j0bS6BMA+nez1UuBeVz/OWUhO2z9B+df6+f0dBZG/+psHyGKw8AHm0dIm9Ikpx7ln4NAhtC//mizI0dEcqrCpNno4oScO+0sSzLqxj86sqpCCCnJpz/Aygq+09Hr7H7/E2kcta5//GvqzSdvrn4D4+sgLO/Fpa5cS9hfaPAJgjlVYWx17Sx3MZKs/4QUZCasGGsnKoAbJ1kvBTnuZjDbDxxo1yXmLz2b6P3J2Y8Im8k/A2/T5O3H5kNtduV6/wCyyGUVxWmsGej9ac+RtHGMi5XquO997tyakL2LTlVQauhSTEVHiauPl4uOQ0Zdn99/LxcITezoMyNGpr2hc5jK3xuQcURyqsKU9iz0brNN8BYIZXtsC8FV4+C1ARvOVVh7wKglMKAFWB8z8byxubJcOtf8KkNjy8Vfi4HQSivKoy9lgdV+Ktfo4mcogByysKVA0Q0KKWDjwlIklSkpE6wrwf8/SMcXyknpz21HLyrV+g6AsshlFcVxlY9G8HY8iorH8skw6btEDlVQdLIqQvZqUUUmDlO+wc/3EGzt7fo38eM6Ai3LsKvk+Qd3SdD2IMmn09gfYTyqsLk2dLyMtguKxG+rGikPEghpyoENpRTFza+wqpREUZDBn62z2T5rt8prIM/JKIePRv7w9qRoM6E+g8UJsoKHAahvKow9os2WsDyAjlV4emv5NSFs7/iciyGyX0Lm7qevJ5m0mmWGfSEBHj9kaawbRYknADPAHhyuShz44AI5VWFseXaxrKc9OWmdjs5dQHg92mMC882+xQfbjlr9L769R1wcIn8ZtAy8KtTQSEF1kAoryqMLZNUzVFdZqu5zmPlFAaNXNF0QLPCmlsrD13hxNU7JR56KcW41M6vwxvChpcKzvsShPcr5iiBIyCUVxXGXtPGsgeX4+SPL5VTGW79y1zP7/QfvbX+FI8v2Vei837i6lj9thItrQ6+DndToVYb6D2r2GMEjoFQXlUYm6ZKVDxBonS8q8upDAol3mfWMEi51+jjnLyivSAB/r5W6Bfb2OYAXNkLbtXg/74GF3drSiyoIEJ5VWFsOW00R3eVW9GFPahvfvGeawxhisI+i2V11Y5QnKHVv8vkN/0XQPVG5ZNBYDOE8qrC2LKqhDkO+wr59ru/gbpuF6opcvjU9VPcyAOg5/xdRYYeL/CF+ZNBjM/ncqfutkOh7eAKCCCwFUJ5VWHsVUnVkmOLoFSR2X8Zt6VqtFZeZrLL6mKHSZLEoCX7AImPXT+nmjoZqjeGRz+uyNUFNkQoryqMTaONtlwO6FuH1/Pk6hMvuGymp/IYACcN/FuPfLIbgOGqP+itOgYqt4IyN9VsKKigIgjlVYWxad9Gs6aNFdN0SgVs13YgJr8vAPNcowkm1Siy+G9yJi0Vl5nmslLe8ch7ENKmQtcV2BahvKow9lqYbcmxxR8vn2Bu/hByarQiUJHJIrclXE7JQKuVOBh3C2/u8qnrYtwV+RD+qNy6TOBUOITyWrJkCWFhYXh4eNC5c2cOHz5c6viffvqJZs2a4eHhQevWrfntt99sJGnlwpZ9G82hojNMXQs0Na4k91lGjsKT+5VnGK/aQOSCXfzni4O86/o1DZWJSL514PElosyNE2L3BVtr1qwhKiqK6OhoOnfuzMKFC+nTpw/nzp0jKCioyPj9+/czZMgQ5syZw2OPPcaqVasYNGgQx44do1WrVna4g9LJzdfwS+wNavt78kDj6kZTonyNltRsNTlqLbn5GnLy5H+1kvxdUqD7TilQKmSLQrdPgUL/fVMo5GmZ4X5TjtVll9vG52U75WC48FsT0Iifak1iWMIHTHT5mYO3mtNOeZOnVHvQokT51Arwqlg5HYF9sLvyWrBgAaNHj2bkyJEAREdHs2nTJmJiYpgyZUqR8YsWLaJv37688Ya8yn/27Nls3bqVzz77jOjoaIvJlZSewzf7L6NSKlAoFKgUshJQKhWolAXbCgVKRcH7gn0qhW5bQZ5Gy/I9ccTdlJVEqzq+NKvly7Xb2Vy7fZeEtBw02vLVWrckDhdtrOi00bA7N3DIpzee1/bytGo3P7oXNoeVekyB+l0rdjGB3bCr8lKr1Rw9epSpU6fq9ymVSiIjIzlw4ECxxxw4cICoqCijfX369GHDhg3Fjs/NzSU3N1f/Pj093STZktNzWXpPtYHyUt3bjWy1hlPX0zl13fj6CgV4uKjwcFXi4arC3UWJUqFAQg7ny/+ChIS2IEn83v3yvwX7Dbd1Y0raj0SjmtUIL6aMsqUxb2G25aw03XVn5I3gPsW/NFLKiasJAR0J6fG6xa4jsD12VV4pKSloNBqCg4ON9gcHB3P27Nlij0lMTCx2fGJiYrHj58yZw6xZ5q9RC6zmxsgHwpAk0GglNJKEJElotBJaCbQF+/TbWgmtpHuhf9+iti8vP9yYfI3Ez0evkZuvoW6AF3UDPKkb4EWQjztKC3Z6dlS6N61BgJcrDWp4lzjGzUWJOl9L0+CKpSu4G0yDg/3cef7BBvz6dwKv5L3COreZZOKJ/3+/BqX1y18LrIfdp43WZurUqUaWWnp6OqGhoWUeV8ffk5kDWlpUltHdG1r0fM6Ej4crh6ZFoi2luunfMx8hN1+Lj4drha6lVCr4Z1YftJKEu4uK9vUC2DP5YWr6uLNmWxsGtA/Ds3rZz4DAsbGr8qpRowYqlYqkpCSj/UlJSdSqVavYY2rVqmXWeHd3d9zdxQJbR6CswICHqwoPV8tYQ97uxo92aKAXAMP7dbPI+QX2x64xcjc3Nzp06MD27dv1+7RaLdu3b6dLl+K7wXTp0sVoPMDWrVtLHC8QCCondp82RkVFMXz4cDp27EhERAQLFy4kKytLH3187rnnqFOnDnPmzAFg4sSJ9OjRg/nz59O/f39Wr17NX3/9xRdffGHP2xAIBDbG7spr8ODB3Lx5kxkzZpCYmEi7du3YsmWL3ikfHx+PUlloIHbt2pVVq1Yxffp0pk2bRpMmTdiwYYND5ngJBALroZDM6Q9VCUhPT8fPz4+0tDR8fX3LPkAgENgUU7+jjrUuRCAQCExEKC+BQOCUCOUlEAicErs77G2NzsVn6jIhgUBgW3TfzbLc8VVOeWVkZACYlGUvEAjsR0ZGBn5+fiV+XuWijVqtlhs3buDj42PRMi26ZUdXr16tNFFMcU/OQWW7J0mSyMjIoHbt2kZpUvdS5SwvpVJJ3bp1rXZ+X1/fSvEAGSLuyTmoTPdUmsWlQzjsBQKBUyKUl0AgcEqE8rIQ7u7uzJw5s1JVsBD35BxUxnsyhSrnsBcIBJUDYXkJBAKnRCgvgUDglAjlJRAInBKhvAQCgVMilFcFSE1N5dlnn8XX1xd/f39GjRpFZmamScdKkkS/fv1QKBQltm2zB+beU2pqKq+88grh4eF4enpSr149JkyYQFpamg2lNqYydmA3556WL19Ot27dCAgIICAggMjIyDL/D5wSSVBu+vbtK7Vt21Y6ePCgtGfPHqlx48bSkCFDTDp2wYIFUr9+/SRAWr9+vXUFNQNz7+nkyZPSk08+KW3cuFG6cOGCtH37dqlJkybSU089ZUOpC1m9erXk5uYmxcTESP/88480evRoyd/fX0pKSip2/L59+ySVSiV99NFH0unTp6Xp06dLrq6u0smTJ20secmYe09Dhw6VlixZIsXGxkpnzpyRRowYIfn5+UnXrl2zseTWRSivcnL69GkJkI4cOaLft3nzZkmhUEjXr18v9djY2FipTp06UkJCgkMpr4rckyE//vij5ObmJuXl5VlDzFKJiIiQXn75Zf17jUYj1a5dW5ozZ06x45955hmpf//+Rvs6d+4svfjii1aV0xzMvad7yc/Pl3x8fKRvvvnGWiLaBTFtLCcHDhzA39+fjh076vdFRkaiVCo5dOhQicdlZ2czdOhQlixZUmK7NntR3nu6F135XhcX2y6d1XVgj4yM1O8zpQO74XiQO7CXNN7WlOee7iU7O5u8vDwCAwOtJaZdEMqrnCQmJhIUFGS0z8XFhcDAwBK7dwNMmjSJrl278vjjj1tbRLMp7z0ZkpKSwuzZsxkzZow1RCzz2iV1YC9JfnM7sNua8tzTvbz55pvUrl27iJJ2doTyuocpU6agUChKfZ09e7Zc5964cSN//vknCxcutKzQZWDNezIkPT2d/v3706JFC955552KCy6oMHPnzmX16tWsX78eDw8Pe4tjUapcSZyyeO211xgxYkSpYxo2bEitWrVITk422p+fn09qamqJ08E///yTixcv4u/vb7T/qaeeolu3buzcubMCkpeMNe9JR0ZGBn379sXHx4f169fj6upaUbHNxhYd2G1Nee5Jx7x585g7dy7btm2jTZs21hTTPtjb6eas6Jzbf/31l37f77//XqpzOyEhQTp58qTRC5AWLVokxcXF2Ur0EinPPUmSJKWlpUn333+/1KNHDykrK8sWopZIRESENH78eP17jUYj1alTp1SH/WOPPWa0r0uXLg7nsDfnniRJkj788EPJ19dXOnDggC1EtAtCeVWAvn37Svfdd5906NAhae/evVKTJk2M0gquXbsmhYeHS4cOHSrxHDhQtFGSzL+ntLQ0qXPnzlLr1q2lCxcuSAkJCfpXfn6+zeVfvXq15O7uLn399dfS6dOnpTFjxkj+/v5SYmKiJEmSNGzYMGnKlCn68fv27ZNcXFykefPmSWfOnJFmzpzpkKkS5tzT3LlzJTc3N2nt2rVGf4+MjAx73YJVEMqrAty6dUsaMmSIVK1aNcnX11caOXKk0QNy6dIlCZB27NhR4jkcTXmZe087duyQgGJfly5dsss9fPrpp1K9evUkNzc3KSIiQjp48KD+sx49ekjDhw83Gv/jjz9KTZs2ldzc3KSWLVtKmzZtsrHEZWPOPdWvX7/Yv8fMmTNtL7gVESVxBAKBUyKijQKBwCkRyksgEDglQnkJBAKnRCgvgUDglAjlJRAInBKhvAQCgVMilJdAIHBKhPISCAROiVBeAodgxIgRxVa76Nu3r71FEzgooqqEwGHo27cvX331ldG+krpA5+XlFalcoVarcXNzM/u65T1OYF+E5SVwGNzd3alVq5bRKyAgAACFQsGyZcsYOHAg3t7evP/++7zzzju0a9eOFStW0KBBA329qvj4eB5//HGqVauGr68vzzzzjFFJmZKOEzgXQnkJnIZ33nmHJ554gpMnT/L8888DcOHCBX7++WfWrVvH8ePH0Wq1PP7446SmprJr1y62bt1KXFwcgwcPNjrXvccJnA8xbRQ4DL/++ivVqlUz2jdt2jSmTZsGwNChQxk5cqTR52q1mm+//ZaaNWsCsHXrVk6ePMmlS5cIDQ0F4Ntvv6Vly5YcOXKETp06FXucwPkQykvgMDz88MMsW7bMaJ9h0wjDxiA66tevb6SAzpw5Q2hoqF5xAbRo0QJ/f3/OnDmjV173HidwPoTyEjgM3t7eNG7cuNTPTdln6rUEzo3weQkqFc2bN+fq1atcvXpVv+/06dPcuXOHFi1a2FEygaURlpfAYcjNzS3SzsvFxYUaNWqYfI7IyEhat27Ns88+y8KFC8nPz2fcuHH06NGj2GmnwHkRlpfAYdiyZQshISFGrwcffNCscygUCn755RcCAgLo3r07kZGRNGzYkDVr1lhJaoG9EGWgBQKBUyIsL4FA4JQI5SUQCJwSobwEAoFTIpSXQCBwSoTyEggETolQXgKBwCkRyksgEDglQnkJBAKnRCgvgUDglAjlJRAInBKhvAQCgVMilJdAIHBK/h90ym0Q9zdQiQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.016554, + "end_time": "2024-03-26T11:22:47.180755", + "exception": false, + "start_time": "2024-03-26T11:22:47.164201", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 690.399603, + "end_time": "2024-03-26T11:22:50.018645", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/tvae/1/mlu-eval-load.ipynb", + "output_path": "eval/treatment/tvae/1/mlu-eval-load.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/treatment/tvae/1", + "path_prefix": "../../../../", + "random_seed": 1, + "single_model": "tvae" + }, + "start_time": "2024-03-26T11:11:19.619042", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/tvae/mlu-eval.ipynb b/treatment/tvae/mlu-eval.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..547d87eacbf6572591990cd3e955c65c759ce84c --- /dev/null +++ b/treatment/tvae/mlu-eval.ipynb @@ -0,0 +1,2285 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:01.674243Z", + "iopub.status.busy": "2024-03-24T02:07:01.673310Z", + "iopub.status.idle": "2024-03-24T02:07:01.714842Z", + "shell.execute_reply": "2024-03-24T02:07:01.712630Z" + }, + "papermill": { + "duration": 0.058185, + "end_time": "2024-03-24T02:07:01.717384", + "exception": false, + "start_time": "2024-03-24T02:07:01.659199", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:01.745524Z", + "iopub.status.busy": "2024-03-24T02:07:01.744574Z", + "iopub.status.idle": "2024-03-24T02:07:01.753033Z", + "shell.execute_reply": "2024-03-24T02:07:01.752175Z" + }, + "papermill": { + "duration": 0.024942, + "end_time": "2024-03-24T02:07:01.755243", + "exception": false, + "start_time": "2024-03-24T02:07:01.730301", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss --depth=1 --single-branch --branch=main\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:01.781492Z", + "iopub.status.busy": "2024-03-24T02:07:01.781114Z", + "iopub.status.idle": "2024-03-24T02:07:01.786010Z", + "shell.execute_reply": "2024-03-24T02:07:01.785032Z" + }, + "papermill": { + "duration": 0.020867, + "end_time": "2024-03-24T02:07:01.788281", + "exception": false, + "start_time": "2024-03-24T02:07:01.767414", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:01.813547Z", + "iopub.status.busy": "2024-03-24T02:07:01.812996Z", + "iopub.status.idle": "2024-03-24T02:07:01.817408Z", + "shell.execute_reply": "2024-03-24T02:07:01.816487Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.019418, + "end_time": "2024-03-24T02:07:01.819382", + "exception": false, + "start_time": "2024-03-24T02:07:01.799964", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:01.844775Z", + "iopub.status.busy": "2024-03-24T02:07:01.844447Z", + "iopub.status.idle": "2024-03-24T02:07:01.850734Z", + "shell.execute_reply": "2024-03-24T02:07:01.849854Z" + }, + "papermill": { + "duration": 0.021432, + "end_time": "2024-03-24T02:07:01.852852", + "exception": false, + "start_time": "2024-03-24T02:07:01.831420", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "bf7731fb", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:01.879631Z", + "iopub.status.busy": "2024-03-24T02:07:01.879307Z", + "iopub.status.idle": "2024-03-24T02:07:01.884967Z", + "shell.execute_reply": "2024-03-24T02:07:01.883961Z" + }, + "papermill": { + "duration": 0.022502, + "end_time": "2024-03-24T02:07:01.887230", + "exception": false, + "start_time": "2024-03-24T02:07:01.864728", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"treatment\"\n", + "dataset_name = \"treatment\"\n", + "single_model = \"tvae\"\n", + "gp = True\n", + "gp_multiply = True\n", + "random_seed = 1\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/treatment/tvae/1\"\n", + "param_index = 0\n", + "allow_same_prediction = True\n", + "log_wandb = False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011632, + "end_time": "2024-03-24T02:07:01.911804", + "exception": false, + "start_time": "2024-03-24T02:07:01.900172", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:01.936951Z", + "iopub.status.busy": "2024-03-24T02:07:01.936623Z", + "iopub.status.idle": "2024-03-24T02:07:01.946265Z", + "shell.execute_reply": "2024-03-24T02:07:01.945357Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.024811, + "end_time": "2024-03-24T02:07:01.948268", + "exception": false, + "start_time": "2024-03-24T02:07:01.923457", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/treatment/tvae/1\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:01.975743Z", + "iopub.status.busy": "2024-03-24T02:07:01.975433Z", + "iopub.status.idle": "2024-03-24T02:07:04.047126Z", + "shell.execute_reply": "2024-03-24T02:07:04.045991Z" + }, + "papermill": { + "duration": 2.08892, + "end_time": "2024-03-24T02:07:04.049755", + "exception": false, + "start_time": "2024-03-24T02:07:01.960835", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:04.079278Z", + "iopub.status.busy": "2024-03-24T02:07:04.078293Z", + "iopub.status.idle": "2024-03-24T02:07:04.100619Z", + "shell.execute_reply": "2024-03-24T02:07:04.099830Z" + }, + "papermill": { + "duration": 0.038909, + "end_time": "2024-03-24T02:07:04.102787", + "exception": false, + "start_time": "2024-03-24T02:07:04.063878", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:04.129184Z", + "iopub.status.busy": "2024-03-24T02:07:04.128526Z", + "iopub.status.idle": "2024-03-24T02:07:04.136486Z", + "shell.execute_reply": "2024-03-24T02:07:04.135555Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.023997, + "end_time": "2024-03-24T02:07:04.138843", + "exception": false, + "start_time": "2024-03-24T02:07:04.114846", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:04.169999Z", + "iopub.status.busy": "2024-03-24T02:07:04.169266Z", + "iopub.status.idle": "2024-03-24T02:07:04.272920Z", + "shell.execute_reply": "2024-03-24T02:07:04.272073Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.122256, + "end_time": "2024-03-24T02:07:04.275534", + "exception": false, + "start_time": "2024-03-24T02:07:04.153278", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:04.302312Z", + "iopub.status.busy": "2024-03-24T02:07:04.301532Z", + "iopub.status.idle": "2024-03-24T02:07:09.213038Z", + "shell.execute_reply": "2024-03-24T02:07:09.211797Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.927642, + "end_time": "2024-03-24T02:07:09.215727", + "exception": false, + "start_time": "2024-03-24T02:07:04.288085", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-24 02:07:06.650200: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-24 02:07:06.650256: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-24 02:07:06.651951: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:09.245489Z", + "iopub.status.busy": "2024-03-24T02:07:09.244163Z", + "iopub.status.idle": "2024-03-24T02:07:09.260341Z", + "shell.execute_reply": "2024-03-24T02:07:09.259437Z" + }, + "papermill": { + "duration": 0.033234, + "end_time": "2024-03-24T02:07:09.262630", + "exception": false, + "start_time": "2024-03-24T02:07:09.229396", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:09.287921Z", + "iopub.status.busy": "2024-03-24T02:07:09.287591Z", + "iopub.status.idle": "2024-03-24T02:07:32.931051Z", + "shell.execute_reply": "2024-03-24T02:07:32.929754Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 23.658952, + "end_time": "2024-03-24T02:07:32.933592", + "exception": false, + "start_time": "2024-03-24T02:07:09.274640", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.preprocessing import DataPreprocessor\n", + "\n", + "preprocessor = DataPreprocessor(\n", + " task,\n", + " target=target,\n", + " cat_features=cat_features,\n", + " mixed_features=mixed_features,\n", + " longtail_features=longtail_features,\n", + " integer_features=integer_features,\n", + " lct_ae_embedding_size=lct_ae_embedding_size,\n", + " lct_ae_params=lct_ae_params,\n", + " lct_ae=lct_ae,\n", + " tab_ddpm_normalization=tab_ddpm_normalization,\n", + " tab_ddpm_cat_encoding=tab_ddpm_cat_encoding,\n", + " tab_ddpm_y_policy=tab_ddpm_y_policy,\n", + " tab_ddpm_is_y_cond=tab_ddpm_is_y_cond,\n", + " realtabformer_embedding=rtf_embed,\n", + " realtabformer_params=rtf_params,\n", + ")\n", + "preprocessor.fit(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a9c9b110", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-03-24T02:07:32.962760Z", + "iopub.status.busy": "2024-03-24T02:07:32.962325Z", + "iopub.status.idle": "2024-03-24T02:07:32.969937Z", + "shell.execute_reply": "2024-03-24T02:07:32.968936Z" + }, + "executionInfo": { + "elapsed": 13, + "status": "ok", + "timestamp": 1696841045411, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "OxUH_GBEv2qK", + "outputId": "76464c90-3baf-4bdc-a955-6f4fddc16b9c", + "papermill": { + "duration": 0.024994, + "end_time": "2024-03-24T02:07:32.972140", + "exception": false, + "start_time": "2024-03-24T02:07:32.947146", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'tvae': 95,\n", + " 'realtabformer': (69, 281, Embedding(281, 768), True),\n", + " 'lct_gan': 75,\n", + " 'tab_ddpm_concat': 12}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "preprocessor.adapter_sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "3cb9ed90", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:32.999062Z", + "iopub.status.busy": "2024-03-24T02:07:32.998769Z", + "iopub.status.idle": "2024-03-24T02:07:33.003783Z", + "shell.execute_reply": "2024-03-24T02:07:33.002951Z" + }, + "papermill": { + "duration": 0.020999, + "end_time": "2024-03-24T02:07:33.005876", + "exception": false, + "start_time": "2024-03-24T02:07:32.984877", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_3_factory\n", + "\n", + "datasetsn = load_dataset_3_factory(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " cache_dir=path_prefix,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad1eb833", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:33.033381Z", + "iopub.status.busy": "2024-03-24T02:07:33.032791Z", + "iopub.status.idle": "2024-03-24T02:07:33.784671Z", + "shell.execute_reply": "2024-03-24T02:07:33.783683Z" + }, + "papermill": { + "duration": 0.768175, + "end_time": "2024-03-24T02:07:33.786864", + "exception": false, + "start_time": "2024-03-24T02:07:33.018689", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/tvae/all inf False\n", + "../../../../ml-utility-loss/aug_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "Caching in ../../../../treatment/_cache_bs_test/tvae/all inf False\n", + "../../../../ml-utility-loss/bs_test/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_synth_test/tvae/all inf False\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "../../../../ml-utility-loss/synthetics/treatment [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n", + "1050\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_dataset_4\n", + "\n", + "test_set = load_dataset_4(\n", + " dataset_dir=os.path.join(path_prefix, \"ml-utility-loss/\"),\n", + " dataset_name=dataset_name,\n", + " preprocessor=preprocessor,\n", + " model=single_model,\n", + " cache_dir=path_prefix,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "14ff8b40", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:33.817033Z", + "iopub.status.busy": "2024-03-24T02:07:33.816647Z", + "iopub.status.idle": "2024-03-24T02:07:34.145727Z", + "shell.execute_reply": "2024-03-24T02:07:34.144774Z" + }, + "executionInfo": { + "elapsed": 588, + "status": "ok", + "timestamp": 1696841049215, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "NgahtU1q9uLO", + "papermill": { + "duration": 0.346972, + "end_time": "2024-03-24T02:07:34.148186", + "exception": false, + "start_time": "2024-03-24T02:07:33.801214", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'gradient_penalty_mode': {'gradient_penalty': True,\n", + " 'forward_once': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'loss_balancer_beta': 0.75,\n", + " 'loss_balancer_r': 0.96,\n", + " 'tf_pma_low': 16,\n", + " 'grad_loss_fn': torch.Tensor>,\n", + " 'pma_ffn_mode': 'none',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'torch',\n", + " 'grad_clip': 0.75,\n", + " 'bias_weight_decay': 0.05,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 2,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.06,\n", + " 'n_warmup_steps': 80,\n", + " 'Optim': torch_optimizer.diffgrad.DiffGrad,\n", + " 'fixed_role_model': 'tvae',\n", + " 'g_loss_mul': 0.2,\n", + " 'd_model': 512,\n", + " 'attn_activation': torch.nn.modules.activation.SELU,\n", + " 'tf_d_inner': 256,\n", + " 'tf_n_layers_enc': 4,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 2048,\n", + " 'ada_n_layers': 6,\n", + " 'ada_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 128,\n", + " 'head_n_layers': 7,\n", + " 'head_n_head': 64,\n", + " 'head_activation': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'single_model': True,\n", + " 'models': ['tvae'],\n", + " 'max_seconds': 3600,\n", + " 'Body': 'twin_encoder',\n", + " 'loss_balancer_log': False,\n", + " 'loss_balancer_lbtw': False,\n", + " 'pma_skip_small': False,\n", + " 'isab_skip_small': False,\n", + " 'layer_norm': False,\n", + " 'pma_layer_norm': False,\n", + " 'attn_residual': True,\n", + " 'tf_n_layers_dec': False,\n", + " 'tf_isab_rank': 0,\n", + " 'tf_layer_norm': False,\n", + " 'tf_pma_start': -1,\n", + " 'head_n_seeds': 0,\n", + " 'dropout': 0,\n", + " 'combine_mode': 'diff_left',\n", + " 'tf_isab_mode': 'separate',\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'synth_data': 2,\n", + " 'bias_lr_mul': 1.0,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 32,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': True,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.5, 'multiply': True, 'forgive_over': True}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "# params = {\n", + "# **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "# }\n", + "params = getattr(PARAMS, dataset_name).BEST_DICT[gp][gp_multiply][single_model]\n", + "if isinstance(params, (list, tuple)):\n", + " params = params[param_index]\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " #params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " #params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:34.178819Z", + "iopub.status.busy": "2024-03-24T02:07:34.178467Z", + "iopub.status.idle": "2024-03-24T02:07:34.303203Z", + "shell.execute_reply": "2024-03-24T02:07:34.302260Z" + }, + "papermill": { + "duration": 0.142572, + "end_time": "2024-03-24T02:07:34.305555", + "exception": false, + "start_time": "2024-03-24T02:07:34.162983", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_train/tvae/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/aug_train/treatment [400, 0]\n", + "Caching in ../../../../treatment/_cache_aug_val/tvae/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/aug_val/treatment [0, 200]\n", + "Caching in ../../../../treatment/_cache_bs_train/tvae/all inf False\n", + "split df ratio is 0\n", + "../../../../ml-utility-loss/bs_train/treatment [100, 0]\n", + "Caching in ../../../../treatment/_cache_bs_val/tvae/all inf False\n", + "split df ratio is 1\n", + "../../../../ml-utility-loss/bs_val/treatment [0, 50]\n", + "Caching in ../../../../treatment/_cache_synth/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/synthetics/treatment [400, 200]\n", + "[900, 450]\n", + "[900, 450]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-03-24T02:07:34.337651Z", + "iopub.status.busy": "2024-03-24T02:07:34.337316Z", + "iopub.status.idle": "2024-03-24T02:07:34.987930Z", + "shell.execute_reply": "2024-03-24T02:07:34.986944Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.669005, + "end_time": "2024-03-24T02:07:34.990054", + "exception": false, + "start_time": "2024-03-24T02:07:34.321049", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['tvae'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:35.020483Z", + "iopub.status.busy": "2024-03-24T02:07:35.019758Z", + "iopub.status.idle": "2024-03-24T02:07:35.024507Z", + "shell.execute_reply": "2024-03-24T02:07:35.023564Z" + }, + "papermill": { + "duration": 0.022454, + "end_time": "2024-03-24T02:07:35.026655", + "exception": false, + "start_time": "2024-03-24T02:07:35.004201", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:35.054831Z", + "iopub.status.busy": "2024-03-24T02:07:35.054547Z", + "iopub.status.idle": "2024-03-24T02:07:35.061913Z", + "shell.execute_reply": "2024-03-24T02:07:35.061004Z" + }, + "papermill": { + "duration": 0.024041, + "end_time": "2024-03-24T02:07:35.063950", + "exception": false, + "start_time": "2024-03-24T02:07:35.039909", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "29729665" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:35.092025Z", + "iopub.status.busy": "2024-03-24T02:07:35.091691Z", + "iopub.status.idle": "2024-03-24T02:07:35.187424Z", + "shell.execute_reply": "2024-03-24T02:07:35.186546Z" + }, + "papermill": { + "duration": 0.112598, + "end_time": "2024-03-24T02:07:35.189768", + "exception": false, + "start_time": "2024-03-24T02:07:35.077170", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 2648, 95] --\n", + "├─Adapter: 1-1 [2, 2648, 95] --\n", + "│ └─Sequential: 2-1 [2, 2648, 512] --\n", + "│ │ └─FeedForward: 3-1 [2, 2648, 2048] --\n", + "│ │ │ └─Linear: 4-1 [2, 2648, 2048] 196,608\n", + "│ │ │ └─LeakyHardtanh: 4-2 [2, 2648, 2048] --\n", + "│ │ └─FeedForward: 3-2 [2, 2648, 2048] --\n", + "│ │ │ └─Linear: 4-3 [2, 2648, 2048] 4,196,352\n", + "│ │ │ └─LeakyHardtanh: 4-4 [2, 2648, 2048] --\n", + "│ │ └─FeedForward: 3-3 [2, 2648, 2048] --\n", + "│ │ │ └─Linear: 4-5 [2, 2648, 2048] 4,196,352\n", + "│ │ │ └─LeakyHardtanh: 4-6 [2, 2648, 2048] --\n", + "│ │ └─FeedForward: 3-4 [2, 2648, 2048] --\n", + "│ │ │ └─Linear: 4-7 [2, 2648, 2048] 4,196,352\n", + "│ │ │ └─LeakyHardtanh: 4-8 [2, 2648, 2048] --\n", + "│ │ └─FeedForward: 3-5 [2, 2648, 2048] --\n", + "│ │ │ └─Linear: 4-9 [2, 2648, 2048] 4,196,352\n", + "│ │ │ └─LeakyHardtanh: 4-10 [2, 2648, 2048] --\n", + "│ │ └─FeedForward: 3-6 [2, 2648, 512] --\n", + "│ │ │ └─Linear: 4-11 [2, 2648, 512] 1,049,088\n", + "│ │ │ └─LeakyHardsigmoid: 4-12 [2, 2648, 512] --\n", + "├─Adapter: 1-2 [2, 661, 95] (recursive)\n", + "│ └─Sequential: 2-2 [2, 661, 512] (recursive)\n", + "│ │ └─FeedForward: 3-7 [2, 661, 2048] (recursive)\n", + "│ │ │ └─Linear: 4-13 [2, 661, 2048] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-14 [2, 661, 2048] --\n", + "│ │ └─FeedForward: 3-8 [2, 661, 2048] (recursive)\n", + "│ │ │ └─Linear: 4-15 [2, 661, 2048] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-16 [2, 661, 2048] --\n", + "│ │ └─FeedForward: 3-9 [2, 661, 2048] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 661, 2048] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-18 [2, 661, 2048] --\n", + "│ │ └─FeedForward: 3-10 [2, 661, 2048] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 661, 2048] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-20 [2, 661, 2048] --\n", + "│ │ └─FeedForward: 3-11 [2, 661, 2048] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 661, 2048] (recursive)\n", + "│ │ │ └─LeakyHardtanh: 4-22 [2, 661, 2048] --\n", + "│ │ └─FeedForward: 3-12 [2, 661, 512] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 661, 512] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-24 [2, 661, 512] --\n", + "├─TwinEncoder: 1-3 [2, 8192] --\n", + "│ └─Encoder: 2-3 [2, 16, 512] --\n", + "│ │ └─ModuleList: 3-14 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-25 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-6 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 2648, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-12 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 2648, 256] 131,328\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 2648, 256] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 2648, 512] 131,584\n", + "│ │ │ └─EncoderLayer: 4-26 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-18 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 2648, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-24 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 2648, 256] 131,328\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 2648, 256] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 2648, 512] 131,584\n", + "│ │ │ └─EncoderLayer: 4-27 [2, 2648, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-30 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 2648, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-36 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 2648, 256] 131,328\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 2648, 256] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 2648, 512] 131,584\n", + "│ │ │ └─EncoderLayer: 4-28 [2, 16, 512] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-7 [2, 2648, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 32, 512] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-20 [2, 32, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 32, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 32, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-42 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-21 [2, 2648, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 32, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 2648, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 2648, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 2648, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-48 [2, 2648, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 2648, 512] --\n", + "│ │ │ │ │ └─Linear: 6-22 [2, 2648, 256] 131,328\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-23 [2, 2648, 256] --\n", + "│ │ │ │ │ └─Linear: 6-24 [2, 2648, 512] 131,584\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-9 [2, 16, 512] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-25 [2, 16, 512] 8,192\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-26 [2, 16, 512] --\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 16, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 2648, 512] 262,144\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 16, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 16, 2648] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 16, 512] 262,656\n", + "│ │ │ │ │ │ └─SELU: 7-54 [2, 16, 512] --\n", + "│ └─Encoder: 2-4 [2, 16, 512] (recursive)\n", + "│ │ └─ModuleList: 3-14 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-29 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-10 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-27 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-28 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-60 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-29 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 661, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-66 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-11 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-30 [2, 661, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-31 [2, 661, 256] --\n", + "│ │ │ │ │ └─Linear: 6-32 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-30 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-12 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-33 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-34 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-12 [2, 32, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-72 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-35 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 661, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-78 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-13 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-36 [2, 661, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-37 [2, 661, 256] --\n", + "│ │ │ │ │ └─Linear: 6-38 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-31 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-14 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-39 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-40 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-84 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-41 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-85 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-86 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-87 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-88 [2, 32, 661, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-15 [2, 32, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-89 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-90 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-15 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-42 [2, 661, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-43 [2, 661, 256] --\n", + "│ │ │ │ │ └─Linear: 6-44 [2, 661, 512] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-32 [2, 16, 512] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-16 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-45 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-46 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-91 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-92 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-93 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-94 [2, 32, 32, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-16 [2, 32, 32, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-95 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-96 [2, 32, 512] --\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-47 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-97 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-98 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-99 [2, 32, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-100 [2, 32, 661, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-17 [2, 32, 661, 32] --\n", + "│ │ │ │ │ │ └─Linear: 7-101 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-102 [2, 661, 512] --\n", + "│ │ │ │ └─DoubleFeedForward: 5-17 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-48 [2, 661, 256] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-49 [2, 661, 256] --\n", + "│ │ │ │ │ └─Linear: 6-50 [2, 661, 512] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-18 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-51 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-52 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-103 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-104 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-105 [2, 661, 512] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-106 [2, 32, 16, 16] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-18 [2, 32, 16, 661] --\n", + "│ │ │ │ │ │ └─Linear: 7-107 [2, 16, 512] (recursive)\n", + "│ │ │ │ │ │ └─SELU: 7-108 [2, 16, 512] --\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-15 [2, 128] --\n", + "│ │ │ └─Linear: 4-33 [2, 128] 1,048,704\n", + "│ │ │ └─LeakyHardsigmoid: 4-34 [2, 128] --\n", + "│ │ └─FeedForward: 3-16 [2, 128] --\n", + "│ │ │ └─Linear: 4-35 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-36 [2, 128] --\n", + "│ │ └─FeedForward: 3-17 [2, 128] --\n", + "│ │ │ └─Linear: 4-37 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-38 [2, 128] --\n", + "│ │ └─FeedForward: 3-18 [2, 128] --\n", + "│ │ │ └─Linear: 4-39 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-40 [2, 128] --\n", + "│ │ └─FeedForward: 3-19 [2, 128] --\n", + "│ │ │ └─Linear: 4-41 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-42 [2, 128] --\n", + "│ │ └─FeedForward: 3-20 [2, 128] --\n", + "│ │ │ └─Linear: 4-43 [2, 128] 16,512\n", + "│ │ │ └─LeakyHardsigmoid: 4-44 [2, 128] --\n", + "│ │ └─FeedForward: 3-21 [2, 1] --\n", + "│ │ │ └─Linear: 4-45 [2, 1] 129\n", + "│ │ │ └─LeakyHardsigmoid: 4-46 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 29,729,665\n", + "Trainable params: 29,729,665\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 116.36\n", + "========================================================================================================================\n", + "Input size (MB): 2.51\n", + "Forward/backward pass size (MB): 1231.11\n", + "Params size (MB): 118.92\n", + "Estimated Total Size (MB): 1352.55\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T02:07:35.222713Z", + "iopub.status.busy": "2024-03-24T02:07:35.222280Z", + "iopub.status.idle": "2024-03-24T03:17:31.752013Z", + "shell.execute_reply": "2024-03-24T03:17:31.750941Z" + }, + "papermill": { + "duration": 4196.549774, + "end_time": "2024-03-24T03:17:31.754626", + "exception": false, + "start_time": "2024-03-24T02:07:35.204852", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 datasets [900, 450, 1050]\n", + "Creating model of type \n", + "[*] Embedding False True\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.2\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.03363065153919415, 'avg_role_model_std_loss': 4.061211425165913, 'avg_role_model_mean_pred_loss': 0.003072351892546645, 'avg_role_model_g_mag_loss': 0.0959731253138226, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.04791451765126276, 'n_size': 900, 'n_batch': 450, 'duration': 580.8831059932709, 'duration_batch': 1.290851346651713, 'duration_size': 0.6454256733258565, 'avg_pred_std': 0.12362771289730214}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.010208010552554698, 'avg_role_model_std_loss': 1.7609046706683165, 'avg_role_model_mean_pred_loss': 0.0012737641513725704, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010208010552554698, 'n_size': 450, 'n_batch': 225, 'duration': 203.08173871040344, 'duration_batch': 0.9025855053795708, 'duration_size': 0.4512927526897854, 'avg_pred_std': 0.09662675946161875}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.010233858526780142, 'avg_role_model_std_loss': 0.7669666670105895, 'avg_role_model_mean_pred_loss': 0.0007310967529919379, 'avg_role_model_g_mag_loss': 0.152462607157989, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010440422089125236, 'n_size': 900, 'n_batch': 450, 'duration': 579.9043016433716, 'duration_batch': 1.288676225874159, 'duration_size': 0.6443381129370795, 'avg_pred_std': 0.20330373977095204}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.010850908685869168, 'avg_role_model_std_loss': 1.70096435137354, 'avg_role_model_mean_pred_loss': 0.0007107436713112964, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.010850908685869168, 'n_size': 450, 'n_batch': 225, 'duration': 196.19050812721252, 'duration_batch': 0.8719578138987223, 'duration_size': 0.43597890694936114, 'avg_pred_std': 0.09977313449461185}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.008459381934637867, 'avg_role_model_std_loss': 1.3255586893503324, 'avg_role_model_mean_pred_loss': 0.0004864757848670276, 'avg_role_model_g_mag_loss': 0.12002694543341123, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.008653573781455684, 'n_size': 900, 'n_batch': 450, 'duration': 573.4812700748444, 'duration_batch': 1.274402822388543, 'duration_size': 0.6372014111942715, 'avg_pred_std': 0.19907333407094685}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.00886493952675291, 'avg_role_model_std_loss': 1.6276623941000343, 'avg_role_model_mean_pred_loss': 0.0003420339755327389, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.00886493952675291, 'n_size': 450, 'n_batch': 225, 'duration': 198.55576705932617, 'duration_batch': 0.8824700758192274, 'duration_size': 0.4412350379096137, 'avg_pred_std': 0.10561421838104858}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.005182997622466448, 'avg_role_model_std_loss': 0.5467010652780068, 'avg_role_model_mean_pred_loss': 0.00016614465866880747, 'avg_role_model_g_mag_loss': 0.1050477642213486, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0052792941385990215, 'n_size': 900, 'n_batch': 450, 'duration': 575.0789859294891, 'duration_batch': 1.2779533020655314, 'duration_size': 0.6389766510327657, 'avg_pred_std': 0.21151418934407248}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.006986802332486501, 'avg_role_model_std_loss': 2.1908077004363427, 'avg_role_model_mean_pred_loss': 0.00018339116417594552, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.006986802332486501, 'n_size': 450, 'n_batch': 225, 'duration': 197.8352234363556, 'duration_batch': 0.8792676597171359, 'duration_size': 0.43963382985856797, 'avg_pred_std': 0.09777659590274804}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.00417525644981459, 'avg_role_model_std_loss': 0.8511908998175017, 'avg_role_model_mean_pred_loss': 0.00011016234336636986, 'avg_role_model_g_mag_loss': 0.09707871191103348, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004249937350797381, 'n_size': 900, 'n_batch': 450, 'duration': 570.9023349285126, 'duration_batch': 1.2686718553966947, 'duration_size': 0.6343359276983473, 'avg_pred_std': 0.19839078328734103}\n", + "Time out: 3681.1604993343353/3600\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tvae', 'n_size': 1050, 'n_batch': 525, 'role_model_metrics': {'avg_loss': 0.005427328764966324, 'avg_g_mag_loss': 0.16761904580252598, 'avg_g_cos_loss': 0.0, 'pred_duration': 10.841743230819702, 'grad_duration': 8.438727617263794, 'total_duration': 19.280470848083496, 'pred_std': 0.23774628341197968, 'std_loss': 0.0005916486261412501, 'mean_pred_loss': 0.000279915431747213, 'pred_rmse': 0.0736704096198082, 'pred_mae': 0.04554838687181473, 'pred_mape': 5041099.0, 'grad_rmse': 0.12793424725532532, 'grad_mae': 0.07811923325061798, 'grad_mape': 1.5463948249816895}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.005427328764966324, 'avg_g_mag_loss': 0.16761904580252598, 'avg_g_cos_loss': 0.0, 'avg_pred_duration': 10.841743230819702, 'avg_grad_duration': 8.438727617263794, 'avg_total_duration': 19.280470848083496, 'avg_pred_std': 0.23774628341197968, 'avg_std_loss': 0.0005916486261412501, 'avg_mean_pred_loss': 0.000279915431747213}, 'min_metrics': {'avg_loss': 0.005427328764966324, 'avg_g_mag_loss': 0.16761904580252598, 'avg_g_cos_loss': 0.0, 'pred_duration': 10.841743230819702, 'grad_duration': 8.438727617263794, 'total_duration': 19.280470848083496, 'pred_std': 0.23774628341197968, 'std_loss': 0.0005916486261412501, 'mean_pred_loss': 0.000279915431747213, 'pred_rmse': 0.0736704096198082, 'pred_mae': 0.04554838687181473, 'pred_mape': 5041099.0, 'grad_rmse': 0.12793424725532532, 'grad_mae': 0.07811923325061798, 'grad_mape': 1.5463948249816895}, 'model_metrics': {'tvae': {'avg_loss': 0.005427328764966324, 'avg_g_mag_loss': 0.16761904580252598, 'avg_g_cos_loss': 0.0, 'pred_duration': 10.841743230819702, 'grad_duration': 8.438727617263794, 'total_duration': 19.280470848083496, 'pred_std': 0.23774628341197968, 'std_loss': 0.0005916486261412501, 'mean_pred_loss': 0.000279915431747213, 'pred_rmse': 0.0736704096198082, 'pred_mae': 0.04554838687181473, 'pred_mape': 5041099.0, 'grad_rmse': 0.12793424725532532, 'grad_mae': 0.07811923325061798, 'grad_mape': 1.5463948249816895}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "del model\n", + "clear_memory()\n", + "\n", + "#opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " #whole_model=model,\n", + " #optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=allow_same_prediction,\n", + " wandb=wandb if log_wandb else None,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:17:31.790820Z", + "iopub.status.busy": "2024-03-24T03:17:31.789952Z", + "iopub.status.idle": "2024-03-24T03:17:31.795018Z", + "shell.execute_reply": "2024-03-24T03:17:31.794120Z" + }, + "papermill": { + "duration": 0.024577, + "end_time": "2024-03-24T03:17:31.797101", + "exception": false, + "start_time": "2024-03-24T03:17:31.772524", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:17:31.828954Z", + "iopub.status.busy": "2024-03-24T03:17:31.828401Z", + "iopub.status.idle": "2024-03-24T03:17:32.005531Z", + "shell.execute_reply": "2024-03-24T03:17:32.004682Z" + }, + "papermill": { + "duration": 0.195959, + "end_time": "2024-03-24T03:17:32.008198", + "exception": false, + "start_time": "2024-03-24T03:17:31.812239", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "#torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:17:32.043651Z", + "iopub.status.busy": "2024-03-24T03:17:32.042998Z", + "iopub.status.idle": "2024-03-24T03:17:32.324684Z", + "shell.execute_reply": "2024-03-24T03:17:32.323735Z" + }, + "papermill": { + "duration": 0.301484, + "end_time": "2024-03-24T03:17:32.326775", + "exception": false, + "start_time": "2024-03-24T03:17:32.025291", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAEXCAYAAAAEFznvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyqklEQVR4nO3deVRTd/4//udNyMK+7wbiAm4gKJvY1o5KRWsXrFXqz6PWaXXajzi11H5aHKvO9PMdpjPt1E711C4zbWdOGa2tOh3r2CpKbQsuLCouMNWyY1gEAgRIILm/P0Ii0YCE7d4kr8c59yg39968EsnTe9953/ebYVmWBSGE8ICA6wIIIcSAAokQwhsUSIQQ3qBAIoTwBgUSIYQ3KJAIIbxBgUQI4Q0KJEIIb1AgEUJ4gwKJEMIbQwqkPXv2QC6XQyqVIiEhAefOnRtw+wMHDmDKlCmQSqWIjIzE0aNHTR5/+umnwTCMybJo0aKhlEYIsWIWB9L+/fuRnp6OHTt2oLCwEFFRUUhOTkZ9fb3Z7XNzc7Fy5Uo888wzKCoqQkpKClJSUnD58mWT7RYtWoSbN28al3/+859De0WEEKvFWHpzbUJCAuLi4rB7924AgE6ng0wmw6ZNm/Dqq6/etX1qaipUKhWOHDliXDd79mxER0dj7969APRnSC0tLTh8+PCQXoROp0NtbS1cXV3BMMyQjkEIGT0sy6KtrQ1BQUEQCPo/D3Kw5KAajQYFBQXIyMgwrhMIBEhKSkJeXp7ZffLy8pCenm6yLjk5+a7wycnJgZ+fHzw9PTF//nz83//9H7y9vc0eU61WQ61WG3+uqanBtGnTLHkphBAOVFVVYdy4cf0+blEgNTY2QqvVwt/f32S9v78/SkpKzO6jUCjMbq9QKIw/L1q0CE888QTGjx+PGzduYOvWrVi8eDHy8vIgFArvOmZmZiZ++9vf3rW+qqoKbm5ulrwkQsgYaG1thUwmg6ur64DbWRRIo+Wpp54y/j0yMhIzZszAxIkTkZOTgwULFty1fUZGhslZl+HFurm5USARwmP3alKxqFHbx8cHQqEQdXV1Juvr6uoQEBBgdp+AgACLtgeACRMmwMfHB9evXzf7uEQiMYYPhRAhtsOiQBKLxYiJiUF2drZxnU6nQ3Z2NhITE83uk5iYaLI9ABw/frzf7QGguroat27dQmBgoCXlEUKsHWuhffv2sRKJhP3kk0/Yq1evshs2bGA9PDxYhULBsizLrl69mn311VeN2//444+sg4MD++abb7LXrl1jd+zYwYpEIra4uJhlWZZta2tjt2zZwubl5bFlZWXsiRMn2FmzZrFhYWFsV1fXoGpSKpUsAFapVFr6cgghY2Cwn1GL25BSU1PR0NCA7du3Q6FQIDo6GseOHTM2XFdWVpp8rTdnzhxkZWVh27Zt2Lp1K8LCwnD48GFEREQAAIRCIS5duoRPP/0ULS0tCAoKwsKFC/H6669DIpGMSOiSgWm1WnR3d3NdBrFiIpHI7BdQlrK4HxIftba2wt3dHUqlktqTLMCyLBQKBVpaWrguhdgADw8PBAQEmG24HuxnlBffshFuGMLIz88PTk5O1KmUDAnLsujo6DDerTGctl+7C6TyRhUuVrfg8ehgrkvhlFarNYZRfx1QCRksR0dHAEB9fT38/PyGfPlmV4FU3qjCL97MgUjI4IEwX3g5i7kuiTOGNiMnJyeOKyG2wvC71N3dPeRAsqvhR+Q+zogIdkO3lsW/LtRwXQ4v0GUaGSkj8btkV4EEAMtjZACAA/nVHFdCCLmT3QXS49FBEAsFuHqzFZdrlFyXQwjpw+4CycNJjIXT9X2mDuRXcVwNsUYMwwx5qBy+2rlzJ6Kjo7kuw/4CCQBWxOov2w5fqEVXt5bjagix3CeffAIPD48RO96WLVvuusWLC3YZSPdN8kGguxTKzm6cuFZ37x0IsVIajWZQ27m4uPCi+4ddBpJQwODJGP0gUZ9T47YRy7Lo0PSM+WLpzQLHjh3D/fffDw8PD3h7e+ORRx7BjRs3AOhvVXrllVdMtm9oaIBIJMLp06cBADdv3sSSJUvg6OiI8ePHIysrC3K5HLt27RrS+1ZcXIz58+fD0dER3t7e2LBhA9rb242P5+TkID4+Hs7OzvDw8MB9992HiooKAMDFixcxb948uLq6ws3NDTExMcjPzx/w+XJycrBu3ToolUrjGPQ7d+4EAMjlcrz++utYs2YN3NzcsGHDBgDAK6+8gvDwcDg5OWHChAl47bXXTG4XuvOS7emnn0ZKSgrefPNNBAYGwtvbGxs3bhz1W4zsqh9SX0/GjMO7J6/j+58aUNvSiSAPR65L4lxntxbTtn8z5s979XfJcBIP/ldRpVIhPT0dM2bMQHt7O7Zv346lS5fiwoULWLVqFf74xz/iD3/4g/Fr6P379yMoKAgPPPAAAGDNmjVobGxETk4ORCIR0tPT+x0TfjC1JCcnIzExEefPn0d9fT2effZZpKWl4ZNPPkFPTw9SUlKwfv16/POf/4RGo8G5c+eMta1atQozZ87Ee++9B6FQiAsXLkAkEg34nHPmzMGuXbuwfft2lJaWAtCf4Ri8+eab2L59O3bs2GFc5+rqik8++QRBQUEoLi7G+vXr4erqiv/93//t93lOnTqFwMBAnDp1CtevX0dqaiqio6Oxfv36Ib1Xg2G3gRTq7YzZE7xw5ucmHCysRtr8MK5LIoO0bNkyk5//9re/wdfXF1evXsWKFSuwefNm/PDDD8YAysrKwsqVK8EwDEpKSnDixAmcP38esbGxAICPPvoIYWFD+/fPyspCV1cX/v73v8PZ2RkAsHv3bjz66KN44403IBKJoFQq8cgjj2DixIkAgKlTpxr3r6ysxMsvv4wpU6YAwKDqEIvFcHd3B8MwZscVmz9/Pl566SWTddu2bTP+XS6XY8uWLdi3b9+AgeTp6Yndu3dDKBRiypQpWLJkCbKzsymQRsvyGBnO/NyEz/Or8T+/mASBwL47CTqKhLj6u2ROntcSP/30E7Zv346zZ8+isbEROp0OgP7DHRERgYULF+Kzzz7DAw88gLKyMuTl5eH9998HAJSWlsLBwQGzZs0yHm/SpEnw9PQcUu3Xrl1DVFSUMYwA4L777oNOp0NpaSnmzp2Lp59+GsnJyXjooYeQlJSEFStWGO/3Sk9Px7PPPot//OMfSEpKwvLly43BNVSGoO1r//79+Mtf/oIbN26gvb0dPT0997wRffr06SY9rgMDA1FcXDys2u7FLtuQDBZHBsBF4oDKpg6cK2/iuhzOMQwDJ7HDmC+W9vB99NFH0dTUhA8//BBnz57F2bNnAdxuwF21ahW++OILdHd3IysrC5GRkYiMjBzx92uwPv74Y+Tl5WHOnDnYv38/wsPDcebMGQD6tpsrV65gyZIlOHnyJKZNm4ZDhw4N6/n6hiOgn2hj1apVePjhh3HkyBEUFRXhN7/5zT0bvO+8dGQYxhj+o8WuA8lJ7IBHo/T/U1HPbetw69YtlJaWYtu2bViwYAGmTp2K5uZmk20ef/xxdHV14dixY8jKysKqVauMj02ePBk9PT0oKioyrrt+/fpdxxisqVOn4uLFi1CpVMZ1P/74IwQCASZPnmxcN3PmTGRkZCA3NxcRERHIysoyPhYeHo4XX3wR3377LZ544gl8/PHH93xesVgMrXZwXVZyc3MRGhqK3/zmN4iNjUVYWJixUZ1v7DqQAODJ3ltJjhbfRFsXDVLGd56envD29sYHH3yA69ev4+TJk3dNs+Xs7IyUlBS89tpruHbtGlauXGl8bMqUKUhKSsKGDRtw7tw5FBUVYcOGDXB0dBzSvVirVq2CVCrF2rVrcfnyZZw6dQqbNm3C6tWr4e/vj7KyMmRkZCAvLw8VFRX49ttv8dNPP2Hq1Kno7OxEWloacnJyUFFRgR9//BHnz583aWPqj1wuR3t7O7Kzs9HY2IiOjo5+tw0LC0NlZSX27duHGzdu4C9/+cuwz8JGi90H0qwQD0z0dUZntxZfX7rJdTnkHgQCAfbt24eCggJERETgxRdfxJ/+9Ke7tlu1ahUuXryIBx54ACEhISaP/f3vf4e/vz/mzp2LpUuXGr9xkkqlFtfj5OSEb775Bk1NTYiLi8OTTz6JBQsWGCdSdXJyQklJCZYtW4bw8HBs2LABGzduxK9+9SsIhULcunULa9asQXh4OFasWIHFixebneLrTnPmzMFzzz2H1NRU+Pr64o9//GO/2z722GN48cUXkZaWhujoaOTm5uK1116z+LWOBRoxEsD7391A5n9KMCvEAwf/575RqJB/urq6UFZWhvHjxw/pg2hLqqurIZPJcOLECbPTbpHBGeh3arCfUbs/QwKApbOCIRQwKKxswfX69nvvQKzayZMn8dVXX6GsrAy5ubl46qmnIJfLMXfuXK5Ls3sUSAD8XKWYN9kXAHCggG64tXXd3d3YunUrpk+fjqVLl8LX19fYSfKzzz6Di4uL2WX69OljVuPixYv7reP3v//9mNUx1uy6H1Jfy2NlOHGtHl8W1GDLwskQCSmrbVVycjKSk833t3rssceQkJBg9rF79aAeSR999BE6OzvNPubl5TVmdYw1CqRe86f4wcdFjMZ2Nb4rbUDSNH+uSyIccHV1vef882MhONg+x3yn04BeIqEAKb0D/9NlGyHcoEDqY3nvOEnZ1+rR2K7muBpC7A8FUh+TA1wRJfNAj47F4SKaBICQsUaBdIcVsYZxkqosHqeHEDI8FEh3eDQqCBIHAf5b145L1TQJACFjiQLpDm5SERZH6MeY+ZwmASBm2OIg/3xBgWSGYRKAry7SJACEn0Z6kH9APzQuwzBoaWkZ0eNaggLJjNkTvDHO0xFtXT345oqC63IIsRsUSGYITCYBsKPLNpYFNKqxX2iQ/xEb5F+tVmPLli0IDg6Gs7MzEhISkJOTY9y3oqICjz76KDw9PeHs7Izp06fj6NGjKC8vx7x58wDoh3hhGAZPP/30kN6P4aCe2v14MmYc3sn+CT9ev4Wqpg7IvJy4Lmn0dXcAvw8a++fdWguIne+9XS8a5L//Qf7T0tJw9epV7Nu3D0FBQTh06BAWLVqE4uJihIWFYePGjdBoNDh9+jScnZ1x9epVuLi4QCaT4csvv8SyZctQWloKNzc3ODqO/cQXFEj9GOfphPsm+uCH6434srAam5PCuS6J9KJB/s0P8l9ZWYmPP/4YlZWVCArS/8eyZcsWHDt2DB9//DF+//vfo7KyEsuWLTMO6TthwgTj/oZ75Pz8/Ea8fWqwKJAGsDx2HH643ogD+dX49fww258EQOSkP1vh4nktQIP8m1dcXAytVovwcNP/PNVqtXESyF//+td4/vnn8e233yIpKQnLli3DjBkzhvR8o4HakAaQPD0ArlIH1LR0Iu/nW1yXM/oYRn/pNNYLDfI/IoP8t7e3QygUoqCgABcuXDAu165dwzvvvAMAePbZZ/Hzzz9j9erVKC4uRmxsLN59990Re63DRYE0AKlIiMej9ae+dtW4zWM0yL+euUH+Z86cCa1Wi/r6ekyaNMlk6XtpJ5PJ8Nxzz+HgwYN46aWX8OGHHxqPCWDQkweMBgqkezD0STp2WQFlJ00CwDUa5F/P3CD/4eHhWLVqFdasWYODBw+irKwM586dQ2ZmJr7++msAwObNm/HNN9+grKwMhYWFOHXqlPH5QkNDwTAMjhw5goaGBpNvCscMawOUSiULgFUqlSN+bJ1Oxy7883ds6CtH2H/klY/48bnS2dnJXr16le3s7OS6FIsdP36cnTp1KiuRSNgZM2awOTk5LAD20KFDxm2OHj3KAmDnzp171/61tbXs4sWLWYlEwoaGhrJZWVmsn58fu3fv3kE9/53PdenSJXbevHmsVCplvby82PXr17NtbW0sy7KsQqFgU1JS2MDAQFYsFrOhoaHs9u3bWa1Wy6rVavapp55iZTIZKxaL2aCgIDYtLW3Q/ybPPfcc6+3tzQJgd+zYwbIsy2o0Gnb79u2sXC5nRSIRGxgYyC5dupS9dOkSy7Ism5aWxk6cOJGVSCSsr68vu3r1araxsdF4zN/97ndsQEAAyzAMu3bt2kHVYTDQ79RgP6NDCqTdu3ezoaGhrEQiYePj49mzZ88OuP3nn3/OTp48mZVIJGxERAT79ddf97vtr371KxYA+/bbbw+6ntEMJJZl2Q9P32BDXznCPvbu96NyfC5YcyCNtKqqKhYAe+LECa5LsWojEUgWX7Lt378f6enp2LFjBwoLCxEVFYXk5OR++3Hk5uZi5cqVeOaZZ1BUVISUlBSkpKTg8uXLd2176NAhnDlzxviVJV8snRkMBwGDi9VKlCrauC6HDBMN8s9fFgfSn//8Z6xfvx7r1q3DtGnTsHfvXjg5OeFvf/ub2e3feecdLFq0CC+//DKmTp2K119/HbNmzTLOW2VQU1ODTZs24bPPPhvTsYsHw9tFggVT/QAAB6hx2+rRIP/8ZVE/JI1Gg4KCAmRkZBjXCQQCJCUlIS8vz+w+eXl5dzU6Jicnm9wtrdPpsHr1arz88suD+kdXq9VQq2+P6Nja2mrJyxiSFbEyfHOlDoeKavC/i6ZA7EDfB1grGuSfvywKpMbGRmi1Wvj7mw6A7+/vj5KSErP7KBQKs9srFLdvWn3jjTfg4OCAX//614OqIzMzc1Cze46kB8N94esqQUObGidL6rEoIuDeOxGrQ4P8c4vz/+YLCgrwzjvv4JNPPhn0164ZGRlQKpXGpapq9C+jHIQCLJulv+H2CxuaBIClUTHJCBmJ3yWLAsnHxwdCoRB1dXUm6+vq6kw6XvUVEBAw4Pbff/896uvrERISAgcHBzg4OKCiogIvvfQS5HK52WNKJBK4ubmZLGNhee/wtqdKG1Df2jUmzzlaDJcfHR0dHFdCbIXhd2k4l7YWXbKJxWLExMQgOzsbKSkpAPTtP9nZ2UhLSzO7T2JiIrKzs7F582bjuuPHjyMxMREAsHr1aiQlJZnsk5ycjNWrV2PdunWWlDfqJvq6ICbUEwUVzThYVIPnHhzaPUd8IBQK4eHhYfx21MnJaUgdAwlhWRYdHR2or6+Hh4cHhELhkI9l8c216enpWLt2LWJjYxEfH49du3ZBpVIZw2PNmjUIDg5GZmYmAOCFF17Agw8+iLfeegtLlizBvn37kJ+fjw8++AAA4O3tbbzxz0AkEiEgIMCk6z1frIgdh4KKZhzIr8Kv5k6w6g+x4Sx1qENvENKXh4dHv1dKg2VxIKWmpqKhoQHbt2+HQqFAdHQ0jh07Zmy4rqyshEBw+0pwzpw5yMrKwrZt27B161aEhYXh8OHDiIiIGFbhXFkyIwg7v7qKGw0qFFa2ICZ0aHeJ8wHDMAgMDISfnx+6u+m2GDJ0IpFoWGdGBgxrA62ara2tcHd3h1KpHJP2pJc+v4gvC6vxVJwMf1jGn6EbCOGrwX5GOf+WzRoZ5m7798VadGh6OK6GENtBgTQE8eO9IPd2gkqjxX+KaRIAQkYKBdIQMIydTgJAyCijQBqiZTHjwDDA2bImlDeq7r0DIeSeKJCGKNDdEXPDfAEAXxRUc1wNIbaBAmkYDD23vyyshlZn9V9WEsI5CqRheGiaPzycRLip7MIP1xu5LocQq0eBNAwSByFSovV3ZVPjNiHDR4E0TIbLtuNX6tDSoeG4GkKsGwXSME0Pcse0QDdotDr86wIHkywSYkMokEaAoec2XbYRMjwUSCPg8ehgiIUCXKltxZVaJdflEGK1KJBGgKezGA9N1492cCCf+iQRMlQUSCNkee+tJIcv1EDdw91UxIRYMwqkEfJAmC8C3KRo6ejGias04BkhQ0GBNEKEgts33B6woUkACBlLFEgjyBBIp//bgJtK83NqEUL6R4E0guQ+zogf7wUdCxwsrOG6HEKsDgXSCFsRKwOgn3LbBkYHJmRMUSCNsIcjA+AsFqL8VgfOlzdzXQ4hVoUCaYQ5iR3wyIwgANRzmxBLUSCNghVx+sbtry/dRLuaJgEgZLAokEbBrBBPTPB1Rme3Fkcv3eS6HEKsBgXSKGAYBstj9I3bdNlGyOBRII2SZbOCIRQwyK9oxo2Gdq7LIcQqUCCNEj83KX4Rrp8EgG64JWRwKJBG0fLePkkHC6vRo9VxXA0h/EeBNIrmT/GDl7MY9W1qnP6pgetyCOE9CqRRJHYQYOnM3kkAztNlGyH3QoE0ygy3kpy4Vodb7WqOqyGE3yiQRtnkAFfMGOeOHh2LwzQJACEDokAaA8vphltCBoUCaQw8FhUEiYMAJYo2FNfQJACE9IcCaQy4O4qwKCIAAPVJImQgFEhjxHAryb8u1KCrmyYBIMQcCqQxMmeiN4I9HNHa1YNvrii4LocQXqJAGiOCvpMA0GUbIWZRII0hQyD9eKMR1c0dHFdDCP9QII0hmZcT5kz0BssCXxbQJACE3GlIgbRnzx7I5XJIpVIkJCTg3LlzA25/4MABTJkyBVKpFJGRkTh69KjJ4zt37sSUKVPg7OwMT09PJCUl4ezZs0MpjfeMkwAUVEGnoz5JhPRlcSDt378f6enp2LFjBwoLCxEVFYXk5GTU15ufrTU3NxcrV67EM888g6KiIqSkpCAlJQWXL182bhMeHo7du3ejuLgYP/zwA+RyORYuXIiGBtu7IXVRRABcpQ6obu7EmbJbXJdDCK8wrIVdhxMSEhAXF4fdu3cDAHQ6HWQyGTZt2oRXX331ru1TU1OhUqlw5MgR47rZs2cjOjoae/fuNfscra2tcHd3x4kTJ7BgwYK7Hler1VCr1Sbby2QyKJVKuLm5WfJyOLH1UDGyzlZi6cxgvJ0azXU5hIw6w2f6Xp9Ri86QNBoNCgoKkJSUdPsAAgGSkpKQl5dndp+8vDyT7QEgOTm53+01Gg0++OADuLu7Iyoqyuw2mZmZcHd3Ny4ymcySl8E5w2Xb0eKbaO3q5rgaQvjDokBqbGyEVquFv7+/yXp/f38oFOb71igUikFtf+TIEbi4uEAqleLtt9/G8ePH4ePjY/aYGRkZUCqVxqWqyrrGrY4a545wfxeoe3T490W64ZYQA958yzZv3jxcuHABubm5WLRoEVasWNFvu5REIoGbm5vJYk0Yhukzwy31SSLEwKJA8vHxgVAoRF1dncn6uro6BAQEmN0nICBgUNs7Oztj0qRJmD17Nv7617/CwcEBf/3rXy0pz6qkzAyGg4DBhaoW/LeujetyCOEFiwJJLBYjJiYG2dnZxnU6nQ7Z2dlITEw0u09iYqLJ9gBw/Pjxfrfve9y+Dde2xsdFgvlT/ADohyUhhAzhki09PR0ffvghPv30U1y7dg3PP/88VCoV1q1bBwBYs2YNMjIyjNu/8MILOHbsGN566y2UlJRg586dyM/PR1paGgBApVJh69atOHPmDCoqKlBQUIBf/vKXqKmpwfLly0foZfKT4bLtUFENumkSAELgYOkOqampaGhowPbt26FQKBAdHY1jx44ZG64rKyshENzOuTlz5iArKwvbtm3D1q1bERYWhsOHDyMiIgIAIBQKUVJSgk8//RSNjY3w9vZGXFwcvv/+e0yfPn2EXiY//WKyL3xcJGhsV+NUST0WTjd/2UuIvbC4HxIfDbaPAx9lHr2G90//jKSp/vhobSzX5RAyKkalHxIZectj9TfcniqtR31bF8fVEMItCiSOTfJzxawQD2h1LA4X0Q23xL5RIPGAYRKAz/OraRIAYtcokHjgkRmBkIoEuF7fjqKqFq7LIYQzFEg84CoV4eHIQADUc5vYNwoknjBMAvDvi7Xo1NAkAMQ+USDxRMJ4L4R4OaFd3YP/XL7JdTmEcIICiScEAgbLe8fc/pxuJSF2igKJR5bFjAPDAGd+bkLlLZoEgNgfCiQeCfJwxP2T9GNAfVFAZ0nE/lAg8YzhhtsvCqqhpUkAiJ2hQOKZh6b5w91RhFplF3683sh1OYSMKQoknpGKhEiJDgIAHCigPknEvlAg8ZDhVpJvrijQ0qHhuBpCxg4FEg9ND3LD1EA3aHp0+IomASB2hAKJh/STAOj7JNGtJMSeUCDx1OPRwRAJGRTXKHG1tpXrcggZExRIPOXlLMZD0/TDAh+gPknETlAg8ZihcftwUQ00PTQJALF9FEg8NjfMFwFuUjR3dCP7Wt29dyDEylEg8ZhQwOCJWcEA6IZbYh8okHjOcNn23X8boFDSJADEtlEg8dx4H2fEy72gY4GDRdQFgNg2CiQr8GSfPkk0CQCxZRRIVmBJZCCcxEKUNaqQX9HMdTmEjBoKJCvgLHHAIzP0kwB8fp4at4ntokCyEoZxkr4uvgmVuofjaggZHRRIViIm1BMTfJzRodHi62KaBIDYJgokK8EwTJ/GbbpsI7aJAsmKLJs1DgIGOF/ejJ8b2rkuh5ARR4FkRfzdpHgw3BeAfsxtQmwNBZKVMTRuf1lYjR4t3XBLbAsFkpVZMNUfXs5i1LWq8f1PNAkAsS0USFZG7CBASrT+hlsaJ4nYGgokK7S899u241fr0KSiSQCI7aBAskJTA90QGeyObi2Lw0U1XJdDyIihQLJShkkAPs+vohtuic2gQLJSj0UFQ+wgQImiDVdoEgBiI4YUSHv27IFcLodUKkVCQgLOnTs34PYHDhzAlClTIJVKERkZiaNHjxof6+7uxiuvvILIyEg4OzsjKCgIa9asQW0tzUc2EHcnEZKnBwCg0SSJ7bA4kPbv34/09HTs2LEDhYWFiIqKQnJyMurr681un5ubi5UrV+KZZ55BUVERUlJSkJKSgsuXLwMAOjo6UFhYiNdeew2FhYU4ePAgSktL8dhjjw3vldkBw2Xb4aIadHVrOa6GkOFjWAsbIBISEhAXF4fdu3cDAHQ6HWQyGTZt2oRXX331ru1TU1OhUqlw5MgR47rZs2cjOjoae/fuNfsc58+fR3x8PCoqKhASEnLX42q1Gmq12vhza2srZDIZlEol3NzcLHk5Vk2rYzH3j6dQ09KJd1fOxKNRQVyXRIhZra2tcHd3v+dn1KIzJI1Gg4KCAiQlJd0+gECApKQk5OXlmd0nLy/PZHsASE5O7nd7AFAqlWAYBh4eHmYfz8zMhLu7u3GRyWSWvAybIRQwWEaTABAbYlEgNTY2QqvVwt/f32S9v78/FAqF2X0UCoVF23d1deGVV17BypUr+03SjIwMKJVK41JVZb8fxidj9GH8w/VG1LR0clwNIcPDq2/Zuru7sWLFCrAsi/fee6/f7SQSCdzc3EwWexXi7YTECd5gWeBLuuGWWDmLAsnHxwdCoRB1daaTFtbV1SEgIMDsPgEBAYPa3hBGFRUVOH78uF2HjKVWxOkbt78oqIZOR32SiPWyKJDEYjFiYmKQnZ1tXKfT6ZCdnY3ExESz+yQmJppsDwDHjx832d4QRj/99BNOnDgBb29vS8qye4umB8JV4oDKpg6cLWviuhxChsziS7b09HR8+OGH+PTTT3Ht2jU8//zzUKlUWLduHQBgzZo1yMjIMG7/wgsv4NixY3jrrbdQUlKCnTt3Ij8/H2lpaQD0YfTkk08iPz8fn332GbRaLRQKBRQKBTQauk9rMBzFQjzS+w0bjSZJrBo7BO+++y4bEhLCisViNj4+nj1z5ozxsQcffJBdu3atyfaff/45Gx4ezorFYnb69Ons119/bXysrKyMBWB2OXXq1KDqUSqVLABWqVQO5eXYhMKKJjb0lSPs5G1H2dZODdflEGJisJ9Ri/sh8dFg+zjYMpZl8dDbp3G9vh2ZT0RiZfzd/bcI4cqo9EMi/MUwjMkNt4RYIwokG7J05jgIBQyKKltwvb6N63IIsRgFkg3xdZVg/hQ/AMCBfOqTRKwPBZKNWR6jv2z7srAG3TQJALEyFEg2Zt4UP/i4iNHYrkZOaQPX5RBiEQokGyMSCvDELJrhllgnCiQbZLhsO1lSj4Y29T22JoQ/KJBsUJi/K6JlHujR0SQAxLpQINkowwy3NAkAsSYUSDbqkahASEUC/FTfjovVSq7LIWRQKJBslJtUhMURgQCo5zaxHhRINswww+2/L9SiU0OTABD+o0CyYbPHe0Pm5Yg2dQ++uWJ+yGBC+IQCyYYJBAyWx9xu3CaE7yiQbNyymHFgGCD3xi1UNXVwXQ4hA6JAsnHBHo64f5IPAOAATQJAeI4CyQ4s7+2T9CVNAkB4jgLJDiyc5g83qQNqWjqRe+MW1+UQ0i8KJDsgFQnxeDTNcEv4jwLJThhuJTl2RQFlRzfH1RBiHgWSnYgIdsOUAFdoenT46lIt1+UQYhYFkp1gGMbYuE3jJBG+okCyIynRQRAJGVyqVqJE0cp1OYTchQLJjni7SJA01R8ATQJA+IkCyc4Ybrg9VFQDTQ9NAkD4hQLJzswN84WfqwRNKg1OltRxXQ4hJiiQ7IyDUIBlMYYZbumyjfALBZIdMkwCkFNaj7rWLo6rIeQ2CiQ7NMHXBbGhntCxwMFCmgSA8AcFkp1a0adPEk0CQPiCAslOPTwjEE5iIX5uVKGwspnrcggBQIFkt1wkDng4sncSgPPUuE34gQLJjhku245cqoVK3cNxNYRQINm1OLkn5N5OUGm0OFp8k+tyCKFAsmcmN9zS8LaEByiQ7NyyWeMgYIBzZU0oa1RxXQ6xcw5cF0BGEMsC3Z1AdwegaQc0HYBGBXSr9H9qetd3967XqBCgUeFTrwq0trag57P3gElTAK8JgNd4/Z8eoYBIyvUrI3aCAokLOt3tULgzLDQqCwLl7oABLO9T9AAACAE0Azj//R2PMoBbcG9AjQc8x98OLM/xgNRt2G8HIQZDCqQ9e/bgT3/6ExQKBaKiovDuu+8iPj6+3+0PHDiA1157DeXl5QgLC8Mbb7yBhx9+2Pj4wYMHsXfvXhQUFKCpqQlFRUWIjo4eSmkjS9sz4NmFZeHRZ/vuMZgfzcEREDsDYidA7AKInHp/7l1EvevFTuhxcMKbp6rQpgHmBagxQVgPb00tXFSVEHa3Aa3V+qX8zrAC4OTTG1B9zqoMoeXkBTDM6L9WYjMsDqT9+/cjPT0de/fuRUJCAnbt2oXk5GSUlpbCz8/vru1zc3OxcuVKZGZm4pFHHkFWVhZSUlJQWFiIiIgIAIBKpcL999+PFStWYP369cN/Vf3RdgM5mWYCpZ+A0apHrxYAANMnHJz7DQz9z86DDhiInACBcNBVOADoab2Kz34ow2cmd5Kw8EQbpkkaEeXchMniW5ALFAjQ3oRHVzUk6iago1G/VJ+7+8AStzvOqvqElksAIKAmTGKKYS28byAhIQFxcXHYvXs3AECn00Emk2HTpk149dVX79o+NTUVKpUKR44cMa6bPXs2oqOjsXfvXpNty8vLMX78eIvPkFpbW+Hu7g6lUgk3twEuIXRa4Hdegz6uESO8IxyGERZ9txc58uYMokPTg39frEVZYweqmztQ3dyJ6uZONLb3H8ou6EAoU49wcSMiHRsxyaEBMtTBt7sGLup7DG3iIO0NKsNZlfx2YLmHAEJqTbAlg/2MWvSvrtFoUFBQgIyMDOM6gUCApKQk5OXlmd0nLy8P6enpJuuSk5Nx+PBhS57ahFqthlp9+4PS2jrI4VgFQiAxDRCKB3k20rsIxbwJjtHiJHZAalzIXes7NVrUtHSahNTtv0twpd0JV9RyHLojtyTQQMbUI5SpQ7ioAVMltzBBWIdAnQKempsQ9HQBDdf0y50EDoC7zPyloKecGtltmEWB1NjYCK1WC39/f5P1/v7+KCkpMbuPQqEwu71CobCw1NsyMzPx29/+dmg7J/+/IT+vPXIUCzHJzwWT/FzMPt7VbQgsc6Hliuy2cchWA+gTWEJoEcQ0Qs7UIZSpwySHBkwWNSCEqYNfTy1EOg3QXKZfbmTf8YwM4BZ091mVIbCokd2qWeV5cUZGhslZV2trK2QyGYcV2S+pSIiJvi6Y6GtpYHmjpDkE37epAS2MgcVABz+0QM7UIURQBzmjwERhAyY61CNYdxNObAfQWqNfBmxkv6OB3Ws84ORt82e61s6iQPLx8YFQKERdnWn7QF1dHQICAszuExAQYNH2gyGRSCCRSIa8Pxk7Qw+sCchp7kRDmxroQW9g6RvZ5UwdQpg6Y2hNFNQjVFAPT7bl3o3sd55VGULLNZAa2XnAokASi8WIiYlBdnY2UlJSAOgbtbOzs5GWlmZ2n8TERGRnZ2Pz5s3GdcePH0diYuKQiya2Y6iBVdncgVxDYPVyQQdCetut+oaWXFCHQOYWoG4FFJf0y50cpLfDytjY3hta1Mg+Zix+l9PT07F27VrExsYiPj4eu3btgkqlwrp16wAAa9asQXBwMDIzMwEAL7zwAh588EG89dZbWLJkCfbt24f8/Hx88MEHxmM2NTWhsrIStbX6GVVLS0sB6M+uhnMmRazfUAPrTHMHvugTWBJoMI5pgJxRIJSpR2ifP8cxjRD1dAENJfrlDiwjBOMh0weWR6j+T0854BmqDy9HT7oUHCEWB1JqaioaGhqwfft2KBQKREdH49ixY8aG68rKSgj6nPrOmTMHWVlZ2LZtG7Zu3YqwsDAcPnzY2AcJAL766itjoAHAU089BQDYsWMHdu7cOdTXRuzAYAKr1hhYt0PrQu+f9W3quxrZ71yk6Aaay/WLOWLXPgElNw0ujxD6VtACFvdD4qNB90Mi5A79BZbhz4a2TvihBSFMPUKYesgE9ZAx9QgXNyGEqYd7T+O9n8Q1sJ+zK7nddBAdlX5IhNgaqUiICb4umDDAGVZ1cycu1yhxrrwJR8ub8N+6dqBb/7jhUnCGczNme7VjurQZIYJ6uHbWgGmuADRtQNtN/VJppq+eUKI/i+obUsbgCgWk7qP10nmJzpAIsVBLhwb55c04X9GE/PJmXKpuQbfW9GPkInHArBAPzB0nQIJnGyZLbkHcWnX70q+lAmipAljtwE/m6Nn/2ZW7DBCKRuU1jrTBfkYpkAgZpq5uLS5WteB8eRPOlzejsKIZbXcMCSwSMogMdkec3Atxci/EhHrCUyrQ37TcXHE7pAyB1Vyh774wEEYAuI3rDSjD2ZX8dnA5+/CmsZ0CiRCOaHUsShStyC9vxrnyJpwva0J92933BIb5uSBuvBfi5J6Ik3sh2MMRTN8AUbcBLZWmIdU3uHruMcmnyKn/syuPUP2tUmOEAokQnmBZFtXNnThX1oT8iiacK2vCjYa7R+cMdJciVu6FeLknYuVemOzvCoGgnzMclgXa6/o/u2qtwT3HxnL266ftSq6/PceCESPuhQKJEB671a5GQUWz8TLvco0SPTrTj6Kr1AGxofpwih/vhchgd0hFgwyJHrW+jaql3PTsyvB3tXLg/QUiwENm/uzKU65v27IABRIhVqRD04MLVS36xvLyJhRWNEOlMW3wFgsFiJK5I1auv8yLCfWCu+MQG7U7m01Dqu8ZVksVoOseeH+J++22qxlPAVMfGXBzCiRCrFiPVodrN9twvtxwmdd819hUDANM9ndFbG8bVJzcC0EejsN/cp0WaK29+zLQEFztd4x1lZwJJP7PgIekQCLEhrAsi4pbHThX3oT8cn13g5/NzBIT7OGIOPnty7xJvi79t0MNlUbV29jeG1LjHwD8pw+4CwUSITauoU2Nggp9G9T58iZcqW2F9o52KA8nkbEdKk7uichgD4gdxr5nOAUSIXZGpe5BUWWL8TKvsKIFnd2m7VASBwGiZB6Il3shVu6JWaGecJOOfudKCiRC7Fy3Voerta293+TpL/NuqTQm2wgYYEqAm74v1Hh9O5S/28jfDEyBRAgxwbIsfm5UIb9c30ieX9GEilt3T8kl83I0NpLHyT0x0dfFtMPmEFAgEULuqb61y9gGdb68CddutuKOZih4Ool6O2zqL/Migt0hElrWDkWBRAixWFtXt7Ed6nx5E4oqW6Du0ZlsIxUJMFPmabzMmxniCRfJwAOHUCARQoZN06PD5VqlyWVeS4dpp8kNcydg68NTBzwOjYdECBk2sYMAs0I8MSvEExvmAjodixsN7Thf3qwPqfImxMmHMPlqPyiQCCGDJhAwCPN3RZi/K/6/BP3EoiN5kWX7Y2cSQkbVcL+B64sCiRDCGxRIhBDeoEAihPAGBRIhhDcokAghvEGBRAjhDZvoh2ToB9Ha2spxJYQQcwyfzXv1WbKJQGprawMAyGQyjishhAykra0N7u79z8ZrE/ey6XQ61NbWwtXV9Z6dtFpbWyGTyVBVVUX3vfVB70v/6L0xz5L3hWVZtLW1ISgoCAJB/y1FNnGGJBAIMG7cOIv2cXNzo18uM+h96R+9N+YN9n0Z6MzIgBq1CSG8QYFECOENuwskiUSCHTt2QCKRcF0Kr9D70j96b8wbjffFJhq1CSG2we7OkAgh/EWBRAjhDQokQghvUCARQniDAokQwht2F0h79uyBXC6HVCpFQkICzp07x3VJnDt9+jQeffRRBAUFgWEYHD58mOuSeCEzMxNxcXFwdXWFn58fUlJSUFpaynVZnHvvvfcwY8YMYw/txMRE/Oc//xmRY9tVIO3fvx/p6enYsWMHCgsLERUVheTkZNTX13NdGqdUKhWioqKwZ88erkvhle+++w4bN27EmTNncPz4cXR3d2PhwoVQqVRcl8apcePG4Q9/+AMKCgqQn5+P+fPn4/HHH8eVK1eGf3DWjsTHx7MbN240/qzVatmgoCA2MzOTw6r4BQB76NAhrsvgpfr6ehYA+91333FdCu94enqyH3300bCPYzdnSBqNBgUFBUhKSjKuEwgESEpKQl5eHoeVEWuhVCoBAF5eIzcxorXTarXYt28fVCoVEhMTh308m7jbfzAaGxuh1Wrh7+9vst7f3x8lJSUcVUWshU6nw+bNm3HfffchIiKC63I4V1xcjMTERHR1dcHFxQWHDh3CtGnThn1cuwkkQoZj48aNuHz5Mn744QeuS+GFyZMn48KFC1Aqlfjiiy+wdu1afPfdd8MOJbsJJB8fHwiFQtTV1Zmsr6urQ0BAAEdVEWuQlpaGI0eO4PTp0xaPu2WrxGIxJk2aBACIiYnB+fPn8c477+D9998f1nHtpg1JLBYjJiYG2dnZxnU6nQ7Z2dkjcu1LbA/LskhLS8OhQ4dw8uRJjB8/nuuSeEun00GtVg/7OHZzhgQA6enpWLt2LWJjYxEfH49du3ZBpVJh3bp1XJfGqfb2dly/ft34c1lZGS5cuAAvLy+EhIRwWBm3Nm7ciKysLPzrX/+Cq6srFAoFAP3Ih46OjhxXx52MjAwsXrwYISEhaGtrQ1ZWFnJycvDNN98M/+DD/8LPurz77rtsSEgIKxaL2fj4ePbMmTNcl8S5U6dOsQDuWtauXct1aZwy954AYD/++GOuS+PUL3/5SzY0NJQVi8Wsr68vu2DBAvbbb78dkWPTeEiEEN6wmzYkQgj/USARQniDAokQwhsUSIQQ3qBAIoTwBgUSIYQ3KJAIIbxBgUQI4Q0KJEIIb1AgEUJ4gwKJEMIb/z9PO4bpJ/yBIwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:17:32.361314Z", + "iopub.status.busy": "2024-03-24T03:17:32.361017Z", + "iopub.status.idle": "2024-03-24T03:26:06.811747Z", + "shell.execute_reply": "2024-03-24T03:26:06.810933Z" + }, + "papermill": { + "duration": 514.470746, + "end_time": "2024-03-24T03:26:06.814392", + "exception": false, + "start_time": "2024-03-24T03:17:32.343646", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:26:06.850502Z", + "iopub.status.busy": "2024-03-24T03:26:06.850098Z", + "iopub.status.idle": "2024-03-24T03:26:06.872575Z", + "shell.execute_reply": "2024-03-24T03:26:06.871609Z" + }, + "papermill": { + "duration": 0.042663, + "end_time": "2024-03-24T03:26:06.874509", + "exception": false, + "start_time": "2024-03-24T03:26:06.831846", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tvae0.00.236190.0054278.2797940.0781191.5463950.1279340.0002810.8866980.0455485041099.50.073670.2377460.00059219.166492
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "tvae 0.0 0.23619 0.005427 8.279794 0.078119 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "tvae 1.546395 0.127934 0.00028 10.886698 0.045548 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "tvae 5041099.5 0.07367 0.237746 0.000592 19.166492 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:26:06.908456Z", + "iopub.status.busy": "2024-03-24T03:26:06.907478Z", + "iopub.status.idle": "2024-03-24T03:26:07.304238Z", + "shell.execute_reply": "2024-03-24T03:26:07.303246Z" + }, + "papermill": { + "duration": 0.416049, + "end_time": "2024-03-24T03:26:07.306448", + "exception": false, + "start_time": "2024-03-24T03:26:06.890399", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:26:07.341789Z", + "iopub.status.busy": "2024-03-24T03:26:07.341021Z", + "iopub.status.idle": "2024-03-24T03:34:57.009575Z", + "shell.execute_reply": "2024-03-24T03:34:57.008726Z" + }, + "papermill": { + "duration": 529.688823, + "end_time": "2024-03-24T03:34:57.012159", + "exception": false, + "start_time": "2024-03-24T03:26:07.323336", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../treatment/_cache_aug_test/tvae/all inf False\n", + "Caching in ../../../../treatment/_cache_bs_test/tvae/all inf False\n", + "Caching in ../../../../treatment/_cache_synth_test/tvae/all inf False\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:34:57.047566Z", + "iopub.status.busy": "2024-03-24T03:34:57.047243Z", + "iopub.status.idle": "2024-03-24T03:34:57.073971Z", + "shell.execute_reply": "2024-03-24T03:34:57.073209Z" + }, + "papermill": { + "duration": 0.046825, + "end_time": "2024-03-24T03:34:57.076280", + "exception": false, + "start_time": "2024-03-24T03:34:57.029455", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:34:57.111180Z", + "iopub.status.busy": "2024-03-24T03:34:57.110402Z", + "iopub.status.idle": "2024-03-24T03:34:57.116376Z", + "shell.execute_reply": "2024-03-24T03:34:57.115463Z" + }, + "papermill": { + "duration": 0.025288, + "end_time": "2024-03-24T03:34:57.118444", + "exception": false, + "start_time": "2024-03-24T03:34:57.093156", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tvae': 0.41946677415673767}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:34:57.154111Z", + "iopub.status.busy": "2024-03-24T03:34:57.153283Z", + "iopub.status.idle": "2024-03-24T03:34:57.507059Z", + "shell.execute_reply": "2024-03-24T03:34:57.506179Z" + }, + "papermill": { + "duration": 0.373956, + "end_time": "2024-03-24T03:34:57.509090", + "exception": false, + "start_time": "2024-03-24T03:34:57.135134", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABApUlEQVR4nO3deXRb9Zn4//fVvtiS9y2x4+wbkIQlECiEAGUdIEOXMG0hKQ3QaShlMnRKes5AgXKgM0ChDJPT3wwk5VAopQ10CmX7UiCl7CQhIYEkJI7j2HG8W9Zibff+/riyEhM7sWTZV7ae1zk+jq+upEeK9fhzP8vzUTRN0xBCCAOZjA5ACCEkEQkhDCeJSAhhOElEQgjDSSISQhhOEpEQwnCSiIQQhpNEJIQwnCQiIYThJBEJIQwniUhk1DvvvMPPfvYzurq6jA5FjCGSiERGvfPOO9x5552SiERKJBEJIQwniUhkzM9+9jN+/OMfAzB58mQURUFRFPLy8liyZMlR56uqyoQJE/j617+ePHb//fdz5plnUlxcjNPp5JRTTuEPf/jDgM/35JNPcsopp+B0OikqKuLqq6+moaFhZF6cGFGKlAERmbJ161buu+8+nn76aX75y19SUlICwJ49e7jrrrtobGykoqIief7GjRtZvHgxzz77bDIZVVdXc8UVVzBnzhwikQi/+93v+OCDD3jhhRe47LLLkve95557+Pd//3e++c1vsnjxYlpbW3nkkUfIy8tj8+bNFBQUjOprF8OkCZFB//mf/6kBWl1dXfLYzp07NUB75JFH+p37gx/8QMvLy9OCwWDy2JH/1jRNi0Qi2gknnKCdd955yWP79u3TzGazds899/Q7d9u2bZrFYjnquMh+cmkmRtyMGTOYP38+zzzzTPJYPB7nD3/4A5dffjlOpzN5/Mh/d3Z20t3dzdlnn82mTZuSxzds2ICqqnzzm9+kra0t+VVRUcH06dN54403RueFiYyxGB2AyA3Lli3jpz/9KY2NjUyYMIE333yTlpYWli1b1u+8F154gZ///Ods2bKFcDicPK4oSvLfu3fvRtM0pk+fPuBzWa3WkXkRYsRIIhKjYtmyZaxZs4Znn32WW265hd///vd4vV4uvvji5Dl/+9vfuOKKKzjnnHP47//+byorK7Faraxbt46nnnoqeZ6qqiiKwksvvYTZbD7qufLy8kblNYnMkUQkMurIlsuRJk+ezMKFC3nmmWe46aab2LBhA0uXLsVutyfP+eMf/4jD4eCVV17pd3zdunX9Hmvq1KlomsbkyZOZMWPGyLwQMaqkj0hklNvtBhhwQuOyZct47733ePzxx2lrazvqssxsNqMoCvF4PHls3759PP/88/3Ou+qqqzCbzdx5551oXxr01TSN9vb2zLwYMWpk+F5k1IcffsjChQu59NJLufrqq7FarVx++eW43W4OHDhATU0NeXl5WK1Wmpub+/Xn/PWvf+X888/n7LPP5lvf+hYtLS08+uijVFRUsHXr1n5J57777mPNmjWceeaZLF26lPz8fOrq6njuuee44YYbuPXWW414+SJdBo7YiXHq7rvv1iZMmKCZTKajhvLPOussDdBWrlw54H0fe+wxbfr06ZrdbtdmzZqlrVu3Trvjjju0gX5V//jHP2pf+cpXNLfbrbndbm3WrFnaqlWrtJ07d47USxMjRFpEQgjDSR+REMJwkoiEEIaTRCSEMJwkIiGE4SQRCSEMJ4lICGG4Mb3EQ1VVmpqayM/PH3RpgRDCOJqm0dPTQ1VVFSbT4O2eMZ2ImpqaqK6uNjoMIcRxNDQ0MHHixEFvH9OJKD8/H9BfpMfjMTgaIcSX+Xw+qqurk5/VwYzpRNR3OebxeCQRCZHFjtd1Ip3VQgjDSSISQhhOEpEQwnBjuo9oKDRNIxaL9Su2JYbGbDZjsVhkaoQYceM6EUUiEQ4ePEgwGDQ6lDHL5XJRWVmJzWYzOhQxjo3bRKSqKnV1dZjNZqqqqrDZbPKXPQWaphGJRGhtbaWuro7p06cfc0KaEMMxbhNRJBJBVVWqq6txuVxGhzMmOZ1OrFYr9fX1RCIRHA6H0SGNij2tfjbVd1LucXDWtBLMJvkDNtLGbSLqI3/FhyfX3r+OQIS/bD1ITNU40Bkiz2Hh5JpCo8Ma98Z9IhIiFR/UdRBTNaxmha9ML+WkCV6jQ8oJkoiESAjH4uw+1APA106ZSKXXeZx7iEzJrXa3OEptbS0PPfSQ0WFkhYaOIDFVo9BlpcKTG/1h2UISkRAJDR0hACYVu1EUhUhM5d097Ty3+QCqKpvdjCS5NBsHIpGIzPPJgMoCB75eN7Ul+m61FpPCpv2dRGIqbYEwZfnSShopOdkiisTUQb9icXXI50aHcG46zj33XG666SZuuukmvF4vJSUl/Pu//3typ9Pa2lruvvturr32WjweDzfccAMAb7/9NmeffTZOp5Pq6mpuvvlmAoFA8nFbWlq4/PLLcTqdTJ48md/+9rdpxTdezarwcOX8CUxOJCKTSaEs3w5Aiy9sZGjjXk62iB5944tBb5tc4mbpggnJn/+/jXuIxgdulk8sdPKNUw8XZnv873WEIv2XkvzLV2ekFeNvfvMbvve97/HBBx/w0UcfccMNN1BTU8P1118PwP3338/tt9/OHXfcAcCePXu4+OKL+fnPf87jjz9Oa2trMpmtW7cOgBUrVtDU1MQbb7yB1Wrl5ptvpqWlJa34ckW5x8GBzhAtPb2AjKCNlJxMRGNBdXU1v/zlL1EUhZkzZ7Jt2zZ++ctfJhPReeedx7/+678mz1+5ciXf/va3ueWWWwCYPn06v/rVr1i8eDFr165l//79vPTSS3zwwQecdtppADz22GPMnj171F9bNuoMRDCZFDyO/mvryjzSIhoNOZmIVi2ZNuhtX55Ee8M5Uwc998srRq47a/JwwurnjDPO6PeBWLRoEQ888EBy8e6pp57a7/xPPvmErVu39rvc0jQtudRl165dWCwWTjnllOTts2bNoqCgIGMxj2Xv7Gln16EezplRwimTipLHyxP9Qq09YVRVwySzrEdETiYim2XoXWMjde5wud3ufj/7/X5uvPFGbr755qPOrampYdeuXaMV2pjU2tMLQEme/fDBSBBvTx3ueC8B8vH1RilwyaDASMjJRDQWvP/++/1+fu+995g+fTpms3nA808++WR27NjBtGkDt/ZmzZpFLBbj448/Tl6a7dy5k66urozGPRbF4ipdoSgAxX2JqLcbNj2BKezntHY/nxRfij88URLRCMnJUbOxYP/+/axevZqdO3fy9NNP88gjj/CjH/1o0PN/8pOf8M4773DTTTexZcsWdu/ezZ/+9CduuukmAGbOnMnFF1/MjTfeyPvvv8/HH3/MypUrcTpl9nBXKIqm6S1aty2R6Pe8AWE/ACdWuFhe9BkTC+S9GimSiLLUtddeSygUYuHChaxatYof/ehHyWH6gZx00km89dZb7Nq1i7PPPpsFCxZw++23U1VVlTxn3bp1VFVVsXjxYq666ipuuOEGysrKRuPlZLXOQASAIneiVExvN7R+rt84/5+wWO0ogVbo3GdckOOcXJplKavVykMPPcTatWuPum3fvn0D3ue0007j1VdfHfQxKyoqeOGFF/odu+aaa4YV53jQkUhEhX2XXYd2gKZBQTUU1kL5CdC0GVo+g6LMDUiIw6RFJHJeZ/BwiwiA1s/07+UnAOD3TuXzZh+ffPKhnqBExkmLSOS8mRUe8h1WqoucEAmCPzHJs1jv+LcUTqItBGath6jvEFZvhYHRjk+SiLLQm2++aXQIOWVyiTu5rIOWz/RWj7sE7HkA2G1Wep3luIMHCLTso0ASUcbJpZkQR+rar38vrE0eUhQFzaPv2x5u329AUOOfJCKR04KRGA0dQXy9+jwifE36d+/Efucphfqawlhnw2iGlzMkEYmc1tgZ4g8fH+AvWw9CPAaBVv2G/P6XX/YCfRpELNAJ0d7RDnPck0QkclpfS8jrtOpJSI2D1QmOgn7n5eXlETbn0RtTISAVCzJNEpHIad2JpR0epxV6Epdl+ZVHrWj2OK1EnCUocHhUTWSMjJqJnOYLxYBEi6jnkH4w/+hRsSklbqacdiJKfQ/4D41miDlBWkQipyVbRA7r4f6hvKOXvSiKgtJ3PNg+WuHlDElEImdpmoYvmYjMEGzTb3CVDHwHZ6JOUbBjFKLLLZKIRM4KROLEVA1FgXxTL8QioJjAVTTg+X+tj7D1QBc9PT59BrbImNxKRJqm/7KN9lcK65OeeOIJiouLCYf7lyZdunSpLFDNMItJ4YLZ5Zw9vQRzKHG55SoC08A1nzrCCp1xhz5yFpJWUSblVmd1PAp/e2D0n/fsfwXL0ApqfeMb3+Dmm2/m//7v//jGN74B6LtvvPjii8dcWS9S57CaOXFioiB+Q2JDBVfxoOfn2c2ErAVEY5365dmXJj2K9OVWi2gMcDqdfOtb30ruvAHw5JNPUlNTw7nnnmtcYONdINE/5B6kfwjIs1vptXgJx6VFlGm51SIyW/XWiRHPm4Lrr7+e0047jcbGRiZMmMD69etZsWJFv2L6YvhaenoJR1WK3Dbcx+uoBvIcFhqsXiIRVTqsMyy3EpGiDPkSyUgLFixg3rx5PPHEE1x44YVs376dF1980eiwxp0t+7vY3uTjrGklLOxLLMe8NLMQNufrG2f2do9SlLkhtxLRGLJy5UoeeughGhsbueCCC6iurj7+nURKAhF9MmOeOQpRfd97nIWDnp9ntxC2SCIaCdJHlKW+9a1vceDAAf7nf/6H6667zuhwxiV/WN8jzqPpRfKxuY/ZYs5zWFAdXixmBS0ahJhsupgpkoiylNfr5Wtf+xp5eXksXbrU6HDGpUBYbxG51R79wDFaQ6C3iP75/DnMm1yBggK9vpEOMWdIIspijY2NfPvb38Zutx//ZJGSWFwlFNFbRK740BJRkiMx5C+XZxljaCJau3YtJ510Eh6PB4/Hw6JFi3jppZeMDCkrdHZ28txzz/Hmm2+yatUqo8MZlwKJJGQ2KdiiXfpBSUSGMbSzeuLEidx3331Mnz4dTdP4zW9+w5VXXsnmzZuZO3eukaEZasGCBXR2dvKLX/yCmTNnGh3OuJS8LLNbUEJd+sEhJKJ397QTrAszjzAlvV0jF2COMTQRXX755f1+vueee1i7di3vvfdeTieiwfYtE5njcVq5YHa5Xnaovks/OIRE5A/HaIs66CUuLaIMyprh+3g8zrPPPksgEGDRokUDnhMOh/utwfL5pLNQpCfPbtGXd8QisCsxauYsOO793DYzjeY8ohFNElEGGd5ZvW3bNvLy8rDb7Xz/+9/nueeeY86cOQOee++99+L1epNfQ5lbo8mGeMMy7t+/vssrq0MvEXscrsRcomhc5hJlkuGJaObMmWzZsoX333+ff/7nf2b58uXs2LFjwHPXrFlDd3d38quhYfAdFaxWfVlFMCjlGoaj7/3rez/Hi6auEA0dQUK+RDG0IXZUu2xmwma3noiiIX0htRg2wy/NbDYb06bpO2qecsopfPjhhzz88MP8+te/Pupcu90+5KFss9lMQUEBLS16fWGXyyVrtVKgaRrBYJCWlhYKCgowmwcujTFWvbe3nfr2IFcWH2QKpJSI4iY7vWrib3i4Z9D6RWLoDE9EX6aq6lG1eNJVUaHXHu5LRiJ1BQUFyfdxPOkbNctTA/qBviH543DbLKAoBBSXfkASUUYYmojWrFnDJZdcQk1NDT09PTz11FO8+eabvPLKKxl5fEVRqKyspKysjGhUmtCpslqt464l1KdveYdTSy0RuexmHFYzZkc+qhbAFPGPVIg5xdBE1NLSwrXXXsvBgwfxer2cdNJJvPLKK3z1q1/N6POYzeZx+4ESqYvFVXqjeiJyxBKJxO4Z0n3tFjP/fO5U2DEZDm3XW0Ri2AxNRI899piRTy9yVCDRGrIoYIkmEsmXNlQ8Lnu+/l0SUUYYPmomxGjzJ8p/eKxxlHhEP+gYWosoqa8FFZa5bJkgiUjknL6O6kJTYg97qzOlKprv7mnnT5/30NoThrD0EWWCJCKRc8rzHXx1Tjknliamcwyxo7pPMBLjUK+F3lhcLs0yJOuG74UYaV6XFa/LCwf0llGql2Uum4VI36TGSABUFUzyN3045N0TuatviUaKLSK33UzU7CSiApoKMoQ/bNIiEjlnf3sQRYHyQCc2AHtqichls4BiIkhibVrEn3pnt+hHEpHIOW/uaqHdH2G5vYMiSLlF5LLpc9ICOIGI9BNlgFyaiZzjT4yaOeKJS6qU+4gSiUhz6Adk5GzYJBGJnBKNq4SjKiY1hk1NbCE0xFnVfZw2M06bGYvTg6pp0keUAXJpJnJKskQsAcwmRZ8/NIQ6REeyW8x8f/FU2NcMdQf0kTMxLNIiEjml77KswBTStwRyePUdgNNhc+vfJRENmyQikVOCid07PKbEZVmKHdX92PL073JpNmxyaSZySl+LyEuicmeK/UN93t7dRnNjJ0sCYYrtkoiGSxKRyCm1xW5sc0yUNW6FAGm3iPzhKIdCZsJRFSJB0LT0L/GEJCKRW4rcNorcNmhJLHhNcyKi02ZJzK7W9NnV0eDhPiORMukjErmpb9/6NC/NXDYzmmImrCRqqEuH9bBIIhI55YsWP/vb/MRCiUSU5qWZ06pPagwducxDpE0uzUROeWV7M/T6uM4cw2KzHh75SpEzMbs6iAOISItomKRFJHJGJKYSianY436sZkXvH0qzfIerXyJClnkMkyQikTOSWwhpASwmU9r9QwAuqwWnzYzJLpMaM0EuzUTO6JtD5FESc4iGUbrD67Lqyzwa2uGLOukjGiZpEYmcEYh8ORENY1Z1H1nmkRGSiETO6Ls0yx/mrOp+kss8JBENh1yaiZzRt7urK8XdXQfz188P0dbSyUWhKF6LFEcbDklEImfMrsyn2GWlYkdYPzDMRNQditIcMhOOxSEWgXg0pW2JxGFyaSZyRlm+gxPKbOT35YphXpo5rWbiio2olvgYSYd12iQRidzSt3OHzQ3m4V0QOG0WUBR6lb7Z1dJPlC5JRCInaJrGZwd9NB06pJd3zcCuG32TGkOSiIZNEpHICZG4ysufNrNx6xdoGhkZuj+83kxmVw+XJCKREwKJETM3Qb1WdQaG7p3JbYUSiSgqLaJ0yaiZyAl9c4g89A3dFwz7Md02fZmHosqkxuGSRCRyQt/yjjz6alUPv0VU4XXoyzwau2HXTklEwyCXZiInHF7w2repYgaWd/SRIvrDJolI5AR/OIZJjeJQIvqBTCzv6JNcbxbM3GPmGLk0EzkhEI5jj/uxmU1gsYPVkZHHfWnbQXyd7fxDOIbbFJAi+mlKq0W0d+/eTMchxIg6eVIB59Xa8TqtGekf6tMRjNAcshCJq6DGIBbO2GPnkrQS0bRp01iyZAlPPvkkvb29mY5JiIyr9DqZ4VVx2SwZGTHr47KZUU0Wwn0XF9JhnZa0EtGmTZs46aSTWL16NRUVFdx444188MEHmY5NiMzqW96Rwf4hp1VPQBGTFNEfjrQS0fz583n44Ydpamri8ccf5+DBg3zlK1/hhBNO4MEHH6S1tTXTcQqRtmhcZXtTN61trWhoGR0x65vUeHg3D2kRpWNYo2YWi4WrrrqKZ599ll/84hd88cUX3HrrrVRXV3Pttddy8ODBTMUpRNp8oSivbj/Ep3sbUFAy2keUXG9GYn+zqIycpWNYieijjz7iBz/4AZWVlTz44IPceuut7Nmzh9dee42mpiauvPLKTMUpRNr6lnfkk5mCaEdKrjdTEqNwcmmWlrSG7x988EHWrVvHzp07ufTSS3niiSe49NJLMSW2Zpk8eTLr16+ntrY2k7EKkRZ/OIaixXHRC1gz2kfktlv0VlFcSsYOR1qJaO3atVx33XWsWLGCysrKAc8pKyvjscceG1ZwQmRCIBLDFg9gMylgsmR0j/rJJW5uXDwVmnpg53ZJRGlKKxG99tpr1NTUJFtAfTRNo6GhgZqaGmw2G8uXL89IkEIMhz8cwx7rwWYx6f1DIzHhUJZ5DEtafURTp06lra3tqOMdHR1Mnjx52EEJkUmBcAx7zJ9IRBlcY3Ykm0v/Lss80pJWi0jTtAGP+/1+HI7MTJ0XIlP8vTF9eYc1M3WIvuz5zY309nSwNBbHocgyj3SklIhWr14NgKIo3H777bhcruRt8Xic999/n/nz5w/58e699142bNjA559/jtPp5Mwzz+QXv/gFM2fOTCUsIY7pK9NLiEctuIOWEWkRtfnD+HvNRFUVh0WFaOhwC0kMSUqJaPPmzYDeItq2bRs2my15m81mY968edx6661Dfry33nqLVatWcdpppxGLxfjpT3/KhRdeyI4dO3C7M9ehKHLbxEIXuKMQMWd0DlEfl81CT6+FiGIHVL3DWhJRSlJKRG+88QYA3/3ud3n44YfxeIb3n/ryyy/3+3n9+vWUlZXx8ccfc8455wzrsYXop9enfx+BFlHfpMaw4gQCiQ7r0ow/z3iWVh/RunXrMh0HAN3d+lqgoqKiAW8Ph8OEw4dXN/t8vhGJQ4wf/nCM+jY/E7va8NrNI9JH1LfMo9eUSEQyuzplQ05EV111FevXr8fj8XDVVVcd89wNGzakHIiqqtxyyy2cddZZnHDCCQOec++993LnnXem/Ngid7X4enljax3ntPdw0sSiEUlERy3zkLlEKRtyIvJ6vSiJkQCvN/PN21WrVvHpp5/y9ttvD3rOmjVrkh3moLeIqqurMx6LGD/84Rj2eI9eEM2eB6bMFyU9alshmUuUsiEnoiMvxzJ9aXbTTTfxwgsvsHHjRiZOnDjoeXa7HbvdntHnFuObfxTmEPUt89Bwg4q0iNKQVh9RKBRC07Tk8H19fT3PPfccc+bM4cILLxzy42iaxg9/+EOee+453nzzTZkMKTIuEI7jiPn0FpGzcESeY3alh9mVHjgYgM+3SiJKQ1rt1CuvvJInnngCgK6uLhYuXMgDDzzAlVdeydq1a4f8OKtWreLJJ5/kqaeeIj8/n+bmZpqbmwmFQumEJcRRAv2WdxSM7JMli+jLpVmq0q7QePbZZwPwhz/8gYqKCurr63niiSf41a9+NeTHWbt2Ld3d3Zx77rlUVlYmv5555pl0whLiKD3h2BEtooKRfTLZzSNtaV2aBYNB8vPzAXj11Ve56qqrMJlMnHHGGdTX1w/5cQZbKiJEpvS1iKwW84i1iFRV44+bDhAN+fimqmKJBkFVR6RjfLxKu3j+888/T0NDA6+88kqyX6ilpWXYkxyFyKRL5pQwpxgcFvOItYhMJoWWnjCHQmaiKvpaM5lLlJK0EtHtt9/OrbfeSm1tLaeffjqLFi0C9NbRggULMhqgEMMxyR2nLM+O2WoD68gtu3BazaCYEss8kA7rFKV1afb1r3+dr3zlKxw8eJB58+Ylj59//vn84z/+Y8aCE2LYQp36d4d3RFfEO21mukNRwiYnEE10WJeP2PONN2nv9FpRUUFFRUW/YwsXLhx2QEJkSrs/TFdDI+XhGHklIzN036f/ejOfXJqlKK1EFAgEuO+++3j99ddpaWlBVdV+t8tOsCIb7GsPsm/XPk7SQkyfWjCiz9U3u7pXcYCGXJqlKK1EtHLlSt566y2uueYaKisrk0s/hMgmgcTQvdU28kP3Lpv+UQom15vJXKJUpJWIXnrpJV588UXOOuusTMcjRMb4wzEcsR5srpGfzOiym3HbzWiWPAgjLaIUpZWICgsLBy3VIUS28IeieOI+7Bb7iLeITq4p5OSaQmgOwWebJBGlKK3h+7vvvpvbb7+dYFA65ET2CgR7MKvRkS2a/2XJ2dWSiFKRVovogQceYM+ePZSXl1NbW4vVau13+6ZNmzISnBDpUlWNWEAfure5vGC2HuceGSKJKC1pJaKlS5dmOAwhMssfiWGP+jApYMsb2aF7gN5onD9/0kSs18/VaCjREKhxMJlH/LnHg7QS0R133JHpOITIKKfVzEVTbJjNeSiu4hF/PotJ4UBnCDQTMYuCVUFvFY1Asf7xKO1VeV1dXfzv//4va9asoaOjA9AvyRobGzMWnBDpsppNlFuClOTZwTnyAysWs0nvi1IUoianflAuz4YsrRbR1q1bueCCC/B6vezbt4/rr7+eoqIiNmzYwP79+5O1ioQwVEj/A4lrdEZ4XTYzkZhK2OTEFQ/J7OoUpNUiWr16NStWrGD37t39dna99NJL2bhxY8aCEyJdda1+mpub6I3GR6VFBIdnV4dNUrs6VWklog8//JAbb7zxqOMTJkygubl52EEJMVzb6w9S19xBV29s5AuiJSS3FUoW0ZdLs6FKKxHZ7fYB9xTbtWsXpaWysZwwXqSnHQCrq2DUhu4PL/OQRJSqtBLRFVdcwV133UU0GgVAURT279/PT37yE772ta9lNEAh0hH364nIlj96KwDcfcs8bHn6Abk0G7K0EtEDDzyA3++ntLSUUCjE4sWLmTZtGvn5+dxzzz2ZjlGIlERiKkqvPpnR4Rm9FvqZU0u44ZypzJ1UmQhEWkRDldaomdfr5bXXXuPvf/87n3zyCX6/n5NPPpkLLrgg0/EJkTJ/OIYz1o3ZpGDNKxn9AKSIfspSTkSqqrJ+/Xo2bNjAvn37UBSFyZMnU1FRgaZpUhJEGK6nN4oj2o3dYhq1oft+ZFuhlKV0aaZpGldccQUrV66ksbGRE088kblz51JfX8+KFSukTKzICr5gFEcskYhGaFPFgfT0Rnn2owae3ZqYvxQLQzw2as8/lqXUIlq/fj0bN27k9ddfZ8mSJf1u++tf/8rSpUt54oknuPbaazMapBCpmOpViVa4UUyjsKniESwmU2KZh4ZqM2HSVL1VNErTB8aylFpETz/9ND/96U+PSkIA5513Hrfddhu//e1vMxacOAZfE2x5GjY9AW1fGB1NVnFFO/E6rXiKykd1bzGH1YTZpICiEJFlHilJ6X9p69atXHzxxYPefskll/DJJ58MOyhxHP4W2PJb6NwH3Y3w6R8kGR0p0KZ/H4XFrkdSFOVwEf2+RCTLPIYkpUTU0dFBefngW6SUl5fT2dk57KDEMWga7HoF4jH8zioOOacQV1XY9RLEIkZHlxV27a2jpaeXmHP0R8z6JjX2Kn0tIumwHoqU+oji8TgWy+B3MZvNxGLSOTeiOvZC9wH2d0f5c++JRM0OFrXvZl5xN46DW6A6t7d06o3GqW/YT3skQOEot4hAn9QIEEI2WkxFSolI0zRWrFiB3W4f8PZwOJyRoMQxHNwCgL36ZLTufKyKwheOE3G1vMtczyZME08b0Y0Es50vGMEZ68RqVrDml43687tlmUdaUkpEy5cvP+45MmI2giKBZF9Q+cyFrLAWoWkaT70To7vrPdpaDlLWtR8KJxkcqHF6ujswq1HsDqshc4jyHBby7BY0qxsiyKXZEKWUiNatWzdScYihaP0cNBXyKyCvjMSKJk6eUsbBlsk0de2htHUnSg4nolD3IQBM7mJDyrSeMaWYM6YUQ0sEtiOzq4do9MY2xfC176Guzc9epZpY/PDuuidN9NKTP4VQNE53w6d6h3aO6k0kIosBl2X9SBH9lEgiGiviUSJteznkC/Nqcx6BSDx5k91iZurME6ku9eLSQtCTuzWhYr5WAGyebElEPTn9h2Go0lr0KgzQtZ/OnhC95jw8xZV4nf1r7JwxrRxCs6FtN3TWgafSoECNFe9pwQQ4CgafZjKSQpE4f97aRCTcy7fRUOIxfamH1XH8O+cwaRGNFZ376AhG6HZMYHpF/sDnFE5OnFs/enFlE03jlFKV2RX5FJZWGRKC1azQ2BmiNagR6ysZG+4xJJaxRBLRGBHvasAXiuJzVDKlxD3gOb15E2nzh2lt/CI3F1uGOnEocQrynDg8BpT/QN/Nw27VP1Zhs0s/GD66mqnoTxLRWBCL4G89gKpBPH8iRW7bgKft73WyvV2lsd0PvgOjHGQW8Lfo392lhm5s2DeXKGxK/MGQFtFxSSIaC3qa8IXCRMxuykvLBq35VFPsxueoIhSNE2zZO8pBGq+pcR8NnUHaldEr/TGQvvVmIVNfi0gS0fFIIhoLug8Qi6v4HRVUFw98WQbgsJqxFtYA0NW8b5SCyx5tzQ0c6AzRFDN2d9U8u94iCiiJRCSTGo9LRs3Ggu4DTC7JY+KUU9HKB+moTvBW1MIBCLQ1gKqOahkMo6k+fdqCs8jYEUNXXyIisfBVWkTHlTu/pWOVpoFP38bbWlSjb2t8DOWVE4krVvyBAATbRyPC7BAJEAv5AAVPsTEjZn3y7PoyD/p285DO6uOSFlG2C3Xq5T1MZnAffySoqtDNbnsp5t4mAm31uPNyY5+5cNdBonGNXouXAs+xW40j7ZRJhZwyqRD8+fAh0iIaAmkRZTt/C3tb/bzfYmZPe+i4pzusZswFEwDwHcqd+UQ9bXqrUXWXHbfVOGrsiYQY7YV41NhYspy0iLKdv5mecIxmq4fiIa4UOPmEE8izf4FN6RjZ2LJIsF1PRGZPhcGRHMFi13eZjUf1VpERO4qMEVnyp0MMJuZrJhSJE7CVUO4ZuA7UlxVV1mIzm/SSqdHjt6LGg2inPm/KUTzB4EhAVTWe/aiBde/sI2qRfqKhkESU5YLtjWiA6i5PDgsfl819eBudnoMjFlvWiASYkhdjfnUhs2fMNDoaTCaF9kCErmBU5hINkSSibBbuIejXR4LySipT2rxyb8TDZ80+OppzoJ/I14SiKDgLyvDmG9tR3afvj8bhRCRziY7F0ES0ceNGLr/8cqqqqlAUheeff97IcLKPv4VAOEbI6qXMm9oH7ECsgK5glK5c6LD2Nenf87On4kC+o28ukbSIhsLQRBQIBJg3bx6PPvqokWFkr55mApEYAWsJZUPsH+rjKdNnWIfbD4z7ejhdh/axu6WHL3qNnVF9pL4WUU9f7WrpIzomQ0fNLrnkEi655JIhnx8Oh/sV6Pf5xvd/ruY/hNNqRnGXU5qfWj2b4vJqGlAIBXz6h8DhHaEoDaZpBNoaaPNH6Il4mGZ0PAnJRKT2za4e37+rwzWm+ojuvfdevF5v8qu6utrokEaU4j/E9LJ8/uGsBUcVQjue8sJ8QrZiwjFVX+4xXgXbCYVCqIqFPINnVB8pL3Fp1t2XiHolER3LmEpEa9asobu7O/nV0DCOP2DRXgh16f/OS73aoM1iwuTVP5hd47nDuvsAgXAMv62Ecq/L6GiSPA4reXYLZldi9DIa0is1igGNqQmNdrt90D3Vxh3/IWKqitlVgGJ1pvUQrpJq1Nat+MdxiyjasZ9QJE6Pp5JyT/aUY60ucnH9OVP0H9526H9Yen2QI0tuUjWmWkQ5xX+Izw728Np+aOhIb0sab3kNZpOCNXhIX4k/DgVa9qIBcc/Eoc+zGm19/XO9XYaGkc2y9H9OxHuaCUZidDmKkkPBqZo2aRKmKeWY1Ki+En+8/TUOdRH0daCh4CrL4r3cHF7oOQS93UZHkrUMTUR+v58vvvgi+XNdXR1btmyhqKiImpoaAyMzXrC9EVWDqKss5Y7qPhaLRd/No2s/9DSNv0TU3UAsrhG0l6U8z2o0vLq9mQOdIf7B7aQMpEV0DIYmoo8++oglS5Ykf169ejWgb229fv16g6LKAvEowS69/rKrsCqlGdVHSSQitbsJU+W8DAWYJbr2U1PkYsLEk4nVFBgdzVGCkTjdoSg9bnciEUmLaDCGJqJzzz0XbZxPtktLoJVgb5SoyUlxUfGwHqo+6qWzoRNrz07mzhr6nK0xoUvvhDcX1WK2GFcsfzAep/7x6tISo3mSiAYlndXZqKeZQDhG0FZMmXeYI0H5VYSiKpGu5vFVEyfUiRbqAMUE3olGRzMgj0O/pO6K980lkkQ0GElEWUjtOUQwEiNgK6YsxRnVX1ZeWkrU7KQ3GqO3sylDEWaBjr3Utwd5t93BFx0Ro6MZUF/fXnss8X8Y7dW/xFEkEWUh1ddMhddJYXk1ha70Oqr7OGwWyNcnNo6rlfgddXSHojRoFVm7lM6TSERdUQWsstTjWCQRZRtVxRJqo6bIxXmnnjS8juoER7F+6eJr2T/sx8oKapxoex2hSJwuZzVVBelN+BxpfS2iQDhOzJ5YkNs3W170I4ko2wTbQY2BxXa4uNkwJVfid4yT3V+7D9DdEyBicuIorMSdpRMZHVYzhS4rlV4HUUtieoH0Ew0oO/8Hc5m/GX84hi2vClsGWkMARRU1dALhnna0SBDFlj1rstLSsZeuUJRuZzW1JXlGR3NMK86arP9jzx7o+gJCuVNHPBXSIsoyWs8hdhz08WKdSrs/M4skSwoKsOYVU+y2EelqzMhjGkbT0Fp30h2M0OmoofYYO99mlb7C+UFJRAORRJRlAh2NxFWNsKOUQpctI49pMimceuJcppbmYQ8eyshjGibYTrCrlbCqEMivpaogexa6HovWd5kd6jQ2kCwliSibaFpyWxxHURUmU2YuzQDwJGr1+MZ4Mf3WnZhMCu6KaUwuL8Jizu5f4S9a/Dz+dh0v701MMQj7xtd8rgyRPqJsEuokFAyiKma8mS7ylV+JqmmE2vbj1jTIUP/TqGvbidNqZt4JC5lXlUV7mA3CbFLoDkWxmm36PmexsD5yNt7W/Q1Tdv85yTU9ekd10FpMqSezQ9K9jhI+2NfFtrqD9PrH6OVBqEtfxa4oUDLd6GiGpCAxhN8diqI5E/1E0mF9FElEWUTrOUggsZliqsXyj8dht6PmlaEBLY11GX3sUXNoO13BCD5HJZp1bIz8eZxWTIpCNK4RsibqEkmH9VEkEWWRQPvhjuoSd+YrUbpL9ImNHc37Mv7YI07TUJu3safVz/8dLGJfe3rF4kab2aRQkJgd71MSc4mkRXQUSUTZQtOwh1qZWupm5vQZme2oTigorwUgOBZLx/qa6Gxrplc1Ey6cTk3R2GgRARS69dHPLi2RiKRFdBRJRNki1IlVi1DmdTN/1shsilNWpU+u03zNBMPZuVB0UIc+pbm7lw5XLbOrSzCPQKIeKUWJaRjtamLOU7B93O81lypJRNnCn5jf4y4F08jU1nF6S3E4nJi0GAebxtByj1gEX/0n+HpjtLtncsKEsbVHW2m+nQqvA4e3VO9oj4YgIltQH0kSUZZQfc00+3rpMBeNXLE4kwlH0QQA2pr2jcxzjISW7TS2dRK25DNhyuxknZ+xYmZFPv+0sIZTppRD38iZv8XYoLKMJKIs4W/bT11bgP/XMLL/JaUTplBd6GS2c4wM4WsanbvfoysY5VD+CZw2ucjoiIanb/5QoM3YOLKMJKJsoKqE2vRLJUdxdUZKfwympHoGEwtdeEKNY6OforsB/C1YrFZKp51KQYaWvRghFleJOUv0HwLSIjqSzKzOBoEWAqEgcZOVwtIR3jbZMxHMFogEINAKeWUj+3zDtf89Cl025k8+lfj07CwJOxT/b8chtjf5uKTSxQzQ33uRJC2ibOBrxN8bw28rp9w7wkW+zBZi+RNpD4T5ZOuWkX2uYdK6G6F9DygmrLWLcFizr0D+UNksJlRN41A8USAt0D5uN71MhySiLBDtbCAYidNjLx+VaoNx7yR2H/LTVLeDrmB2DuMHIzHe+eufaPWHoXzu4TIaY1RJnj5B9WDYDmarXvxOJjYmSSLKAj0t9WgAnqpR2TbZXjYNr8uKJ3yQzw5kX6dpJKby2rubMHXspaEzRGTCGUaHNGyl+XoiagtE0NyJDmv/GC/JkkGSiIwWCRDo0pOBNzHzecS5SykqLsOkxWnauyOr9pbrjcZ5blMD7v1/xWJSmD7vLGyeEqPDGrYitw2zSSEcVQnaE/1yY70kSwZJIjJadyNVBU5mTZnM/Mnlo/OcikJx7YmYTQq2zi840Bkanec9jnZ/mN9/1ECs8RO8sXZmTiwhf9Z5RoeVEWaTQlFiqUe7KbFpZs842t5pmCQRGa2rHrOiUDhhOmWe0as2aCmfRUmejcLeejbvM34EZ3tTN09/sJ+erg6m97zP3CoPnllLwJ7dNalTUZa4PGvWEtUaew6BGjcwouwhichonfv074WTRvd5PROoKC3FokbpaNhJW4bqY6crHFOJxlQWht9jfpULV0kNTDjV0JgyrbbEzcyKfIpKysHq0DusZYY1IPOIjBX209J8gEBExamUMqr1BhUF14QTKGpuYbpaRzR+9qg9dTSusrc1gNWsMKVUb/HMn1hAYct71Fq7UEw2mPUPYBpffydnlOczozyxAr+5Cjr26pdnnkpjA8sCkoiM1FVPa0+Yg2oBZb2m0U1EAJXzmFr6AdPNnSiOkbtE0DSNzmCUA51BGjpC7GsPEImpFLqs1Ba7MZkUTK2fMblnC6DAzIvBXTxi8WQFTyIRdTXAhFOMjsZwkogMFG2vwx+O0Z1XxalG1NfJK8VcOBG6G6F5K0w6E1XV0q6FFFc1fKFosv4OwMufHmT3IT8xtf/InMdpZUZFPnFNw9S6Gz77s37DxNOg4sS0X1K260vKJnsVBQBd9fpSm7FaQzxDJBEZRdPwNe5E1SBeMCm5PfGoq1oA3Y2oDR/yTm8tTT1xrpxfhd1y7FnMqqrR5g/T2BWiqauXNn+YrqC+O8VN5007ol6QQkzVsJgUKrwOqotc1BS5qPQ69DV1TZth16ugqVBxAkw7f4RfsLE+qOvgnT3tzC5zcrHZApGgvgA2x4vpSyIySk8zvq4O4oqV4qppI7rQ9ZjK5kDd3wj3dND8+fs0uuby9Pv7OWdGafKy6cve2dPG5v1dRGJHL1GwWUz4e2N4E+VRz5hSxBlTivA4rP0fKxaGPX+Fpi36zxUnwMzLxn3LoG/mfH1XBM07EaVzn94qkkQkjKC17aIzGKXbOYk5FQXGBWIyw6RFOHe+zMXmPfzeNIPOIPxpSxM2i4k8u4VoXGXpggnJZQpWs4lITMVmMVFV4KDK66Tc46A4z0ae3dIvqR61Wl7ToHUn7H1D35UDYPI5MOnMcZ+EQE9ENouJYCROl72KQvbpI6cTx9cIYaokERnEd+AzwjGVnqLJVBeO/PqyY6o4CQ58RF6gje+U7uVd2xnsaPLRG43TEdPXonWHoslENKsin0nFLkrc9qH3J6lxaP0cDnx4eEaxwwuzLoXC2hF4UdnJbFKYWOhkb2uAeq2cQoDOOn3TRfPYKviWSZKIjBBoQ+05hM1ixjthlvG7lZrMMONi2PwkttZPWTx7MmdPm0tHMEJvNN5vVjBAvsNK/lCqJGoadB/QW0At2/X+ENA/cNULofp0fdPBHFNb7GZva4DdARfzHV7o7YaOOiidYXRohpFEZIRDn1LosnHygtlE5ozyRMbBFFTDpEVQ/y7sfAmTxU5JOpsYRoJ6n0fnPmj/AsJH1Ga25+md41ULwObOWOhjTW2x/tqbusOEJ07DfvBjaNspiUiMIk2DQzsAUCpOPO7o1KiqPUff6qZ1J3z6R30ofdKZYB3k0lHT9L3cfQfB16gnny/PFLbY9V1ZS2dB0ZQR2xhgLPG6rJR57LT4wtQxkVl8DG27IR7Ti9bloNx81UbqrMPf3Y7L5cJUPDLbBqXNZII5S2H3K/poVsMH0LQJCiaBqxgsiWUJ0SCEOvVh50jg6MfJK4WCWiiarPf/SPI5yplT9YoCNYUO6PRAr09vFZXPNTgyY0giGmXhfe+zvambtoJJnBOF/Gz7jJpMMPMSKJ4OdW/pLZz2PfrXQBQTuEv0mcIFNXrSGkcLVUfK5JIjLk0r50Hd3/Q5VZKIxIgLtNOybweqphAqXTAqRdDSVjINiqfqiai7AXq7IBYBk0VfsOks1L/yynN6tCcTtIoTUfa9rS/38Lfm5JyiLP4kjD+RPW/pu5U6J3HSjFrjJjEOlaJAfrn+JTIuGlf5sK6DnYd6+HbhNGwdu6H+bZj7j0aHNurG1/LmbOZronn3ZmIq+CvPYFqpXL7kOotJYU+rn65glM3KCfrBls/1OkU5RhLRaFDj+Le9wMHuEK3u6Zx6wqy0F5aK8UNRFM6cpndaf9BqJlCQmC6x6+Wc2+FDEtEoiO95i7q6vUQUO6Yp5zKlJHfn0Ij+ppS4qS5yEVM1Xg3NQTXbwNcEDe8bHdqokkQ00g5+gtLwHk6riabSczh33pTs7xsSo0ZRFL46pxybxcS+gIWPLAvQ0PQRy5bPjQ5v1GRFInr00Uepra3F4XBw+umn88EHHxgd0vBpGux/Dz7/CyZFYeqpF3LhkiXZPVImDOF1WrlobjmKAn/3T2CbNlX//dnxJ31IP4t2WRkphieiZ555htWrV3PHHXewadMm5s2bx0UXXURLy9it5at1NdC88XF2vfN/qJoGE05BmbK4X8EwIY40rSyf82eVYzabcM29RJ9PpKmw82XY9uy433pI0Qze1Or000/ntNNO47/+678AUFWV6upqfvjDH3Lbbbcd874+nw+v10t3dzcej2c0wu1PVVEjQUJBH71dh/C3NdJ7aBeBjmbCMRVVsVB04oXMPnVJTpS4EMPXHYrqRfI0Dfa/y4HNrxKNRslzWLF5K7GVTcVWUIk5r1Rfr2dxZnVt76F+Rg29TohEInz88cesWbMmecxkMnHBBRfw7rvvHnV+OBwmHD6824TP5xvyczV2hdj0wduUdm5GQdM3FUzmYI3aYhcVHn0luC8UYUdTN6ChaJq+CyuJ8zWN6iIXE7w2iEXw90bZ3tQ/Dk0x0eWdTfHsc5kys1aSkBiyZKVORSE84XRe2WmlrONjilv2oLTsht27ATArehWE2ZUesNhAMfNJk59IXP/901D6/d45bWZOmlCQ/HnrgS5C0YFH5uxWE/MnFiTj2Nboo8k2iZaiwzWTvn7yRJy2zC0LMDQRtbW1EY/HKS/vP2GuvLyczz8/uqPu3nvv5c4770zruaIxlR6/nxJ/CwM1ATVn7+HFnb1RtF49uQzYXIwqENP/aTEpxM12YvZCLPll2EonU1ozi4VlxcaX9xBjmklRWDx/Ng0dkzjY3YXWvger7wCuaCeOWDcuLbHhQaJmVDzkIzZA1UwAzWaGQCz5s+rvQo0OvGGCZjH1O1fzdxGyemmzHm4EaAN/MtI2pnpO16xZw+rVq5M/+3w+qqurh3Tfco+Dsxedibl3FqCgJObxKIqCgkKewwp2C6Bgj6tMC8UgcRtH1F9WFAWnzQI2K1gdOC0OzsQk84JExlnNJqaV5TGtLA8oA2agqhqRuEpvNI6iaWCN6WV31TgTZgSIx+OgxY/q4LaaFMjvq/2kUTW1l9iAeUjT643nH64TVTklTJHZxcnOw1t/Z7pqhKGJqKSkBLPZzKFD/WeSHjp0iIqKozfXsdvt2O3pFdJy2sxUV5YDx1+uYAeGutpHSXwJMRpMJgWHyYzD2pcI7MnaTuUprFErKxz6c5YWpRBgmgy9drDZbJxyyim8/vrryWOqqvL666+zaNEiAyMTQowmwy/NVq9ezfLlyzn11FNZuHAhDz30EIFAgO9+97tGhyaEGCWGJ6Jly5bR2trK7bffTnNzM/Pnz+fll18+qgNbCDF+GT6PaDgMn0ckhDimoX5GZXxZCGE4SURCCMNJIhJCGM7wzurh6OveSmWphxBi9PR9No/XFT2mE1FPTw/AkGdXCyGM0dPTg9frHfT2MT1qpqoqTU1N5Ofn52Sxsb4lLg0NDTJqeBzyXg1dJt8rTdPo6emhqqoK0zGqBIzpFpHJZGLixIlGh2E4j8cjH64hkvdq6DL1Xh2rJdRHOquFEIaTRCSEMJwkojHMbrdzxx13pF2RIJfIezV0RrxXY7qzWggxPkiLSAhhOElEQgjDSSISQhhOEpEQwnCSiLJcqrvgPvvss8yaNQuHw8GJJ57IX/7yl1GK1HipvFfr16/XN0444svhcIxitMbZuHEjl19+OVVVVSiKwvPPP3/c+7z55pucfPLJ2O12pk2bxvr16zMakySiLJbqLrjvvPMO//RP/8T3vvc9Nm/ezNKlS1m6dCmffvrpKEc++tLZMdjj8XDw4MHkV319/ShGbJxAIMC8efN49NFHh3R+XV0dl112GUuWLGHLli3ccsstrFy5kldeeSVzQWkiay1cuFBbtWpV8ud4PK5VVVVp995774Dnf/Ob39Quu+yyfsdOP/107cYbbxzROLNBqu/VunXrNK/XO0rRZS9Ae+655455zr/9279pc+fO7Xds2bJl2kUXXZSxOKRFlKX6dsG94IILkseOtQsuwLvvvtvvfICLLrpo0PPHi3TeKwC/38+kSZOorq7myiuvZPv27aMR7pgzGr9Xkoiy1LF2wW1ubh7wPs3NzSmdP16k817NnDmTxx9/nD/96U88+eSTqKrKmWeeyYEDB0Yj5DFlsN8rn89HKBTKyHOM6dX3QqRr0aJF/fbOO/PMM5k9eza//vWvufvuuw2MLDdJiyhLpboLLkBFRUVK548X6bxXX2a1WlmwYAFffPHFSIQ4pg32e+XxeHA6nRl5DklEWSqdXXAXLVrU73yA1157bdzvmpuJHYPj8Tjbtm2jsrJypMIcs0bl9ypj3d4i4373u99pdrtdW79+vbZjxw7thhtu0AoKCrTm5mZN0zTtmmuu0W677bbk+X//+981i8Wi3X///dpnn32m3XHHHZrVatW2bdtm1EsYNam+V3feeaf2yiuvaHv27NE+/vhj7eqrr9YcDoe2fft2o17CqOnp6dE2b96sbd68WQO0Bx98UNu8ebNWX1+vaZqm3Xbbbdo111yTPH/v3r2ay+XSfvzjH2ufffaZ9uijj2pms1l7+eWXMxaTJKIs98gjj2g1NTWazWbTFi5cqL333nvJ2xYvXqwtX7683/m///3vtRkzZmg2m02bO3eu9uKLL45yxMZJ5b265ZZbkueWl5drl156qbZp0yYDoh59b7zxhgYc9dX3/ixfvlxbvHjxUfeZP3++ZrPZtClTpmjr1q3LaExSBkQIYTjpIxJCGE4SkRDCcJKIhBCGk0QkhDCcJCIhhOEkEQkhDCeJSAhhOElEQgjDSSISY8r69espKChI/vyzn/2M+fPnJ39esWIFS5cuHfW4xPBIIhIDWrFiBYqi8P3vf/+o21atWoWiKKxYsaLf+ZlOALW1tTz00EP9ji1btoxdu3YNep+HH364Xz3lc889l1tuuSWjcYnMk0QkBlVdXc3vfve7fsWvent7eeqpp6ipqTEkJqfTSVlZ2aC3e73efi0mMTZIIhKDOvnkk6murmbDhg3JYxs2bKCmpoYFCxYM67EHaqksXbo02co699xzqa+v51/+5V+Su2zA0ZdmX3Zky2zFihW89dZbPPzww8nHqKurY9q0adx///397rdlyxYURZF6RAaRRCSO6brrrmPdunXJnx9//HG++93vjvjzbtiwgYkTJ3LXXXcld9lI1cMPP8yiRYu4/vrrk49RU1Nz1GsCWLduHeeccw7Tpk3L1EsQKZBEJI7pO9/5Dm+//Tb19fXU19fz97//ne985zsj/rxFRUWYzWby8/OpqKhIq8qk1+vFZrPhcrmSj2E2m1mxYgU7d+5M7nsWjUZ56qmnuO666zL9MsQQSc1qcUylpaVcdtllrF+/Hk3TuOyyyygpKTE6rGGpqqrisssu4/HHH2fhwoX8+c9/JhwO841vfMPo0HKWtIjEcV133XWsX7+e3/zmNxlrNZhMJr5cCisajWbksYdi5cqVyY74devWsWzZMlwu16g9v+hPEpE4rosvvphIJEI0GuWiiy7KyGOWlpb26/eJx+NH7Uhrs9mIx+PDep7BHuPSSy/F7Xazdu1aXn75ZbksM5hcmonjMpvNfPbZZ8l/D6a7u5stW7b0O1ZcXEx1dfVR55533nmsXr2aF198kalTp/Lggw/S1dXV75za2lo2btzI1Vdfjd1uT+uSsLa2lvfff599+/aRl5dHUVERJpMp2Ve0Zs0apk+fPu43GMh20iISQ+LxePB4PMc8580332TBggX9vu68884Bz73uuutYvnw51157LYsXL2bKlCksWbKk3zl33XUX+/btY+rUqZSWlqYV96233orZbGbOnDmUlpayf//+5G3f+973iEQiozIKKI5NalaLnPW3v/2N888/n4aGhqN2MhWjSxKRyDnhcJjW1laWL19ORUUFv/3tb40OKefJpZnIOU8//TSTJk2iq6uL//iP/zA6HIG0iIQQWUBaREIIw0kiEkIYThKREMJwkoiEEIaTRCSEMJwkIiGE4SQRCSEMJ4lICGG4/x+vrH0GcbH73gAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:34:57.546243Z", + "iopub.status.busy": "2024-03-24T03:34:57.545310Z", + "iopub.status.idle": "2024-03-24T03:34:57.866864Z", + "shell.execute_reply": "2024-03-24T03:34:57.865924Z" + }, + "papermill": { + "duration": 0.342434, + "end_time": "2024-03-24T03:34:57.868991", + "exception": false, + "start_time": "2024-03-24T03:34:57.526557", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEmCAYAAADGL52gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/OUlEQVR4nO3deXxcdb34/9eZfUlmsm9N0rRNW7ovQLFFliKyilTuVVSElkVRwPuo/XGReu+FC+qt3itcq3LRe7WtXKkIyvL1qlREFtmXLkBb2pImadrs6+zrOb8/zsy0aZI2M5mZM8l8no9HHmlmzsy8M03e+azvj6QoioIgCIKGdFoHIAiCIBKRIAiaE4lIEATNiUQkCILmRCISBEFzIhEJgqA5kYgEQdCcSESCIGjOoHUAEyHLMu3t7RQWFiJJktbhCIJwEkVRcLvd1NTUoNON3e6Z1Imovb2duro6rcMQBOE02traqK2tHfP+SZ2ICgsLAfWbdDgcGkcjCMLJXC4XdXV1id/VsUzqRBTvjjkcDpGIBCGHnW7oRAxWC4KgOZGIBEHQnEhEgiBoblKPEY2HoihEIhGi0ajWoUxper0eg8EgllEIKZnSiSgUCtHR0YHP59M6lLxgs9morq7GZDJpHYowyUzZRCTLMs3Nzej1empqajCZTOKvdYYoikIoFKKnp4fm5mZmz559ysVrgnCyKZuIQqEQsixTV1eHzWbTOpwpz2q1YjQaaW1tJRQKYbFYtA4pZU09Hna2DlDpsHBuYxl6nfgDlmlTNhHFib/M2TMV3ut+b4g/vtdBRFY4OuCnwGJgeX2x1mFNeVM+EQlCMt5q7iciKxj1Eh+fXc7iaU6tQ8oLIhEJQkwwEuVQlxuAvzuzlmqnVeOI8sfkb0sLQpq09fuIyArFNiNVjsk7xjUZiUSUgy688ELWr1+vdRh5p63fD8D0UjuSJBGKyLze1MdTu44iy+L4v0wSXTNBiKkusuAK2GkoswNg0EnsPDJAKCLT6w1SUShaSZmSly2iUEQe8yMSlcd9bXgc1yZr3bp1vPTSS2zevBlJkpAkidraWh5++OFh1+3atQudTkdraysADz74IIsWLcJut1NXV8dtt92Gx+MZ9phXXnmF8847D6vVSl1dHf/wD/+A1+tNOsap6owqB1cvncaMWCLS6SQqCs0AdLuCWoY25eVli+ihFz4a874ZZXbWLJuW+Pq/X24iHB29WV5bbOWzZx0vzLbl1Wb8oeFbSb7xyTlJxbZ582YOHjzIwoULuf/++wF44IEH2L59O1/72tcS1z366KOce+65TJ8+HVCnzn/0ox8xY8YMDh8+zG233cZdd93Ff/3XfwHQ1NTEZZddxne+8x22bNlCT08Pd9xxB3fccQdbt25NKsZ8UumwcHTAT7c7AIgZtEzJyxZRLnM6nZhMJmw2G1VVVVRVVXHdddfx6quvcuTIEUBdNf7YY49x3XXXJR63fv16Vq9eTUNDAxdddBHf+c53ePzxxxP3b9q0ieuuu47169cze/ZsVq1axY9+9CMeeeQRAoFA1r/PXDPgDTHkD6Mow//oVDhEiygb8rJFdPvqxjHvO3kR7VfOnzXmtSfvGLnp3BkTCWtMS5cuZd68eWzfvp27776bl156ie7ubj772c8mrvnLX/7Cpk2b+PDDD3G5XEQiEQKBAD6fD5vNxp49e3jvvfd49NFHE49RFCWxFWbevHkZiX2yeK2pj4Ndbs6fU8aZ00sSt1fGxoV63EFkWUEnVllnRF62iEwG3ZgfBr1u3Ncax3Ftulx33XVs374dgO3bt3PZZZdRWloKQEtLC5/61KdYvHgxv/vd73j33Xd56KGHAHWrC4DH4+HWW29l9+7diY89e/Zw6NAhZs0aO9nmix632iosKzAfvzHkw+k+hD3qJiIruAJhjaKb+vKyRZTrTCbTiLIlX/ziF/nnf/5n3n33XX7729/y05/+NHHfu+++iyzLPPDAA4ltFid2ywCWL1/Ovn37aGwcuzWYryJRmUG/mmRK44koMAQ7H0EX9HB2n4c9pVfgCdZSZBOVBTIhL1tEua6hoYE333yTlpYWent7kWWZhoYGVq1axc0330w0GuXTn/504vrGxkbC4TA//vGPOXz4MP/7v/87LFEBfPOb3+S1117jjjvuYPfu3Rw6dIhnnnmGO+64I9vfXs4Z9IdRFLVFazfp1RubXoCgOuu4qMrG2pL91BaJldaZIhJRDrrzzjvR6/XMnz+f8vLyxCD1ddddx549e/jMZz6D1Xr8l2LJkiU8+OCDfP/732fhwoU8+uijbNq0adhzLl68mJdeeomDBw9y3nnnsWzZMu655x5qamqy+r3logGv2n0tscdKxQSGoOdD9c6lX8BgNCN5e2CgRbsgpzhJOXmaYBJxuVw4nU6GhoZGnOIRCARobm5mxowZk7okxWQyWd/zNw/38VpTH/OqHVy2sApaX4fDL0JRHSz7Ehx4Ftp3QfUSOOMKrcOdVE71O3oi0SIS8t6A73iLCICe/ernyoUAeJyz+LDTxZ49b8Pk/bud08RgtZD35lY5KLQYqSuxQsgHnm71jlJ1YN9QPJ1eP+gVN2FXF0ZnlYbRTk2iRSTkvRllds5tLFPLfgy2qq0eexmYCwAwm4wErJUAeLtbNIx06hKJSBBONKhODFDckLhJkiQUh3pue7DviAZBTX0iEQl5zReK0NbvO75Y0dWufnbWDrtOKlb3FEYG2rIZXt4QiUjIa8cG/Pz23aP88b0OiEbA26PeUTh8HMhcpC5ziHgHICz25qWbSERCXou3hJxWo5qE5CgYrWApGnZdQUEBQX0BgYgM3m4NIp3aRCIS8tpQbGuHw2oEd6xbVlg9Ykezw2okZC1DguOzakLaiOl7Ia+5/BEg1iJyd6k3Fo6cnp9ZZmfm2YuQWt3g6cpmiHlBtIhyUK7VrM61eNIp0SKyGI+PDxVUjLhOkiSk+O2+vmyFlzdEIpqi4uU/hLEpioIrkYj04OtV77CVjf4Aa6xOka8/C9HlF5GIcsxoNaubmpq4+eabmTFjBlarlblz57J58+YRj1uzZg3f/e53qampYe7cuQC89tprLF26FIvFwllnncXTTz+NJEns3r078dgPPviAyy+/nIKCAiorK7n++uvp7e0dM56WlpZsvR0Z5Q1FicgKkgSFugBEQiDpwFYy6vV/bQ3x3tFB3G6XugJbSJv8GiNSFIhqUNxKbxxZznEMo9WsLi4upra2lieeeILS0lJee+01vvKVr1BdXc3nPve5xGOff/55HA4Hzz33HKBuOLzqqqu44oor2L59O62trSO6WIODg1x00UXccsst/Od//id+v59vfvObfO5zn+Ovf/3rqPGUl5en4U3RnkEncfG8SkLRKHp/rLtlKwGdftTr+4MS+qiF6ohMob8fTLYsRju15Vciiobhbw9k/3XP+//AML6CWifXrI677777Ev+eMWMGr7/+Oo8//viwRGS32/n5z3+OyaS+1k9/+lMkSeJ//ud/sFgszJ8/n2PHjvHlL3858Zif/OQnLFu2jH/7t39L3LZlyxbq6uo4ePAgc+bMGTWeqcBi1LOoNlYQvy12oIKtdMzrC8x6/MYiwpEBtXt20qJHIXX5lYgmsYceeogtW7Zw5MgR/H4/oVCIpUuXDrtm0aJFiSQEcODAARYvXjysJMeKFSuGPWbPnj288MILFBQUjHjNpqYm5sxJ7hSSScsbGx+yjzE+BBSYjfQanASjfeAX40TplF+JSG9UWydavO4EPPbYY9x555088MADrFy5ksLCQv7jP/6DN998c9h1drs96ef2eDxcddVVfP/73x9xX3V1dcoxTwbd7gDBsEyJ3YT9dAPVQIHFQJvRSSgkiwHrNMuvRCRJ4+4iaenkmtWvvvoqq1at4rbbbkvc1tTUdNrnmTt3Lr/61a8IBoOYzWot5rfffnvYNcuXL+d3v/sdDQ0NGAyj/ziMVkN7Kth9ZJC97S7ObSxjRTyxnLJrZiCoL1QPzgwMZSnK/CBmzXLQyTWrZ8+ezTvvvMOOHTs4ePAg//Iv/zIioYzmi1/8IrIs85WvfIX9+/ezY8cOfvCDHwDquhiA22+/nf7+fr7whS/w9ttv09TUxI4dO7jxxhsTyWe0GtpTgTekLmYs0IchrJ57j7V4zOsLzAaCBpGIMkEkohx0cs3qSy+9lGuuuYZrr72Wc845h76+vmGto7E4HA5+//vfs3v3bpYuXco//dM/cc899wAkxo1qamp49dVXiUajXHLJJSxatIj169dTVFSUOBFkrBrak50nqCZahxI7mttkP2WLucBiQLY4MegllLAPIuLQxXQRNavzzKOPPsqNN97I0NDQsAL86TDZ3vOfvtSEPxRlXaOP4pY/qbNgy68//QNf+U91B/7Zt0DB1FjKkCnjrVmdX2NEeeiRRx5h5syZTJs2jT179iTWCKU7CU02kaiMP6S2iGxRt3rjKbplw1icaiIKDIlElCaads0efvhhFi9ejMPhwOFwsHLlSv70pz9pGdKU09nZyZe+9CXmzZvHN77xDT772c/y3//931qHpTlvLAnpdRKm8KB6YzKJCMQ4URpp2iKqra3le9/7HrNnz0ZRFH75y19y9dVXs2vXLhYsWKBlaFPGXXfdxV133aV1GDnHG1QHqu1mA5J/UL1xHIno9aY+fM1BlhCkLDCYuQDzjKaJ6Kqrrhr29Xe/+10efvhh3njjDZGIhIxyWI1cPK9S3XnTOqjeOI5E5AlG6A1bCBAVLaI0ypkxomg0yhNPPIHX62XlypWjXhMMBgkGj89UuFyubIUnTDEFZoO6vSMSgoOxWTNr0WkfZzfpOaYvIBxSRCJKI82n799//30KCgowm8189atf5amnnmL+/PmjXrtp0yacTmfio66u7rTPP4knBSedSflex7tXRotaIvY0bLG1ROGoWEuUTponorlz57J7927efPNNvva1r7F27Vr27ds36rUbN25kaGgo8dHWNvaJCkajuq3C5xPlGrIl/l7H3/tc1j7op63fh98VK4Y2zoFqm0lPUG9XE1HYr001hylI866ZyWSisVE9UfPMM8/k7bffZvPmzfzsZz8bca3ZbE5sVTgdvV5PUVER3d1qfWGbzZZYTSykl6Io+Hw+uru7KSoqQq8fvYxGLnnjcB+tfT6uLu1gJiSViKI6MwE59jc86B6zfpEwfponopPJsjxsHGgi4mUr4slIyKyioqJJUyokPmtWIHvVG+JT8qdhNxlAkvBKsVpEIhGlhaaJaOPGjVx++eXU19fjdrvZvn07L774Ijt27EjL80uSRHV1NRUVFYTDogmdSUajcVK0hOLi2zusSnKJyGbWYzHq0VsKkRUvupAnUyHmFU0TUXd3NzfccAMdHR04nU4WL17Mjh07+OQnP5nW19Hr9ZPql0TIrEhUJhBWE5ElEksk5rG3H5zIbNDztQtnwb4Z0LVXbREJE6ZpIvrFL36h5csLecobaw0ZJDCEY4nkpAMVT8tcqH4WiSgtNJ81E4Rs88TKfziMUaRo7LQTy/haRAnxFlRQrGVLB5GIhLwTH6gu1sXOsDdak6qi+XpTH8986KbHHYSgGCNKB5GIhLxTWWjhk/MrWVQeW84xzoHqOF8oQlfAQCASFV2zNMm56XtByDSnzYjT5oSjasso2W6ZzWQgFF/UGPKCLINO/E2fCPHuCfkrvkUjyRaR3awnrLcSkgFFBjGFP2GiRSTknSN9PiQJKr0DmADMySUim8kAkg4fsb1pIU/yg93CMCIRCXnnxYPd9HlCrDX3UwJJt4hsJnVNmhcrEBLjRGkgumZC3vHEZs0s0ViXKukxolgiUmJ1ucXM2YSJRCTklXBUJhiW0ckRTHLsCKFxrqqOs5r0WE16DFYHsqKIMaI0EF0zIa8kSsTiRa+T1PVD46hDdCKzQc9XL5gFLZ3QfFSdORMmRLSIhLwS75YV6fxISOr4UKrlYUyxI75FIpowkYiEvOKLnd7h0MW6ZUkOVA9jKlA/i67ZhImumZBX4i0iJ7HKnUmOD8W9cqiXzmMDrPYGKTWLRDRRIhEJeaWh1I5pvo6KY++Bl5RbRJ5gmC6/nmBYhpAPFCX1Lp4gEpGQX0rsJkrsJuiObXhNcSGi1WSIra5W1NXVYd/xMSMhaWKMSMhPgVj5jhS7ZjaTHkXSE5RiNdTFgPWEiEQk5JWPuj0c6fUQ8ccSUYpdM6tRXdToP3Gbh5Ay0TUT8sqOvZ0QcHGTPoLBZDw+85Uka2x1tQ8LEBItogkSLSIhb4QiMqGIjDnqwaiX1PGhFMt32IYlIsQ2jwkSiUjIG4kjhBQvBp0u5fEhAJvRgNWkR2cWixrTQXTNhLwRX0PkkGJriCZQusNpM6rbPNr64KNmMUY0QaJFJOQNb+jkRDSBVdVxYptHWohEJOSNeNescIKrqodJbPMQiWgiRNdMyBvx011tSZ7uOpa/fthFb/cAl/rDOA2iONpEiEQk5I151YWU2oxU7QuqN0wwEQ35w3T69QQjUYiEIBpO6lgi4TjRNRPyRkWhhYUVJgrjuWKCXTOrUU9UMhFWYr9GYsA6ZSIRCfklfnKHyQ76iXUIrCYDSBIBKb66WowTpUokIiEvKIrC/g4X7V1dannXNJy6EV/U6BeJaMJEIhLyQigq8+wHnbz83kcoCmmZuj++30ysrp4okYiEvOCNzZjZ8am1qtMwdW9NHCsUS0Rh0SJKlZg1E/JCfA2Rg/jUfdGEn9NuUrd5SLJY1DhRIhEJeSG+vaOAeK3qibeIqpwWdZvHsSE4eEAkogkQXTMhLxzf8Bo/VDEN2zviRBH9CROJSMgLnmAEnRzGIoXUG9KxvSMusd/Ml77nzDOiaybkBW8wijnqwaTXgcEMRktanvdP73fgGujjU8EIdp1XFNFPUUotosOHD6c7DkHIqOXTi7iowYzTakzL+FBcvy9Ep99AKCqDHIFIMG3PnU9SSkSNjY2sXr2aX/3qVwQCgXTHJAhpV+20MscpYzMZ0jJjFmcz6ZF1BoLxzoUYsE5JSolo586dLF68mA0bNlBVVcWtt97KW2+9le7YBCG94ts70jg+ZDWqCSikE0X0JyKlRLR06VI2b95Me3s7W7ZsoaOjg49//OMsXLiQBx98kJ6ennTHKQgpC0dl9rYP0dPbg4KS1hmz+KLG46d5iBZRKiY0a2YwGLjmmmt44okn+P73v89HH33EnXfeSV1dHTfccAMdHR3pilMQUubyh/nz3i4+ONyGhJTWMaLEfjNi55uFxcxZKiaUiN555x1uu+02qqurefDBB7nzzjtpamriueeeo729nauvvjpdcQpCyuLbOwpJT0G0EyX2m0mxWTjRNUtJStP3Dz74IFu3buXAgQNcccUVPPLII1xxxRXoYkezzJgxg23bttHQ0JDOWAUhJZ5gBEmJYiMAGNM6RmQ3G9RWUVSUjJ2IlBLRww8/zE033cS6deuorq4e9ZqKigp+8YtfTCg4QUgHbyiCKerFpJNAZ0jrGfUzyuzcesEsaHfDgb0iEaUopUT03HPPUV9fn2gBxSmKQltbG/X19ZhMJtauXZuWIAVhIjzBCOaIG5NBp44PZWLBodjmMSEpjRHNmjWL3t7eEbf39/czY8aMCQclCOnkDUYwRzyxRJTGPWYnMtnUz2KbR0pSSkSKoox6u8fjwWIZ/9L5TZs2cfbZZ1NYWEhFRQVr1qzhwIEDqYQkCGPyBCKx7R3pqUN0sqd3HeOx3X0EIlG1azbG74cwtqS6Zhs2bABAkiTuuecebDZb4r5oNMqbb77J0qVLx/18L730Erfffjtnn302kUiEb33rW1xyySXs27cPuz19/Xghv318dhnRsAG7z5CRFlGvJ4gnoCcsy1gMMoT9x1tIwrgklYh27doFqC2i999/H5PJlLjPZDKxZMkS7rzzznE/37PPPjvs623btlFRUcG7777L+eefn0xogjCm2mIb2MMQ0qd1DVGczWTAHTAQksyArLaKRCJKSlKJ6IUXXgDgxhtvZPPmzTgc6f1PHRpSl+CXlJSk9XkFgYBL/ZyBFlF8UWNQsgLe2IB1edpfZypLadZs69at6Y4DWZZZv3495557LgsXLhz1mmAwSDB4fHezy+VKexzC1OIJRmjt9VA72IvTrM/IGFF8m0dAF0tEYnV10sadiK655hq2bduGw+HgmmuuOeW1Tz75ZNKB3H777XzwwQe88sorY16zadMm7rvvvqSfW8hf3a4AL7zXzPl9bhbXlmQkEY3Y5iHWEiVt3InI6XQixdZfOJ3pbd7ecccd/N///R8vv/wytbW1Y163cePGxIA5qC2iurq6tMYiTC2eYARz1K0WRDMXgC79RUlHHCsk1hIlbdyJ6MTuWLq6Zoqi8PWvf52nnnqKF1988bRrkMxmM2azOS2vLeQHTxbWEMW3eSjYQUa0iFKQ0hiR3+9HUZTE9H1raytPPfUU8+fP55JLLhn389x+++1s376dZ555hsLCQjo7OwG1xWW1WlMJTRCG8QajWCIutUVkLc7Ia8yrdjCv2gEdXvjwPZGIUpBSO/Xqq6/mkUceAWBwcJAVK1bwwAMPcPXVV/Pwww+P+3kefvhhhoaGuPDCC6murk58/OY3v0klLEEYwTtse0dRZl8sUURfdM2SlXKFxvPOOw+A3/72t1RVVdHa2sojjzzCj370o3E/j6Ioo36sW7culbAEYQR3MHJCi6gosy8mTvNIWUpdM5/PR2FhIQB//vOfueaaa9DpdHzsYx+jtbU1rQEKwkTEW0RGgz5jLSJZVvjdzqOE/S4+J8sYwj6Q5YwMjE9VKRfPf/rpp2lra2PHjh2JcaHu7u60L3IUhIm4fH4Z80vBYtBnrEWk00l0u4N0+fWEZdS9ZmItUVJSSkT33HMPd955Jw0NDZxzzjmsXLkSUFtHy5YtS2uAgjAR0+1RKgrM6I0mMGZu24XVqAdJF9vmgRiwTlJKXbO///u/5+Mf/zgdHR0sWbIkcfsnPvEJPvOZz6QtOEGYMP+A+tnizOjBh1aTniF/mKDOCoRjA9aVGXu9qSblk16rqqqoqqoadtuKFSsmHJAgpEufJ8hg2zEqgxEKyjIzdR83fL+ZS3TNkpRSIvJ6vXzve9/j+eefp7u7G1mWh90vToIVckFLn4+Wgy0sVvzMnlWU0deKr64OSBZQEF2zJKWUiG655RZeeuklrr/+eqqrqxNbPwQhl3hjU/dGU+an7m0m9VfJl9hvJtYSJSOlRPSnP/2JP/zhD5x77rnpjkcQ0sYTjGCJuDHZMr+Y0WbWYzfrUQwFEES0iJKUUiIqLi4WNYOEnOfxh3FEXZgN5oy3iJbXF7O8vhg6/bB/p0hESUpp+v7b3/4299xzDz6fGJATcpfX50YvhzNbNP9kidXVIhElI6UW0QMPPEBTUxOVlZU0NDRgNBqH3b9z5860BCcIqZJlhYhXnbo32ZygN57mEWkiElFKUkpEa9asSXMYgpBenlAEc9iFTgJTQWan7gEC4Si/39NOJODh8yhIYT/IUdDpM/7aU0FKiejee+9NdxyCkFZWo55LZ5rQ6wuQbKUZfz2DTuLogB8UHRGDhFFCbRVloFj/VJTyrrzBwUF+/vOfs3HjRvr7+wG1S3bs2LG0BScIqTLqdVQafJQVmMGa+YkVg16njkVJEmFdrJaW6J6NW0otovfee4+LL74Yp9NJS0sLX/7ylykpKeHJJ5/kyJEjiVpFgqApv/oHElt2ZnhtJj2hiExQZ8UW9YvV1UlIqUW0YcMG1q1bx6FDh4ad7HrFFVfw8ssvpy04QUhVc4+Hzs52AuFoVlpEcHx1dVAnalcnK6VE9Pbbb3PrrbeOuH3atGmJcq+CoKW9rR00d/YzGIhkviBaTOJYoUQRfdE1G6+UEpHZbB71TLGDBw9SXi4OlhO0F3L3AWC0FWVt6v74Ng+RiJKVUiL69Kc/zf333084HAZAkiSOHDnCN7/5Tf7u7/4urQEKQiqiHjURmQqztwPAHt/mYSpQbxBds3FLKRE98MADeDweysvL8fv9XHDBBTQ2NlJYWMh3v/vddMcoCEkJRWSkgLqY0eLIXgt91awyvnL+LBZMr44FIlpE45XSrJnT6eS5557j1VdfZc+ePXg8HpYvX87FF1+c7vgEIWmeYARrZAi9TsJYUJb9AEQR/aQlnYhkWWbbtm08+eSTtLS0IEkSM2bMoKqqCkVRREkQQXPuQBhLeAizQZe1qfthxLFCSUuqa6YoCp/+9Ke55ZZbOHbsGIsWLWLBggW0traybt06USZWyAkuXxhLJJaIMnSo4mjcgTBPvNPGE+/F1i9FghCNZO31J7OkWkTbtm3j5Zdf5vnnn2f16tXD7vvrX//KmjVreOSRR7jhhhvSGqQgJGOWUyZcZUfSZeFQxRMYdLrYNg8F2aRDp8hqqyhLywcms6RaRL/+9a/51re+NSIJAVx00UXcfffdPProo2kLTjgFVzvs/jXsfAR6P9I6mpxiCw/gtBpxlFRm9Wwxi1GHXieBJBES2zySktT/0nvvvcdll1025v2XX345e/bsmXBQwml4umH3ozDQAkPH4IPfimR0Im+v+jkLm11PJEnS8SL68UQktnmMS1KJqL+/n8rKsY9IqaysZGBgYMJBCaegKHBwB0QjeKw1dFlnEpVlOPgniIS0ji4nHDzcTLc7QMSa/Rmz+KLGgBRvEYkB6/FIaowoGo1iMIz9EL1eTyQiBucyqv8wDB3lyFCY3wcWEdZbWNl3iCWlQ1g6dkNdfh/pFAhHaW07Ql/IS3GWW0SgLmoE8CMOWkxGUolIURTWrVuH2Wwe9f5gMJiWoIRT6NgNgLluOcpQIUZJ4iPLImzdr7PAsRNd7dkZPUgw17l8IayRAYx6CWNhRdZf3y62eaQkqUS0du3a014jZswyKORNjAVVzl3BOmMJiqKw/bUIQ4Nv0NvdQcXgESiernGg2nEP9aOXw5gtRk3WEBVYDBSYDShGO4QQXbNxSioRbd26NVNxCOPR8yEoMhRWQUEFsR1NLJ9ZQUf3DNoHmyjvOYCUx4nIP9QFgM5eqkmZ1o/NLOVjM0uhOwR7Eaurxyl7c5vCxPU10dzr4bBURyR6/HTdxbVO3IUz8YejDLV9oA5o56lALBEZNOiWDSOK6CdFJKLJIhom1HuYLleQP3cW4A1FE3eZDXpmzV1EXbkTm+IHd/7WhIq4egAwOXIlEbnz+g/DeKW06VXQwOARBtx+AvoCHKXVOK3Da+x8rLES/POg9xAMNIOjWqNAtRV1d6MDLEVjLzPJJH8oyu/faycUDHAdClI0om71MFpO/+A8JlpEk8VAC/2+EEOWacyuKhz9muIZsWtbsxdXLlEUziyXmVdVSHF5jSYhGPUSxwb89PgUIvGSsUG3JrFMJiIRTRLRwTZc/jAuSzUzy+yjXhMoqKXXE6Tn2Ef5udnSP4BFilJUYMXi0KD8B+ppHmaj+msV1NvUG4Mjq5kKw4lENBlEQnh6jiIrEC2spcRuGvWyIwEre/tkjvV5wHU0y0HmAE+3+tlerunBhvG1REFd7A+GaBGdlkhEk4G7HZc/SEhvp7K8YsyaT/WldlyWGvzhKL7uw1kOUnvtx1poG/DRJ2Wv9Mdo4vvN/Lp4i0gkotMRiWgyGDpKJCrjsVRRVzp6twzAYtRjLK4HYLCzJUvB5Y7ezjaODvhpj2h7umqBWW0ReaVYIhKLGk9LzJpNBkNHmVFWQO3Ms1AqxxiojnFWNcBR8Pa2gSxntQyG1mSXumzBWqLtjKEtnoiIbXwVLaLTyp+f0slKUcClHuNtLKlXjzU+hcrqWqKSEY/XC76+bESYG0JeIn4XIOEo1WbGLK7ArG7zIH6ahxisPi3RIsp1/gG1vIdOD/bTzwTVFNs5ZC5HH2jH29uKvSA/zpkLDnYQjioEDE6KHKduNWbamdOLOXN6MXgK4W1Ei2gcRIso13m6Odzj4c1uPU19/tNebjHq0RdNA8DVlT/ridy9aqtRtlecttWYNeZYQgwHIBrWNpYcJ1pEuc7TiTsYodPooHScOwWWL1xIgfkjTFJ/ZmPLIb4+NRHpHVUaR3ICg1k9ZTYaVltFWpwoMknkyJ8OYSwRVyf+UBSvqYxKx+h1oE5WUt2ASa9TS6aGT9+KmgrCA+q6KUvpNI0jAVlWeOKdNra+1kLYIMaJxkMkohzn6zuGAsj2ysS08GmZ7MeP0XF3ZCy2nBHyMrMgwtK6YubNmat1NOh0En3eEIO+sFhLNE6aJqKXX36Zq666ipqaGiRJ4umnn9YynNwTdOPzqDNBBWXVSR1eeTjkYH+ni/7OPBgncrUjSRLWogqchdoOVMfF/2gcT0RiLdGpaJqIvF4vS5Ys4aGHHtIyjNzl6cYbjOA3OqlwJvcLdjRSxKAvzGA+DFi72tXPhblTcaDQEl9LJFpE46HpYPXll1/O5ZdfrmUIuc3diTcUwWsso36c40Nxjop6/M0Q7DuqrkWawnWsB7ta6Ol2IxU5aNQ6mJh4i8gdr10txohOaVLNmgWDwWEF+l2uqf2fq3i6sBr1SPZKyguTq2dTWllHGxJ+r0v9JbA4MxSlxhQFb28bvZ4Q7lAOJiI5vrp6av+sTtSkGqzetGkTTqcz8VFXV6d1SBklebqYXVHIp85dNqIQ2ulUFhfiN5USjMjqdo+pyteH3+9HlgwUaLyi+kQFsa7ZUDwRBUQiOpVJlYg2btzI0NBQ4qOtbQr/goUD4B9U/12QfLVBk0GHzqn+Yg5O5QHroaN4gxE8pjIqnTato0lwWIwUmA3obbHZy7BfrdQojGpSdc3MZvOYZ6pNOZ4uIrKM3laEZLSm9BS2sjrknvfwTOEWUbj/CP5QFLejmkpH7pRjrSux8eXzZ6pfvGJR/7AEXJAnW26SNalaRHnF08X+DjfPHYG2/tSOpHFW1qPXSRh9XepO/CnI230YBYg6ase/zirb4uNzgUFNw8hlmv7PeTwePvroo8TXzc3N7N69m5KSEurr6zWMTHtRdye+UIRBS0liKjhZjdOno5tZiU4Oqzvxp9pfY/8gPlc/ChK2ihw+y83iBHcXBIa0jiRnaZqI3nnnHVavXp34esOGDYB6ouy2bds0iio3+PqOISsQtlUkPVAdZzAY1NM8Bo+Au33qJaKhNiJRBZ+5Iul1Vtnw572dHB3w8ym7lQoQLaJT0DQRXXjhhSjizKeRomF8g2r9ZVtxTVIrqkeIJSJ5qB1d9ZI0BZgjBo9QX2JjWu1yIvVFWkczgi8UZcgfxm23xxKRaBGNRYwR5SJvD75AmLDOSmlJ6YSeqjXsZHfbAPsPHkhTcDlkUB2E15c0YDZoVyx/LA6r+nd+UInN5olENCaRiHKRuxNvMILPVEqFc4IzQYU1+MMyocHOqVUTxz+A4u8HSQfOWq2jGZXDonapB6PxtUQiEY1FJKIcJLu78IUieE2lVCS5ovpkleXlhPVWAuEIgYH2NEWYA/oP09rn4/U+Cx/1h7SOZlTxsb2+SOz/MBxQP4QRRCLKQbKrkyqnleLKOoptqQ1Ux1lMBihUFzZOqZ34/c0M+cO0KVU5e7S8I5aIBsMSGMVWj1MRiSjXyDIGfy/1JTYuOmvxxAaqYyylatfF1X1kws+VE+Qo4b5m/KEog9Y6aopSW/CZafEWkTcYJWKOHXEUXy0vDCMSUa7x9YEcAYPpeHGzCXJUqGuygv1T5PTXoaMMub2EdFYsxdXYc3Qho8Wop9hmpNppIWyILS8Q40Sjys3/wXzm6cQTjGAqqMGUptIdJVX1DABBdx9KyIdkyp09WSnpP8ygP8yQtY6GsgKtozmldefOUP/R1ASDH4E/f+qIJ0O0iHKM4u5iX4eLPzTL9HnSs0myrKgIY0EppXYTocFjaXlOzSgKSs8BhnwhBiz1NJzi5NucEi+c7xOJaDQiEeUYb/8xorJC0FJOsc2UlufU6STOWrSAWeUFmH1daXlOzfj68A32EJQlvIUN1BTlzkbXU1Hi3Wz/gLaB5CiRiHKJoiSOxbGU1KDTpbGqoiNWq8c1yYvp9xxAp5OwVzUyo7IEgz63f4Q/6vaw5ZVmnj0cW2IQdE2t9VxpIsaIcol/AL/Phyzpcaa7yFdhNbKi4O89gn0yl47tPYDVqGfJwhUsqcmhM8zGoNdJDPnDGPUm9ZyzSFCdOZtq+/4mKLf/nOQbtzpQ7TOWUu5I75R0wFLGWy2DvN/cQcAzSbsH/kF1F7skQdlsraMZl6LYFP6QP4xijY0TiQHrEUQiyiGKuwNv7DDFiiSL5Z+OxWxGLqhAAbqPNaf1ubOmay+DvhAuSzWKcXLM/DmsRnSSRDiq4DfG6hKJAesRRCLKId6+4wPVZfb0V6K0l6kLG/s7W9L+3BmnKMid79PU4+H/dZTQ0pdasbhs0+skimKr411SbC2RaBGNIBJRrlAUzP4eZpXbmTt7TnoHqmOKKhsA8E3G0rGudgZ6OwnIeoLFs6kvmRwtIoBiuzr7OajEEpFoEY0gElGu8A9gVEJUOO0sPSMzh+JU1KiL6xRXJ75gbm4UHVPXB3QOBei3NTCvrgx9BhJ1ppTElmH0ybE1T74+cnaDnEZEIsoVntj6Hns56DJTW8fqLMdisaJTInS0T6LtHpEQrtY9uAIR+uxzWThtcp3RVl5opsppweIsVwfaw34IiSOoTyQSUY6QXZ10ugL060syV7VSp8NSMg2A3vaWzLxGJnTv5VjvAEFDIdNmzkvU+Zks5lYV8oUV9Zw5sxLiM2eebm2DyjEiEeUIT+8Rmnu9/KUts/8l5dNmUldsZZ51kkzhKwoDh95g0Bemq3AhZ88o0TqiiYmvH/L2ahtHjhGJKBfIMv5etatkKa1LS+mPsZTVzaG22IbDf2xyjFMMtYGnG4PRSHnjWRSladuLFiJRmYi1TP3CK1pEJxIrq3OBtxuv30dUZ6S4PMPHJjtqQW+AkBe8PVBQkdnXm6gjb1BsM7F0xllEZ+dmSdjx+Mu+Lva2u7i82sYcUN97IUG0iHKB6xieQASPqZJKZ4aLfOkNRApr6fMG2fPe7sy+1gQpQ8egrwkkHcaGlViMuVcgf7xMBh2yotAVjRVI8/ZN2UMvUyESUQ4ID7ThC0VxmyuzUm0w6pzOoS4P7c37GPTl5jS+LxThtb8+Q48nCJULjpfRmKTKCtQFqh1BM+iNavE7sbAxQSSiHODubkUBcNRk5dhkc0UjTpsRR7CD/Udzb9A0FJF57vWd6PoP0zbgJzTtY1qHNGHlhWoi6vWGUOyxAWvPJC/JkkYiEWkt5MU7qCYDZ2zlc8bZyykprUCnRGk/vC+nDrkMhKM8tbMN+5G/YtBJzF5yLiZHmdZhTViJ3YReJxEMy/jMsXG5yV6SJY1EItLa0DFqiqycMXMGS2dUZuc1JYnShkXodRKmgY84OuDPzuueRp8nyOPvtBE5tgdnpI+5tWUUnnGR1mGlhV4nURLb6tGnix2a6Z5CxztNkEhEWhtsRS9JFE+bTYUje9UGDZVnUFZgojjQyq4W7Wdw9rYP8eu3juAe7Ge2+00W1DhwnLEazLldkzoZFbHuWacSq9bo7gI5qmFEuUMkIq0NtKifi6dn93Ud06gqL8cgh+lvO0BvmupjpyoYkQlHZFYE32BpjQ1bWT1MO0vTmNKtoczO3KpCSsoqwWhRB6zFCmtArCPSVtBDd+dRvCEZq1ROVusNShK2aQsp6exmttxMOHpe1l46HJU53OPFqJeYWa62eJbWFlHc/QYNxkEknQnO+BToptbfyTmVhcypjO3A76yB/sNq98xRrW1gOUAkIi0NttLjDtIhF1ER0GU3EQFUL2FW+VvM1g8gWTLXRVAUhQFfmKMDPtr6/bT0eQlFZIptRhpK7eh0Erqe/cxw7wYkmHsZ2EszFk9OcMQS0WAbTDtT62g0JxKRhsJ9zXiCEYYKajhLi/o6BeXoi2th6Bh0vgfTVyHLSsq1kKKygssfTtTfAXj2gw4OdXmIyMNn5hxWI3OqCokqCrqeQ7D/9+odtWdD1aKUv6VcF0/KOnMNRQCDrepWm8laQzxNRCLSiqLgOnYAWYFo0fTE8cRZV7MMho4ht73Na4EG2t1Rrl5ag9lw6lXMsqzQ6wlybNBP+2CAXk+QQZ96OsUdFzWeUC9IIiIrGHQSVU4LdSU26ktsVDst6p669l1w8M+gyFC1EBo/keFvWFtvNffzWlMf8yqsXKY3QMinboDN82L6IhFpxd2Ja7CfqGSktKYxoxtdT6liPjT/jaC7n84P3+SYbQG/fvMI588pT3SbTvZaUy+7jgwSiozcomAy6PAEIjhj5VE/NrOEj80swWExDn+uSBCa/grtu9WvqxbC3CunfMsgvnK+dTCE4qxFGmhRW0UiEQlaUHoPMuALM2SdzvyqIu0C0elh+kqsB57lMn0Tj+vmMOCDZ3a3YzLoKDAbCEdl1iybltimYNTrCEVkTAYdNUUWapxWKh0WSgtMFJgNw5LqiN3yigI9B+DwC+qpHAAzzofpq6Z8EgI1EZkMOnyhKIPmGoppUWdOa6fWDGGyRCLSiOvofoIRGXfJDOqKM7+/7JSqFsPRdyjw9vKl8sO8bvoY+9pdBMJR+iPqXrQhfziRiM6oKmR6qY0yu3n840lyFHo+hKNvH19RbHHCGVdAcUMGvqncpNdJ1BZbOdzjpVWppBhgoFk9dFE/uQq+pZNIRFrw9iK7uzAZ9DinnaH9aaU6Pcy5DHb9ClPPB1wwbwbnNS6g3xciEI4OWxUMUGgxUjieKomKAkNH1RZQ9151PATUX7i6FVB3jnroYJ5pKLVzuMfLIa+NpRYnBIagvxnK52gdmmZEItJC1wcU20wsXzaP0PwsL2QcS1EdTF8Jra/DgT+hM5gpS+UQw5BPHfMYaIG+jyB4Qm1mc4E6OF6zDEz2tIU+2TSUqt97+1CQYG0j5o53ofeASERCFikKdO0DQKpadNrZqaxqOF896qbnAHzwO3UqffoqMI7RdVQU9Sx3Vwe4jqnJ5+SVwgazeipr+RlQMjNjBwNMJk6bkQqHmW5XkGZqOYN3ofcQRCNq0bo8lJ/ftZYGmvEM9WGz2dCVZubYoJTpdDB/DRzaoc5mtb0F7TuhaDrYSsEQ25YQ9oF/QJ12DnlHPk9BORQ1QMkMdfxHJJ8RVs1SKwrUF1tgwAEBl9oqqlygcWTaEIkoy4Itb7K3fYjeoumcH4bCXPsd1elg7uVQOhuaX1JbOH1N6sdoJB3Yy9SVwkX1atKaQhtVM2VG2Qld0+ol0Pw3dU2VSERCxnn76G7Zh6xI+MuXZaUIWsrKGqF0lpqIhtogMAiREOgM6oZNa7H6UVCZ17M96aBULUJqeUXd7uHpycs1RTn8mzD1hJpeUk8rtU5n8ZwG7RYxjpckQWGl+iGkXTgq83ZzPwe63FxX3Iip/xC0vgILPqN1aFk3tbY35zJXO52HdhGRwVP9MRrLRfcl3xl0Ek09HgZ9YXZJC9Ubuz9U6xTlGZGIskGO4nn//+gY8tNjn81ZC89IeWOpMHVIksSqRnXQ+q0ePd6i2HKJg8/m3QkfIhFlQbTpJZqbDxOSzOhmXsjMsvxdQyMMN7PMTl2JjYis8Gf/fGS9CVzt0Pam1qFllUhEmdaxB6ntDaxGHe3l53Phkpm5PzYkZI0kSXxyfiUmg44Wr4F3DMtQUNQZy+4PtQ4va3IiET300EM0NDRgsVg455xzeOutt7QOaeIUBY68AR/+EZ0kMeusS7hk9ercnikTNOG0Grl0QSWSBK96pvG+Mkv9+dn3jDqln0OnrGSK5onoN7/5DRs2bODee+9l586dLFmyhEsvvZTu7slby1cZbKPz5S0cfO3/ISsKTDsTaeYFwwqGCcKJGisK+cQZlej1OmwLLlfXEykyHHgW3n9iyh89JCkaH2p1zjnncPbZZ/OTn/wEAFmWqaur4+tf/zp33333KR/rcrlwOp0MDQ3hcDiyEe5wsowc8uH3uQgMduHpPUag6yDe/k6CERlZMlCy6BLmnbU6L0pcCBM35A+rRfIUBY68ztFdfyYcDlNgMWJyVmOqmIWpqBp9Qbm6X89gzena3uP9HdW0nxAKhXj33XfZuHFj4jadTsfFF1/M66+/PuL6YDBIMHj8tAmXyzXu1zo26GfnW69QPrALCUU9VDCRgxUaSm1UOdSd4C5/iH3tQ4CCpCjqKazErlcU6kpsTHOaIBLCEwizt314HIqkY9A5j9J5FzJzboNIQsK4JSp1ShLBaeew44CRiv53Ke1uQuo+BIcOAaCX1CoI86odYDCBpGdPu4dQVP35U5CG/dxZTXoWTytKfP3e0UH84dFn5sxGHUtrixJxvH/MRbtpOt0lx2sm/f3yWqym9G0L0DQR9fb2Eo1GqawcvmCusrKSDz8cOVC3adMm7rvvvpReKxyRcXs8lHm6Ga0JqFgDxzd3BsIoATW5jNpcDEsQUf9p0ElE9WYi5mIMhRWYymdQXn8GKypKtS/vIUxqOknigqXzaOufTsfQIEpfE0bXUWzhASyRIWxK7MCDWM2oqN9FZJSqmQCKSQ/eSOJr2TOIHB79wATFoBt2reIZxG900ms83ghQRv/NSNmkGjnduHEjGzZsSHztcrmoq6sb12MrHRbOW7kKfeAMQEKKreORJAkJiQKLEcwGQMIclWn0RyB2HyfUX5YkCavJACYjGC1YDRZWoRPrgoS0M+p1NFYU0FhRAFQAc5BlhVBUJhCOIikKGCNq2V05yrQ5XqLRKCjREQPcRp0EhfHaTwo1swJERs1DilpvvPB4najqmUFK9DaWW48f/Z3uqhGaJqKysjL0ej1dXcNXknZ1dVFVNfJwHbPZjNmcWiEtq0lPXXUlcPrtCmZgvLt9pNiHIGSDTidh0emxGOOJwJyo7VSZxB61iuLxv2Z5SRIBpkjTvoPJZOLMM8/k+eefT9wmyzLPP/88K1eu1DAyQRCySfOu2YYNG1i7di1nnXUWK1as4Ic//CFer5cbb7xR69AEQcgSzRPRtddeS09PD/fccw+dnZ0sXbqUZ599dsQAtiAIU5fm64gmQvN1RIIgnNJ4f0fF/LIgCJoTiUgQBM2JRCQIguY0H6yeiPjwVjJbPQRByJ747+bphqIndSJyu90A415dLQiCNtxuN06nc8z7J/WsmSzLtLe3U1hYmJfFxuJbXNra2sSs4WmI92r80vleKYqC2+2mpqYG3SmqBEzqFpFOp6O2tlbrMDTncDjEL9c4ifdq/NL1Xp2qJRQnBqsFQdCcSESCIGhOJKJJzGw2c++996ZckSCfiPdq/LR4ryb1YLUgCFODaBEJgqA5kYgEQdCcSESCIGhOJCJBEDQnElGOS/YU3CeeeIIzzjgDi8XCokWL+OMf/5ilSLWXzHu1bds29eCEEz4sFksWo9XOyy+/zFVXXUVNTQ2SJPH000+f9jEvvvgiy5cvx2w209jYyLZt29Iak0hEOSzZU3Bfe+01vvCFL3DzzTeza9cu1qxZw5o1a/jggw+yHHn2pXJisMPhoKOjI/HR2tqaxYi14/V6WbJkCQ899NC4rm9ububKK69k9erV7N69m/Xr13PLLbewY8eO9AWlCDlrxYoVyu233574OhqNKjU1NcqmTZtGvf5zn/uccuWVVw677ZxzzlFuvfXWjMaZC5J9r7Zu3ao4nc4sRZe7AOWpp5465TV33XWXsmDBgmG3XXvttcqll16atjhEiyhHxU/BvfjiixO3neoUXIDXX3992PUAl1566ZjXTxWpvFcAHo+H6dOnU1dXx9VXX83evXuzEe6kk42fK5GIctSpTsHt7Owc9TGdnZ1JXT9VpPJezZ07ly1btvDMM8/wq1/9ClmWWbVqFUePHs1GyJPKWD9XLpcLv9+flteY1LvvBSFVK1euHHZ23qpVq5g3bx4/+9nP+Pa3v61hZPlJtIhyVLKn4AJUVVUldf1Ukcp7dTKj0ciyZcv46KOPMhHipDbWz5XD4cBqtablNUQiylGpnIK7cuXKYdcDPPfcc1P+1Nx0nBgcjUZ5//33qa6uzlSYk1ZWfq7SNuwtpN1jjz2mmM1mZdu2bcq+ffuUr3zlK0pRUZHS2dmpKIqiXH/99crdd9+duP7VV19VDAaD8oMf/EDZv3+/cu+99ypGo1F5//33tfoWsibZ9+q+++5TduzYoTQ1NSnvvvuu8vnPf16xWCzK3r17tfoWssbtdiu7du1Sdu3apQDKgw8+qOzatUtpbW1VFEVR7r77buX6669PXH/48GHFZrMp//iP/6js379feeihhxS9Xq88++yzaYtJJKIc9+Mf/1ipr69XTCaTsmLFCuWNN95I3HfBBRcoa9euHXb9448/rsyZM0cxmUzKggULlD/84Q9Zjlg7ybxX69evT1xbWVmpXHHFFcrOnTs1iDr7XnjhBQUY8RF/f9auXatccMEFIx6zdOlSxWQyKTNnzlS2bt2a1phEGRBBEDQnxogEQdCcSESCIGhOJCJBEDQnEpEgCJoTiUgQBM2JRCQIguZEIhIEQXMiEQmTyrZt2ygqKkp8/a//+q8sXbo08fW6detYs2ZN1uMSJkYkImFU69atQ5IkvvrVr4647/bbb0eSJNatWzfs+nQngIaGBn74wx8Ou+3aa6/l4MGDYz5m8+bNw8qYXnjhhaxfvz6tcQnpJxKRMKa6ujoee+yxYTVnAoEA27dvp76+XpOYrFYrFRUVY97vdDqHtZiEyUEkImFMy5cvp66ujieffDJx25NPPkl9fT3Lli2b0HOP1lJZs2ZNopV14YUX0trayje+8Y1EcXsY2TU72Ykts3Xr1vHSSy+xefPmxHM0NzfT2NjID37wg2GP2717N5IkiTIgGhGJSDilm266ia1btya+3rJlCzfeeGPGX/fJJ5+ktraW+++/P1HcPlmbN29m5cqVfPnLX048R319/YjvCWDr1q2cf/75NDY2putbEJIgEpFwSl/60pd45ZVXaG1tpbW1lVdffZUvfelLGX/dkpIS9Ho9hYWFVFVVpVTczel0YjKZsNlsiefQ6/WsW7eOAwcOJI4bCofDbN++nZtuuind34YwTqJUrHBK5eXlXHnllWzbtg1FUbjyyispKyvTOqwJqamp4corr2TLli2sWLGC3//+9wSDQT772c9qHVreEi0i4bRuuukmtm3bxi9/+cu0tRp0Oh0nV6AJh8Npee7xuOWWWxID8Vu3buXaa6/FZrNl7fWF4UQiEk7rsssuIxQKEQ6HufTSS9PynOXl5cPGfaLR6IiDIE0mE9FodEKvM9ZzXHHFFdjtdh5++GGeffZZ0S3TmOiaCael1+vZv39/4t9jGRoaYvfu3cNuKy0tpa6ubsS1F110ERs2bOAPf/gDs2bN4sEHH2RwcHDYNQ0NDbz88st8/vOfx2w2p9QlbGho4M0336SlpYWCggJKSkrQ6XSJsaKNGzcye/bsKV/XO9eJFpEwLg6HA4fDccprXnzxRZYtWzbs47777hv12ptuuom1a9dyww03cMEFFzBz5kxWr1497Jr777+flpYWZs2aRXl5eUpx33nnnej1eubPn095eTlHjhxJ3HfzzTcTCoWyMgsonJooFSvkrb/97W984hOfoK2tbcQBgkJ2iUQk5J1gMEhPTw9r166lqqqKRx99VOuQ8p7omgl559e//jXTp09ncHCQf//3f9c6HAHRIhIEIQeIFpEgCJoTiUgQBM2JRCQIguZEIhIEQXMiEQmCoDmRiARB0JxIRIIgaE4kIkEQNCcSkSAImvv/ARHy39bn4WMuAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:34:57.905592Z", + "iopub.status.busy": "2024-03-24T03:34:57.905220Z", + "iopub.status.idle": "2024-03-24T03:34:58.121381Z", + "shell.execute_reply": "2024-03-24T03:34:58.120464Z" + }, + "papermill": { + "duration": 0.237106, + "end_time": "2024-03-24T03:34:58.123625", + "exception": false, + "start_time": "2024-03-24T03:34:57.886519", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAqp0lEQVR4nO3df1xT970/8BfhRyIg+AMFRWyqrv6oFRQG4uyt3fjh7Y9p23VU24Kpl+5q6WxTO8tuB7N2pV0V6VYqt53oauvktrM/HptTWJRef+BUUKdWsP7Eyg9BqxFoQyCf7x9+ObsxCQQlnJzk9Xw8eGBOPid5Hz/hlXM+OfkcHyGEABGRwqjkLoCI6GYwvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJF8pO7gP5msVhQV1eHgQMHwsfHR+5yiOgGQghcu3YNI0eOhErleP/K68Krrq4OUVFRcpdBRD04f/48Ro0a5fB+rwuvgQMHArj+HxMSEiJzNa5jNptRWlqKlJQU+Pv7y10O9QFv6VOj0YioqCjpb9URrwuvrkPFkJAQjw+vwMBAhISEePQL3Zt4W5/2NKzDAXsiUiSGFxEpEsOLiBTJLcKrsLAQWq0WGo0GCQkJ2Ldvn8O2s2bNgo+Pj83P/fff348VE5HcZA+vkpIS6PV65ObmoqqqCtHR0UhNTcXFixfttt+8eTPq6+uln6NHj8LX1xePPvpoP1dORHKSPbzy8/ORmZkJnU6HSZMmoaioCIGBgSguLrbbfsiQIYiIiJB+ysrKEBgYyPAi8jKynirR3t6OyspKZGdnS8tUKhWSkpJQUVHh1GOsXbsWjz32GIKCguzebzKZYDKZpNtGoxHA9Y+dzWbzLVTvXtra2lBTUyPdbvnWhD1HTmHgoL0IHqC2ajt+/HgEBgb2d4l0i7per570urXH2e2TNbyam5vR2dmJ8PBwq+Xh4eGorq7ucf19+/bh6NGjWLt2rcM2eXl5WL58uc3y0tJSj/oDPnXqFF544QWb5b+103bVqlUYO3as64silygrK5O7BJdqa2tzqp2iT1Jdu3Yt7rrrLsTHxztsk52dDb1eL93uOns3JSXFo05SbWtrw8yZM6XbJ+qv4sVPvsSbD03CHSNCrdpyz0uZzGYzysrKkJyc7NEnqXYdHfVE1vAKCwuDr68vGhsbrZY3NjYiIiKi23VbW1uxadMmvPLKK922U6vVUKvVNsv9/f096gUQGhpqFeIB5y5BXdGOyTHTEHPbUBkro77maa/dGzm7bbIO2AcEBCA2NhYGg0FaZrFYYDAYkJiY2O26H330EUwmE5544glXl0lEbkj2w0a9Xo+MjAzExcUhPj4eBQUFaG1thU6nAwCkp6cjMjISeXl5VuutXbsWc+fOxdCh3Ksg8kayh1daWhqampqQk5ODhoYGxMTEYOvWrdIgfm1trc2cPjU1Ndi1axdKS0vlKJnI5dra2mw+tOr6BHlw2AGrT5AnTJjglWOYPt52xWyj0YjQ0FBcvXrVowbsb3To3CXMXbMXny6azjEvBaqqqkJsbKxTbSsrKzFt2jQXV9R/nP0blX3Pi4hsTZgwAZWVlVbLauqvQP/REeQ/ehfGjxhk1dYbMbyI3FBgYKDN3pTq3CWod36LiZOjuTcNN/h6EBHRzWB4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJEYXkSkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRZI9vAoLC6HVaqHRaJCQkIB9+/Z12/7KlSt45plnMGLECKjVatxxxx3YsmVLP1VLRO5C1mmgS0pKoNfrUVRUhISEBBQUFCA1NRU1NTUYPny4Tfv29nYkJydj+PDh+PjjjxEZGYlz585h0KBB/V88EclK1vDKz89HZmamdI3GoqIi/PWvf0VxcTFeeuklm/bFxcW4fPky9uzZI11VV6vV9mfJRC5zprkVraYOh/efamqVfvv52f/TDVL74fawIJfU525kC6/29nZUVlYiOztbWqZSqZCUlISKigq763z++edITEzEM888g88++wzDhg3D/PnzsWzZMvj6+tpdx2QywWQySbeNRiMAwGw2w2w29+EWuZeOjg7ptydvp6c4e6kVyQW7nWr7wsdHur2/7LkfQDtUuQHm7OtVtvBqbm5GZ2endHHZLuHh4TYX2+xy+vRpbN++HY8//ji2bNmCkydPYvHixTCbzcjNzbW7Tl5eHpYvX26zvLS01KMv1Hm+BQD8sHfvXlw4Knc11JOu/npyXCfCB9i/lKrZAlw2AUPUgL+d0erGb32w4aQvthm+QFSwS8t1qba2NqfaKerSZxaLBcOHD8e7774LX19fxMbG4sKFC3jzzTcdhld2djb0er1022g0IioqCikpKR590dnDtZeBIwcwffp0RI8eInc51INjdUasPLIXjyT/AHeOtP+6NJvNKCsrQ3JysjRscuNjbDi5FzNnznT4GErQdXTUE9nCKywsDL6+vmhsbLRa3tjYiIiICLvrjBgxAv7+/laHiBMnTkRDQwPa29sREBBgs45arYZarbZZ7u/vb/cFoCTdjZGc+8Yk/dZoHL+TedMYiTvrGsPy8/Pr8XXp6LXbm8dwZ87WLlt4BQQEIDY2FgaDAXPnzgVwfc/KYDAgKyvL7jo/+MEPsHHjRlgsFqhU1/ebT5w4gREjRtgNLk92prkV964s77FdT+MjALBj6SwGGCmOrIeNer0eGRkZiIuLQ3x8PAoKCtDa2ip9+pieno7IyEjk5eUBABYtWoS3334bS5YswbPPPouvvvoKr732Gn7+85/LuRmy6NrjKkiLwbjhtgMcrd+a8JfyCjwwKxFBA2z3PAHg5MUWPFdyqNtPuIjclazhlZaWhqamJuTk5KChoQExMTHYunWrNIhfW1sr7WEBQFRUFLZt24bnn38eU6ZMQWRkJJYsWYJly5bJtQmyGzc8GJMjQ22Wm81mNAwDpt02WNGHEESOyD5gn5WV5fAwsby83GZZYmIi9u7d6+KqiMjdyf71ICKim8HwIiJFYngRkSIxvIhIkRheRKRIsn/aSESAqfM7qDQXcMZYA5XG/hcTOzo6UNdRh+OXj9udVeKMsQUqzQWYOr8DYHv6jKdheBG5gbrWcwi6/ff4ZfdzcQIA3tn6jsP7gm4H6lpjEItwh208BcNLoXp6p+7pXRrwvndqdzYy6Da0nnkWb6XFYKydb0wA1/t0967d+MHMH9jt01MXW7Ck5BBG3nubq8t1CwwvhXL2nbq7d2nAu96p3ZnaVwPLd5G4PWQ8Jg21/0ZiNptxxu8MJg6ZaPdbE5bvrsLyXRPUvhpXl+sWGF4K1dM7dU/v0oD3vVOTZ2F4KVRP79Q9vUsD3vdOTZ6Fp0oQkSIxvIhIkRheRKRIDC8iUiSGFxEpEsOLiBSJ4UVEisTwIiJFcovwKiwshFarhUajQUJCAvbtc/ydl/Xr18PHx8fqR6PhSZZE3kb28CopKYFer0dubi6qqqoQHR2N1NRUXLx40eE6ISEhqK+vl37OnTvXjxUTkTuQPbzy8/ORmZkJnU6HSZMmoaioCIGBgSguLna4jo+PDyIiIqSfrkulEZH3kPW7je3t7aisrER2dra0TKVSISkpCRUVFQ7Xa2lpwW233QaLxYJp06bhtddew5133mm3rclkgslkkm4bjUYA17/7Zzab+2hL+l9HR4f02952dC3rbht7egzqP870RU996in96WztsoZXc3MzOjs7bfacwsPDUV1dbXed8ePHo7i4GFOmTMHVq1excuVKzJgxA8eOHcOoUaNs2ufl5WH58uU2y0tLSxEYGNg3GyKD8y0A4Iddu3bhnP3pnwAAZWVlt/wY5Hq96QtHfeop/dnW1uZUO8XNKpGYmIjExETp9owZMzBx4kT893//N1asWGHTPjs7G3q9XrptNBoRFRWFlJQUhISE9EvNrnCszoiVR/Zi5syZuHOk7XaYzWaUlZUhOTnZ4awSPT0G9R9n+qKnPvWU/uw6OuqJrOEVFhYGX19fNDY2Wi1vbGxERESEU4/h7++PqVOn4uTJk3bvV6vVUKvVdtdz9EetBF1zdPn5+XW7Hd1tp7OPQa7Xm75w1Kee0p/O1i7rgH1AQABiY2NhMBikZRaLBQaDwWrvqjudnZ04cuQIRowY4aoyicgNyX7YqNfrkZGRgbi4OMTHx6OgoACtra3Q6XQAgPT0dERGRiIvLw8A8Morr2D69OkYN24crly5gjfffBPnzp3Df/zHf8i5GUTUz2QPr7S0NDQ1NSEnJwcNDQ2IiYnB1q1bpUH82tpaqFT/2kH85ptvkJmZiYaGBgwePBixsbHYs2cPJk2aJNcmEJEMZA8vAMjKykJWVpbd+8rLy61ur169GqtXr+6HqojIncl+kioR0c1geBGRIjG8iEiRGF5EpEgMLyJSJIYXESkSw4uIFKnX4XX69GlX1EFE1Cu9Dq9x48bh3nvvxQcffIDvvvvOFTUREfWo1+FVVVWFKVOmQK/XIyIiAj/72c+6nXOeiMgVeh1eMTExeOutt1BXV4fi4mLU19dj5syZmDx5MvLz89HU1OSKOomIrNz0gL2fnx8efvhhfPTRR3jjjTdw8uRJLF26FFFRUUhPT0d9fX1f1klEZOWmw+vAgQNYvHgxRowYgfz8fCxduhSnTp1CWVkZ6urqMGfOnL6sk4jISq9nlcjPz8e6detQU1OD++67D++//z7uu+8+adqa22+/HevXr4dWq+3rWomIJL0OrzVr1uCpp57CggULHM5eOnz4cKxdu/aWiyMicqTX4VVWVobRo0dbTRAIAEIInD9/HqNHj0ZAQAAyMjL6rEgiohv1esxr7NixaG5utll++fJl3H777X1SFBFRT3odXkIIu8tbWlqg0WhuuSAiImc4fdjYde1DHx8f5OTkWF2wtbOzE//4xz8QExPT5wUSEdnj9J7XwYMHcfDgQQghcOTIEen2wYMHUV1djejoaKxfv/6miigsLIRWq4VGo0FCQoLTZ+xv2rQJPj4+mDt37k09LxEpl9N7Xjt27AAA6HQ6vPXWW312temSkhLo9XoUFRUhISEBBQUFSE1NRU1NDYYPH+5wvbNnz2Lp0qW4++67+6QOIlKWXo95rVu3rs+CC7h+3lhmZiZ0Oh0mTZqEoqIiBAYGori42OE6nZ2dePzxx7F8+XKMGTOmz2ohIuVwas/r4Ycfxvr16xESEoKHH36427abN292+snb29tRWVmJ7OxsaZlKpUJSUhIqKiocrvfKK69g+PDhWLhwIXbu3Nntc5hMJphMJum20WgEAJjNZpjNZqdrdTcdHR3Sb3vb0bWsu23s6TGo/zjTFz31qaf0p7O1OxVeoaGh8PHxkf7dV5qbm9HZ2SldYLZLeHg4qqur7a6za9curF27FocOHXLqOfLy8rB8+XKb5aWlpVYfOijN+RYA8MOuXbtwLthxu7Kyslt+DHK93vSFoz71lP5sa2tzqp1T4bVu3Tq7/+5v165dw5NPPon33nsPYWFhTq2TnZ0tfVIKXN/zioqKQkpKSp8e/va3Y3VGrDyyFzNnzsSdI223w2w2o6ysDMnJyfD397+px6D+40xf9NSnntKfXUdHPZH1itlhYWHw9fVFY2Oj1fLGxkZERETYtD916hTOnj2LBx98UFpmsVgAXJ/loqamBmPHjrVaR61WQ61W2zyWv7+/wz9qJfDz85N+d7cd3W2ns49BrtebvnDUp57Sn87W7lR4TZ06VTps7ElVVZVT7QAgICAAsbGxMBgM0ukOFosFBoMBWVlZNu0nTJiAI0eOWC17+eWXce3aNbz11luIiopy+rmJSNmcCi9Xnkel1+uRkZGBuLg4xMfHo6CgAK2trdDpdACA9PR0REZGIi8vDxqNBpMnT7Zaf9CgQQBgs5yIPJtT4ZWbm+uyAtLS0tDU1IScnBw0NDQgJiYGW7dulQbxa2trbb4ETkQk65hXl6ysLLuHiQBQXl7e7bo3e1Y/ESmbU+E1ZMgQnDhxAmFhYRg8eHC341+XL1/us+KIiBxxKrxWr16NgQMHSv92dvCeiMhVnAqv/zux4IIFC1xVCxGR03o9Eu7r64uLFy/aLL906RJ8fX37pCgiop702WSEJpMJAQEBt1wQEZEznP608Xe/+x2A65MR/uEPf0Bw8L++PNXZ2Yn//d//xYQJE/q+QiIiO5wOr9WrVwO4vudVVFRkdYgYEBAArVaLoqKivq+QiMgOp8PrzJkzAIB7770XmzdvxuDBg11WFBFRT3p9kmrXjKpERHLqdXg99dRT3d7f3QyoRER9pdfh9c0331jdNpvNOHr0KK5cuYIf/vCHfVYYEVF3eh1en3zyic0yi8WCRYsW2cylRUTkKn0yXYNKpYJer5c+kSQicrU+m2vm1KlT0gUAiIhcrdeHjf93Pnjg+nlf9fX1+Otf/2r1HUgiIlfqdXgdPHjQ6rZKpcKwYcOwatWqHj+JJCLqKzzPi4gUifMrE5EiMbyISJHcIrwKCwuh1Wqh0WiQkJCAffv2OWy7efNmxMXFYdCgQQgKCkJMTAw2bNjQj9USkTuQPbxKSkqg1+uRm5uLqqoqREdHIzU11e6Eh8D1+fT/67/+CxUVFfjnP/8JnU4HnU6Hbdu29XPlRCSnPguvr7/+Gk8//XSv18vPz0dmZiZ0Oh0mTZqEoqIiBAYGOvyO5KxZs/DQQw9h4sSJGDt2LJYsWYIpU6Zg165dt7oJRKQgfXbps0uXLmHt2rV49913nV6nvb0dlZWVyM7OlpapVCokJSWhoqKix/WFENi+fTtqamrwxhtv2G1jMplgMpmk20ajEcD172SazWana3U3XScEd3R02N2OrmXdbWNPj0H9x5m+6KlPPaU/na1d1us2Njc3o7OzU7rAbJfw8HBUV1c7XO/q1auIjIyEyWSCr68v3nnnHSQnJ9ttm5eXh+XLl9ssLy0tRWBg4K1tgIzOtwCAH3bt2oVzwY7blZWV3fJjkOv1pi8c9amn9GdbW5tT7dziorO9NXDgQBw6dAgtLS0wGAzQ6/UYM2YMZs2aZdM2Ozvb6lsBRqMRUVFRSElJQUhISD9W3beO1Rmx8shezJw5E3eOtN0Os9mMsrIyJCcnw9/f/6Yeg/qPM33RU596Sn92HR31RNbwCgsLg6+vLxobG62WNzY2IiIiwuF6KpUK48aNAwDExMTg+PHjyMvLsxtearUaarXaZrm/v7/DP2ol8PPzk353tx3dbaezj0Gu15u+cNSnntKfztbudHg9/PDD3d5/5coVZx9KEhAQgNjYWBgMBsydOxfA9el1DAYDsrKynH4ci8ViNa5FRJ7P6fAKDQ3t8f709PReF6DX65GRkYG4uDjEx8ejoKAAra2t0Ol0AID09HRERkYiLy8PwPUxrLi4OIwdOxYmkwlbtmzBhg0bsGbNml4/NxEpl9PhtW7dOpcUkJaWhqamJuTk5KChoQExMTHYunWrNIhfW1sLlepfZ3S0trZi8eLF+PrrrzFgwABMmDABH3zwAdLS0lxSHxG5J7cYsM/KynJ4mFheXm51+9VXX8Wrr77aD1URkTtzOrycne6GF+Agov7gdHitX78et912G6ZOnQohhCtrIiLqkdPhtWjRIvzpT3/CmTNnoNPp8MQTT2DIkCGurI2IyCGnw6uwsBD5+fnYvHkziouLkZ2djfvvvx8LFy5ESkoKfHx8XFknkUf71twJADh64arDNq3fmnCgCYg49w2CBtieu3jyYovL6nNHvRqwV6vVmDdvHubNm4dz585h/fr1WLx4MTo6OnDs2DEEByv4OwlEMjr1/4Pnpc1Hemjphw0n93fbIkjtFp/DudxNb6VKpYKPjw+EEOjs7OzLmoi8Tsqd179RMnZ4MAb4+9ptU1N/FS98fASrfnIXxo+wf95lkNoPt4cFuaxOd9Kr8DKZTNJh465du/DAAw/g7bffxuzZs63OxSKi3hkSFIDH4kd326Zr1oixw4IwObL7k8a9gdPhtXjxYmzatAlRUVF46qmn8Kc//QlhYWGurI260dMYSU/jI4D3jZGQZ3E6vIqKijB69GiMGTMGX3zxBb744gu77TZv3txnxZFjzo2R9Dw+AnjPGAl5Fqdftenp6fxE0Y30NEbizPgI4F1jJORZenWSKrmPnsZIOD5Cno6j7ESkSAwvIlIkhhcRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbyISJHcIrwKCwuh1Wqh0WiQkJCAffv2OWz73nvv4e6778bgwYMxePBgJCUlddueiDyT7OFVUlICvV6P3NxcVFVVITo6Gqmpqbh48aLd9uXl5Zg3bx527NiBiooK6erXFy5c6OfKiUhOsodXfn4+MjMzodPpMGnSJBQVFSEwMNDhhTw+/PBDLF68GDExMZgwYQL+8Ic/SBeqJSLvIet0Au3t7aisrER2dra0TKVSISkpCRUVFU49RltbG8xms8P59E0mk9XVtI1GIwDAbDbDbDbfQvXureu7jR0dHR69nd7EW/rU2W2TNbyam5vR2dkpXWC2S3h4OKqrq516jGXLlmHkyJFISkqye39eXh6WL19us7y0tBSBgYG9L1ohzrcAgB/27t2LC0flrob6grf0aVtbm1PtFD2R0+uvv45NmzahvLwcGo3Gbpvs7Gzo9XrpttFolMbJQkJC+qvUfne49jJw5ACmT5+O6NG8ypMn8JY+7To66oms4RUWFgZfX180NjZaLW9sbERERES3665cuRKvv/46/v73v2PKlCkO26nVaqjVtjOJ+vv7w9/f/+YKVwA/Pz/ptydvpzfxlj51dttkHbAPCAhAbGys1WB71+B7YmKiw/V++9vfYsWKFdi6dSvi4uL6o1QicjOyHzbq9XpkZGQgLi4O8fHxKCgoQGtrK3Q6HYDrM7hGRkYiLy8PAPDGG28gJycHGzduhFarRUNDAwAgODiYl14j8iKyh1daWhqampqQk5ODhoYGxMTEYOvWrdIgfm1trdWVidasWYP29nb85Cc/sXqc3Nxc/PrXv+7P0olIRrKHFwBkZWUhKyvL7n3l5eVWt8+ePev6gojI7cl+kioR0c1geBGRIjG8iEiRGF5EpEgMLyJSJIYXESkSw4uIFInhRUSKxPAiIkVieBGRIjG8iEiRGF5EpEgMLyJSJIYXESkSw4uIFInhRUSKxPAiIkVieBGRIjG8iEiRZA+vwsJCaLVaaDQaJCQkYN++fQ7bHjt2DI888gi0Wi18fHxQUFDQf4USkVuRNbxKSkqg1+uRm5uLqqoqREdHIzU1FRcvXrTbvq2tDWPGjMHrr7/e40VpicizyRpe+fn5yMzMhE6nw6RJk1BUVITAwEAUFxfbbf/9738fb775Jh577DG7V8EmIu8h26XP2tvbUVlZiezsbGmZSqVCUlISKioq+ux5TCYTTCaTdNtoNAIAzGYzzGZznz2Pu+no6JB+e/J2ehNv6VNnt0228GpubkZnZ6d0cdku4eHhqK6u7rPnycvLw/Lly22Wl5aWIjAwsM+ex92cbwEAP+zduxcXjspdDfUFb+nTtrY2p9q5xUVnXSk7Oxt6vV66bTQaERUVhZSUFISEhMhYmWsdrr0MHDmA6dOnI3r0ELnLoT7gLX3adXTUE9nCKywsDL6+vmhsbLRa3tjY2KeD8Wq12u74mL+/P/z9/fvsedyNn5+f9NuTt9ObeEufOrttsg3YBwQEIDY2FgaDQVpmsVhgMBiQmJgoV1lEpBCyHjbq9XpkZGQgLi4O8fHxKCgoQGtrK3Q6HQAgPT0dkZGRyMvLA3B9kP/LL7+U/n3hwgUcOnQIwcHBGDdunGzbQUT9T9bwSktLQ1NTE3JyctDQ0ICYmBhs3bpVGsSvra2FSvWvncO6ujpMnTpVur1y5UqsXLkS99xzD8rLy/u7fCKSkewD9llZWcjKyrJ7342BpNVqIYToh6qIyN3J/vUgIqKbwfAiIkVieBGRIjG8iEiRGF5EpEgMLyJSJIYXESkSw4uIFInhRUSKxPAiIkVieBGRIjG8iEiRGF5EpEgMLyJSJIYXESkSw4uIFInhRUSKxPAiIkVyi/AqLCyEVquFRqNBQkIC9u3b1237jz76CBMmTIBGo8Fdd92FLVu29FOlROQuZA+vkpIS6PV65ObmoqqqCtHR0UhNTcXFixfttt+zZw/mzZuHhQsX4uDBg5g7dy7mzp2Lo0c9+BLCRGRD9vDKz89HZmYmdDodJk2ahKKiIgQGBqK4uNhu+7feeguzZ8/Giy++iIkTJ2LFihWYNm0a3n777X6unIjkJGt4tbe3o7KyEklJSdIylUqFpKQkVFRU2F2noqLCqj0ApKamOmxPRJ5J1kufNTc3o7OzU7pOY5fw8HBUV1fbXaehocFu+4aGBrvtTSYTTCaTdNtoNAIAzGYzzGbzrZTvVtra2lBTUyPdPlF/FaaGkzh6KADtjaFWbcePH4/AwMD+LpF64cb+BBz3qaf1p7N/l7Jft9HV8vLysHz5cpvlpaWlHtXhp06dwgsvvGCz/Mk/2rZdtWoVxo4d2w9V0c1y1J+AbZ96Wn+2tbU51U7W8AoLC4Ovry8aGxutljc2NiIiIsLuOhEREb1qn52dDb1eL902Go2IiopCSkoKQkJCbnEL3EdbWxtmzpwp3W751oRtO/cj9e7vI3iA2qqtp71Te6Ib+xNw3Kee1p9dR0c9kTW8AgICEBsbC4PBgLlz5wIALBYLDAaDw6toJyYmwmAw4LnnnpOWlZWVITEx0W57tVoNtVpts9zf3x/+/v63vA3uIjQ0FPHx8dJts9mMa1cu4+4Z0z1qO73Fjf0JeE+fOrttsh826vV6ZGRkIC4uDvHx8SgoKEBrayt0Oh0AID09HZGRkcjLywMALFmyBPfccw9WrVqF+++/H5s2bcKBAwfw7rvvyrkZRNTPZA+vtLQ0NDU1IScnBw0NDYiJicHWrVulQfna2lqoVP/6UHTGjBnYuHEjXn75Zfzyl7/E9773PXz66aeYPHmyXJtARDLwEUIIuYvoT0ajEaGhobh69apHjXndyGw2Y8uWLbjvvvs8+hDDm3hLnzr7Nyr7SapERDeD4UVEisTwIiJFYngRkSLJ/mljf+v6fMLZE+GUymw2o62tDUaj0aMHd72Jt/Rp199mT58lel14Xbt2DQAQFRUlcyVE1J1r164hNDTU4f1ed6qExWJBXV0dBg4cCB8fH7nLcZmur0GdP3/eo08J8Sbe0qdCCFy7dg0jR460OsfzRl6356VSqTBq1Ci5y+g3ISEhHv1C90be0Kfd7XF14YA9ESkSw4uIFInh5aHUajVyc3PtzqhBysQ+teZ1A/ZE5Bm450VEisTwIiJFYngRkSIxvIhIkRheCjBr1iyrOftJOdyt79ytnlvB8CJyc+3t7XKX4J4EubWMjAwBwOonMjJSvPPOO1btqqqqhI+Pjzh79qwQQohVq1aJyZMni8DAQDFq1CixaNEice3aNat1du7cKWbOnCk0Go0YNWqUePbZZ0VLS0u/bZuns9d3J0+eFE899ZTQarVCo9GIO+64QxQUFNisN2fOHPHqq6+KESNGCK1WK4QQYvfu3SI6Olqo1WoRGxsrPvnkEwFAHDx4UFr3yJEjYvbs2SIoKEgMHz5cPPHEE6KpqclhPWfOnOmv/44+x/Byc1euXBGJiYkiMzNT1NfXi/r6erF06VIxc+ZMq3YvvPCC1bLVq1eL7du3izNnzgiDwSDGjx8vFi1aJN1/8uRJERQUJFavXi1OnDghdu/eLaZOnSoWLFjQb9vm6ez13XfffSdycnLE/v37xenTp8UHH3wgAgMDRUlJibReRkaGCA4OFk8++aQ4evSoOHr0qLh69aoYMmSIeOKJJ8SxY8fEli1bxB133GEVXt98840YNmyYyM7OFsePHxdVVVUiOTlZ3HvvvQ7r6ejokOO/pk8wvBTgnnvuEUuWLJFuHzx4UPj4+Ihz584JIYTo7OwUkZGRYs2aNQ4f46OPPhJDhw6Vbi9cuFA8/fTTVm127twpVCqV+Pbbb/t2A7zYjX1nzzPPPCMeeeQR6XZGRoYIDw8XJpNJWrZmzRoxdOhQq7557733rMJrxYoVIiUlxeqxz58/LwCImpoap+tRCo55KVBMTAwmTpyIjRs3AgC++OILXLx4EY8++qjU5u9//zt+9KMfITIyEgMHDsSTTz6JS5cuSZdSP3z4MNavX4/g4GDpJzU1FRaLBWfOnJFlu7xFYWEhYmNjMWzYMAQHB+Pdd99FbW2tVZu77roLAQEB0u2amhpMmTIFGo1GWnbjRWkPHz6MHTt2WPXphAkTAACnTp1y4RbJg+GlUI8//rgUXhs3bsTs2bMxdOhQAMDZs2fxwAMPYMqUKfjzn/+MyspKFBYWAvjX4G9LSwt+9rOf4dChQ9LP4cOH8dVXX2Hs2LHybJQX2LRpE5YuXYqFCxeitLQUhw4dgk6nsxmUDwoK6vVjt7S04MEHH7Tq00OHDuGrr77Cv/3bv/XVJrgNr5vPS4kCAgLQ2dlptWz+/Pl4+eWXUVlZiY8//hhFRUXSfZWVlbBYLFi1apU0mdv//M//WK0/bdo0fPnllxg3bpzrN8CL3dh3u3fvxowZM7B48WJpmTN7RePHj8cHH3wAk8kkfTF7//79Vm2mTZuGP//5z9BqtfDzs/+nbe+1pFTc81IArVaLf/zjHzh79iyam5thsVig1WoxY8YMLFy4EJ2dnfjxj38stR83bhzMZjN+//vf4/Tp09iwYYNVuAHAsmXLsGfPHmRlZUnvzp999hmysrL6e/M82o19973vfQ8HDhzAtm3bcOLECfzqV7+yCSF75s+fD4vFgqeffhrHjx/Htm3bsHLlSgCQZgR+5plncPnyZcybNw/79+/HqVOnsG3bNuh0Oimw7L2WFEvuQTfqWU1NjZg+fboYMGCA1cfb77zzjgAg0tPTbdbJz88XI0aMEAMGDBCpqani/fffFwDEN998I7XZt2+fSE5OFsHBwSIoKEhMmTJF/OY3v+mnrfION/ZddXW1WLBggQgNDRWDBg0SixYtEi+99JKIjo6W1uk6VeJGu3fvFlOmTBEBAQEiNjZWbNy4UXrMLidOnBAPPfSQGDRokBgwYICYMGGCeO6554TFYrFbj5JPleCUOEQK9eGHH0Kn0+Hq1asYMGCA3OX0O455ESnE+++/jzFjxiAyMhKHDx/GsmXL8NOf/tQrgwtgeBEpRkNDA3JyctDQ0IARI0bg0UcfxW9+8xu5y5INDxuJSJH4aSMRKRLDi4gUieFFRIrE8CIiRWJ4EZEiMbzIpRYsWAAfHx/4+PjA398f4eHhSE5ORnFxca++mrJ+/XoMGjTIdYU6sGDBAsydO7ffn5d6xvAil5s9ezbq6+tx9uxZ/O1vf8O9996LJUuW4IEHHkBHR4fc5ZFSyfvtJPJ0jr6nZzAYBADx3nvvCSG6n7Z6x44dNtMX5+bmCiGEeP/990VsbKwIDg4W4eHhYt68eaKxsVF6nsuXL4v58+eLsLAwodFoxLhx40RxcbF0f21trXj00UdFaGioGDx4sPjxj38sfd8vNzfX5nl37Njhkv8n6j3ueZEsfvjDHyI6OhqbN28GAKhUKvzud7/DsWPH8Mc//hHbt2/HL37xCwDAjBkzUFBQgJCQENTX16O+vh5Lly4FAJjNZqxYsQKHDx/Gp59+irNnz2LBggXS8/zqV7/Cl19+ib/97W84fvw41qxZg7CwMGnd1NRUDBw4EDt37sTu3bsRHByM2bNno729HUuXLsVPf/pTac+xvr4eM2bM6N//KHJM7vQkz+Zoz0sIIdLS0sTEiRPt3nfjtNXr1q0ToaGhPT7f/v37BQBpr+3BBx8UOp3ObtsNGzaI8ePHSzMuCCGEyWQSAwYMENu2beuxfpIX97xINkIIaS6qnqatdqSyshIPPvggRo8ejYEDB+Kee+4BAGla5UWLFmHTpk2IiYnBL37xC+zZs0da9/Dhwzh58iQGDhwoTZs8ZMgQfPfddx45bbKnYXiRbI4fP47bb7/dqWmr7WltbUVqaipCQkLw4YcfYv/+/fjkk0+s1vv3f/93nDt3Ds8//zzq6urwox/9SDrkbGlpQWxsrM20ySdOnMD8+fNdvPV0qzirBMli+/btOHLkCJ5//nmnpq22N31xdXU1Ll26hNdffx1RUVEAgAMHDtg817Bhw5CRkYGMjAzcfffdePHFF7Fy5UpMmzYNJSUlGD58OEJCQuzW6UnTJnsa7nmRy5lMJjQ0NODChQuoqqrCa6+9hjlz5uCBBx5Aenq6U9NWa7VatLS0wGAwoLm5GW1tbRg9ejQCAgKk9T7//HOsWLHCar2cnBx89tlnOHnyJI4dO4a//OUvmDhxIoDrFzEJCwvDnDlzsHPnTpw5cwbl5eX4+c9/jq+//lp63n/+85+oqalBc3MzzGZz//ynUc/kHnQjz/Z/r9Ls5+cnhg0bJpKSkkRxcbHo7OyU2jkzbfV//ud/iqFDh1qdKrFx40ah1WqFWq0WiYmJ4vPPP7e5luHEiRPFgAEDxJAhQ8ScOXPE6dOnpcesr68X6enpIiwsTKjVajFmzBiRmZkprl69KoQQ4uLFi9JU2eCpEm6F83kRkSLxsJGIFInhRUSKxPAiIkVieBGRIjG8iEiRGF5EpEgMLyJSJIYXESkSw4uIFInhRUSKxPAiIkVieBGRIv0/c/vCB9vc+vwAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-03-24T03:34:58.161356Z", + "iopub.status.busy": "2024-03-24T03:34:58.161057Z", + "iopub.status.idle": "2024-03-24T03:34:58.444811Z", + "shell.execute_reply": "2024-03-24T03:34:58.443946Z" + }, + "papermill": { + "duration": 0.305075, + "end_time": "2024-03-24T03:34:58.447045", + "exception": false, + "start_time": "2024-03-24T03:34:58.141970", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS8AAAEmCAYAAAAz0RYQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABbx0lEQVR4nO2dd3wU5dbHf1uym947BJIA0gnNhICIQBC4NPW9VwQuTQSlCBJBigICarhSRJByQQELCOpFQUEQIiAlgJTQA4QkJJR0SM/W5/1jMpPdZLYlW5Pn+/msZmefmXmGJCfnnOf3nCMghBBQKBSKgyG09QQoFAqlLlDjRaFQHBJqvCgUikNCjReFQnFIqPGiUCgOCTVeFArFIaHGi0KhOCTUeFEoFIdEbOsJWBu1Wo1Hjx7Bw8MDAoHA1tOhUCg1IISgpKQEoaGhEAp1+1eNzng9evQIYWFhtp4GhUIxQFZWFpo2barz80ZnvDw8PAAw/zCenp42ng2FQqlJcXExwsLCuN9VXTQ648WGip6entR4USh2jKG0Dk3YUygUh4QaLwqF4pBQ40WhUBySRpfzojgWhBAolUqoVCpbT4ViJkQiEcRicb2lStR4UewWuVyOx48fo7y83NZToZgZV1dXhISEQCKR1Pka1HjpQ60ClJWAxM3WM2l0qNVqpKenQyQSITQ0FBKJhIqKGwCEEMjlcuTl5SE9PR2tWrXSK0TVBzVeusi5CeyfAQS0AV7aaOvZNDrkcjnUajXCwsLg6upq6+lQzIiLiwucnJxw//59yOVyODs71+k6NGGvC3kp8PASkLwTSD9p69k0Wur6V5li35jj+0p/MnQRFg10f535+rd3AKXMptOhUCjaUOOlj/6LAfcgoCAVOPWZrWdDoVA0oMZLHy7ewKAVzNcnVwP5d206HQrFUZgwYQJeeukli96DGi9DtH8ZaDkAUMmB32YDtM0lxQAvvPAC3nnnHVtPo8FDjZchBAJgyCpA7AJknASufG/rGVEoVkGhUNh6CnqhxssYfMKBF+YzXx9+HygrsOl0GiOEEJTLlTZ5mdJUfsKECThx4gQ+//xzCAQCCAQCNG3aFJs2bdIad/nyZQiFQty/fx8AsGbNGnTs2BFubm4ICwvDtGnTUFpaqnXOqVOn0Lt3b7i4uCAsLAwzZ85EWVmZUfN6/PgxhgwZAhcXF0RERGDXrl0IDw/H2rVruTECgQCbNm3C8OHD4ebmho8//hgqlQqTJk1CREQEXFxc0Lp1a3z++eda11apVIiPj4e3tzf8/Pzw3nvvmfRvVleozstYYqcDV38Acm8ARxZR7ZeVqVCo0G7xYZvc++aygXCVGPer8vnnn+POnTvo0KEDli1bBgBYvXo1du3ahalTp3Ljdu7ciV69eqF58+YAGOnAunXrEBERgbS0NEybNg3vvfceNm5kfs7u3buHQYMG4aOPPsK2bduQl5eHGTNmYMaMGdi+fbvBeY0bNw75+fk4fvw4nJycEB8fj9zc3FrjPvzwQ6xYsQJr166FWCyGWq1G06ZN8eOPP8LPzw9nzpzBlClTEBISgldffZV7vh07dmDbtm1o27YtVq9ejZ9//hn9+vUz6t+srtjU8/rrr78wbNgwhIaGQiAQ4JdfftE7fu/evRgwYAACAgLg6emJ2NhYHD5spR9okRMwbC0AAdV+UXTi5eUFiUQCV1dXBAcHIzg4GGPGjMHp06eRmZkJgNk9sHv3bowZM4Y775133kHfvn0RHh6Ofv364aOPPsIPP/zAfZ6QkIAxY8bgnXfeQatWrdCzZ0+sW7cO33zzDSorK/XOKSUlBUePHsXWrVsRExODrl274ssvv0RFRUWtsaNHj8bEiRMRGRmJZs2awcnJCUuXLkX37t0RERGBMWPGYOLEiVpzW7t2LRYsWIBXXnkFbdu2xebNm+Hl5VXff0qD2NTzKisrQ1RUFF5//XW88sorBsf/9ddfGDBgAD755BN4e3tj+/btGDZsGM6dO4cuXbpYfsKs9uvCV0zyfuppQCy1/H0pcHES4eaygTa7d33o3Lkz2rZti127dmH+/Pk4ceIEcnNz8a9//Ysbc/ToUSQkJCAlJQXFxcVQKpWorKxEeXk5XF1dceXKFVy9ehU7d+7kziGEcNuo2rZtq/P+t2/fhlgsRteuXbljLVu2hI+PT62x3bt3r3Vsw4YN2LZtGzIzM1FRUQG5XI7OnTsDAIqKivD48WPExMRw48ViMbp3727x0NGmxmvw4MEYPHiw0eM143MA+OSTT7Bv3z78+uuv1jFeAKP9SvkNKLgLnFoLvDDPOvdt5AgEAqNDN3tkzJgxnPHatWsXBg0aBD8/PwBARkYGhg4diqlTp+Ljjz+Gr68vTp06hUmTJkEul8PV1RWlpaV48803MXPmzFrXbtasmdnm6eamvY939+7dmDNnDlavXo3Y2Fh4eHhg5cqVOHfunNnuWVccOmGvVqtRUlICX19fnWNkMhmKi4u1XvVCS/u1CshPrd/1KA0OiURSq4TP6NGjcf36dVy8eBE//fSTVsh48eJFqNVqrF69Gj169MAzzzyDR48eaZ3ftWtX3Lx5Ey1btqz1MlSZoXXr1lAqlbh8+TJ3LDU1FU+ePDH4LKdPn0bPnj0xbdo0dOnSBS1btsS9e/e4z728vBASEqJlzJRKJS5evGjw2vXFoY3XqlWrUFpayiUO+UhISICXlxf3MkvnIC3t1ztU+0XRIjw8HOfOnUNGRgby8/OhVqsRHh6Onj17YtKkSVCpVBg+fDg3vmXLllAoFFi/fj3S0tLw7bffYvPmzVrXnDdvHs6cOYMZM2YgOTkZd+/exb59+zBjxgyD82nTpg3i4uIwZcoUnD9/HpcvX8aUKVPg4uJisFJHq1atcOHCBRw+fBh37tzBokWL8Pfff2uNmTVrFlasWIFffvkFKSkpmDZtGp4+fWr8P1hdIXYCAPLzzz8bPX7nzp3E1dWVHDlyRO+4yspKUlRUxL2ysrIIAFJUVFS/CRemE7I8iJAlnoRc3lW/a1FqUVFRQW7evEkqKipsPRWTuX37NunRowdxcXEhAEh6ejohhJCNGzcSAGTcuHG1zlmzZg0JCQkhLi4uZODAgeSbb74hAMiTJ0+4MefPnycDBgwg7u7uxM3NjXTq1Il8/PHHRs3p0aNHZPDgwUQqlZLmzZuTXbt2kcDAQLJ582ZuDN/vYGVlJZkwYQLx8vIi3t7eZOrUqWT+/PkkKiqKG6NQKMisWbOIp6cn8fb2JvHx8WTcuHFkxIgROuej7/tbVFRk1O+oQxqv77//nri4uJDffvvN5PsY+w9jFCfXMMZrRTghpfn1vx6Fw5GNlyPA/hE/evSoTe5vDuPlcGHj999/j4kTJ+L777/HkCFDLHafqw+eYsSG0xjz5Vndg2JnAIHtgIpC4Mhii82FQqkvf/75J/bv34/09HScOXMGr732GsLDw/H888/bemp1xqbGq7S0FMnJyUhOTgYApKenIzk5mdPDLFiwAOPGjePG79q1C+PGjcPq1asRExOD7OxsZGdno6ioyOxzc3YS4UrWU1zNKtK95CtyAoZVqY2Tv6PaL4pNOHnyJNzd3XW+AGarz8KFC9G+fXu8/PLLCAgI4ASrjopN154vXLiAvn37cu/j4+MBAOPHj8eOHTvw+PFjzpABwJYtW6BUKjF9+nRMnz6dO86ONyfNfF0hEAAlMiUKyuTwd9eh5+K0X9uo9otiE7p37845ALoYOHAgBg60jU7OUtjUeL3wwgt6hWw1DdLx48ctOyENnJ1ECPVywcOnFUjPL9NtvACg/xLgFtV+UWyDi4sLWrZsaetpWB2Hy3lZk8gARrCXnm9g86uLNzCYar8oFGtCjZcewv2MNF4A0P4VoGUc1X5RKFaCGi89RPhXGa88I4yXQAAMWa1R92u3hWdHoTRuqPHSA2u8MgqMq5nE1P2qyncdXkjrflEoFoQaLz1wnld+GdRqI8NAqv2iUKwCNV56aOrjArFQAJlSjexi/TWTOKj2i1IP+KqbGqpz11ihxksPYpEQzXyZbs1GJe1ZtHo+zqY9Hyl15vHjxyaVjWpMUONlgHB/E1YcNem/BHALrNZ+USh1IDg4GFKpbUXPhBAolUqbzoEParwMEFFX40W1X+aFEEBeZpuXibKXkpISjBkzBm5ubggJCcFnn31W53ZommFjRkYGBAIB9u7di759+8LV1RVRUVFISkrSOsdQo45vv/0W3bt3h4eHB4KDgzF69GitevbHjx+HQCDA77//jm7dukEqleLUqVMmz93SOG5pSivBrTiaarwARvuVvAtIPcpov8b/ykgqKKajKAc+CbXNvRc+AiRuhsdVER8fj9OnT2P//v0ICgrC4sWLcenSJa50cn15//33sWrVKrRq1Qrvv/8+Ro0ahdTUVIjFYqMadSgUCixfvhytW7dGbm4u4uPjMWHCBBw8eFDrPvPnz8eqVasQGRnJWzLa1lDjZYA6e15AtfZrQ49q7VfnUWaeIcWeKCkpwddff41du3ahf//+AIDt27cjNNR8hnfOnDlcRZWlS5eiffv2SE1NRZs2bbQadQBMMcF169ahT58+2LRpE5ydnfH6669z14qMjMS6devw7LPPorS0lNvIDQDLli3DgAEDzDZvc0ONlwFY45VZWA6lSg2xyMRIm9V+Hf2Q0X61ehFw8zP7PBs8Tq6MB2SrextJWloaFAoFoqOjuWNeXl5o3bq12abTqVMn7uuQkBAAQG5uLtq0aWNUo46LFy/iww8/xJUrV/DkyROo1WoAQGZmJtq1a8edx9eMw56gxssAwZ7OkIqFkCnVePCkgkvgm0TsjKqejzcZ7ddLG8w/0YaOQGBS6NaQ0Sxjw5ZxZg2QoUYdZWVlXIWJnTt3IiAgAJmZmRg4cCDkcrnW+JrNOOwNmrA3gFAoqF/oCFDtVyMiMjISTk5OWnXei4qKcOfOHavc31CjjpSUFBQUFGDFihXo3bs32rRpw9t81hGgxssITNqgrQuq/WoUeHh4YPz48Zg7dy6OHTuGGzduYNKkSRAKhQabXZgDQ406mjVrBolEwjX72L9/P5YvX27xeVkCaryMIMLY0jiGoNqvRsGaNWsQGxuLoUOHIi4uDr169ULbtm3h7Oxs8Xt36tQJJ06cwJ07d9C7d2906dIFixcv5hYMAgICsGPHDvz4449o164dVqxYgVWrVll8XpZAQPRVA2yAFBcXw8vLC0VFRfD09DTqnB/+zsJ7/7uK3q388e2kGMMn6OP6/4CfXgdEEmBqEuDf+IrIGUNlZSXS09MRERFhlV96S1JWVoYmTZpg9erVmDRpkq2nYxfo+/4a+ztKPS8jYD2vNGNK4xhCs+7Xgdm07lcD5PLly/j+++9x7949XLp0iWswO2LECBvPrGFBjZcRsAn7R0UVqFSoDIw2gGbdr/S/gKt7zDBDir2xatUqREVFIS4uDmVlZTh58iRu3bplsFEGxXioVMII/Nwk8JCKUSJTIrOwHM8EedTvgnzaL1dfc0yVYgd06dKFt919RUWFwUYZFOOhxssIBAIBIgLccPVBEdLyyupvvIAa2q9FwAiq/WroNNZGGZaCho1GwsoljK6qaghN7dfl74AM+9v4ag80svWkRoM5vq82NV5//fUXhg0bhtDQUKOLrh0/fhxdu3aFVCpFy5Ytzd6vURcm1bM3Fqr90gmrIi8vL7fxTCiWgP2+1qfprU3DxrKyMkRFReH111/HK6+8YnB8eno6hgwZgrfeegs7d+5EYmIi3njjDYSEhFi8oSZnvMzlebGwPR/z7wCnPwf6vGfe6zsoIpEI3t7enPrb1dXVKiJPimUhhKC8vBy5ubnw9vaGSCSq87VsarwGDx5sUpXIzZs3IyIiAqtXrwYAtG3bFqdOncJnn31mPeNVX6FqTVy8gUEJwP8mAX+tYqQUVPsFgCnEB8Bht69QdOPt7c19f+uKQyXsk5KSEBcXp3Vs4MCBeou8yWQyyGTV4VhxcXGd7s1uyM4rkaFUpoS71Iz/dB3+j6n7dS+R0X6N20/rfoFZKAkJCUFgYCAUCoWtp0MxE05OTvXyuFgcynhlZ2cjKChI61hQUBCKi4tRUVEBFxeXWuckJCRg6dKl9b63l4sT/NwkKCiTIyO/DB2aeNX7mhys9mtjj2rtV9Rr5ru+gyMSiczyw05pWDT41cYFCxagqKiIe2VlZdX5WhYLHQHANwLoo9HzsbzQ/PegUBoQDmW8goODkZOTo3UsJycHnp6evF4XAEilUnh6emq96kqdm3EYS8+3mZ6P5QWM9otCoejEoYxXbGwsEhMTtY4dOXIEsbGxVrm/RT0vgNF+DV3LfE21XxSKXmxqvEpLS5GcnMxtmUhPT0dycjIyMzMBMCHfuHHjuPFvvfUW0tLS8N577yElJQUbN27EDz/8gNmzZ1tlvhY3XgDQLAboNpH5mmq/KBSd2NR4XbhwAV26dEGXLl0AMF1X2PpDANNwkzVkABAREYEDBw7gyJEjiIqKwurVq/Hll19aXCbB3d8axgsA4qrqfrHaLwqFUgtaz8sEKuQqtF18CABwedEA+LhJLDFFhms/MdovkRSYeoZqvyiNBlrPywK4SEQI8WIKp6VZ2vvq8H9Ai/6ASkbrflHshoJSGXaeu4+SStvr7qjxMpF6NaE1Ba7ulzOt+0WxG/77Vxre//k6vjubaXiwBnKl2uyb7KnxMhGLyyU0odovip3B/tG+b+Ie3xW/p2DklrO4/rDIbHOhxstEIq1pvACq/aLYFTklzOp3TnGl0edkFZbju7P3cT69EIVlcsMnGEm9jFdpaSmKi4u1Xg0ds7RBMwWq/aLYEblVRiu72HgJz2dH7kCuUqNXSz/0buVvtrmYbLzYsjRubm7w8vKCj48PfHx84O3tDR8fH7NNzF5hm3FkFJRZr1Ae1X5R7AC1miC3yvPKNdLzuvW4GD8nPwQAzBvUxqxljUzemP3vf/8bhBBs27YNQUFBja7GUpiPK4QCoFyuQm6JDEGeVmrLFbcESDlA635RbEZBmRwqNeG+livVkIj1+z+fHkoBIcCQjiHo1NTbrPMx2XhduXIFFy9eROvWrc06EUdBIhYizNcV9wvKkZZXZj3j5eJD635RbErNPFduSSWa+rjqHH8urQDHbudBJBRgzkDz2wuTw8Znn322XpUZGgJmr2dvLHai/fr58gP8nUFXPhsbuSXaxitHT96LEIIVh1IAAK89G8ZJjMyJyZ7Xl19+ibfeegsPHz5Ehw4datWg7tSpk9kmZ69E+LvhxJ086yXtWeyg7ldWYTlm77mCJt4uOD2/n1XvTbEtNY2VvhXHwzdycDnzKVycRJjVv5VF5mOy8crLy8O9e/cwceJE7phAIAAhBAKBACpVPZuyOgCRAVZecdSE1X4lLrVJz8eiCkZZnVtSyX3PKQ0LQghm7LqMcrkS2yY8y32PaxorXcZLqVJj5WHG65r0XAQCLZRaMdl4vf766+jSpQu+//77RpmwB2wgl6hJz7eBaz9W9XxcDIz4wmq3linVAACFiqBSoYaLhFY4bWg8KVfgwLXHAID8UjkCPKQAante2TqM1/8uPcC9vDL4uDphSp9Ii83TZON1//597N+/v1E3z2Tj98yCcqjUBCKhlQ04q/3a9iJw+VsgahQQ3ssqt5ZXGS8AKKlUUOPVACksqzZSElF1WpyVR4T7uSKjoBy5PDmvSoUKnx25CwCY3rclPJ3r3trMECYn7Pv164crV65YYi4OQ6i3CyQiIeQqNR49rbDNJLS0X+9YTfslV1Ubr2I72JxLMT8Fpfwq+JyqhD0reeALG3ecyUB2cSWaeLvg3z2aW2yOQB08r2HDhmH27Nm4du0aOnbsWCthP3z4cLNNzl4RCQVo7ueKu7mlSMsvQ5iv7uVii6Kl/VoH9Jlr8Vtqel7FlUqL349ifZ6UV/9RkilVAJjfcTZs7NTUC/uvPKoVNhaVK7DxWCoAYPaAZ+DsZFmv3GTj9dZbbwEAli1bVuuzxpKwB5gN2ndzS5GRX4Y+zwTYZhJa2q+VQIdXAL8WFr2llvGqoJ5XQ+RJebXnlV8qR6CnM5QqNfJLWePlDQC1wsaNJ1JRXKlE6yAPvNylicXnaXLYqFardb4ai+ECbLBBWxea2q/fLK/9kmt8j6nn1TDR3DydV2WwCsrkIISJOtqGeAAASmVKlMqYn4HHRRXYcToDAPDeoNZWyQObZLwUCgXEYjGuX79uqfk4DFYtjaMPrbpfJ4CrP1j0djUT9pSGB+thAUyTZaA6vxXoIYWHsxM8qpous8fXHrkLmVKN6HBf9GsTaJV5mmS8nJyc0KxZs0blYenCavXsjUGr7tcCi9b90g4bqefVENHyvDjjxfyf1WwFelbJJ4oqkZpbgh8vMrtu5g027+ZrfZgcNr7//vtYuHAhCgsb9/YQNmx88KRc6xfaZmjV/VpssdvIqOfV4OE3XoyHFVSl+QquKoeeU1KJlYdvQ02AAe2C0K259SrLmJyw/+KLL5CamorQ0FA0b94cbm7ae5YuXbpktsnZMwEeUrhJRCiTq5BZWI6Wge62nZCVtF9UKtHwyS+tnfNiNV5sIYIgD+b/v1/Lxh83cyAUAO9ZYPO1Pkw2Xi+99JIFpuF4CAQChPu74cajYqTnl9neeAFV2q8JwMUdjPbrrVOAWGrWW2jnvGjY2BDRFKnmVWm72LAxqCpcZMPHP24yHez/2a0pWgV5WHOaphuvJUuWmHUCGzZswMqVK5GdnY2oqCisX78e0dHROsevXbsWmzZtQmZmJvz9/fHPf/4TCQkJcHa2UmkaDVjjZfFmHKYQ96FFtV9UKtGwIYTwh40lbMKe+T0L9qz+oygRC/FO3DNWnCVDnctAX7x4Ed999x2+++47XL58uU7X2LNnD+Lj47FkyRJcunQJUVFRGDhwIHJzc3nH79q1C/Pnz8eSJUtw69YtfPXVV9izZw8WLlxY18eoF2zey+Jt0EzBxQcYtIL5+q+VQME9s16eel4Nm+JKJRSqarlN7YQ9Y7Q069hN7BmOUG8XK86SwWTjlZubi379+uHZZ5/FzJkzMXPmTHTr1g39+/dHXl6eSddas2YNJk+ejIkTJ6Jdu3bYvHkzXF1dsW3bNt7xZ86cQa9evTB69GiEh4fjxRdfxKhRo3D+/HlTH8MscHW97Ml4AVXar34W0X7RnFfDpmaDjOJKJSoVqlo5L1Yq5OEsxtQXLCuM1oXJxuvtt99GSUkJbty4gcLCQhQWFuL69esoLi7GzJkzjb6OXC7HxYsXERcXVz0ZoRBxcXFISkriPadnz564ePEiZ6zS0tJw8OBB/OMf/9B5H5lMZrEmIRG2LI2jDwtqv6jn1bBh811hvi5ciedHTytQUGXUWOPVNsQTn7/WGbve6AFvVwt2jteDycbr0KFD2LhxI9q2bcsda9euHTZs2IDff//d6Ovk5+dDpVIhKChI63hQUBCys7N5zxk9ejSWLVuG5557Dk5OTmjRogVeeOEFvWFjQkICvLy8uFdYWJjRczRERJXnlV1ciXK5nf0i+0ZW17k3o/aL5rwaNuxKo5+bFAHuTIh48zHzB99JJICPa/Ve5hGdm6BjUy/rT7KKOm0PqrkZG2AErGq1ZfVOx48fxyeffIKNGzfi0qVL2Lt3Lw4cOIDly5frPGfBggUoKiriXuYsYe3jJoF31TczI7/cbNc1G7FvAwFtzar9kmmEjWVyFZQqO9C4UcwGGzb6uUm4Ol7XHzLGK9DD2a7q99WpJM6sWbPw6NEj7tjDhw8xe/Zs9O/f3+jr+Pv7QyQSIScnR+t4Tk4OgoODec9ZtGgRxo4dizfeeAMdO3bEyy+/jE8++QQJCQk6DadUKoWnp6fWy5ywSnur17M3BrEEGLaW+fryt0DG6XpfsqYgl93bRmkYcMbLvdp43XjEdLkO8qyH7OZpJvDkfr3np4nJxuuLL75AcXExwsPD0aJFC7Ro0QIREREoLi7G+vXrjb6ORCJBt27dkJiYyB1Tq9VITExEbGws7znl5eUQCrWnLBIxZTes1kOxBhG2rqpqiGY9GO0XYJa6XzWNF90i1LBg9zX6ukk1PC/WeNVRjkQIsH8msCEGuLnPLPME6qDzCgsLw6VLl3D06FGkpDB1qtu2bauVeDeW+Ph4jB8/Ht27d0d0dDTWrl2LsrIyrj7+uHHj0KRJEyQkJABgaomtWbMGXbp0QUxMDFJTU7Fo0SIMGzaMM2LWhvW80vLs1HgBZtV+1TJedMXRoSgskyM9vwxdm3nzhoCaYaO0KmHP1veqs/G69hOQdoxZQAruWLdr8GCy8QIYdfmAAQMwYMCAet185MiRyMvLw+LFi5GdnY3OnTvj0KFDXBI/MzNTy9P64IMPIBAI8MEHH+Dhw4cICAjAsGHD8PHHH9drHvUh3J7DRhZW+2WGul9yFTVejszgz/9CTrEMe6b0QEykX63P2Sqqfu6SWiW+A+sSNpYXMgtGAPD8XGYhyUzUyXglJiYiMTERubm5tXJNujRaupgxYwZmzJjB+9nx48e13ovFYixZssTsKv/6YFfVJfTR4f+A5J3AvT8Z7de4fYykwkRqel5ULuE4KFVqTmx6Nq2Q33hVeV6+bhK4SbW/1+x+RpM4+iFQlgcEtAF6Gi+lMgaTc15Lly7Fiy++iMTEROTn5+PJkydar8YG63kVlslRVG7HXoiZtF+s8WIbM1C5hOOQmlfKfR3izW+IWJ2Xn0bOi8XksPF+EnDpa+broWuZBSQzYrLntXnzZuzYsQNjx44160QcFXepGIEeUuSWyJBeUIbOrt62npJuWO1X4jLGlW81wOSej2zY6OcuweOiSup5ORBXHxRxX/OVcdLc1+jnLoFKrb0IZtJqo1LOLBABQNfxQHP+Rbj6YLLnJZfL0bNnT7NPxJHh8l72HjoC9dZ+sT/0fu7MX1Ga83Icrj54yn1dqahdUFRzX6Ovhs6LxaTmsWfWAXkpgFsAs2BkAUw2Xm+88QZ27dplibk4LHa5QVsX9dR+scUI/avU11Qq4Thc0/C8ZDyeF+t1uUvFcHYSwdlJBA9nJjiTioXwdDYyUCu4xywMAcDABIt1dDc5bKysrMSWLVtw9OhRdOrUqZbafs2aNWabnKPgMEl7Flb7dXEHk7x/65TR+Qi5kvmLzRovWk3VMZAr1bj1uIR7z+d5FXAar+qfhQAPKUoqlQjyNFJdTwhw4F1AWQlE9gU6/rP+k9eBycbr6tWr6Ny5MwDUasRhT1sHrIlDhY0snPbrNnDmc2YZ2wg0c14ADRsdhRuPirRkLrzGS2OlkSXAXYq0vDLj812amq6ha+q0om0sJhuvY8eOWWIeDo1mGzRCiGMYcU3t14mVQHvjtF9sziuA87xo2OgI/O/SA633lQrdYaO/u7bnBRiZ77KgpouPOhcjpFQT5usKgYDZ55evo1W6XaJZ9+tAvMG6X0qVGuwClCme181HxRi37TxuPjJfOSKK8ZTKlPj50kMAQO9W/gCMDxuDq4xWqJcRxsuCmi4+qPEyA85OIjSpqiTpMHkvQFv7lXYcuPaj3uGaYYe/CZ7X3ksP8NedPCT8fqte06XUjX3JD1EmVyHS343r7l7Jk7CvDhurQ8Sxsc0xtkdzjO0Rrv8mFtZ08UGNl5mIcMS8F6Bd9+uQ/rpfmtogPzd2tdGw51VR9Vf+5N18ZBXaYemgBgwhBN+dzQQAjI5pxm354fO8+MLG5n5uWP5SBzTzc9V9EytouvigxstMRDiSXKImnPYrHziqe+sVa7wEgurQoqRSabCih+ay/A8XzFdPjWKYy1lPcetxMaRiIf7ZrSmcxbqNF7uvUTNsNAoraLr4MNl4/fXXX1Aqa4cKSqUSf/31l1km5Yg4rOcFaGu/Ln0D3D/DO4wNGyUiITxdmLUepZpwnpUuNI3Xjxce0AKGVoIQgpWHbgMAhnQKgberBM5OjPGS8STsCzh1vQlKeitpuvgw2Xj17duXt1t2UVER+vbta5ZJOSLhjqb1qolm3a9f32FCgRpw+xrFQrg4iSASMquqhoSqMg3jll1ciRN3TGvUQjGd7KJKDF1/CklpBQCA/m2YSi3OTsyvfKWSL2xk9zUa6XlZUdPFh8nGS5cUoKCgoFb37MZEpEZpHLXaNoUR603ch4zrz2q/asB6XlKxEAKBgFNcGxKqsp4Xu+y+40yGzYpHNhY2Hk/FjUfFcJOIMO2FFhjQjjVe/GGj5r5Go8NGK2q6+DBa5/XKK68AYISoEyZMgFRa7VqqVCpcvXq1Ue95bOLtArFQAJlSjcfFldzqo0NhQPtVs6KEp4sTnpQrDMolZFV/5cf2aI71f97Fybv52HwizWYtsxoDf2cwFV4+/WcUhnQK4Y5znleNsLHmvkaDWFnTxYfRnhfbfYcQAg8PD62OPMHBwZgyZQq+++47S87VrhGLhNyKTLo9V1U1hB7tl2bYCIDb91ZsQC7Bel5tgj2wZFh7AMCnh1No+GghCCFIqyp/0z5Uu2eDVEfCvua+RoNYWdPFh9Ge1/bt2wEA4eHhmDNnTqMOEXUR4eeGtLwypBeU4bkqMaDDwWq/NsZWa786vQqgtvHydGb2tRqSS7DJYamTCGNimuH6wyLs/jsLb++6hF/ffg7N/ejPkjm5X1AOmVINkVBQq5O1rrCRFaj6uRvhddlA08WHyTmvJUuWUMOlA26DtiN7XgATArB7HTW0X2zbs5qelyGhKhs2srmypSPao0szbxRXKjHlm4u8y/aUuvPHTabvaY9IX+57xVKdsNcOG/n2NfKipekaZzVNFx8mG6+cnByMHTsWoaGhEIvFEIlEWq/GjEPUszeWnjOZkEBD+1Ur58V6XkYm7NmGDlKxCJv/3Q1+bhLczinBqbv5FnmExgghBAeuMcZrUPvaLQRZz0uuVGstLHG16w0ZL1bT5eoPxC0106zrhskbsydMmIDMzEwsWrQIISEhjrEJ2UpEOrpcQhOxhAkJtg9itF9RoyBXhgPQ9LwY42XY82KNV/UftyBPZ/SI9MOBa49xn6ruzcap1HxcyXoKJ5EAA/UYL4D5vrCKe83yzzrR1HQNsq6miw+TjdepU6dw8uRJriwOpZqIAMZ4ZRWWQ6FSw0nk4BsYmscy2z0ufQ38+g4UMbsBAJIqI8QKVQ3nvKrCRiftf4+mvkw+hm4ZMh9sd+sX2wXzVoJw1ggjKxUquEhEKJMp8dvVxwCAIF0bsLU0XS8AHf9l9rmbism/XWFhYVSjo4MgD2c4OwmhVBM8eFJh6+mYBw3tV4u7TGcoNmw03fPS/nEL82FWZx88ocbLXOSWVAKo/sNQE7FICHGVuLhSqYJaTTDnxytIyS6Bv7sEo6LD+C/MarpEUmCI9TVdfJhsvNauXYv58+cjIyPDLBPYsGEDwsPD4ezsjJiYGJw/f17v+KdPn2L69OkICQmBVCrFM888g4MHD5plLvVFKBQgnOugXWpgtIPg6sts+wDQIfW/aC7I5oyQJyeV0O15EUJ4w0aAKSUEAFmFDcTQ2xCFSo28Ehlyq1qbBeppU1a94qjGuj/v4vfr2XASCfDfsd0Q4sVj9DQ1XX3m1rnnp7kxOWwcOXIkysvL0aJFC7i6utYqA823dUgXe/bsQXx8PDZv3oyYmBisXbsWAwcOxO3btxEYGFhrvFwux4ABAxAYGIiffvoJTZo0wf379+Ht7W3qY1iMCH83pGSXID2/AXkTHf8JJO+EKO0YPhJvw17RFwAYkSqgP2zULKNTM2wM86kKG5+UO04RRztl1u7LOFiVqAeAyADdigBnJyFKZcDPlx9iXeJdAMDHL3VEt+Y6clispsu/NdBzljmnXS9MNl5r1641283XrFmDyZMnY+LEiQCYtmoHDhzAtm3bMH/+/Frjt23bhsLCQpw5c4YzmuHh4WabjzmormffQDwvgAkRhq6B8ose6I3reFCaCKCLUVIJzU3ZNcPGJj4uEAiAcrkKhWVy0zYEU7TQNFxioQDR4bqT6awHzBqu13tF4NVndYSLmpquYWttpuniw2TjNX78eLPcWC6X4+LFi1iwYAF3TCgUIi4uDklJSbzn7N+/H7GxsZg+fTr27duHgIAAjB49GvPmzdMp05DJZJDJZNz74mLLVvOsrmffgDwvAPCNxNmmk/Bc5kYMy14PlL9ulFRCs3qBpMYChlQsQpCHM7KLK5H1pIIaLzPROcwbblLdv9rOGh5w71b+WPiPNvwDa2m67Gv7X52Ww+7du4cPPvgAo0aNQm5uLgDg999/x40bN4y+Rn5+PlQqFYKCgrSOBwUFITs7m/ectLQ0/PTTT1CpVDh48CAWLVqE1atX46OPPtJ5n4SEBK2tTGFhOv7CmIkGJZeowZmg0bijbgJ35VPg6BLOeOn3vLQFqjUJoyuO9aZmiaGeLfXv7mDlERH+bvhiVFeIda2K25Gmiw+TjdeJEyfQsWNHnDt3Dnv37kVpKRMeXblyBUuW6C5kZw7UajUCAwOxZcsWdOvWDSNHjsT777+PzZs36zxnwYIFKCoq4l5ZWZYthsd6Xo+KKhqccrxSLcJCxSTmzaVv4JP/NwAm7FPoqNGla6WRhV1xzKIrjnXmaY2cY88WfnrH/7NrU0Q19cLWcd3h5erEP8jONF18mGy85s+fj48++ghHjhyBRFId//br1w9nz541+jr+/v4QiUTIycnROp6Tk4Pg4NriOgAICQnBM888oxUitm3bFtnZ2ZDL+RtfSKVSeHp6ar0siZ+bBB7OYhDC7DFrSMhVKlwgbXAt6CUAgPuRuXAC43WV6vC+qvdD8of1TemKY715Wl5tvJr6uKBLM2+94yf0isC+Gc+hZaA7/wA71HTxYbLxunbtGl5++eVaxwMDA5Gfb/w2D4lEgm7duiExMZE7plarkZiYiNhY/v1SvXr1QmpqKtTq6r/yd+7cQUhIiJYhtSUCgaDBho6sITrXYibgFgBB/m3MkBwAoDvvZdjzYsLG789nYvgXp2gfyDrwtJz5w+3j6oQDM3vXkqSYjB1quvgw2Xh5e3vj8ePHtY5fvnwZTZo0Mela8fHx2Lp1K77++mvcunULU6dORVlZGbf6OG7cOK2E/tSpU1FYWIhZs2bhzp07OHDgAD755BNMnz7d1MewKA5fVVUHXAMOFx9O+/WWcC+aC7J15r10qetZWK0XAFx9UIRD1/jznRR+KhUqrpxNM19XeLnoCAONxU41XXyYvNr42muvYd68efjxxx8hEAigVqtx+vRpzJkzB+PGjTPpWiNHjkReXh4WL16M7OxsdO7cGYcOHeKS+JmZmRAKq3/ow8LCcPjwYcyePRudOnVCkyZNMGvWLMybN8/Ux7AoDVIuAY0a9mIhp/2Sph3DcvF2FJcP5z1Hl0CVRdN4AcDv1x/rXranaJFbUonoj6sjF29XM0Qfdqrp4sNk48V6OmFhYVCpVGjXrh1UKhVGjx6NDz74wOQJzJgxAzNmzOD97Pjx47WOxcbGmpRbswURDVQuwXpeTiIhp/2SrYvB86JruHL3Z6DVlFrnGAobgz2d4SQScFU8T6Xmo7hSwa1kUnSz7/Ijrfc+upLvxmLHmi4+TA4bJRIJtm7dinv37uG3337Dd999h5SUFHz77beNviQOi0O3QdODrEZJHPhG4lfvMQCA1skJvD0fNaUSfNQsmKdQESTeysGlzCdYdfi2zlVMCqCqsce4Xp6XUg78Npv52g41XXyY7HmxNGvWDM2aNTPnXBoMbM4rv1SGkkoFt4HZ0alZSRVgtF+dCv/AM/KHTMgxfJ3WOZpVVHUR5uOqtTK799JD3HpcgvxSGdqEeGBop1AzPkXDQV3LeNXj5yxpPZB3y241XXwYZbzi4+OxfPlyuLm5IT4+Xu/YNWvWmGVijoynsxP83SXIL5UjI78cHZt62XpKZkGuqm28XF1csFAxCT9JlzEhR9QoreqahsJGALWalZzUKE6Y5uhVaS1IzS5VPnX1vArTgBOfMl/bqaaLD6OM1+XLl6FQKLivdUE31lYT7ueG/FI50gvKGo7x4vG8PJ2dcIG0wUX/4eiWv5/ZTvLmSS5fYihsBIAQ7+oKCP7uUuSXVm/namhaOXNw8NpjNPF2Qc2Iuk6el4NouvgwyngdO3aM92uKbiL83XDh/hPHr2evAWu8pBrbSdiQ+Be/KehWkcRsJzmzDnh+DgDDq40Ak7RnGdopBDvOZEAoANQEyCxsOP9+5iDpXgGm7bwEAJjZr6XWZ3XKeV3/H3DvT7vXdPHh4KU+7Re2qmqDqGdfBV/YyFZTzVG4cNov/LWS2V4CzZyX7h+1YI3qnTP6tcT42OZIeKUjACCjoBwqR23iawHY5hoAcDdXW4pj8mpjxRPgUFX1FjvXdPFhlOfFNpw1hr1799Z5Mg2JCL+Gt+LIFzZqVVOt0n4h7RgTioz92aiwsV1I9ZYtPzcJlo7ogKJyBeb97xrySmRo/cHvmPZCC8S/2NoSj+VQ3HpcXRXlXLr26q7JOS8H0nTxYZTnpVmVwdPTE4mJibhw4QL3+cWLF5GYmAgvr4aR2zEHrOeVnlfaYMpm8+e8NKqpVmm/IHZmDNi1n4wKGwM9nfHDm7HYN70Xlzf1cnXi1OJKNcG6P1Mt8kyOxpOy6u1TrLKexaScV+ZZ4OIO5msH0HTxYZTnxTacBYB58+bh1VdfxebNmzldl0qlwrRp0yy+6dmRaO7LGK/iSiWelCuMa6Fu59RsfQZUe17cnkS25+Ofy4HDCyCMYESP+jwvAIiOqL3CFeHvhuSsp9z7CrmKK+fSWCko4y9AIBYK4K6nhpcWSjnw6zvM1w6i6eLD5JzXtm3bMGfOHC1BqkgkQnx8PLZt22bWyTkyLhIRQqtyOQ1lm1DNprMA4OXCU02V7flYlof+DzcB0J/z0kXnMG+t93dySky+RkOCEIIn5fzGy9vVyfjVfgfUdPFh8k+UUqlESkpKreMpKSla1R4omhu0HX+5nxCiUyoBMMaLC4/Zno8Aejz5Fd0FKXWqdBBboy7V7ezGa7yyiyqRXyrnFi+eCdIuZ2P0SqODarr4MFlhP3HiREyaNAn37t1DdHQ0AODcuXNYsWIFVw2CwhDh74Yz9woahOfF7j0EAKmG182GjSo1QblcVV1+WKPn4ydOX+GCcITJ9+zzTIDW+1vZli3hba+cupuPf391DjFVobWbRIReLf1xJ6f658rbmGoSDqzp4sNk47Vq1SoEBwdj9erVXGmckJAQzJ07F++++67ZJ+jINKQN2ppdgDQ9L2cnIbexurhSoV07Pe5DFCfvwzN4CNn9r4Gey026p7OTCF2beeNS5lMAjdfzSvj9FoDq1UUfNwmiw32x/XQGN8Yoz8uBNV18mBw2CoVCvPfee3j48CGePn2Kp0+f4uHDh3jvvffoxuwaNKQN2nIlv/ESCAS6m8+6+uIbzzcBAO3u/pfTfpnCzjd6cGLMlOySBrNyawqaHZgAwNdNgu41ugMZ1HhparqedzxNFx/1Eqlao6yyI1PteZU5/C8da7xEQgFEQu2/2Jxcgqd/459Oz+OkqgNEahkTspj47+AiEWHqCy0hFDDSgDyNrUONhZq9EHxcJQjw0O605GNoNVtT09VrpplnaBvqZLx++uknvPrqq+jRowe6du2q9aJUE+brCpFQgAqFCjnFjv1LxyeTYNHpeQGQqQg+UL4OtVDCab9MxUUi4jqRpzxufKFjTT0XK7tpH1rtOOitoKqp6Rr6GSBuGC3mTDZe69atw8SJExEUFITLly8jOjoafn5+SEtLw+DBgy0xR4fFSSTkarSnOXjSXq5i/vpLePRa7BYhvvrzMqUa90kwHnR6mzlweAFv3S9DtAnxANA4817lcm3PizVe/+gYwh3Tqa7X1HR1GQuE97LEFG2CycZr48aN2LJlC9avXw+JRIL33nsPR44cwcyZM1FUVGSJOTo0DaUJrYxHJsHiIa0SqvKEjez2oIKoN5mQpSyPCWFMpHUQ42U0thVHvnQDa7xeaF29Gqsz56Wp6RqwzCJztBUmG6/MzEz07Mkocl1cXFBSwvwlHDt2LL7//nvzzq4B0FDq2esLG6s9L56wsWpjtkTqzGxDAZi6X/f5u6LrorF6XiWy2v+mfF4Wb/9FTU3XwE8cWtPFh8nGKzg4GIWFjNvfrFkzrp58enq6wyelLUFEAxGqyvUUFay1RUgDrb2NzXsy21EApu6Xkl8tzge7eft2dkmjao+WX1I7V+rrxvx7C1C9cFLLoGlquiL6AJ1eteg8bYHJxqtfv37Yv38/AEawOnv2bAwYMAAjR47k7efY2GkwnhfP1iAWT30J+5pVJeKWMiEMW/fLSMJ8XREZ4AalmuBMaj7Xq7Chw7eXkTVUBKTWMQ5NTdfQzxxe08WHySLVLVu2cNuApk+fDj8/P5w5cwbDhw/Hm2++afYJOjqs8cosZOpS1ZQZOAp8W4NYuLCxRs6LEFLtebF7G119mW0peyczdb/av2y05ijS3x1peWV46zumGN++6b0QVWP/Y0OD3/NiDFWZrDqRr1VRogFquvgwyfNSKpX46KOPkJ1dXRDttddew7p16/D222/XuWv1hg0bEB4eDmdnZ8TExOD8+fNGnbd7924IBAK89NJLdbqvNQj1coFELIRCRfDwieO2tK+LVEKhIpysS2tvY8d/MdtTlJUmab9q3nrDsYZfJiefz/OqMl6a3qezZoOTBqjp4sMk4yUWi/Hpp59CqeTvjlwX9uzZg/j4eCxZsgSXLl1CVFQUBg4ciNzcXL3nZWRkYM6cOejdu7fZ5mIJhEIBwv2YxqqOLJfQHzbySyXYkBGokSsTCJjtKSKpSdovsVD73n/czDHqPEeCEIKvz2Qg6V4BAH7Pi93HyHqdLpqGq4FquvgwOefVv39/nDhxwmwTWLNmDSZPnoyJEyeiXbt22Lx5M1xdXfWW11GpVBgzZgyWLl2KyMhIs83FUrACywwH3iakVyqhw/PS3NZSK9Hv14IpPQww2q+KJwbnIHTQkNsUTqXmY8n+Gxi1lVkIy6+xo8DLxQniKhc0yNMZZ+b3w98fxDEfNmBNFx8m57wGDx6M+fPn49q1a+jWrRvc3Ny0Ph8+nL/tOx9yuRwXL17EggULuGNCoRBxcXFIStK9lL5s2TIEBgZi0qRJOHnypN57yGQyyGTVPwDFxdbXCXFVVR3YeBkllaio6XlVGzzeWlM9ZwFXfwTybzOhzrDP9c5BxHMJQkiD6lp1r0Zd+oJSfnU9i2bD3oas6eLDZOM1bdo0APz9GQUCAVQqVa3jusjPz4dKpUJQUJDW8aCgIN6aYQBw6tQpfPXVV0hOTjbqHgkJCVi61LYF19h69ukO3MZLb8Jel+dVtSdPymPwADB1v4atBbYPZkKdqFFAsx465yAS1r5OZmE5mvu58Yx2TGr2GqnpeekUozZwTRcfJoeNarVa58sUw1UXSkpKMHbsWGzduhX+/v5GnbNgwQIUFRVxr6ysLIvOkY+GIJcwRipRoVBpVZ+otdLIh6b269d39Gq/+Gxgn5XHG5S+UPNJrj8sqiWV8HXjyWE1Ak0XHyZ7XubE398fIpEIOTnaidecnBwEBwfXGn/v3j1kZGRg2LBh3DFWtiEWi3H79m20aKG9LCyVSiGV2jZpyRqvh08qIFOq6lRV1NboE6m6O1f/GJVUKuDnzvx7G9N8AwCj/Uo5yIQ8SeuB3vx14fg8LwB49uOjGNg+GB+/3NHgc9gbT8vlePv7y/i/rk3xUpcmWoZ46PpTtcazAlUtGoGmiw+jPa+Kigr89ttv3PsFCxYgPj6ee82dOxeVlZUm3VwikaBbt25ITEzkjqnVaiQmJiI2NrbW+DZt2uDatWtITk7mXsOHD0ffvn2RnJyMsLAwk+5vLQI8pHCTiKAmQFahY4aOCpXunJdIo/mDZuioz+BpwWq/ACb0KUzjHaYr+swvlWPnuUz997BT1h69i5N38/HOnmSjxtcqfdNINF18GO15ff311zhw4ACGDh0KAPjiiy/Qvn17uLgwCcOUlBSEhoZi9uzZJk0gPj4e48ePR/fu3REdHY21a9eirKyMKyk9btw4NGnSBAkJCXB2dkaHDh20zvf29gaAWsftCYFAgIgAN1x/WIz0/HK0DPSw9ZRMRl/OCwA8nMUolSm15BKsVELXOVp0/FdVz8fjTAj07721PAiRAY+iUqHS1js5AJo5LUII1AZCYN+aSvpGouniw2jPa+fOnZgyZYrWsV27duHYsWM4duwYVq5ciR9++MHkCYwcORKrVq3C4sWL0blzZyQnJ+PQoUNcEj8zM5MrN+3IsHIJR8176ZNKAPxJ++pu2UYYFE3t170/mVCoBrrCRpavTqUbvo+doWmqtp3OQKVCfxMbLc+rEWm6+DDaeKWmpqJjx+qcgrOzM4QaP0zR0dG4efNmnSYxY8YM3L9/HzKZDOfOnUNMTAz32fHjx7Fjxw6d5+7YsQO//PJLne5rTSL9HVsuwSXsdZT65pNLyIwNG1k0tV+H5tfSfukKG1lWHr6Ncrn5BNTWQDPH9dmRO6hQ1F700tz6w3lejUzTxYfRxuvp06daeqm8vDyEh4dz79VqtdbnFG3CHd14GQwbeTyvmpuyjaHnLJ11v4wRqSYc5JfY2CuaUaJELESFvLbxah3kgQ5NPCERCbnSQI1N08WH0T9VTZs2xfXr13V+fvXqVTRt2tQsk2qIRDRw48W3Rcjo1UZNWO0XwIREmWerPzLCeJ25l2/8vewAzRyXQqnmNV4ezmL8Mq0Xzr/fH019XBulposPo43XP/7xDyxevJh3RbGiogJLly7FkCFDzDq5hgRrvHKKZSjjKTBn7xjreRVr5byqPC9Tu2Xr0H4ZStgD2jkkR0BTlCpTqnnDRleJGGKRkGlv1kg1XXwY/VO1cOFCFBYWonXr1li5ciX27duHffv24dNPP0Xr1q3x5MkTLFy40JJzdWi8XSWcOjqjwPG8LzbnpUstb5aclyZc3a8q7RcMJ+wBIC3PsTo1HdHYXC5XqWvVqwcAV4mG59pINV18GP1TFRQUhDNnzqBt27aYP38+Xn75Zbz88stYsGAB2rVrh1OnTtXa5kPRxpHr2RvvedUzbGTh0X4ZStiz/H492/AgO6VmmzOA8bwAVGm6qvYBNzJNFx8m/UmMiIjAoUOHkJeXh7Nnz+Ls2bPIy8vDoUOHHKK6g61x5G1ChnNeZkrYa1Kj7pexRSVO3M6r2/3sAL6w0YndkX50KVCW2yg1XXzUaXuQr68voqOjzT2XBk+kA9ezl+lR2ANMUhmoETZyOq86Gi9W+7UxFrj3J1pL/wAQbvA0I6JLm5ORX4aTd2sbWb6w0cvVCcg8B1zczhxohJouPhzg29xwCG/InpcLn+dVj7CRRUP71fPuKnjCmH87+88DvbDqOBbtu1HreM2wcWD7ILzSKYhpWAIAXf7dKDVdfFDjZUUcWS4hN7DVh18qUc+wkaVK++WiKMR88W6Dwx25ZmFNqcSG0V0RfGMrkHsTcPUDBiy30czsD2q8rAi7RehJucLhut/oK4kD6BKp1mO1UROxhAmVAIwW/4lugtt6h9v7Ju3NJ+7p/IzdIeDsJMT2Cc9CXHQfOPEf5sNGrOnigxovK+ImFSPIk8lVOJr3pa+SKlAtlSipVHBSBZP2NhoivBdSmzKt9T5x+gpO0K+VK65UoFyuRBFPF29bs+J33bsA2L2Nf777Avq2DtDQdD0PdBpprSk6BNR4WRmunr2Dab0MlbdhVxvVBCirCn3MFjZWca3tu8gnnmgtfIDJogN6xz4tU6Dd4sOIWvoHr/zAXmE9XDeJuErTlchouoY0bk0XH9R4WZlItp59nmMaL11ho1Qs5LwydsXRbGFjFUqpNz5S/BsAMFO8F80EOTr7YD6/8hj3dc1SyrbkXFqBwTHdmvvAS1CqoemaA/i3tPDMHA9qvKxMuIPWszeU8xIIBNVyicqaxss8NbZEQgF+UffCKVV7OAsU+Ei8DU29nQ2eZ6jMjLXIKizHyC1nDY57f0hbDU3XM0CvWVaYneNBjZeVcUShqlpNoFAxeSxdOS+gtlyCCxvrqvOqAeNlCfCB8nXIiBOeF11Df5X+7lEA8MnBW2a5f33ZeFx3op7lxXZB6Io7GpqutVTTpQNqvKyMZtjoKHvwWK8L0F8VtaZQlUvYmylsZEPEDBKC9cqXAAAzZF8Z1H79maK/gbE1UKrU+P68/lXQVoHu2DImimq6jIQaLysT5usKoYBJaufZUS5GH8Yar5pbhMweNmokrLeohuKuugl8UWSU9utypuGmtpbkSbnhVc/IADfgzHqq6TISarysjFQsQhMfpu6/oyTtNduZ6Qsba+e8zLvaqJmcl8MJ7yteB1Ct/Vo3qovOcyds/9ssc6grxnjZz/mVUk2XCVDjZQMcTS6hqfHS151al+flbNacVzXnSVscEDOt7j9x+grD2+vu5VlUocB9G/17F1cqMPmbC3rHCAQEY/I/p5ouE6DGywawG7TTHESoakgmwVKzpld1zst8q4012eI8gdN+IekLvV5en5XHzTIPU1mfeBdXHhTpHZPyajmEaX9STZcJUONlA6rrejmI8arKeXGlWXSgWU2VEGLRsJGlVODBab9w4j9YFeep9xqXbJD7qtn1uiZfvBwO6dH3mTdU02U01HjZAEfboG2056WR81KqCVfi2BIJexaBoFr7BWUlhmathr5i0K9tOWv9faV60l2tgzwwNHcr1XTVAbswXhs2bEB4eDicnZ0RExOD8+fP6xy7detW9O7dGz4+PvDx8UFcXJze8fZIpL87ACCjoBxqtf3LJQz1bGThPK8KBXcOYG6dlzaCqv9+oHwdEEkhuJeICy8V67yGXKnG3J+ummU+xrD01xvYe/mhzs8P/5+UarrqiM2N1549exAfH48lS5bg0qVLiIqKwsCBA5Gby6/NOX78OEaNGoVjx44hKSkJYWFhePHFF/Hwoe4fEHsj1NsZTiIB5Eo1HhVV2Ho6BjG0KZtFU6Qq09hPaOg8Y+E1XlWHMkgIUxoZgP/JJYgJ1n3P47eto/t6XFSB7aczdH4uhpJquuqBzY3XmjVrMHnyZEycOBHt2rXD5s2b4erqim3btvGO37lzJ6ZNm4bOnTujTZs2+PLLL6FWq5GYmGjlmdcdsUiIZr6uABwjdKzeGqQ//NOUSrCel5NIYFS/RWPg97w0jvWaWdXzMRdfNdW9cVvXfkhzM2TdKb2fTxYdpJquemBT4yWXy3Hx4kXExcVxx4RCIeLi4pCUlGTUNcrLy6FQKODry6+JkclkKC4u1nrZAxEOlLQ3Puel4XmZWaAKGGF0xFKu7pf79W/RVXCHd1ilQo29lx5AZeGQvVBPoj5MkIP3XH5h3lBNV52wqfHKz8+HSqWq1XUoKCgI2dnGdYCZN28eQkNDtQygJgkJCfDy8uJeYWFh9Z63OYhwILkEVw7HQPinuT3I3CuNACA0Rj4Q3gvoMhYAo/0S66j7Ff/DFewysF2nPuQW1+5vWg3Biba/QUA1XfXC5mFjfVixYgV2796Nn3/+Gc7O/NUFFixYgKKiIu6VlZVl5Vny40hyCblKfwloFjbnJVOqOaGqOY2XmEeqwWvPBiwDXP3RRpjFhGY6MKY8TV05radz9zBhEoRptE5XfbGp8fL394dIJEJOTo7W8ZycHAQHB+s9d9WqVVixYgX++OMPdOrUSec4qVQKT09PrZc94EhyCWPDRg+pmPs9zC9h9m2apYpqFcZ0zAbAhGADPwEAzBL/D2GCHN5hv119bK6pGY0nSrHWew/zhmq66oVNjZdEIkG3bt20ku1s8j02NlbneZ9++imWL1+OQ4cOoXv37taYqtlhjVfWkwooVPZRb0oXxq42CoUCuFc1SGULAJo1bORdbdRh0Dq9qlH3azv0iq3MzIZjqZi95wrvZ392PgFReR7VdJkBm4eN8fHx2Lp1K77++mvcunULU6dORVlZGSZOnAgAGDduHBYsWMCN/89//oNFixZh27ZtCA8PR3Z2NrKzs1Fa6jj1sQAgyMMZLk4iqNQEWYX2XZjQWJ0XUB065pUyyWqzho06dV48CKrrfvURXcUwIf8CkCXKEq08zN8gpKvgDvxTdjJvqKar3tjceI0cORKrVq3C4sWL0blzZyQnJ+PQoUNcEj8zMxOPH1e795s2bYJcLsc///lPhISEcK9Vq1bZ6hHqhFAoqM572fkGbUNVVDVhk/bVnpf5wka+hL2+SFKz7tdip295634ZUyDQFHSJjsVQYrP3t8wbqukyC3XqmG1uZsyYgRkzZvB+dvz4ca33GRkZlp+QlYjwd8Wtx8VIyytDvza2no1ujM15AdVyiTwu52XZhL0htqiGYoToDFoJH2KeeA/eV07S+nzl4duY3td8ead/rOOv7DpZdBCBFfeopsuM2Nzzasw4StLe2JwXUO15ccbLjGEj/95G/edo1v0aI07k1X6ZM3RMyS6pdSxMkINZ4v8xb6imy2xQ42VDHKWul6G2Z5qwOS+LhI2GFPY6OE/aYrfyBQD82i/LalUJPhJvh7NAQTVdZoYaLxviKG3Q6pLzsoTnxZuwNzKSTFCORoWTD6/2a8J282zsn70nudaxYcIk9BFdBUQSqukyM9R42RDW83pUVGnXjVFNCRvZnBe3PciMOS9+z8s4iuCOUy3iAdTWfp28q1tQaiwKlRo/16ge4YlSLHaqStL3ppouc0ONlw3xdZNwNbDsOXQ0JWHPel4sZt3bWE+vJcV/EBDRh1f79dqWpHo1p939d+2dG/PEexAgKAL8WgHPvVPna1P4ocbLhggEAofYoC0zIWxkc14sZk3Y823MNsGgqQBgyBpAJK2l/TqbVohPD6XUeW6Lfrmu9b6r4A7GiKvE18PWUk2XBaDGy8Y4wgbtukglWCxuvExArSZM6Pb8HAC1tV8/XHhQr+uziKHEJ05fMW86/xsIf84s16VoQ42XjYlgq6o6gvEyQSrBYum9jaaYMxUrieg1C/B/BgGCIswT76n3vNYn3tV6P1l0EG2EWSCufsCLVNNlKajxsjHh/vZflNAkz8uCYSP/3kbjz1eymgixlNmeg9rar7rsM119pPr8MEEOZor3MnN78WOq6bIg1HjZGLaevV0bL5XxOq/aCXvL/oiZ4nlpbd0J78Vs04G29uu7s/dNuv9Xp9I13jGaLheBHAjvDUS9ZtK1KKZBjZeNYT2v/FI512na3qhfzst8YSMf+prg1qSWUzVgOQqIh5b2a+mvN/FtUobR11z+203u66HCs+gjugoikjCeHdV0WRRqvGyMh7MT/N2ZlSh7zXtV57wMG6LaOS878rxqbgNy9eV6Pmpqvxbtu4EKuWHdneYYT5RhidM3zJyopssqUONlB0TYed7LFIW9s5NIa5zFw0aTcl6181k/q5/DaZ66X7UMHQ9rE6tzXe+Jd1NNl5WhxssOsPcN2qaEjUB181nACmGjCb4Xfy5eu+7XUOFZAECZjL/2vSb/PZEGgNF0/ZtquqwONV52gL3Xs5eZIJUAtPNelva8TEFXra10EoIrEW8AAJY4fQNPlCH6k0Sc1VPj/u+MQgBU02VL7OcnqxETaeeel8KEsBEAPDTkEpbOeZmCSk8oeLnZeJR7tkCAoAjviXcDAF7bclbn+H9tZtT5b1RpukqEXlTTZWXs5yerEROhIZewRFni+mJKSRzAumGjKRl7XZ4XAKiFEri+sh4A8G8ddb9qwtTpYjRdHsP/QzVdVoYaLzuguR+TsC+uVOptVGorTEnYA9YNG01ZbVTqMV4CAYDwXqhoPwpAtfYr6V7t0PFpuRyamq4cv2iq6bIB1HjZAc5OIjTxdgFgf6GjSk24ztLG5rw8rJmwN8F66QsbuTFxyzjt1xuigxi19ayWN6xSE3RedoTTdMmIGEGjNlFNlw2gxstOsNdtQmzICJjgeVkx52XKaqO+sJFDS/u1F2GCHJxPL+Q+TryVo6XpUj/3LtV02QhqvOwEe5VL1MV4uWhsxrYnnZfKCONFCOG0Xy4COT4Sb8fILdWlc6Z8e5HTdGUgFC59363LtClmgBovO8Fe69nLVIyKXCDgL8PMh6a3ZazBswZGGS8AurRf+aUyLU1X03FbqKbLhtjFT9aGDRsQHh4OZ2dnxMTE4Px5/TXFf/zxR7Rp0wbOzs7o2LEjDh48qHe8I8DWs0+zs3r2muVwjN1HqJkbMzZPVlfMlfNiL8MOSSchuNFyMgBG+7X0xzMYsOoop+n6QdkH4sjedZkyxUzY3Hjt2bMH8fHxWLJkCS5duoSoqCgMHDgQubm5vOPPnDmDUaNGYdKkSbh8+TJeeuklvPTSS7h+/TrveEeBlUvcLyg3LjcDxpN4Wi7HE54VSkIIlHUo71ITU9X1gHYNL7GljZdJCnsj/l01hjQZMh+p6lAECIrQ4soqjFTsRxthFgqIB/61YHsdZksxJzZvOrtmzRpMnjwZEydOBABs3rwZBw4cwLZt2zB//vxa4z///HMMGjQIc+fOBQAsX74cR44cwRdffIHNmzdbde7mpKmPC0RCASoUKqz/MxWuEhFKZUoUVypQVKFAcQXzdXGFAiWVShRVKFCqsYWlibcLOjfzRqS/G1KyS5Cc9RR5JTJIxUJ4OIvhLhXD3VkMkVAIoYDpPi0UMFUZhALGCAiFzHH2mFAgQLmcuYcpuSuphQ2WJqZ4XsbsVyQa1ksglmKhYhJ+kC6v3v4D4GH0+/Bz8zNpnhTzY1PjJZfLcfHiRSxYsIA7JhQKERcXh6SkJN5zkpKSEB8fr3Vs4MCB+OWXX3jHy2QyyGTVjRWKi4vrP3EL4CQSItzPFffyyvDZUcMCyZo8fFqBh08rah2XKdWQlcqRX1o//Vigh7PRY59r5Q8AcJNYWKAKINTLxeixxiXsNd4ImJ6Pe5QvYKT4OADglnNndPrHWybOkmIJbGq88vPzoVKpEBQUpHU8KCgIKSn8zRCys7N5x2dnZ/OOT0hIwNKlS80zYQuz/KUO+PnSQyjVBIQQuEnF8HRxgpeLEzydneDpIq76P3tMDA9nJ8hValzNeorLWU+RkV+GZ4I80KWZNyID3FEuV6JUpkRJJfN/tZpATRgvhJDqr9WECTWJxnt2DCFAr5b+Rj9HqLcLjsY/D29Xidn/jd58PhL//YvZEP2PjsGYP7gNyuRK/Kt7WK2xE3qGY8eZDET4uyE9vwyTe0fWGvNiuyAcvZWDl7s2AaCtUfOpmn+CchT6ipLhjgqIhtHei3YDsSEPHz4kAMiZM2e0js+dO5dER0fznuPk5ER27dqldWzDhg0kMDCQd3xlZSUpKiriXllZWQQAKSoqMs9DUKyOQqkieSWVRK1WGxwrU6iISqUmj56W836uVqtJpUKpdayoQk6elssJIYQ8LZeTCxmFZP/pZJKScrP+k6cYpKioyKjfUZt6Xv7+/hCJRMjJydE6npOTg+DgYN5zgoODTRovlUohldLl7IaEWCTkCjgagl1oCNERXgoEglq7ADS3N3m5OKFbcx90a+5Tx9lSLIVNVxslEgm6deuGxMTqZKharUZiYiJiY2N5z4mNjdUaDwBHjhzROZ5CoTRMbL7aGB8fj/Hjx6N79+6Ijo7G2rVrUVZWxq0+jhs3Dk2aNEFCQgIAYNasWejTpw9Wr16NIUOGYPfu3bhw4QK2bNliy8egUChWxubGa+TIkcjLy8PixYuRnZ2Nzp0749ChQ1xSPjMzE0JhtYPYs2dP7Nq1Cx988AEWLlyIVq1a4ZdffkGHDh1s9QgUCsUGCAixwwJSFqS4uBheXl4oKiqCp6enradDoVBqYOzvqM0V9hQKhVIXqPGiUCgOCTVeFArFIbF5wt7asCk+e90mRKE0dtjfTUPp+EZnvEpKSgAAYWG1t5NQKBT7oaSkBF5eXjo/b3SrjWq1Go8ePYKHh4fR9anqQ3FxMcLCwpCVldVgVzfpMzYM7OUZCSEoKSlBaGiolkyqJo3O8xIKhWjatKnV7+vp6dlgf+hZ6DM2DOzhGfV5XCw0YU+hUBwSarwoFIpDQo2XhZFKpViyZEmDrmxBn7Fh4GjP2OgS9hQKpWFAPS8KheKQUONFoVAcEmq8KBSKQ0KNF4VCcUio8TIDhYWFGDNmDDw9PeHt7Y1JkyahtLRU7zkvvPACBFU9EtnXW29pt9TKzMzEkCFD4OrqisDAQMydOxdKpVLHFS2Lqc9YWFiIt99+G61bt4aLiwuaNWuGmTNnoqioSGtczX8DgUCA3bt3W/pxAJi/UzshBIsXL0ZISAhcXFwQFxeHu3fvWvIR9GLK823duhW9e/eGj48PfHx8EBcXV2v8hAkTan2vBg0aZOnH0I1F24A0EgYNGkSioqLI2bNnycmTJ0nLli3JqFGj9J7Tp08fMnnyZPL48WPupdktRalUkg4dOpC4uDhy+fJlcvDgQeLv708WLFhg6cfhxdRnvHbtGnnllVfI/v37SWpqKklMTCStWrUi//d//6c1DgDZvn271r9DRUWFpR+H7N69m0gkErJt2zZy48YNMnnyZOLt7U1ycnJ4x58+fZqIRCLy6aefkps3b5IPPviAODk5kWvXrnFjVqxYQby8vMgvv/xCrly5QoYPH04iIiKs8jw1MfX5Ro8eTTZs2EAuX75Mbt26RSZMmEC8vLzIgwcPuDHjx48ngwYN0vpeFRYWWuuRakGNVz25efMmAUD+/vtv7tjvv/9OBAIBefjwoc7z+vTpQ2bNmqXz84MHDxKhUEiys7O5Y5s2bSKenp5EJpOZZe7GUtdnrMkPP/xAJBIJUSgU3DEA5OeffzbndI0iOjqaTJ8+nXuvUqlIaGgoSUhI4B3/6quvkiFDhmgdi4mJIW+++SYhhGmhFhwcTFauXMl9/vTpUyKVSsn3339vgSfQj6nPVxOlUkk8PDzI119/zR0bP348GTFihLmnWmdo2FhPkpKS4O3tje7du3PH4uLiIBQKce7cOb3n7ty5E/7+/ujQoQMWLFiA8vJyret27NhRq8HuwIEDUVxcjBs3bpj/QfRQn2fUhC3rKxZrb6mdPn06/P39ER0djW3bthkshVJf2E7tcXFx3DFjOrVrjgeY7wc7Pj09HdnZ2VpjvLy8EBMTo/OalqIuz1eT8vJyKBQK+Pr6ah0/fvw4AgMD0bp1a0ydOhUFBQVmnbspNLqN2eYmOzsbgYGBWsfEYjF8fX11dvEGgNGjR6N58+YIDQ3F1atXMW/ePNy+fRt79+7lrsvXGZz9zJrU9Rk1yc/Px/LlyzFlyhSt48uWLUO/fv3g6uqKP/74A9OmTUNpaSlmzpxptvnzzcXcndrZ/5vSzd1S1OX5ajJv3jyEhoZqGcBBgwbhlVdeQUREBO7du4eFCxdi8ODBSEpKgkgk0nM1y0CNlw7mz5+P//znP3rH3Lp1q87X1/wl7tixI0JCQtC/f3/cu3cPLVq0qPN1TcHSz8hSXFyMIUOGoF27dvjwww+1Plu0aBH3dZcuXVBWVoaVK1da1HhR9LNixQrs3r0bx48fh7OzM3f8tdde477u2LEjOnXqhBYtWuD48ePo37+/1edJjZcO3n33XUyYMEHvmMjISAQHByM3N1fruFKpRGFhoc4u3nzExMQAAFJTU9GiRQsEBwfXWu1hO4Wbcl19WOMZS0pKMGjQIHh4eODnn3+Gk5OT3vExMTFYvnw5ZDKZxfbYWaJTO/v/nJwchISEaI3p3LmzGWdvmLo8H8uqVauwYsUKHD16FJ06ddI7NjIyEv7+/khNTbWJ8aIJ+3rCJrMvXLjAHTt8+LDJyexTp04RAOTKlSuEkOqEvebq0H//+1/i6elJKisrzfcARlDXZywqKiI9evQgffr0IWVlZUbd66OPPiI+Pj71nrMhoqOjyYwZM7j3KpWKNGnSRG/CfujQoVrHYmNjayXsV61axX1eVFRk04S9Kc9HCCH/+c9/iKenJ0lKSjLqHllZWUQgEJB9+/bVe751gRovMzBo0CDSpUsXcu7cOXLq1CnSqlUrLRnBgwcPSOvWrcm5c+cIIYSkpqaSZcuWkQsXLpD09HSyb98+EhkZSZ5//nnuHFYq8eKLL5Lk5GRy6NAhEhAQYFOphCnPWFRURGJiYkjHjh1Jamqq1vK6UqkkhBCyf/9+snXrVnLt2jVy9+5dsnHjRuLq6koWL15s8efZvXs3kUqlZMeOHeTmzZtkypQpxNvbm1vdHTt2LJk/fz43/vTp00QsFpNVq1aRW7dukSVLlvBKJby9vcm+ffvI1atXyYgRI2wqlTDl+VasWEEkEgn56aeftL5XJSUlhBBCSkpKyJw5c0hSUhJJT08nR48eJV27diWtWrWy+h9TFmq8zEBBQQEZNWoUcXd3J56enmTixIncN50QQtLT0wkAcuzYMUIIIZmZmeT5558nvr6+RCqVkpYtW5K5c+dq6bwIISQjI4MMHjyYuLi4EH9/f/Luu+9qyQysianPeOzYMQKA95Wenk4IYeQWnTt3Ju7u7sTNzY1ERUWRzZs3E5VKZZVnWr9+PWnWrBmRSCQkOjqanD17lvusT58+ZPz48Vrjf/jhB/LMM88QiURC2rdvTw4cOKD1uVqtJosWLSJBQUFEKpWS/v37k9u3b1vjUXgx5fmaN2/O+71asmQJIYSQ8vJy8uKLL5KAgADi5OREmjdvTiZPnqwl5bE2tCQOhUJxSKjOi0KhOCTUeFEoFIeEGi8KheKQUONFoVAcEmq8KBSKQ0KNF4VCcUio8aJQKA4JNV4UCsUhocaLYhfwlRi2eZlhil1Dq0pQ7IZBgwZh+/btWsd0VZZQKBS1KlTI5XJIJBKT71vX8yi2hXpeFLtBKpUiODhY6+Xj4wOAadSxadMmDB8+HG5ubvj444/x4YcfonPnzvjyyy8RERHB1Z7KzMzEiBEj4O7uDk9PT7z66qta5WF0nUdxLKjxojgMH374IV5++WVcu3YNr7/+OgCm/tn//vc/7N27F8nJyVCr1RgxYgQKCwtx4sQJHDlyBGlpaRg5cqTWtWqeR3E8aNhIsRt+++03uLu7ax1buHAhFi5cCIApnT1x4kStz+VyOb755hsEBAQAAI4cOYJr164hPT0dYWFhAIBvvvkG7du3x99//41nn32W9zyK40GNF8Vu6Nu3LzZt2qR1TLMBhGYDEJbmzZtrGaBbt24hLCyMM1wA0K5dO3h7e+PWrVuc8ap5HsXxoMaLYje4ubmhZcuWej835pix96I4NjTnRWlQtG3bFllZWcjKyuKO3bx5E0+fPkW7du1sODOKuaGeF8VukMlktdqEicVi+Pv7G32NuLg4dOzYEWPGjMHatWuhVCoxbdo09OnThzfspDgu1POi2A2HDh1CSEiI1uu5554z6RoCgQD79u2Dj48Pnn/+ecTFxSEyMhJ79uyx0KwptoKWgaZQKA4J9bwoFIpDQo0XhUJxSKjxolAoDgk1XhQKxSGhxotCoTgk1HhRKBSHhBovCoXikFDjRaFQHBJqvCgUikNCjReFQnFIqPGiUCgOCTVeFArFIfl/zJ9X7x8vyesAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.018104, + "end_time": "2024-03-24T03:34:58.483395", + "exception": false, + "start_time": "2024-03-24T03:34:58.465291", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 5281.145473, + "end_time": "2024-03-24T03:35:01.224529", + "environment_variables": {}, + "exception": null, + "input_path": "eval/treatment/tvae/1/mlu-eval.ipynb", + "output_path": "eval/treatment/tvae/1/mlu-eval.ipynb", + "parameters": { + "allow_same_prediction": true, + "dataset": "treatment", + "dataset_name": "treatment", + "debug": false, + "folder": "eval", + "gp": true, + "gp_multiply": true, + "log_wandb": false, + "param_index": 0, + "path": "eval/treatment/tvae/1", + "path_prefix": "../../../../", + "random_seed": 1, + "single_model": "tvae" + }, + "start_time": "2024-03-24T02:07:00.079056", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/treatment/tvae/model.pt b/treatment/tvae/model.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5f66d27fe898a00fca9b4f1788179f78373476d --- /dev/null +++ b/treatment/tvae/model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19b6e979000ccc08d99a524afa70e9b57bdc0776c05d6f333358764ef5b3eb10 +size 118970453 diff --git a/treatment/tvae/params.json b/treatment/tvae/params.json new file mode 100644 index 0000000000000000000000000000000000000000..4e249d6db10aa6039124321d439ec4f3e9207374 --- /dev/null +++ b/treatment/tvae/params.json @@ -0,0 +1 @@ +{"Body": "twin_encoder", "loss_balancer_meta": true, "loss_balancer_log": false, "loss_balancer_lbtw": false, "pma_skip_small": false, "isab_skip_small": false, "layer_norm": false, "pma_layer_norm": false, "attn_residual": true, "tf_n_layers_dec": false, "tf_isab_rank": 0, "tf_lora": false, "tf_layer_norm": false, "tf_pma_start": -1, "ada_n_seeds": 0, "head_n_seeds": 0, "tf_pma_low": 1, "gradient_penalty_kwargs": {"mag_loss": true, "mse_mag": true, "mag_corr": false, "seq_mag": false, "cos_loss": false, "mse_mag_kwargs": {"target": 1.0, "multiply": true, "forgive_over": true}, "mag_corr_kwargs": {"only_sign": false}, "cos_loss_kwargs": {"only_sign": true, "cos_matrix": false}}, "dropout": 0, "combine_mode": "diff_left", "tf_isab_mode": "separate", "grad_loss_fn": "mae", "single_model": true, "bias": true, "bias_final": true, "pma_ffn_mode": "none", "patience": 10, "inds_init_mode": "torch", "grad_clip": 0.75, "gradient_penalty_mode": "ALL", "synth_data": 2, "bias_lr_mul": 1.0, "bias_weight_decay": 0.05, "loss_balancer_beta": 0.75, "loss_balancer_r": 0.96, "tf_pma_low_exp_2": 4, "dataset_size_exp_2": 11, "batch_size_exp_2": 1, "epochs": 100, "lr_mul": 0.06, "n_warmup_steps": 80, "Optim": "diffgrad", "fixed_role_model": "tvae", "mse_mag_target": 0.5, "g_loss_mul": 0.2, "d_model_exp_2": 9, "attn_activation": "selu", "tf_d_inner_exp_2": 8, "tf_n_layers_enc": 4, "tf_n_head_exp_2": 5, "tf_activation": "leakyhardtanh", "tf_activation_final": "leakyhardtanh", "tf_num_inds_exp_2": 5, "ada_d_hid_exp_2": 11, "ada_n_layers": 6, "ada_activation": "leakyhardtanh", "ada_activation_final": "leakyhardsigmoid", "head_d_hid_exp_2": 7, "head_n_layers": 7, "head_n_head_exp_2": 6, "head_activation": "leakyhardsigmoid", "head_activation_final": "leakyhardsigmoid", "mse_mag": true, "mse_mag_multiply": true, "models": ["tvae"], "max_seconds": 3600} \ No newline at end of file