{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e90f90dd-be7b-4706-8fa6-fff9abe22192",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6c055b4d-336a-4963-9c87-3a0da5935e96",
   "metadata": {},
   "outputs": [],
   "source": [
    "import subprocess\n",
    "from concurrent.futures import ThreadPoolExecutor, as_completed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7ce02c2d-750e-4405-af82-495ae08b59a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['python3 train.py --run_number -1 --task_name var_copy --layer1 TF --layer2 TF --embed_dim 12 --window 20 --num_heads 1 --state_dim 1 --num_epochs 100 --save_path interpret.pt']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "command= ['python3 train.py --run_number -1 --task_name var_copy --layer1 TF --layer2 TF --embed_dim 12 --window 20 --num_heads 1 --state_dim 1 --num_epochs 100 --save_path interpret.pt']\n",
    "command"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8b875c1d-ea18-4e4d-80a3-f9bbc8880ef4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['python3 train.py --run_number -1 --task_name var_copy --layer1 TF --layer2 TF --embed_dim 12 --window 20 --num_heads 1 --state_dim 1 --num_epochs 100 --save_path interpret.pt'],\n",
       " 0,\n",
       " 'Generating Data\\n- Sequence Num: 0\\n- Sequence Num: 8000\\n- Sequence Num: 16000\\n- Sequence Num: 24000\\n- Sequence Num: 32000\\n- Sequence Num: 40000\\n- Sequence Num: 48000\\n- Sequence Num: 56000\\nStarting work for results/data_100_5_30/run-1.pt\\nLoss: 2.7080, Acc: 0.2483\\n',\n",
       " '')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def run_command(cmd):\n",
    "    \"\"\"Run a single shell command and return (cmd, returncode, stdout, stderr).\"\"\"\n",
    "    result = subprocess.run(cmd, shell=True, capture_output=True, text=True)\n",
    "    return cmd, result.returncode, result.stdout, result.stderr\n",
    "\n",
    "run_command(command)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f92e83f8-a718-412c-9a46-0afd43319018",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "9c0b1f58-6964-44e8-888a-d03c77da2484",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import matplotlib.pyplot as plt\n",
    "from generate import generate_data, set_task_specific_parameters\n",
    "from models.transformer import generate_mask, scaled_dot_product_attention, SimpleHandmadeTFLayer\n",
    "from models.ssm import SimpleSSMLayer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2fafc39f-b2f9-436e-824d-bf39b55f41db",
   "metadata": {},
   "outputs": [],
   "source": [
    "class args:\n",
    "    batch_size = 1\n",
    "    batches_per_epoch = 1\n",
    "    task_name = \"var_copy\"\n",
    "    min_num_token = 5\n",
    "    num_numbers = 5\n",
    "    num_vocab = 30\n",
    "    sequence_len = 100\n",
    "\n",
    "set_task_specific_parameters(args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "bedcc2e9-3a59-4f90-b119-a9b8b948ade7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating Data\n",
      "- Sequence Num: 0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[18, 17, 11, 19,  6, 22,  6, 25, 38,  5, 39, 39, 24, 38,  9, 29, 20, 14,\n",
       "          5,  5,  5, 21, 14,  5, 36, 35, 28,  8,  5,  8,  6, 25, 38,  9,  7, 16,\n",
       "         30, 14, 13,  7, 18, 14, 30,  5,  7, 26,  6, 27, 39, 35, 17, 10,  9, 22,\n",
       "         35, 21, 22, 24, 39, 39, 18, 30,  5,  8,  5,  8, 25, 39,  5,  6, 28,  8,\n",
       "         34, 19, 31,  7,  6, 12, 36, 11,  8, 23, 10, 30, 35, 17, 19, 14,  8, 10,\n",
       "         38, 13, 13, 38, 17,  9, 29,  5, 23, 23]], dtype=torch.int32)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "in_data, out_data = generate_data(args)\n",
    "in_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d74c8b86-6493-4bdf-914a-4131e24b56a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = torch.load(\"saved_models/interpret.pt\", weights_only=False).to('cuda:0')\n",
    "mask = generate_mask(args.sequence_len).to('cuda:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e8466863-faa0-425b-a3e0-64931805f276",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HybridModel(\n",
       "  (embedding): Embedding(41, 12)\n",
       "  (layers): ModuleList(\n",
       "    (0-1): 2 x SimpleHandmadeTFLayer(\n",
       "      (transformer_encoder): TransformerHead(\n",
       "        (mha): MultiHeadAttention(\n",
       "          (q_proj): Linear(in_features=12, out_features=12, bias=True)\n",
       "          (k_proj): Linear(in_features=12, out_features=12, bias=True)\n",
       "          (v_proj): Linear(in_features=12, out_features=12, bias=True)\n",
       "          (out_proj): Linear(in_features=12, out_features=12, bias=True)\n",
       "          (dropout): Dropout(p=0.2, inplace=False)\n",
       "        )\n",
       "        (norm1): LayerNorm((12,), eps=1e-05, elementwise_affine=True)\n",
       "        (norm2): LayerNorm((12,), eps=1e-05, elementwise_affine=True)\n",
       "        (ffn): Sequential(\n",
       "          (0): Linear(in_features=12, out_features=12, bias=True)\n",
       "          (1): ReLU()\n",
       "          (2): Dropout(p=0.2, inplace=False)\n",
       "          (3): Linear(in_features=12, out_features=12, bias=True)\n",
       "        )\n",
       "        (dropout): Dropout(p=0.2, inplace=False)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (decoder): Linear(in_features=12, out_features=41, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9d61b9cf-38d0-4a18-97fc-75eed286fbee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4193"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(p.numel() for p in model.parameters() if p.requires_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "0bdb809b-fdaf-4961-87fa-1ce2ec444ff1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGdCAYAAABU0qcqAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjMsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvZiW1igAAAAlwSFlzAAAPYQAAD2EBqD+naQAAGbpJREFUeJzt3X1sVPe95/HPeMYeGzDmKbZxMMFNuSI8hEIMiDjbNsUKQoQFZW9UVk5lkVWoUlNwkJLgtsBmKRhoy0UQZBKkBKLylD8KSVmFCrk8iC3PhjQoLRCFS6ZQ26ELM8aGsT3z2z+yeNcJFHJzZr4e+/2Szh8+c/D3NzCeN8cen/E555wAAEiyNOsFAAB6JgIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMBKwX8GXxeFxXrlxRdna2fD6f9XIAAF+Tc05NTU0qKChQWtrdz3O6XICuXLmiwsJC62UAAL6hUCikIUOG3PX2Lheg7OxsSdLYd34if69g0uf3CUaTPvO2xht9TOZa3ufnH/pfZrMvRR8wm33kH0Vms9vjdt95v3K1n8ncPn1umcyVpKz0NrPZD2aHTea2Nbfqf87a1vF8fjddLkC3v+3m7xWUv3fyAxRI/sgO/rjN8ECmyVhJUlYfu4dgMD3dbHbgluEDzTBAaS02DzZ/L7tLXgYy7P6+03tnmM2WdM8fo/AiBACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMJCxAGzZs0LBhw5SZmalJkybp+PHjiRoFAEhBCQnQzp07tXDhQi1dulR1dXUaO3aspk6dqsbGxkSMAwCkoIQEaM2aNXrhhRc0Z84cjRw5Uhs3blSvXr301ltvJWIcACAFeR6g1tZWnTp1SqWlpf9vSFqaSktLdeTIka8cH41GFYlEOm0AgO7P8wBdvXpVsVhMeXl5nfbn5eWpvr7+K8dXV1crJyenY+OtGACgZzB/FVxVVZXC4XDHFgqFrJcEAEgCz6+FP2jQIPn9fjU0NHTa39DQoPz8/K8cHwwGFQwaXpoeAGDC8zOgjIwMPfbYY6qtre3YF4/HVVtbq8mTJ3s9DgCQohLybmALFy5UeXm5iouLNXHiRK1du1bNzc2aM2dOIsYBAFJQQgL0wx/+UJ9//rmWLFmi+vp6fec739HevXu/8sIEAEDPlbD3Q543b57mzZuXqE8PAEhx5q+CAwD0TAQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYSNgvon5TfYJRBQyuUdrSlp78of/XkJywydxwNNNkriT996MzzWa7uM9sdvaAZrPZVo8zSfrBt8+ZzD13vWdeheXc1VyTubGW6H0dxxkQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgImA9QLuxu+Ly++LJ33u59eykz7ztl65bSZzfSZTv5DTv9lsdszZ3fNeGTb/1pLUN+OW2ezaCyNM5g7od8NkriRlBtrNZt9ozjSZG2+5v+M4AwIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAE54HqLq6WhMmTFB2drZyc3M1a9YsnTt3zusxAIAU53mADh48qIqKCh09elT79u1TW1ubnnrqKTU3213zCwDQ9Xh+MdK9e/d2+njz5s3Kzc3VqVOn9N3vftfrcQCAFJXwnwGFw2FJ0oABAxI9CgCQQhL6dgzxeFyVlZUqKSnR6NGj73hMNBpVNBrt+DgSiSRySQCALiKhZ0AVFRU6e/asduzYcddjqqurlZOT07EVFhYmckkAgC4iYQGaN2+e9uzZo/3792vIkCF3Pa6qqkrhcLhjC4VCiVoSAKAL8fxbcM45/fSnP9WuXbt04MABFRUV/dPjg8GggsGg18sAAHRxngeooqJC27Zt03vvvafs7GzV19dLknJycpSVleX1OABAivL8W3A1NTUKh8P6/ve/r8GDB3dsO3fu9HoUACCFJeRbcAAA3AvXggMAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADARELfjuGbaGoNKpCe/GvE5Q+wezuIm23pZrOt9A62ms0OpMXNZrfH7f7vF2rqZzY7f2DYZK7lr8fHnM9s9oODrpvMbW+O6t/v4zjOgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMBKwXcDf1/z5QaVmZSZ+bX/SPpM+87YdDT5nM3X35OyZzJSm2Mc9sduiZVrPZ8Ui62Wyf85nNfnpSncncg3/7tslcScrKaDObPW7gZZO5reltOnIfx3EGBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwETCA7Ry5Ur5fD5VVlYmehQAIIUkNEAnTpzQG2+8oUcffTSRYwAAKShhAbpx44bKysq0adMm9e/fP1FjAAApKmEBqqio0PTp01VaWvpPj4tGo4pEIp02AED3l5D3A9qxY4fq6up04sSJex5bXV2t1157LRHLAAB0YZ6fAYVCIS1YsEBbt25VZua931CuqqpK4XC4YwuFQl4vCQDQBXl+BnTq1Ck1NjZq/PjxHftisZgOHTqk119/XdFoVH6/v+O2YDCoYDDo9TIAAF2c5wGaMmWKPvroo0775syZoxEjRujVV1/tFB8AQM/leYCys7M1evToTvt69+6tgQMHfmU/AKDn4koIAAATCXkV3JcdOHAgGWMAACmEMyAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATCTlF1H/QzLiX2xJduOW3YVRe6W1msyNO5/JXElyhv8F6nsky2x2eGTMbHZg4E2z2Vdu5pjMbW4x/LoO2nxdS9If/jj+3gclQPzWLUnv3vM4zoAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATASsF3A3Dz74vxXoHUz63AGZLUmfedvOy8Umc2POZzJXkooWnjOb3Rr3m81O8zmz2ef/8YDZ7CG9rpvMjQ+xe4w3tmSbzX7mqSMmc6M32rR+yb2P4wwIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACAiYQE6PLly3ruuec0cOBAZWVlacyYMTp58mQiRgEAUpTnFyO9du2aSkpK9OSTT+qDDz7QAw88oAsXLqh///5ejwIApDDPA7Rq1SoVFhbq7bff7thXVFTk9RgAQIrz/Ftw77//voqLi/Xss88qNzdX48aN06ZNm+56fDQaVSQS6bQBALo/zwP06aefqqamRsOHD9cf/vAHvfjii5o/f762bNlyx+Orq6uVk5PTsRUWFnq9JABAF+R5gOLxuMaPH68VK1Zo3Lhxmjt3rl544QVt3LjxjsdXVVUpHA53bKFQyOslAQC6IM8DNHjwYI0cObLTvkceeUSfffbZHY8PBoPq27dvpw0A0P15HqCSkhKdO9f5bZbPnz+vhx56yOtRAIAU5nmAXnrpJR09elQrVqzQJ598om3btunNN99URUWF16MAACnM8wBNmDBBu3bt0vbt2zV69GgtW7ZMa9euVVlZmdejAAApzPPfA5Kkp59+Wk8//XQiPjUAoJvgWnAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmEjIL6J64fNwH6W1ZSZ9blp/l/SZt+X2ajKZG2rqZzJXko5fsrtG4H8b/Sez2W99PNlsdtvVLLPZtbF/MZn7yAMNJnMlKZAWN5t9qWWAydy2m633dRxnQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmAtYLuJv//O2PFOyTnvS5x/4xLOkzbztxvshkbv7gayZzJem9yTVms9d//gOz2VO+dd5s9s2Hkv91dVtvf6vJ3L6BmyZzJamxJdtsdtz5uvRczoAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwITnAYrFYlq8eLGKioqUlZWlhx9+WMuWLZNzzutRAIAU5vm14FatWqWamhpt2bJFo0aN0smTJzVnzhzl5ORo/vz5Xo8DAKQozwP0pz/9STNnztT06dMlScOGDdP27dt1/Phxr0cBAFKY59+Ce/zxx1VbW6vz57+42u+HH36ow4cPa9q0aXc8PhqNKhKJdNoAAN2f52dAixYtUiQS0YgRI+T3+xWLxbR8+XKVlZXd8fjq6mq99tprXi8DANDFeX4G9O6772rr1q3atm2b6urqtGXLFv3617/Wli1b7nh8VVWVwuFwxxYKhbxeEgCgC/L8DOjll1/WokWLNHv2bEnSmDFjdOnSJVVXV6u8vPwrxweDQQWDQa+XAQDo4jw/A2ppaVFaWudP6/f7FY/HvR4FAEhhnp8BzZgxQ8uXL9fQoUM1atQonT59WmvWrNHzzz/v9SgAQArzPEDr16/X4sWL9ZOf/ESNjY0qKCjQj3/8Yy1ZssTrUQCAFOZ5gLKzs7V27VqtXbvW608NAOhGuBYcAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACY8/0VUr5y5NkSB1p51kdK09JjJXL/P7u3S4/KZzT4XzjWbParf381mN7X1MZt9pTnHZO6NtgyTudYaW7JN5ra3RO/rOM6AAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwErBdwNzfb0xVoT0/63La4XZPzHwibzbZScf6/ms2+1W738K+7Wmg2uyfy+5zZ7Ljzmc22eozH2mP3dRxnQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEx87QAdOnRIM2bMUEFBgXw+n3bv3t3pdueclixZosGDBysrK0ulpaW6cOGCV+sFAHQTXztAzc3NGjt2rDZs2HDH21evXq1169Zp48aNOnbsmHr37q2pU6fq1q1b33ixAIDu42tfKnXatGmaNm3aHW9zzmnt2rX6xS9+oZkzZ0qS3nnnHeXl5Wn37t2aPXv2N1stAKDb8PRnQBcvXlR9fb1KS0s79uXk5GjSpEk6cuTIHf9MNBpVJBLptAEAuj9PA1RfXy9JysvL67Q/Ly+v47Yvq66uVk5OTsdWWMj7pABAT2D+KriqqiqFw+GOLRQKWS8JAJAEngYoPz9fktTQ0NBpf0NDQ8dtXxYMBtW3b99OGwCg+/M0QEVFRcrPz1dtbW3HvkgkomPHjmny5MlejgIApLiv/Sq4Gzdu6JNPPun4+OLFizpz5owGDBigoUOHqrKyUr/85S81fPhwFRUVafHixSooKNCsWbO8XDcAIMV97QCdPHlSTz75ZMfHCxculCSVl5dr8+bNeuWVV9Tc3Ky5c+fq+vXreuKJJ7R3715lZmZ6t2oAQMrzOeec9SL+f5FI5IuXbu+ar0DvYNLnt8XtXpeR5rP5p/CZTP1CIC1uNvtW+9f+/5dnLO93T2T1tSVJcWf3FdZu9HwWa46q7l//TeFw+J/+XN/8VXAAgJ6JAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABN2v4l3D7fa0+VvS0/63PANuys2/I9xvzeZ+8Zn3zWZK0lZ/+Wa2eyg4XtPnX+r2Gx2Ru9Ws9nn/tM7JnNnX/yByVxJ+vT6QLPZWTX9Tea2t93fO2BzBgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMB6wV8mXNOkhRriZrMj7eYjJUktdyImcxtb7b5u5akdtdqNjvm2sxmx2/espvts/s7jzTFTea2NRs+zoyeyySpvc3mcdbe/sXc28/nd+Nz9zoiyf72t7+psLDQehkAgG8oFAppyJAhd729ywUoHo/rypUrys7Ols/n+9p/PhKJqLCwUKFQSH379k3ACruennifJe53T7rfPfE+S6l7v51zampqUkFBgdLS7v6Tni73Lbi0tLR/Wsz71bdv35T6B/NCT7zPEve7J+mJ91lKzfudk5Nzz2N4EQIAwAQBAgCY6HYBCgaDWrp0qYLBoPVSkqYn3meJ+92T7ndPvM9S97/fXe5FCACAnqHbnQEBAFIDAQIAmCBAAAATBAgAYKJbBWjDhg0aNmyYMjMzNWnSJB0/ftx6SQlVXV2tCRMmKDs7W7m5uZo1a5bOnTtnvaykWrlypXw+nyorK62XknCXL1/Wc889p4EDByorK0tjxozRyZMnrZeVULFYTIsXL1ZRUZGysrL08MMPa9myZfe8xlgqOXTokGbMmKGCggL5fD7t3r270+3OOS1ZskSDBw9WVlaWSktLdeHCBZvFeqzbBGjnzp1auHChli5dqrq6Oo0dO1ZTp05VY2Oj9dIS5uDBg6qoqNDRo0e1b98+tbW16amnnlJzc7P10pLixIkTeuONN/Too49aLyXhrl27ppKSEqWnp+uDDz7Qxx9/rN/85jfq37+/9dISatWqVaqpqdHrr7+uv/zlL1q1apVWr16t9evXWy/NM83NzRo7dqw2bNhwx9tXr16tdevWaePGjTp27Jh69+6tqVOn6tYtuwvaesZ1ExMnTnQVFRUdH8diMVdQUOCqq6sNV5VcjY2NTpI7ePCg9VISrqmpyQ0fPtzt27fPfe9733MLFiywXlJCvfrqq+6JJ56wXkbSTZ8+3T3//POd9j3zzDOurKzMaEWJJcnt2rWr4+N4PO7y8/Pdr371q459169fd8Fg0G3fvt1ghd7qFmdAra2tOnXqlEpLSzv2paWlqbS0VEeOHDFcWXKFw2FJ0oABA4xXkngVFRWaPn16p3/z7uz9999XcXGxnn32WeXm5mrcuHHatGmT9bIS7vHHH1dtba3Onz8vSfrwww91+PBhTZs2zXhlyXHx4kXV19d3epzn5ORo0qRJ3eK5rctdjPQ/4urVq4rFYsrLy+u0Py8vT3/961+NVpVc8XhclZWVKikp0ejRo62Xk1A7duxQXV2dTpw4Yb2UpPn0009VU1OjhQsX6mc/+5lOnDih+fPnKyMjQ+Xl5dbLS5hFixYpEoloxIgR8vv9isViWr58ucrKyqyXlhT19fWSdMfnttu3pbJuESB8cUZw9uxZHT582HopCRUKhbRgwQLt27dPmZmZ1stJmng8ruLiYq1YsUKSNG7cOJ09e1YbN27s1gF69913tXXrVm3btk2jRo3SmTNnVFlZqYKCgm59v3uKbvEtuEGDBsnv96uhoaHT/oaGBuXn5xutKnnmzZunPXv2aP/+/Z68lUVXdurUKTU2Nmr8+PEKBAIKBAI6ePCg1q1bp0AgoFjM5l1lE23w4MEaOXJkp32PPPKIPvvsM6MVJcfLL7+sRYsWafbs2RozZox+9KMf6aWXXlJ1dbX10pLi9vNXd31u6xYBysjI0GOPPaba2tqOffF4XLW1tZo8ebLhyhLLOad58+Zp165d+uMf/6iioiLrJSXclClT9NFHH+nMmTMdW3FxscrKynTmzBn5/X7rJSZESUnJV15if/78eT300ENGK0qOlpaWr7yhmd/vVzxu89beyVZUVKT8/PxOz22RSETHjh3rHs9t1q+C8MqOHTtcMBh0mzdvdh9//LGbO3eu69evn6uvr7deWsK8+OKLLicnxx04cMD9/e9/79haWlqsl5ZUPeFVcMePH3eBQMAtX77cXbhwwW3dutX16tXL/fa3v7VeWkKVl5e7Bx980O3Zs8ddvHjR/e53v3ODBg1yr7zyivXSPNPU1OROnz7tTp8+7SS5NWvWuNOnT7tLly4555xbuXKl69evn3vvvffcn//8Zzdz5kxXVFTkbt68abzyb67bBMg559avX++GDh3qMjIy3MSJE93Ro0etl5RQku64vf3229ZLS6qeECDnnPv973/vRo8e7YLBoBsxYoR78803rZeUcJFIxC1YsMANHTrUZWZmum9961vu5z//uYtGo9ZL88z+/fvv+HVcXl7unPvipdiLFy92eXl5LhgMuilTprhz587ZLtojvB0DAMBEt/gZEAAg9RAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJv4PJ7tglwfNFjgAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if type(model.layers[0]) == SimpleHandmadeTFLayer:\n",
    "    layer0 = model.layers[0].transformer_encoder\n",
    "    q_proj = layer0.mha.q_proj.weight.to('cpu').detach()\n",
    "    k_proj = layer0.mha.k_proj.weight.to('cpu').detach()\n",
    "\n",
    "    plt.imshow(torch.matmul(q_proj, k_proj.transpose(-2, -1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "b3945736-3161-4840-b4b8-d2878771ed94",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGgCAYAAADsNrNZAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjMsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvZiW1igAAAAlwSFlzAAAPYQAAD2EBqD+naQAAPC1JREFUeJzt3X9wXNV58PHn3l1pJdv6Ycu2ZGEZK4TUBMgbgsEYaJsGT5mUtBA8aZk6rUMypUnkgPFMCE5jOiEBkXQmoWQcKEzqkBZCw/sWkpKWvLwmYYZgftgpJA7BdoLBxrbkn5Js/dbe8/4hd/ec51q7XlvSuSt/PzOa2bv37u79sdLRPc85zxMYY4wAADDJQt87AAA4M9EAAQC8oAECAHhBAwQA8IIGCADgBQ0QAMALGiAAgBc0QAAAL2iAAABe0AABALyYsAZo/fr1snDhQqmqqpIlS5bIyy+/PFEfBQAoQ8FE5IL7t3/7N/nrv/5reeCBB2TJkiVy7733yuOPPy7btm2TuXPnFnxtFEWyd+9eqampkSAIxnvXAAATzBgjR48elebmZgnDAvc5ZgJceumlpq2tLbeczWZNc3OzaW9vL/ra3bt3GxHhhx9++OGnzH92795d8O99WsbZ0NCQbNmyRdauXZt7LgxDWbZsmWzatCm2/eDgoAwODuaWzfEbsrd/sVBqZ4y2nB99z4XjvZsAgAkyIsPyvPyn1NTUFNxu3BuggwcPSjablcbGRuf5xsZGeeONN2Lbt7e3y5e//OXY87UzQqmtGW2A0kHFeO8mAGCijN5HFA2jjHsDVKq1a9fKmjVrcss9PT3S0tIin3r796VieqWIiPQuf09u/fT/89Kk7yMAYPyNewM0e/ZsSaVS0tnZ6Tzf2dkpTU1Nse0zmYxkMpnx3g0AQMKN+zDsyspKufjii2Xjxo2556Ioko0bN8rSpUvH++MAAGVqQrrg1qxZIytXrpTFixfLpZdeKvfee6/09vbKjTfeeNLv8ZdzNsn0mpSIiNz7y5m557PjvrcAAB8mpAH6i7/4Czlw4IDccccd0tHRIe9///vl6aefjg1MAACcuSZsEMKqVatk1apVE/X2AIAyRy44AIAX3odhj+U9lV1SU3m8fTx4xO/OAADGHXdAAAAvaIAAAF7QAAEAvEhsDKgpVS21qdH2Mdvd43lvAADjjTsgAIAXNEAAAC8S2wX3z90LpSp7fPcuPT+/4sVf+tkhAMC44g4IAOAFDRAAwAsaIACAF4mNAf3vd94v6emjheoGL5iee77hRV97BAAYT9wBAQC8oAECAHhBAwQA8CKxMaD+pxslVVklIiI1B0dyzx/9i8uc7eq3dDrL0Vu7c4/NyIgAAJKJOyAAgBc0QAAAL2iAAABeJDYGVH0gK+mKrIiIZA4O5Z7vW1TtbDc8r95ZTncezD02R49O3A4CAE4Ld0AAAC9ogAAAXiS2C27GE5slHVSIiEiQyeSen9PT6mx3cPFMZ3noAxfmHs//0R5nXbT/oLNsBgfd5chYG2dL32kAwEnjDggA4AUNEADACxogAIAXiY0BhdOqJQwqRx/PrM893980w9muotc4y/Xb+3OPTcptX4NUyv0QK7YkIhJk83Efk3W3NcNDAgAYP9wBAQC8oAECAHhBAwQA8CKxMSAzNCwmOP64tzf3fKrPLbGQrlZxHisklJ3lxovS2cj9kENH3M+0YkBi1LYAgHHFHRAAwAsaIACAF8ntghsZEROM9sFFvfmh1RVv7Ha2S7U2O8sDc6tyj0eq3GHWM1QXXGpApeIx+f47MzR8CnsNADhZ3AEBALygAQIAeEEDBADwIrExIJs9PDp7uMtZl1JxnWnds3KPB+fXO+uijHu4vUvPdpZrtuXfO7Wnw1mX7eoeewePx6ryO2xOvB0AIIc7IACAFzRAAAAvaIAAAF6URQwoCPMxlrC+zl05q37M1x1dUOksVx1xSyzUvvyOsxx19+QemyHKLwDAROIOCADgBQ0QAMCLsuiCMyP5DNjZg4fclXrZMnP77wq+7+G/vMxZ7p2Xb4+r96tKq/+yacz3CdIVzjLVUwGgOO6AAABe0AABALygAQIAeFEWMaCJMvM1tyJqz9kN+cetbnqd+gLvE6TcdtxQyQEAiuIOCADgBQ0QAMALGiAAgBdndAwo2r7TWT7ruXw577c+Mu2k36f36vc5yzNecN/X9PXlP7O3t5RdBIApizsgAIAXNEAAAC9ogAAAXiQ2BpSaPVtS4Wg5hezBg/kV41juWudsC154Lfe49cWU3nxM/Q3uthXvW+Au9+Q/J33oqPviIXfSkOlx19tlIaKBgZPeJwBIOu6AAABe0AABALxIbBdckE5JEB7v2hrHbreTFmVPetMZe0cKrh+qy1dmzQy7w7uDY/3Osjnifm40RF4fAFMTd0AAAC9ogAAAXpTUALW3t8sll1wiNTU1MnfuXLnuuutk27ZtzjYDAwPS1tYmDQ0NMmPGDFm+fLl0dnaO604DAMpfSTGg5557Ttra2uSSSy6RkZER+eIXvyh//Md/LK+//rpMnz5dRERuvfVW+fGPfyyPP/641NXVyapVq+T666+Xn//85yXtmKmvEZPKjC7s6yjptZOt+nduWXBTXeksR9Pyy4Nz3RhQOLPKWa481OW+eT9DrwFMTSU1QE8//bSz/N3vflfmzp0rW7ZskT/4gz+Q7u5u+c53viOPPvqofOhDHxIRkQ0bNsh5550nL774olx22WWx9xwcHJTBwcHcck9Pz6kcBwCgzJxWDKi7u1tERGbNmiUiIlu2bJHh4WFZtmxZbptFixbJggULZNOmTSd8j/b2dqmrq8v9tLS0nM4uAQDKxCk3QFEUyerVq+WKK66QCy64QEREOjo6pLKyUurr651tGxsbpaPjxN1oa9eule7u7tzP7t27T3WXAABl5JTnAbW1tcnWrVvl+eefP60dyGQykslkCm8UWOWxfcwJKkLHfCRS662S3RVdg+6mlSrlT9q9JEEqv96UMDcJAJLulO6AVq1aJU899ZT89Kc/lfnz5+eeb2pqkqGhIenq6nK27+zslKamptPaUQDA1FJSA2SMkVWrVskTTzwhzz77rLS2tjrrL774YqmoqJCNGzfmntu2bZvs2rVLli5dOj57DACYEkrqgmtra5NHH31UfvjDH0pNTU0urlNXVyfV1dVSV1cnn/rUp2TNmjUya9Ysqa2tlc997nOydOnSE46AK8RUpsWkRncvrK7OPR9Z1UWTIti1z1229ldEZLA5f5dYeUil1gnUJZhV5yyGI/k0P9kjbvZuAChnJTVA999/v4iIfPCDH3Se37Bhg3ziE58QEZFvfvObEoahLF++XAYHB+Xqq6+Wb3/72+OyswCAqaOkBsicxACAqqoqWb9+vaxfv/6UdwoAMPWRCw4A4EVyyzH0DUrwPyOQq610NQmMAZmsGncducv9DfnTXPWOGwMKj7nDsiUMnMWgdkZ+4ciRU99JAEgY7oAAAF7QAAEAvKABAgB4kdgYkFSkRY7PAwqq8jGgVMMsZ7PoWK+zbAZVTGUSREePuk+o5drHDuQXZs92X7vQzRBhUm4MyNTnyzekK9TlOtLtLGbtZdL2AEg47oAAAF7QAAEAvEhuF9zQsMjxLNJmIF8VNKiZ4WwWBm6XVZTNdz0ZK42NV/YEXtVFGAyqYdkD7rKpqsg/nuZmDQ+Hp7vvZXVHmkG64AAkG3dAAAAvaIAAAF7QAAEAvEhsDMh07BcTjFYaDaZNG3M7e4i2iEhQ2Z9/j6TEgCxR/4CzHPxOlSBXx5p9z1n511a4/y+krPiQiEiq51j+dR6GowNAKbgDAgB4QQMEAPCCBggA4EViY0BRX79EwWgMJwzz7WR0dqOzXdjT7yzL4QnftdNiRoYLLgcqbpU+XJ973HvOTPfN3ClQkp5mlQKndAOAhOMOCADgBQ0QAMALGiAAgBeJjQGN5k8bzaHmlDvYvNXZrFDGs1DNEar5f27utO4rD53OHp4aOy/ciVar+TvZ3+zIPa76TeG3tqNHoZpPNHjle53liv+7ufCbAcAE4w4IAOAFDRAAwIvkdsFZgorK3ONUgzsU2cyqczcesoY173e72Lp/X43RVqUc3Dcu3FU2WVIzreNtcqupDje4XYqp/vyxh3vdY888/7qzHFnHHqRSzrokpjACMPVwBwQA8IIGCADgBQ0QAMCLsogBmeGh3OORjk53pV62pBpmOcvRle93lo8scodpNz61M/85+zpK3MuJkbVT6qj0Ovq/Bztqlc2o8t1qWLa56D25x+m97vuOvLVLvXEy4mEAphbugAAAXtAAAQC8oAECAHhRFjGggkJ3DktYmS9Trct1V/52n7M8K+uWdpBMfr5RWFPjrHLSAWl6PtF4xkwKzFUK0m5Jbgnz24bVqlT5DHfOUPpA/niiOjWfSB179liv+zlRoQRIAHByuAMCAHhBAwQA8KL8u+BUd1A0kF+O9uwt/NLfa3aWR+bm0/qkqt1hzPKbAl1wE6lAd549PF3Lqqza0tXtLKbPbsk9Hmp20xlVzpvrbtvrVp2NDueHbUd9fWPuAwAUwh0QAMALGiAAgBc0QAAAL8o/BlSIGsL89pcvc5an73E3n/svb+QeZ/vduEdBZZCqJlCpecz06tzj9KZfO+ti8SMAmADcAQEAvKABAgB4QQMEAPBiaseAVGzm7Ds2Fdw8aMzPf0m1tjjrsr/eNubrQpXyJ9IxlATEiIzap+zr23OPU7/3bmfdUHOts9w7r9JZrttmzYn65Q73cwrMTQIAG3dAAAAvaIAAAF4ktwsuCPLDqCepCyt74FDucSp78hmfzYXnOsupt9xqqmZgIPc4OnastJ0KrP8RJigLdfTWbmc5c6zBWU71usvZafkuuYqF8511Zo977IGVnTzbo46drNrAGY07IACAFzRAAAAvaIAAAF4kNwbkgxWTiFT5goJUjCqw0tyIiATpfNVWo1L8mGKxpkmIk8SGaB885Cyn027V2YFz8sPVzXR3CHpY5w7hljD/P06gyjoYE6kd8T9cHcDk4Q4IAOAFDRAAwAsaIACAF4mNAYWZSgmD0fkmkTWPZrKYkZGT3jYYcLc1Feq0TsvHSUIdAxpWr1XxGCdGNEkxEjPkptOJDh52ljPHevMLjbPdF1vzfkREzIxpucehuo6m1y3n7aQwIh4ETHncAQEAvKABAgB4QQMEAPAisTGgoLZWgvB4zrGExwbCIz3qCbddN/ZcmJn1zrpgxJ3nEx0+4r6234qbmEnKnabOcdTvxm4C63qEoTtHyOh5S5l83rhAzxFKu1+/4HBX/n0o6wBMedwBAQC8oAECAHiR3C646owEYeb4gtVOTlY3VAlG9nU6y2FVxlkOavPVVaNqt7poMOweTzg87CybofxyrHtrsqjPtTPoZA8edLdV3Xcpq5st2zzHWRdWqXNxNF+uwYy45yGJXa8ATg93QAAAL2iAAABenFYDdM8990gQBLJ69erccwMDA9LW1iYNDQ0yY8YMWb58uXR2do79JgCAM9Ipx4BeeeUV+ad/+id53/ve5zx/6623yo9//GN5/PHHpa6uTlatWiXXX3+9/PznPy/tA7JRLtiQbszHDkb2dYz1isTQJRYCK91O2F2kJPc0t5RDqiKf2mZkzz71QQkoZxCo/2F0jM4awp060KVeG7jLVhqf1PTpzqrsgQOnuocAEuqU7oCOHTsmK1askIceekhmzpyZe767u1u+853vyDe+8Q350Ic+JBdffLFs2LBBXnjhBXnxxRdP+F6Dg4PS09Pj/AAApr5TaoDa2trkmmuukWXLljnPb9myRYaHh53nFy1aJAsWLJBNmzad8L3a29ulrq4u99PS0nLC7QAAU0vJDdBjjz0mv/jFL6S9vT22rqOjQyorK6W+vt55vrGxUTo6Ttx1tnbtWunu7s797N69u9RdAgCUoZJiQLt375ZbbrlFnnnmGamqqir+gpOQyWQkk8nEV4TB6I+4ZazDmhpns1iJ6xLKKIwbPU9m0F3Obv9d7nGqVqWjmTfXXY5UXCeVnyuTnuuWPoiOuvGkqM8qbzBZ8aAic5OyVpdqqK5NqNIS2al5gow7Ryg1u8H9WHXsTgkJ5gwBZaGkO6AtW7bI/v375QMf+ICk02lJp9Py3HPPyX333SfpdFoaGxtlaGhIurq6nNd1dnZKU1PTeO43AKDMlXQHdNVVV8mvfvUr57kbb7xRFi1aJF/4whekpaVFKioqZOPGjbJ8+XIREdm2bZvs2rVLli5dOn57DQAoeyU1QDU1NXLBBRc4z02fPl0aGhpyz3/qU5+SNWvWyKxZs6S2tlY+97nPydKlS+Wyyy4rbc8Gh3NdcLZAVdwUNeTZRwXRUuhqo2Gv24WohyabmnxFUT1EO8i63XV2lmovXZFF6OHpUc9RZzmwj1118QY6c3al20UnUf5ak0kbKA/jngvum9/8poRhKMuXL5fBwUG5+uqr5dvf/vZ4fwwAoMyddgP0s5/9zFmuqqqS9evXy/r160/3rQEAUxi54AAAXiS2HMNI536R4Hi8x6q6maqd4WwXnOWOrgvtcgfb33LWRXZl1ROZhJhRNOBWF43e2VNw+9BKSRPoYdgL3WNPWfGx6NBhZ52OCel4jLtyYs6DUedfLzuO9TqLsRIXKlVPOCc/TDu7e6/7OfpY7RRGCYwTAmcK7oAAAF7QAAEAvKABAgB4kdgYkC1I5WNAQX2du3JE9e+/le//z6p4S1lQ84CCGflYx3DzTGedSbvbpqw0PjrWVG6CCverGda6KZikwp0PZo7mY0bMAwLKA3dAAAAvaIAAAF6URRecGRnOP1ZdM0ZlTQ6tCpw6+3KkMmcncgiu2qfsgUO5xxW626lKpaOxpObMcZ9QQ56zCS/8p1MW6WHkdresiEhgDdPWGcf1e5V79yQwVXAHBADwggYIAOAFDRAAwIuyiAHZcZHsjjcLbmoXKEjPc1PV/G7d/3KWz/nyf7uvTWJswKo4OlIkbY8tpYarm3fNd5aDrdvz6xJYuiEWCzt4aIwN42LxL33sv8lXqC2YDgjAhOIOCADgBQ0QAMALGiAAgBflEQOyyjEEqkx3OG2au211vpSzidyS1e/+2uvuttPd14Zhvj0uNg/FodLnjOf8oiCTn98Sm/tS7ZatdsqTB+7/FmGHG0OJqvPlvUNV5jzqOeYs2/OwRp+YhPlT+pyq44mn6snP/TFHVanvt9UcIuucGnVOC173Ytd5Ar8HwFTEHRAAwAsaIACAFzRAAAAvEhsDCioqJThektvu74/10atlsZajIRW7iNzSDXYsQESVfVDrCsWAQiueIlKk3HVUOC4QpFSsw3rvWElrfXzW58bOU7f7uc6x6s9U5a+l0PQo48bZ9LGHdrwlW3hb+710bK/YfB3T15d7HJvPpV4bVOZz6MXiavq6W9dLb6tjY6k6Nwddtqu74D4DZzrugAAAXtAAAQC8SGwXnBkeEhOY3OMxtzuNNDLhuxc6y0GvVa5BdRdFvb0ylvEs82BUr5pMUHqgcE5DfqHOrTYaHnWPVZe8MMfy66PBwkO0TzW9UaHzLSIi6roX7KJT+5SaWZ9fUN2NutxHqvNgfp3+TPW9zHYnu8QFkDTcAQEAvKABAgB4QQMEAPAisTGgSWH174uImPr8MNpAp1WZYszRfLqdYPZMd6WK+QQqVY8ZtGIfSSxhUURkHXs4zR1CL2l3qLWd2inQcaY+d5HUO0BpuAMCAHhBAwQA8IIGCADgxRkdA9JlnkNrfkvQMOuk3yc1e7azHKkULE7KmahAmp5JFFmpa1J797sr692UMmaaW/bBjgmFUZH5UlYpjcQcu72Pe1XqoNkN7sZWLNCc5ZZ4F1W2IqW+M9kDB059J4EzAHdAAAAvaIAAAF6c0V1wqTlznOXAGoJrqjN68zFFR444y6eTHmiyhNOn5x4HM6a7K1WW7WBEdZ1Z3W7Z/iLDsBPS7Wazjz2sr3PWmVr3XASHre7U/YfdN1KZwLOH1HoABXEHBADwggYIAOAFDRAAwIszOgZUcJhsCal4Ui1nOct2mpvRD7JiJjplv4ojTFY6F3sosh46HaTdr4WOk8jc/FDltC7VcERVAbWOLzqmyjx4ipUVOnbZ6173VI1VquKsRmddWlWSFZXWZ+TNt3KPgwp1nkZ03Q17JSl9cGbgDggA4AUNEADACxogAIAXiY0BBRWVEgSjKV8KleSeMKX0w6t0NJJKjbkc6hLQWXeejBlWcREP82hi+6RiN4FdrlzNIQpqZ7hvZpVuCPSxBe7/P05cxFccRJcUt1IWhfvd1E1GxcoKRQ3D6W58KOpXW0f5z/XyfQc84A4IAOAFDRAAwIvEdsGF06skDEaHrma7kt0lobuotKDCqiiqhuoGqrsulkl7yOrumqxuKd0NpVLz2Kl6VGejiKqeGoT5/3GCap1V2x2aHB09mt+FhKQzsvcjUtmvAzUM27nOSjDD7ZoM1XU31jmlCw5nCu6AAABe0AABALygAQIAeJHYGJCEqVw1TTuNSRL7x40qSaBT2ZhGq2LqAbd0gz38VkQkVHESY6UEigaKlD4YL8XSEFlDw82Qez2iri5nOTXXKnmhh6frmJa9Xp2XRJR1UGmTogH32IPBEr6boXsugrR1vPr8k5oHUxR3QAAAL2iAAABe0AABALxIbgzImFwcIJxVn3s627nf0w6NLRoYdJaDlDuHJdXbn3tsdCxDxTqCajVPqCofE4r2q/k4E1XKQb+PGTv+YqeqOdFr7dIUeh5Q7Nit2EdYV+usy6rYko+4SLG5ScaaxxSjX6u/B1bcMBU7dlXiwo4RER9CGeMOCADgBQ0QAMALGiAAgBcJjgFFIjIa4zB9+RhKWKXmyeiYg53yf7LmjqhYjBlxl7N79uUeBxm3HEOgyzMMj12qOaVKH8TKJvjIJ1YkBqHnCdn0fCkJx/5/KJw2zX1CH7u1H2bQjclNlkIxIvs7LBL/3gZV1rwgVaYinO5ed+d9VJ6+JM6TA8bCHRAAwAsaIACAF4ntgjMDg2KC490UTkp/NUxZp0eRfLoap5SBiLchq8YebqxLG+h9UmlYggrrEulSB9mxU9s41UVP9DmTxDl2VRE1Vv3VvpYqbY8u3aC7PQPrc7K+uuB0+iB7neqK1F1whcSO3S4DobugE3LdgZPBHRAAwAsaIACAFyU3QHv27JGPf/zj0tDQINXV1XLhhRfK5s2bc+uNMXLHHXfIvHnzpLq6WpYtWyY7duwY150GAJS/kmJAR44ckSuuuEL+6I/+SP7rv/5L5syZIzt27JCZM2fmtvn6178u9913nzz88MPS2toq69atk6uvvlpef/11qVJDqAsxRsTIaP91UKw8gM3uh/dUwjq+3ipfoNeFKuajhiYHs/Ln1uhtdUyl0xrqm5S+/6jAsetUQtbw47By7PLWIiIyrGIf2QSUaygw7L+kEuP6WPTvjf09iJIR5wRORUkN0Ne+9jVpaWmRDRs25J5rbW3NPTbGyL333itf+tKX5NprrxURke9973vS2NgoTz75pNxwww2x9xwcHJRBK2jc09NT8kEAAMpPSV1wP/rRj2Tx4sXysY99TObOnSsXXXSRPPTQQ7n1O3fulI6ODlm2bFnuubq6OlmyZIls2rTphO/Z3t4udXV1uZ+WlpZTPBQAQDkpqQF688035f7775dzzz1XfvKTn8hnPvMZufnmm+Xhhx8WEZGOjg4REWlsbHRe19jYmFunrV27Vrq7u3M/u3fvPpXjAACUmZK64KIoksWLF8vdd98tIiIXXXSRbN26VR544AFZuXLlKe1AJpORjEpPIzKaTsUEUe5xbofnzna3S6m0JUFXfn+P6hQ5JfTDTxQVJzCDetmdwxK9lZ/XlG6c46wTHS+yUvWEah5NIlK2FEuNZMXKsofd0uWp+np3W1XawZ4HFOhj1TEVD+W9T6eUQ2qaLtFhpSUacVMUhRVu7Mz0qxRASfgdAI4r6Q5o3rx58t73vtd57rzzzpNdu3aJiEhTU5OIiHR2djrbdHZ25tYBACBSYgN0xRVXyLZt25zntm/fLmeffbaIjA5IaGpqko0bN+bW9/T0yEsvvSRLly4dh90FAEwVJXXB3XrrrXL55ZfL3XffLX/+538uL7/8sjz44IPy4IMPisjocOnVq1fLV7/6VTn33HNzw7Cbm5vluuuuG5cdHtn59klvq4c072+73Fmeu/6FcdmnCWV1F43sO3Ec7UT0sYcL3cEd2d/uPL39mmhqOHH2yJExNoxLzz/LWR5ucbttg02vnfp+eZA9eMh9Qi9bdObsgT++yFnO/Ocr47ZfwOkqqQG65JJL5IknnpC1a9fKnXfeKa2trXLvvffKihUrctvcdttt0tvbKzfddJN0dXXJlVdeKU8//XRJc4AAAFNfyclIP/KRj8hHPvKRMdcHQSB33nmn3Hnnnae1YwCAqY1ccAAALxJbjsER5ocU66qgwXR3GKpdUTQ6esxZVxYxH8WuBBrW1TrrzAz32O3hx9HhLmdd9FaB+VU61VES0rmE7jDyUFWODfQwbKtkwcheN1YW7Nk7zjs3uXQ12HBOQ35BlYCIDh12ljP/tVmApOIOCADgBQ0QAMALGiAAgBflEQOy5sJkdbZsvWzHM4rEMlLvOWfMdeadfe4u9PWN/UYTGEOxP7fgPpQo3ZTP1zf8LjdLRcVON5NF1O2e4/HcjzGpdDmxzzyNfbDnCZkqt9y12bff/dz+fCqkQJXDiKW1maDvgT726O2JOf/plvnOcrYjfy68pG7ClMcdEADACxogAIAX5dEFVwqr28PuZhIRGTzPTdESvvSGsxz1n2JF0SQMW1ZSc9zM2SPvbnaWs7/IH3vw4kF3Ww/ZoseTTkcj57hpiKLtb+Ufq+zjha6lLuAa38D/9yDVONdZjlrcZbN565ivHdn9zoTsEzAW7oAAAF7QAAEAvKABAgB4MfViQJaRDnc4cUotB+e+y11v9+Ef7XXWZTvd4blJlz1wwFkO1HLYMCu/0OiWK5BDXc6i6VNVNa1YWRIrbEa97rWTX7qxvtTsfCqb9Dw3TijD7vFEVukDo2I8unptEsS+pyV8b9NnuXFCY6WysoejizAsG+ODOyAAgBc0QAAAL2iAAABeTOkYkE7pH6TcZT1vI+i1YhvZEubCqM/RaWS80GlhAvW/hlWyO+hxYyaxuE6kJsDY51GfpwTMhYnR34OMVdpBxzZOZ/+T8D0odt0L7JOOaTnnItTvq5bdFxbaQyCHOyAAgBc0QAAAL2iAAABeTO0YkOrvNmo5+9udY740SJ/8qUkvcHPMRYeOuBvYZcJLyD02uiMnX16i4PsadewF5oc4MRIRCXUZ9LS1rOJF2SPd7msrK3KPSz728aKu+0iBEt36utvLqTnufKlsp5pb9a4F7vptvx3zfQuWcjid81Lkuhd86TE3FmiXOQ9q3XLw+pwGVtnwkXf2lLSP9vctiXOrMHG4AwIAeEEDBADwYmp3wZ2GkoZhK7HuFnthaFjclQkYsq0YlY7GDLjdIkFVCW9Wke+CK4tjL3TdY91b7vD0qLZ6zJcW7YJLgEhdnzDM/38am8KgmEzF2Cv1UHB13QOr+5EB3GcW7oAAAF7QAAEAvKABAgB4QQxoPKgYSSw9ja1oXWe9vf9ecd3/H1RbsQ51rGGVO4S73DmxG5V+JrSGHouIBHsPOct2lKdozCcB1znGuu7BNBXfUtfddPXkt7WGb4vEY4pabHg+zhjcAQEAvKABAgB4QQMEAPCCGNBYTqdPvtKdE+HEUHr71Ockby6MjlPF4hcD+RIGOm2PnkcTWusDldK/1HDYpNBlt7PWTqoYkBlSZamrC8S/9DyaJJax0BfESiFlBtyyFTrO46xTx1osBoQzF3dAAAAvaIAAAF4ktgsurJkhYTB6mx8dPep5bwqLdLeaEqTy7XyqTmUVTrn/A0Td7rGaESs9ymR106jPifr73fXWcljtDs8Nq1WeHqvbLaypUZ+jUtlY2ZiTkqrGDOe72Ub27HPW6S7FoPvYmO8T1te5T+jqo1YanKiv8Pdpwujrbne7qf0NVddrYF1bnXYoNWeGs6wzsaes12Z7egRnDu6AAABe0AABALygAQIAeJHYGFAQBLk07XafclJiAzY9HDeWumb2rPy2XaqPO+vGQQKdysbqhrfjEV5ZsQI97DrqduNFqfr6sd9Hp+m3l0M1bDlK3nD12HexlOujjj2wh+73u7GlZAzRVsPT9fKRfBVgXUG3YGoqEXeIehlcd4wf7oAAAF7QAAEAvKABAgB4kdgYkKTTIuHo7tl9ytmubl97NKZYShadiuSYNa9DzR2RyO1L12nv7fhXtkuXtPYzL8hZpY9dxwasOUPBdLd8gWbPl9KxsETMBSsSj4j6Bwqud+jvgeS/M7rMQ9TbK0lT6LpHvW4cMJw+dqlyEXHmwoUqjVU0QAxoKuMOCADgBQ0QAMCLxHbBBZWVEoTHU/Ec6fK7M8Xobic1PDd74EDucapWpeKZ2+AsBqpLTtJWV0dX1ynv4oQp0g1op3MJQzX0uMZN0eJ0u+kh2mXAFKrsqdYFVVVjbOim5UmsQt2yaji6GSr8Z8bumjWDqmtvoIRuTZSd8vstBwBMCTRAAAAvaIAAAF4kNgY00tEpEowOyQwq8tUX0/PPcrYzqhSCPVw3iWl7Yunm9bKqummnskm99z3upn1u/3jUmY81eUvpX0Bsnwrso660mm5qdJb1tTVWKYcogXGD+PQBtWxfdxX/0sdub6vjo0k89mLfxawqc2FLz2ty38u6zjrFTxK/8yiMOyAAgBc0QAAAL2iAAABeJDYGFFRUSpCLAeV3M+pWMZPILWeQeCrdfKx0Q4V7SYJaa65Mj0rJouMgxdLe+xbo9DNj06W9jYptmOHkxfcK0WWqjZrv5Ryvuo762O1UT7osQiLp66722S7VrlP86BhvLAUQyhp3QAAAL2iAAABe0AABALxIbAzIDA+JCUzu8f/Q84DsVO4iItHhrvx7qP7jRJT3Vftg9LLKoxXtyqe2TzXMctbpuIIdRzC6JITuW/cxR6qEeIWeN6Nz6MVKl1txk0CVOY/Fxjx8D4qd70IlF1LqWIOKfMmCUM2X0tc9GlD56Xz8DhS57lmrnLemr3tYX5d/2z637IP+fXDKYyThdx8x3AEBALygAQIAeJHYLrgx6a6MwK2gGFhDPvUQZ93dVXZ02YdCXRt66Ks6F7HzmHC6G01f20LlGwJVfdSU2ch9o7oUCw1m191QQai6Xsvu2NV1L/C9jU1hGMyfqXI77jMFd0AAAC9ogAAAXpTUAGWzWVm3bp20trZKdXW1nHPOOfKVr3zF6Qoyxsgdd9wh8+bNk+rqalm2bJns2LFj3HccAFDeSooBfe1rX5P7779fHn74YTn//PNl8+bNcuONN0pdXZ3cfPPNIiLy9a9/Xe677z55+OGHpbW1VdatWydXX321vP7661JVoAxxQVb6mkgNrY6VNtaxgTLnDLPVaXz0UGSr5LUZUimLCqXpKZIqxQt1rHoYcywGZJ2n2LDrMg8AxNLPzJieexhMq3ZWRWr4euLTM4m4378i5e1Nd35YeapxrrMuOnTY3bYcjv0MV1ID9MILL8i1114r11xzjYiILFy4UL7//e/Lyy+/LCKjdz/33nuvfOlLX5Jrr71WRES+973vSWNjozz55JNyww03xN5zcHBQBgfzX6oeXR8HADAlldQFd/nll8vGjRtl+/btIiLy2muvyfPPPy8f/vCHRURk586d0tHRIcuWLcu9pq6uTpYsWSKbNm064Xu2t7dLXV1d7qelpeVUjwUAUEZKugO6/fbbpaenRxYtWiSpVEqy2azcddddsmLFChER6ejoEBGRxka3gmNjY2NunbZ27VpZs2ZNbrmnp4dGCADOACU1QD/4wQ/kkUcekUcffVTOP/98efXVV2X16tXS3NwsK1euPKUdyGQyklHpRGKs+Tt2yW0RkaBfpeOw3iu0SxmIiOgU/pXuHCJjpe5w0niofZhMkdU9KUPDzrqgZ+zLF1RWOst2ynsRceMiOn2LKnnhJW2PTlE06C5nVVwktJZjJS4qq9Ry/tzo+EpSrrvNDLrpdLIHDuYXYseqrnuF+x3Xvz/Oa3XJiMm67gVijvrYbdmDh5zl2LHb50Z9hk5RFJszOEzZh8lQUgP0+c9/Xm6//fZcLOfCCy+Ut99+W9rb22XlypXS1DRav72zs1PmzZuXe11nZ6e8//3vH7+9BgCUvZJiQH19fRKG7ktSqZREx4vCtba2SlNTk2zcuDG3vqenR1566SVZunTpOOwuAGCqKOkO6E//9E/lrrvukgULFsj5558v//3f/y3f+MY35JOf/KSIjKbBWb16tXz1q1+Vc889NzcMu7m5Wa677rqJ2P/4UEvrlr3QUF0REalzu+jsNB92Gg8RjyN57a4Do7qlhtROWeloAj20WnWvxM7FGO9z/M3G3idfdJeK6p60BaHKJm0PXy+S4ieJ6ZucrrEiQ411ZdmC9O+L/d5JuOaK7j6Nfeet7ke9LqxUGdP18RUYGo7xU1ID9K1vfUvWrVsnn/3sZ2X//v3S3Nwsf/u3fyt33HFHbpvbbrtNent75aabbpKuri658sor5emnnz71OUAAgCkpMAkrKt/T0yN1dXXyQblW0irR6Amp/2zs/2DDadPcbYvcAcnB/ES26Jhbn8VLIL4Y/R+fdecSqgEWQbU7YVEPwLDZNZVERMyIurtI1ldmVDj2BORwuvs9CK2JnLpmlB6EkPhgtP7+60C8ugPSdZac1+raQvYdRhKvub6r0b/fBe6AYvWx9ATYpB97wo2YYfmZ/FC6u7ulVtV0spELDgDgRfmVY9AKpO7IFsmqkNapbGbk74hC1R+eVWk+EkH/Z2bFiKIBFRsYcP+ztytNBrPqnXX2HYJIfCis/d9hYu4MC8RqYkP3rSq6gb47UncMTmqbSH3XknB3pL//esh2gWHMWljj9gjYFUcTUVFXiw2tVkPorWXdGxI2znHfqk/d+VoVao2avpGI6z5FcAcEAPCCBggA4AUNEADAi/KPAZ0Go9OS2HNlCoyqikniPJki7D79QKUlEVXCWo+Okig/hyIRsYASGft4dQlrdaz2yKpYHGTsqUflKZbWxxopGal5M2V23XUcR0b0/C/1v7j9PdDHPtWuu0fcAQEAvKABAgB4cUZ3welJeUFFfhKfHopciJ4AF+mhrwnsknOGrPa4/4cEaihyrPKq1W0VZHX3RPKHqNrHrvc/rFXXyuqODHRGcXWd9STQQpmcE0mnJbKzl1S4xxa77vZrk5i+SH0vR/buc5ZTc91h2UHGOl7dTaurtNrD8xN47EnGHRAAwAsaIACAFzRAAAAvzugYkGb3E0fHTv51Op1LoGM+Vv94EoevRqqqbBjpMg9qmLk1XFfHh3R5jMCKoSTx2HVsIOpxL3xQkf8VCVVG9yDtJnTVqWycNDhlMFTfqAS8Yiew1clt1XUPrePL6ukNxY7VnvIwWTEUncLoqPqFt+K6TjxI4kP17d/3RKYsSjDugAAAXtAAAQC8oAECAHhBDGgMsSJsheiiYLr8tRUziZUQT0IsQKe1V/NXwlDNE5qenyMVqNhAGI1d+daocgZJnDMRm8dkxk47FCt3rUuZ26t0qe8ExgZi1916rFPVxL7j9nwpFRuLfef1dfdW7z6vYBxUH3uFe+zGmhMVmwtWqGR6En73PeMOCADgBQ0QAMALGiAAgBfEgMZSSv+s2lb3ETuxAV36wCQvDhKbI6HmNjhxHx0L0POArD7xQJV5SEDXf1FO3KrIeQlrVa44m4oB6fOUiHhAgfL2wZCKica+41YMSK9TFzqR173QsRd5aWAdu9HXWccFE3nw/nAHBADwggYIAOBFYrvgwmnVEgaj3TdRX5/nvSnM9A+4T+hh2VaXRKpImQd9rM4wTk/dNHrIcLa7J/dYDzvVw43t4bnhNDdlkX7fyO6eTMoQbWs/ske6C2woEgwMjLkurK11lo3a1u7OS0oZB/v66PQ6+ro75UvU9z+sc489e+iwu976XkS9Kh2QJ86xH+5y1unvuN0lrdeFupSJPSVDDf1O4tD8icYdEADACxogAIAXNEAAAC8SGwMKKiskCI73rfZbfcpJGK6q6OG4esitm7pexTZ0ud8hlebDGQackLiIfQ3UcOJo2O3HTtnHp4ZhB9Oq3WXrtUYPV03CdS+yT7FYYIHXxlLZWKlfkhIDchQoMSIiEtlxQVWuwCm3cAJO3CSJZSti111NJ7CHqFeo/dV/C+zrntYpfRI4NH+CcQcEAPCCBggA4AUNEADAi8TGgCQyIsFoH2holQZO4pygWBlqvYHu17apFCdBtRsXCSvycwwiXeo4AYr1W9vxMV3GWR+7LVRxhKjAHJtJU6RPvqR5HPo7Y80VCypUSn9dIiIBSolXxOaGafax61IOSTj2Aml6RKTg73dQaJ0ua67KniTxb9144w4IAOAFDRAAwIvkdsGl0yLh6O45w1ITOUyz8C169uCh3GOdjkZnULarK4q43XmxrplilSYnQ5Hzb3edBWrbUKUlCky+SyLWdaG76xI4TLvQ8GnT56Zd0ZVk7azJutJqNtbdZR27r+Mu8Ln6+x+bpqBZUw30eUnEd7wY61wU6zJ0UhipCsGx6sLi/q1IZKqq08QdEADACxogAIAXNEAAAC+SGwOKsvnUM3YfsRrSaVQ/atL7Ro1KVaNjPqL7kK24T3wYs7utGSyzYy+QukZXltRVNvV7iSQgLlJAfH9ddvoaHUMJY3ERK21PEoYpFxH7jmt2Shp17PHrbi0k/HddJB7Dcq5tpOK9ofpzrEs7WN+DSF/2MjgXJ8IdEADACxogAIAXNEAAAC8SGwOKjvVJFBzv8LX6QoNAlS/Qr7P7RhPYL2pG3PksUZdb5jlIuf8ThDU1Y64zGXdeUGT1NyeyvK+auxOpksT2XJhY6XIdE1LzhBIfF9HHPuDOGQqt49HxIj0/JLDSM+n3TeJ119/5GPv3VMdFdBofO6aiYiZJPPbY9bGvuy5Por/j+vc9yq8Pq6ZG2h7ugAAAXtAAAQC8SGwXXJBO5brbjJ3qokhlxiR2u9lit9m6e0UPM7eHWqtUL0alp4kNSU+YolmRbWGBDOISH9obS9mSNLHuFbXeyoSshx7HhpXb1VMTfs1FTvCd1l1l0dgVdrVyOF5H4P6P7/y+6+ta7DvvDOEus/MwBu6AAABe0AABALygAQIAeJHYGFA0MChREE/hkZo5Uz2hhipa6V1iMZIEDM+NpaovNnTUqoKaqnFLNwSqamho97XrFCA6bY+HIaulfGZWDU+PlbFQJQtsekhqEtI1FSrVUGy9PnY7JZOuolmO1z175MiY62LXfbq1HBVO8WOfU29DtNV3LertHXtbNZI6duxq2WZG3L8FxprikMjh6cdxBwQA8IIGCADgBQ0QAMCLxMaAxlKov1hEnJLdsT7U97S62+oY0e69ucd2KWmvrLkC2Z6ek35ZkFYpi3S8qL4u/xGqXHQsRU4CyhuUkmpEH7sTN1DL2SNdzrpisRofYsde4FzEjl39DhT8DiWw3P3pXHf7O586a56zLrtnn7OchDihdjrX3S79nZrf7L7v/oPusv37PsnXnDsgAIAXNEAAAC9ogAAAXpRdDEgr1PcZK+f71jtjbht/44T0h9v7ofNK6Zxh9jp9XvS5GMzPD9H56ESVCsiVRk8SdX2ca6nziQ2rWJ8V84qVdUjKdS8ktMqTFCtdXsocEPX9SuR1twQV7u9v7Htsx0879rur1Hcklq+uSBVx33RMV/++28dujh4r8mbWdZ/ka84dEADACxogAIAXZd8FF0vDPzx2l0Os9IGqKGoPwQ3S7rbe0vjYXUDq9tgUKjSph5WqlC262qK7TnVH6P6IJHRLFSjLERtSq7YN7LT3sfIYY38nEsO6HsV6TAp10xbb1gxZ1z0J11wpWmnVus6xLmnd1arf2/67ksRj13/ndJeifby6ey5WGiS/HPtdj33w+J4L7oAAAF7QAAEAvEhcF9z/VDwdkWGRk7rbU6OhjP04UutUV0zktr+RyXezGd3dVbC/y5MCt8vxY9fnaez/PSJ1rLFjT2CXRFCgaq6+7qF17Mao7OTlcN2d77zqWo1dd3dZX1t3W5VZ3t42gde80O++iIhY33l9Hor1XUZJP/aiv9/59UGkMqIbvTx8wscn/tyTOxcjMnx888LbB6bYFpPsnXfekZaWFt+7AQA4Tbt375b58+ePuT5xDVAURbJ3714xxsiCBQtk9+7dUltb63u3Equnp0daWlo4T0Vwnk4O5+nkcJ4KM8bI0aNHpbm5OV6zypK4LrgwDGX+/PnSczxpYm1tLRf4JHCeTg7n6eRwnk4O52lsdXV1RbdhEAIAwAsaIACAF4ltgDKZjPz93/+9ZFTOI7g4TyeH83RyOE8nh/M0PhI3CAEAcGZI7B0QAGBqowECAHhBAwQA8IIGCADgBQ0QAMCLxDZA69evl4ULF0pVVZUsWbJEXn75Zd+75E17e7tccsklUlNTI3PnzpXrrrtOtm3b5mwzMDAgbW1t0tDQIDNmzJDly5dLZ2enpz1OhnvuuUeCIJDVq1fnnuM8jdqzZ498/OMfl4aGBqmurpYLL7xQNm/enFtvjJE77rhD5s2bJ9XV1bJs2TLZsWOHxz2efNlsVtatWyetra1SXV0t55xzjnzlK19xEmxynk6TSaDHHnvMVFZWmn/+5382v/71r83f/M3fmPr6etPZ2el717y4+uqrzYYNG8zWrVvNq6++av7kT/7ELFiwwBw7diy3zac//WnT0tJiNm7caDZv3mwuu+wyc/nll3vca79efvlls3DhQvO+973P3HLLLbnnOU/GHD582Jx99tnmE5/4hHnppZfMm2++aX7yk5+Y3/72t7lt7rnnHlNXV2eefPJJ89prr5k/+7M/M62traa/v9/jnk+uu+66yzQ0NJinnnrK7Ny50zz++ONmxowZ5h//8R9z23CeTk8iG6BLL73UtLW15Zaz2axpbm427e3tHvcqOfbv329ExDz33HPGGGO6urpMRUWFefzxx3Pb/OY3vzEiYjZt2uRrN705evSoOffcc80zzzxj/vAP/zDXAHGeRn3hC18wV1555ZjroygyTU1N5h/+4R9yz3V1dZlMJmO+//3vT8YuJsI111xjPvnJTzrPXX/99WbFihXGGM7TeEhcF9zQ0JBs2bJFli1blnsuDENZtmyZbNq0yeOeJUd3d7eIiMyaNUtERLZs2SLDw8POOVu0aJEsWLDgjDxnbW1tcs011zjnQ4Tz9D9+9KMfyeLFi+VjH/uYzJ07Vy666CJ56KGHcut37twpHR0dznmqq6uTJUuWnFHn6fLLL5eNGzfK9u3bRUTktddek+eff14+/OEPiwjnaTwkLhv2wYMHJZvNSmNjo/N8Y2OjvPHGG572KjmiKJLVq1fLFVdcIRdccIGIiHR0dEhlZaXU19c72zY2NkpHR4eHvfTnsccek1/84hfyyiuvxNZxnka9+eabcv/998uaNWvki1/8orzyyity8803S2VlpaxcuTJ3Lk70O3gmnafbb79denp6ZNGiRZJKpSSbzcpdd90lK1asEBHhPI2DxDVAKKytrU22bt0qzz//vO9dSZzdu3fLLbfcIs8884xUVVX53p3EiqJIFi9eLHfffbeIiFx00UWydetWeeCBB2TlypWe9y45fvCDH8gjjzwijz76qJx//vny6quvyurVq6W5uZnzNE4S1wU3e/ZsSaVSsZFJnZ2d0tTU5GmvkmHVqlXy1FNPyU9/+lOnymBTU5MMDQ1JV1eXs/2Zds62bNki+/fvlw984AOSTqclnU7Lc889J/fdd5+k02lpbGzkPInIvHnz5L3vfa/z3HnnnSe7du0SEcmdizP9d/Dzn/+83H777XLDDTfIhRdeKH/1V38lt956q7S3t4sI52k8JK4BqqyslIsvvlg2btyYey6KItm4caMsXbrU4575Y4yRVatWyRNPPCHPPvustLa2OusvvvhiqaiocM7Ztm3bZNeuXWfUObvqqqvkV7/6lbz66qu5n8WLF8uKFStyjzlPIldccUVsGP/27dvl7LPPFhGR1tZWaWpqcs5TT0+PvPTSS2fUeerr64tV80ylUhJFkYhwnsaF71EQJ/LYY4+ZTCZjvvvd75rXX3/d3HTTTaa+vt50dHT43jUvPvOZz5i6ujrzs5/9zOzbty/309fXl9vm05/+tFmwYIF59tlnzebNm83SpUvN0qVLPe51Mtij4IzhPBkzOkQ9nU6bu+66y+zYscM88sgjZtq0aeZf//Vfc9vcc889pr6+3vzwhz80v/zlL8211157xg0vXrlypTnrrLNyw7D//d//3cyePdvcdtttuW04T6cnkQ2QMcZ861vfMgsWLDCVlZXm0ksvNS+++KLvXfJGRE74s2HDhtw2/f395rOf/ayZOXOmmTZtmvnoRz9q9u3b52+nE0I3QJynUf/xH/9hLrjgApPJZMyiRYvMgw8+6KyPosisW7fONDY2mkwmY6666iqzbds2T3vrR09Pj7nlllvMggULTFVVlXnXu95l/u7v/s4MDg7mtuE8nR7qAQEAvEhcDAgAcGagAQIAeEEDBADwggYIAOAFDRAAwAsaIACAFzRAAAAvaIAAAF7QAAEAvKABAgB4QQMEAPDi/wOHiMQ8zjOdKwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if type(model.layers[0]) == SimpleHandmadeTFLayer:\n",
    "    x = model.embedding(in_data.to('cuda:0')) * (model.embedding.embedding_dim ** 0.5)\n",
    "            \n",
    "    if model.positional_encoding == \"sine\":\n",
    "        x = x + model.pos_encoder(x) # For the sinusoidal positional encoding class\n",
    "    if model.positional_encoding == \"learned\":\n",
    "        x = x + model.pos_encoder # For the learned positional encodings\n",
    "    \n",
    "    # xx = x.permute(1, 0, 2)\n",
    "    xx = x\n",
    "    \n",
    "    x_norm = model.layers[0].transformer_encoder.norm1(xx)\n",
    "    \n",
    "    mha = model.layers[0].transformer_encoder.mha\n",
    "    B, T, _ = x_norm.size()\n",
    "    q = mha.q_proj(x_norm).view(B, T, mha.num_heads, mha.head_dim).transpose(1, 2)\n",
    "    k = mha.k_proj(x_norm).view(B, T, mha.num_heads, mha.head_dim).transpose(1, 2)\n",
    "    v = mha.v_proj(x_norm).view(B, T, mha.num_heads, mha.head_dim).transpose(1, 2)\n",
    "    \n",
    "    attn_output, attn_weights = scaled_dot_product_attention(q, k, v, mask)\n",
    "\n",
    "    head_number = 0\n",
    "    plt.imshow(attn_weights[0,head_number].to('cpu').detach())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "36e16938-890a-42fc-8902-f83f4b48a012",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGgCAYAAADsNrNZAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjMsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvZiW1igAAAAlwSFlzAAAPYQAAD2EBqD+naQAAPelJREFUeJzt3X9wXNV98P/P3ZW0kq0fxgZLVrBA5OEZ8ysPxAZjYNo0+BsmJS0UT1pmnNYhmdIkdsB4nhDcxnRKAiLpdxJKxoHCpA6ZQmiYKSSh35LhaxKeIZhfJiY4BOMEgg1GMgYk2bJ+7j3PHyK753yu9l5d7UpnZb1fM5rR3ftjz7278vE9n3M/n8AYYwQAgBmW8d0AAMDcRAcEAPCCDggA4AUdEADACzogAIAXdEAAAC/ogAAAXtABAQC8oAMCAHhBBwQA8GLaOqCtW7fKySefLPX19bJy5Up55plnpuutAACzUDAdueD+4z/+Q/7mb/5G7rzzTlm5cqXcdttt8sADD8iePXtk8eLFsfuGYSgHDhyQpqYmCYKg0k0DAEwzY4wcPnxY2tvbJZOJuc8x0+C8884z69evLyzn83nT3t5uurq6Evfdv3+/ERF++OGHH35m+c/+/ftj/72vkQobGRmRnTt3yubNmwuvZTIZWb16tezYsSOy/fDwsAwPDxeWzfs3ZK8/f7I0N473nH/xP8+qdDMBANNkTEblCfn/pKmpKXa7indAhw4dknw+L62trc7rra2t8vLLL0e27+rqkn/6p3+KvN7cmJHmpvEOqCaorXQzAQDTZfw+IjGMUvEOKK3NmzfLpk2bCsv9/f2ydOlSWf7klZKZVy8iInX/u7Gwvv3/fXLG2wgAqLyKd0DHH3+8ZLNZ6enpcV7v6emRtra2yPa5XE5yuVylmwEAqHIVn4ZdV1cny5cvl+3btxdeC8NQtm/fLqtWrar02wEAZqlpGYLbtGmTrFu3TlasWCHnnXee3HbbbTIwMCBXXXXVpI/xge/WSE3NePPq3jpYeH34/+9wtqtZva8yjQYAzKhp6YD+6q/+St5++2258cYbpbu7W84++2x55JFHIhMTAABz17RNQtiwYYNs2LBhug4PAJjlyAUHAPDC+zTsUgJjJHj/oVRTW2zmvudPcLY76U+Od5azP3t++hsHACgbd0AAAC/ogAAAXtABAQC8qNoYkITv/4hI2FhXePmE5427nV688OzC78Evdk1L0wAA5eMOCADgBR0QAMCLqh2Cq+kfkprs+Pha5t3DhdcPX9TobNe8Z9hZzhwZLPxuzj7dWRfueqnSzQQATBF3QAAAL+iAAABe0AEBALyo2hiQhCLyfjVXMzpaeLlmyN0sMzzqLAcj1nJN1lmXPe1UZzn/m71lNxMAMDXcAQEAvKADAgB4QQcEAPCiamNApj4rJjvevCA0pTcMAnc5W4z7BIePusdsyLm71rinb8bGptBSAMBUcAcEAPCCDggA4AUdEADAi6qNAUkQFOI7QX0xdtP8exWnMW58yC7fHah1UuuebtDQ4C7n84Xfw6Nu/AgAUFncAQEAvKADAgB4UbVDcJmBEclk359ibaXiyb034m44qobkMsU+1aghN71tUFer1hendNd0nuSsGnvt9Um0GgAwWdwBAQC8oAMCAHhBBwQA8KJqY0DSfVAkqBMRkfxgsQZDzSuhs5kZcuszBFZqHmNNqx4/kLusU+8YK+VPRk3h7v/vDzrLzR//XVzrAQAJuAMCAHhBBwQA8IIOCADgRfXGgBYeJ5IdT8GTeae38HLQNN/ZLPIsj03Fh8yIW75bVAwoyBTjR7pUw+h/LnaWs/+jGIvK//a10m0AAEyIOyAAgBd0QAAAL6p3CK5/QCQzPmRmhocLLwcqnY6ehm2n2zE6G3aopnDHVFrVU7T7T3HXL37CyrpdW+fuO6rSBQEAIrgDAgB4QQcEAPCCDggA4EXVxoDMkcNi3k/FY/LF2I3p63e3G3HjLXbcx07LIyJOqQYRkSCbdfe1U/WoeFG+wY0XDX2gufB77jXVjxv3sup4EgCAOyAAgCd0QAAAL+iAAABeVG0MSGprRYLxNDuBWCl0dOod9axPoGI3zrq4mI+IBJItue28N9y+Ojtk7au2jbTJWm8/0wQAcxl3QAAAL+iAAABeVO0QnBkZFfOHWdRWypxAZ7TWVU9jhuAiqXlSCNSuw4uK6Xdq9dCennZttz+Xc7dlSA7AHMUdEADACzogAIAXdEAAAC+qNgbkMFYqnqQSC3GpeHR8SJdjiHmfvFtxoeR7JlLvSSkHAHMVd0AAAC/ogAAAXtABAQC8qN4YUD4vEoz3j07pbFV+IfIcUFDsU42o2ExGl2dQy/ah1HEz6vGjoQXFZ38as24/HlfqO1FGpfUJ8xNvBwCzHHdAAAAv6IAAAF7QAQEAvKjaGJDJ58UE0f7RjKo8a0bnfivGTHRJBQlVSW5V2iEudpN7z1030F6MHx1foy5jpE0plLMvAMwi3AEBALygAwIAeFG1Q3DjVUXfH/ayU+qkGKLSQ2qBGpELGue7Lxw+Uvxdpe1pft2d/t17enH4TpdYSNOm6AZlTOEGgFmEOyAAgBd0QAAAL1J1QF1dXXLuuedKU1OTLF68WC6//HLZs2ePs83Q0JCsX79eFi1aJI2NjbJmzRrp6empaKMBALNfqg7o8ccfl/Xr18tTTz0ljz76qIyOjsrHPvYxGRgYKGxz3XXXyU9+8hN54IEH5PHHH5cDBw7IFVdcUbkWBxn3pwzm6KD7MzpW+NGOttU6P9nBoPBjhoedn7IEgfsDAMeoVJMQHnnkEWf5e9/7nixevFh27twpf/RHfyR9fX3y3e9+V+677z756Ec/KiIi27Ztk9NOO02eeuopOf/88yPHHB4elmHrH+3+/v6pnAcAYJYp6xair69PREQWLlwoIiI7d+6U0dFRWb16dWGbZcuWSUdHh+zYsWPCY3R1dUlLS0vhZ+nSpeU0CQAwS0y5AwrDUDZu3CgXXnihnHnmmSIi0t3dLXV1dbJgwQJn29bWVunu7p7wOJs3b5a+vr7Cz/79+6faJADALDLl54DWr18vu3fvlieeeKKsBuRyOclN9BxNJht9cEei6XVMTLUCvW2gyiZESnTbat00Pfk6Nx7TGNdPxsSmJjglR/R8rDbqZ4TiYkQ8TwSgyk3pDmjDhg3y8MMPy89+9jM58cQTC6+3tbXJyMiI9Pb2Otv39PRIW1tbWQ0FABxbUnVAxhjZsGGDPPjgg/LYY49JZ2ens3758uVSW1sr27dvL7y2Z88e2bdvn6xataoyLQYAHBNSDcGtX79e7rvvPvnRj34kTU1NhbhOS0uLNDQ0SEtLi3z2s5+VTZs2ycKFC6W5uVm++MUvyqpVqyacATcdAl3l1JZR2bDrS6fQ0cN1zb93U/H0ddYVt50/z9336GDpNqhhP6OGygLjtj9uiDEWlVUBVLlUHdAdd9whIiIf+chHnNe3bdsmn/70p0VE5Fvf+pZkMhlZs2aNDA8PyyWXXCLf+c53KtJYAMCxI1UHpP+3PpH6+nrZunWrbN26dcqNAgAc+8gFBwDwoorLMYTiTEEuyE6wnbVohzqC+Du2cKiv5LGCujpn1b6Pucste63jHHpXHbeMdDy63ETcXWfsHak6jp6yzTRtAJ5xBwQA8IIOCADgBR0QAMCLqo0BZZsaJRuMx13CwaHC65kFLc525uhRd8d8MQhk8gnlu+OejVGlszsedZ8Dyu38bfEtrfaJRJ9FsstwB7XxlzyocdfbsSjT4WaTCAbdNpk33rL2c1MJ6RIToVVCAwB84A4IAOAFHRAAwIuqHYKTjFURVE9NtqnpxM7DsnH7JVH71h1yh/rsYcHIVPBQ9ev2+jBh+rOeHm0NyQ22Nzqr6rvdNjkDfyoTuExQ5RUAfOIOCADgBR0QAMALOiAAgBfVGwMKzcSpdOLKLYhIYKWcMToOkhR/cQ7k9s0DnU3OcuOB5sLv+XfcVDzRadjFY0WqskbeV53fcDGtz/w9b7vr9NRqq9SDGRmd/PuQlgeAB9wBAQC8oAMCAHhBBwQA8KJqY0DGGDEyQWwiIY7jPAekty3juaDRhphne3QbYt43qahfEPdcU40b0wrC0m0IVCwp8q7EfQB4xh0QAMALOiAAgBd0QAAAL6o3BjQyKmaCR36MKiNg8jElFSLPDKkYSlwD1L76kSTTfkJx3eEj7spIm4rvG0SeTdIlxdW+9rJVbkFExOg4jx3X0cdRMR+n7EOQEN9S6+1nmcIhtxQFAEwWd0AAAC/ogAAAXlTtEFyQzUgQjA9XhVbKmUwu52wXqYhqS5qyHTNMFahhp8Hj3eXjjhSHnvLqOJFp2La4IcMJ2NOpA3XumpN+R0+zjh3qS2qTuz6obSguZPSQYrrzAzB3cQcEAPCCDggA4AUdEADAi6qNAYVDIxL+Ye6zncpmcNDZzqQoNa3LJMRSU5F7zx5xlk/Y2VI87u/3x+5rx2NMQpWESDPsYyWVWLCvU8pYUywVD4u75s70bhExY5QCBzAx7oAAAF7QAQEAvKADAgB4UbUxoPF4RrTcQJqYQiTmo2IZmVxMvEKlzFn6sLvvyMK6wu8NDfXOunBQpaexmhFJxaPEPZskRq3T5bud/eLT6WSbrBLjqv1GpRYK6mqdZfv8MrXquSwdH6L0N4ASuAMCAHhBBwQA8KJ6h+CmyBl201me9ZBcRq23ttcVRRsOuNO/RxYWh62CujpnXaCnS9vDYWVUZY3QQ1pxQ3KadS0iU6f1MKFadoYR9XvGTXXX2zIkB8xp3AEBALygAwIAeEEHBADwonpjQMaISDRGECl1EOpSCNaCjjkkpZRxUtm42/7+zxud5ebfFX/PHVFVWsd0DMhqc1Kcppy4SOy+buwptNqsY1ZmZCR+2bpueqp75PMhzgOgBO6AAABe0AEBALygAwIAeFG9MaAgmNxzLQlxHneVu61+fseJCaltlzzpxovCOus5mnqVjmZAp8wp3QYtNhWPlua5H3Vd7PLeQeN8d1sV0wpq3a9JKMVUPPp5Kd1eMzL5557s54so4wAc+7gDAgB4QQcEAPCiiofgMhMOp0Wn/U5+yC1ueC6yrxpayoy5Q0s1g9ZQWegOM8UOsyW1IdBDblNM3aOH3OLSECVVT1VTq51ro9MZlVOJ1W5zRqUDCitY4RVAVeAOCADgBR0QAMALOiAAgBfVGwMqVRE1RShAx4eCjIrVqDIEdizH5N1ta4646WqGFxanMdeqOEgkHU1MGyJ0uYY0qWyc6qPucfS1yFjnHpmGradA63INVkXUoJwYkJ6ybe9bybIVAKoSd0AAAC/ogAAAXtABAQC8qOIYkFWOIU3KmTgpngPSqXiy/cPu8jwrhpLQvqT0OxUTV/ZBx1Tqaourcm5KIh3zCWprpSRdvjuuTQBg4Q4IAOAFHRAAwAs6IACAF9UbAypRjiGoceMRkfLXKYRHj7rHsp5D0aUa3llxnLOc6yvGVHIJpQPingvyJeztK/weHB101ukS3JHnpeyY1qi6/jrO5jybZEqvE7fsQ7RcOrnggGMNd0AAAC/ogAAAXlTxENzE5RgimyVNA056D+dYpY87r8cdEqrrc4ep3MNOfdp1tLxETEoafX3sqdZJ184eFtTDaJFGqXIM9rXRpSiyKi1R3l5W5xK5/vZxVZqeSJtKX5fMvHnOcjgwUGJLAD5xBwQA8IIOCADgRVkd0K233ipBEMjGjRsLrw0NDcn69etl0aJF0tjYKGvWrJGenp5y2wkAOMZMOQb07LPPyr/+67/Khz70Ief16667Tv7rv/5LHnjgAWlpaZENGzbIFVdcIb/4xS9SHT9TVyOZYHzKdThSjFFk5jc424VWaYDxF6xoQaS0gSpREDd9WsUn5u095O57oNiphkNump5o2fBim5JiVpm6mBLjLc3uC6r9Yf8R60AqDqVjKqMj1u+xTRKjy5M3FD8D+7MRcadSR5ZVGzIN9e621vnZ08RF3PIRIiJm2L3m9tRx3QYA1WlKd0BHjhyRtWvXyt133y3HHVd8Pqavr0+++93vyje/+U356Ec/KsuXL5dt27bJk08+KU899dSExxoeHpb+/n7nBwBw7JtSB7R+/Xq59NJLZfXq1c7rO3fulNHRUef1ZcuWSUdHh+zYsWPCY3V1dUlLS0vhZ+nSpVNpEgBglkndAd1///3y/PPPS1dXV2Rdd3e31NXVyYIFC5zXW1tbpbu7e8Ljbd68Wfr6+go/+/fvT9skAMAslGqwfP/+/XLttdfKo48+KvX19ck7TEIul5NcLhd53RgR84enP+zYjU5ro5etbSMpcFKUeTaqtHT++CZnOfPGW1Yb3G2N0aUQiu1Im5THea6myS2dHRxxUwnZ5xcEKmaS+p3tnXX8yIo9qXOX/OSfgTL6+SInXhQfv9Prnc9apw4CUJVS3QHt3LlTDh48KB/+8IelpqZGampq5PHHH5fbb79dampqpLW1VUZGRqS3t9fZr6enR9ra2irZbgDALJfqv4oXX3yxvPjii85rV111lSxbtky+/OUvy9KlS6W2tla2b98ua9asERGRPXv2yL59+2TVqlWVazUAYNZL1QE1NTXJmWee6bw2f/58WbRoUeH1z372s7Jp0yZZuHChNDc3yxe/+EVZtWqVnH/++akaNp4Me3w4J83gkYmbhp2G2remu9dZHhuZYhbuxDapadrWtO3RthZnVe1BdQN76J0y3nfy9PBkLHs6u4lP2+MMq+n3qFfDtJHp7NbnMdXPBsCMqvhg+be+9S3JZDKyZs0aGR4elksuuUS+853vVPptAACzXNkd0M9//nNnub6+XrZu3Spbt24t99AAgGMYueAAAF5U7XxVexq2nb4mVClYtNg0LGmmZSeUM8guWlg8rEobE3dcXV1UT0WOpMyx0u3UPPeyuy5y8Jg0PnFTk+PKOkyw3o7dmLyKxegUQHYsRx0nHHArsWZiKseGR1RJhTBmCnckHZBb3daREBuLTdcEoCzcAQEAvKADAgB4QQcEAPCiemNAIyNigj+k4ildzsAuKzCdFt//nrPc/f9MrQ1lxRQSUszYx04qsVAOM2bFeXQMKyidhihCb1vbWPg1PHw4oRExx1WpnWbqOwIgHe6AAABe0AEBALyo2iG4TK5OMsH49Fm76mZkOnGabNiazuTsNMAd6ju4xs2GbQYPFhf0UFKMoKbWfUGnp9HnZw05Bh3t7rb97tTkvJWKJzLdWw396YqiadjDoDotT6Tiqz31Wk3RDurc6dFBYzHbd0ZXmVXbRs7H3r6ciqhphhABlIU7IACAF3RAAAAv6IAAAF5UbQwoHBmTcILYSjg4FLtfYMUZAp0WRm9bP08d3Iof5eMrcGbbWourVBmE2KnWuk2h+3+ASGzDLi2w59XSx9US4l+Zeda567hNQrkFuyJqUKf21amErGMFqlJseOSIu60dx1GxsTSfu1Hxo2xzc+k2qXhRJOWPFcMK6tz4nVFlH5juDaTDHRAAwAs6IACAF3RAAAAvqjYGFGSCQkluW+KzPTEloBPVWmP8eTeOMNbztvs2Vuwg0s648gX6uZJI+QJ9KCu2kXDukWdwbPpa2M8Xqfabcp6F0c812eenyz4EMfEi/cxQQjzPPrZ+ximYN09vXaSuWeT5KTsWmIn//1qmyX1WLDGdEDDHcQcEAPCCDggA4AUdEADAi+qNAdVkJQjGm2c/GxOJBei4gr1exVN0DMUMuiWhnWc+1Ptk9DMgVmwgzLvP7uh9zViKWJSOH9UXSwtk57uxDDM8opaLsY9ILGPE3VY/g5OGnc9OH1fHoeJKgduxMRFRMTj14SXFauzPstYtx5CPi8UMuM/9xOXQM2Pucz+ZxkZ3W3Ut7GetwqNHS7cBmKO4AwIAeEEHBADwomqH4JxUPNbU3khFVD1UYy0mT+VVx7KH6PRQ2Eknust22p79B1TbyylH6p6PsYbKyhk2i5yPVd4gWmVWDSmqoTJ7KnlSeYkwRdmHIG+nQoqMn076OPp/VbqNTvodNeVcp/wJVHVVp0kq5Y8eopPlZxR/3/lrtTNlHgDugAAAXtABAQC8oAMCAHhRtTGg8TH/6Lh/JDYQe4j4/lWn4g/s+ItOKXPUjQ2YxoZJtyMVHeuYaqwgkk5HnY+dikfFOSLXWKegsacbq9hSJF1QivY775v2Oljnq0tpRD5na0p9MH++s05Pl3bSFNW6saRwIGFqdYpS7cBcxB0QAMALOiAAgBd0QAAAL6o3BlSKTr0T93yIXqf3VbEOO36hnxkKF7llnWUspnx3XJt0GzzRz/o4dMoindPIWRlTfiG6ekZEnlvS52rHZg7HP1vllM8YSSi5reJUo83FZ60if2jllLwAjhHV8a8hAGDOoQMCAHgx+4bg0kg73OWMF7npacI6dzk7XEy74m3wJM0037hrESYNIcZUWk1838m30Z4aHskgnmbISk8bH1NDcNbQa6RCrRZThTZxuDHNF8M+P4bjMEdwBwQA8IIOCADgBR0QAMCL6o0BGSMTDqKbyafimSiVjyNb5yzaI/o6pUzNW++5zWh2U7hMWtK85DTj/6m2da+bXWk1aKh3t9UpZlS5htCajpxYHiNNKh67nEHaOIi1faRKq0o15JzvwgXucfa+6m5rp/HR1yGhPEZ22E4tRFwH0LgDAgB4QQcEAPCCDggA4EX1xoBscc9IlPEsjF2WWjMqDpLvPuju22vFFXSpb/0+1vpIuYKIMsoxpHnmxipJIDW6XEHp6zK+gXV+Ki6i0xvFtinu3NKmqglKX+NMo4rXWevD+W78S8e07GekdOkGUTGgoNa9btkXi/GkfEYdN02OIuJHOEZxBwQA8IIOCADgxewYgrOHIJKGZlIMQ5mhYXfZGj7SaVYynUvdbRus4ZaX3am7engl1czxcoZb0kx5ts5dX7Fw2L0uQU3M10QPKeqUP2Gaky+Dde5BVmUy7+1zljPWNOzsQXd6/ZgaQrSHI83R+AqoZtSd/j268vTiez6xK3ZfYC7iDggA4AUdEADACzogAIAX1RsDCoLJxXPitomZDi0SrZwZ6+AhZzHT2Fj4PdRTd/WyZVqnYduSrp3dRtVeHfPRU5Od66ivqVElCszMlxmITMPOqa+5nZpHlW6ITMO2lp2p6yIig0Pusrrkda8XvzNjaR4X0JiGjWMUd0AAAC/ogAAAXtABAQC8qN4YkM1+xkOl1tep993x8vjncYwu1WxTY/avf2GFs3zy918vvsvgYEwbUoqNaSXEEeLeV60LBwaKC/bvk2GnlRkbLr0uqU3qfJy4mi4JkSI2lql3vyOhPr8U52sOHy65TsfK9Pfp6BlLCr/X7X9D7Zwi1dBU0xkBVY47IACAF3RAAAAv6IAAAF5UbQwoqKuTIBh/7sKMWmPr6hmPyHMblsgzN2lS4KtniE66a4+znO8rHRtI82ySblPkmRsrzhC0NLv7jow6i6EVr4jEJ/IqHqbylqXhlpcovU5E3LLn+lz180b2udaWXjd+LPezdWKBMd+JRCliM5F4pMoj1/Bk8TuTV8eNlLyw9tXHyTY1udva11/lM9TCIfdZpUx9fcl1wEzjDggA4AUdEADAi6odgispzTBaBd9Hp/R3hkkqWKU1wkkF4w7bRKaR20NPKsWM5Ct33eLSCel1Qa11LcL4c41MqbePo4fV1FCfM0w76g5NTpvI5+6eX2iXb9BpodR3xNjrA3XcSHoguyJtzKMEE3BKbaStOgtUGHdAAAAv6IAAAF6k7oDefPNN+dSnPiWLFi2ShoYGOeuss+S5554rrDfGyI033ihLliyRhoYGWb16tezdu7eijQYAzH6pYkDvvfeeXHjhhfInf/In8t///d9ywgknyN69e+W4444rbPONb3xDbr/9drnnnnuks7NTtmzZIpdccom89NJLUm9NAU2UzxfHza14TFDX4GwWxk1DLSdepMbs8+ef6SzXvPC7YhuOHIk/Vkx6oMimKk5i7JT/b7slIUzMVGSTFFsqQ9w0bC0uriPiljfIzJ9X3K9fTTFXpQ/0VGW7IZkGNW05zXTjNHGQUJ28Kj+e+eAphd/ze92y7WZMxYBiYoqhLgVuxdki1yFBTceJxX2PuCmJ8u+8m+pYQLlSdUBf//rXZenSpbJt27bCa52dnYXfjTFy2223yVe+8hW57LLLRETk+9//vrS2tspDDz0kV155ZeSYw8PDMmwFRvv7+1OfBABg9kn13+Qf//jHsmLFCvnkJz8pixcvlnPOOUfuvvvuwvrXXntNuru7ZfXq1YXXWlpaZOXKlbJjx44Jj9nV1SUtLS2Fn6VLl07xVAAAs0mqDujVV1+VO+64Q0499VT56U9/Kp///OflmmuukXvuuUdERLq7u0VEpLW11dmvtbW1sE7bvHmz9PX1FX72798/lfMAAMwyqYbgwjCUFStWyC233CIiIuecc47s3r1b7rzzTlm3bt2UGpDL5SSnUpqIyHgMphDHKI5zR9K3ZNwYQ9ox8ZJUcMPUqNiMHduo4PMTkVQ29roGN/4VqHPNW23Sx0kuBT5N7FiUDhjpct61quR1zLZSoY+5HDoGpwVHdEkJe6X6v5/97I+qG6KfgTJh8dmftJ+z6S/GK40uIwLMsFR3QEuWLJHTTz/dee20006Tffv2iYhIW1ubiIj09PQ42/T09BTWAQAgkrIDuvDCC2XPHjcp5yuvvCInnXSSiIxPSGhra5Pt27cX1vf398vTTz8tq1atqkBzAQDHilRDcNddd51ccMEFcsstt8hf/uVfyjPPPCN33XWX3HXXXSIynl5k48aN8rWvfU1OPfXUwjTs9vZ2ufzyy1M1zIyOiNEpSUQkr6aO6qmvsdKkyNHtUUMdmaZi9c58bFXWlO8TmV5cPFa+tzdh56lPz03DOXYkncvkK5eaYXcKff6d94rrxlQ6nRTXNH7qd+U46X8mMHpyMRYavOXGQNOcX2QadpyE7/jwOcWp4XXvqOO+8JvJvw9QAak6oHPPPVcefPBB2bx5s9x0003S2dkpt912m6xdu7awzfXXXy8DAwNy9dVXS29vr1x00UXyyCOPpHsGCABwzEudjPQTn/iEfOITnyi5PggCuemmm+Smm24qq2EAgGMbueAAAF5UbTmGoNaqiGqNl2fq3Km6kZQm9jTUCpZuyL24z1nOv9trvU9COYYU8YvYiqhL2911appv3k7Vo1P4q5hQpJRDGvYUYh2Dy6j2Z2Omlavp90Hj/OJhVfmLuAqiIiolk57OPU2VP/V3MRxSlUx/WZywE6rvREZXU7U+D/3ZZBctdLe1K+Gq0hOR9Ewqzlb7f14otinN1HxKN2AacAcEAPCCDggA4AUdEADAi6qNAZV6Digc0c9PlFNyIeaZCV1CuXGes1xTXxzDz+tnPPQzOCmeP9Lj/86ySukfKynGY8VqIqleVPtj072omI8WF2uKrLNLLuhUSMMxZTf0cXWKGXX9g5rSKX/08zmRUuCWyHdRCU7pKC689Iq7b4q4VFllEtS5h+eeUfg9M6S+a7temvxhdQwrxecD/AF3QAAAL+iAAABeVO0QnARBcfhguqZ8xh7XHQLKL3KrbGZ+/1bxMDHpcyIShqxiK6YmVTm1h61SbGvG4tPnGKOGEOMyXEeyPE9++DGoLX4dI+l00hxHDw8NuOmbnGG2hO9WXKVSnZk9koWozlqf1P64dqRJIaWvv67San3WmSH3GkeSN8WlUUrz+TBlGyVwBwQA8IIOCADgBR0QAMCL6o0BGSMiE4wdVzC9TiQeExNDyb6lpsJa2b2TpjGXfI8J18eMl5sUJRYStnXiF7r9qsxA7PnpmEOKcgxabBmFNMfRU4J1eiAr1qTPLVRTuJ316lyTSl4EA/a08jLiIGn21Z+7js2EVuwvp6ajR+J39nHdzzWjqvM6JSMif1dVUL4WVYk7IACAF3RAAAAv6IAAAF5UbwwokxUJJnhmJuG5EzttjNHp5tW+OsWMBLUlt42UB4hJ0ZL4DE6shOdqJik2fU7kLdXzLfq6xFy3yHHj4giaPq51TaPHjTmOpkta6I/KOrYJVTqgmDhPUKO+a/o7oK9b72FrXcLzX3GxQf0dt6+TTpuk43UqtZD9/JrUpPjz19dFlyOPOz+1Ltvc6Czn1d8W5g7ugAAAXtABAQC8oAMCAHhRvTGgMD+5+Id6xqCcx4TicoTp/GLSUHwOyBxxc41FylSXY4rPUCReh2yxxHVQr3KnRcoZ6PF/63mdCpZqLqtMuH2cERVbUuW87fPNnLDIWTf26u/dbWut65R1r0NSSYUgZ71vOd+JyHe89LHi1omIhKd8oPB7vt7988+8/Y6z7MSaRt1ntDILFzjL+UPuvrFtGBhM3ghzAndAAAAv6IAAAF5U7xDcDEgz5BPMcyuiOqlsKpkeaIbYw41mQJeT0OcTl1qoClPt63ISOsWPNXU5POBWs40cyr5OKUcITfP8dDvMgN7/WWxTRp1PUzj5ob7w8JEpt8FOhSQSHd7D3MEdEADACzogAIAXdEAAAC/mdAwothyDjiOoKbd6Su5sY0+x1dOUwyG3nEEkrY8dO6vgNOyKUW0KatyyA/Znl1m00FkXvnlAbRtTjiEhdhEMDseu92H+W8WYVr4+oXy6fb66tLeKieZ1CYwYkTQ+mLNm97+iAIBZiw4IAOAFHRAAwIs5HQOKlpq2l9SzMKNuWns3DqLLUs+CEsRODEjFSCKp9lVsoNrPXadRUvE6OxWPme+Wlo6wr1NNuudXzNEqSDmj4jq5nuLzO+E8N/ZndNkH63kdM6w+V/WdSWUWPjeH6cEdEADACzogAIAXx/YQnJ5WqqWpQqkyRjtDM3qasklTvlO3aYamMdtVQdNOi7Wua+K5pzkf+/Mq5zqo6fVGH8s63+Cd90q3QVPVR5OmoAe1ZQxTTRerjcGwO6wcGWq1z1ef62BMJvCk66Km/VcqCzpmH+6AAABe0AEBALygAwIAeDH7YkBJcZ3YfVUqFT2mH7Nt0NTkrg+naSrpDKW2cc59JGadiAQ6rhNXqbacadnOvur6llNpVZdjsONfIyoOolnXItKChHM19W6soxqYTLHNwZh7jcNQnWFMjDRMkXpHC5oa3ReOHp3ysTC7cQcEAPCCDggA4AUdEADAi9kXA9LSxAbSxCPUtgNnLnGW5z3/enHT2fgcg5VeP6708vh69YIVpzJjpuS6slQy9qVLa6QoAR372Sacq5mXi10/I9S5jy2oL/xe293nbpvwPXAOGxcDSvjsxk5x/5aCnoPWQhWW98C04Q4IAOAFHRAAwIvqHYLLZEWC91Oq2EMDkSnAlZuuG6fhZy86y/mhmFQk5dDVPK2UP9kTjnfWmQF3+mr+8OEJ9xOZYMr5TA1txAxT6TYGueKQVahSvdiZmcc3UMNqY8Xp1JFzTzG0lErCNOxw997Su+rM2vbnoz4bXX3U2TbuUQKJDiFmfvFCcVdP34HaN95xlo1VlTbsbHfWZfrUFO3e4nc87HWHEO3vwPgLDN9VO+6AAABe0AEBALygAwIAeFG9MSBbXPr/MCEm5GxcRjoXve9USwckTVPWcQVr2YypVC+RNk1TipxypGmTs0rFwtR1M9HEOEW6rMA0SYo12eeQWKLDvhYJn419LRLTA8UfyF0uJ2YS9/egU2ANqSncVhxLx3yCQXdbYz8TMEOfM6YPd0AAAC/ogAAAXtABAQC8mB0xIHtMWY27S37yaVVSjXmrbbPtbe7bHuguHqaM1PRJ7Gc+wv5+d2VM+vxIbCyu/HhacfGKSKwpLianPo9M6f8P6edZ4kppRONF0yThmmYWtBR+zx9Sz76kKH0QKZluf86RMuHxZ5ttaS7uq0pRhHFlEZL+duzlyLbuueU73b+l4IVXiu07PODuenTQbaP9t+br2TZUDHdAAAAv6IAAAF7QAQEAvKjeGJAJZcJnevTYeWS/Co0Dxz33IxOMy0/1uBGlYwGR2Ib+74Nz7GkqGS6SENcp4zms0I5hudvq54Lijh15PmqaROI4ev1wivhkpd43IS5lP4MTJpUjd3ZMU/Ykftsw5/6zU2N9r0dVfKh239vOcmCVV4+UEMeswx0QAMALOiAAgBfVOwQXZIpTetOkjZlqipy444iIDFao/ELaVDw2Xb5ATUM19rHTTIeeKQnnbqe2iQy5RVIU6c+2eH4zNQ07mhZKrY8rRaH3jfuK67REUrxOiX8aMRtUtGxFbCoe9UjDoBr6s6bf1/aoEgv6EQfnO6Km5lfBVxzpcAcEAPCCDggA4EWqDiifz8uWLVuks7NTGhoa5IMf/KB89atfdWYdGWPkxhtvlCVLlkhDQ4OsXr1a9u4tXRkSADA3pYoBff3rX5c77rhD7rnnHjnjjDPkueeek6uuukpaWlrkmmuuERGRb3zjG3L77bfLPffcI52dnbJlyxa55JJL5KWXXpL6+vpJv1eQCQpj6LGp7ONiHUnxlhQxIjPiTql1Uu3rsee4tCVp0uXLJKYfuxtPfl055RjSpOJx3jNhinBCeWn3bWJiKDEpfSoq4bM0celp4o6V4rNJ+n5ULC5SRhorfZ0yvSrdjr2gp67n1QnE/S1VsrwEZkSqDujJJ5+Uyy67TC699FIRETn55JPlBz/4gTzzzDMiMv4Hd9ttt8lXvvIVueyyy0RE5Pvf/760trbKQw89JFdeeWXkmMPDwzJsBRr7db4zAMAxKdV/FS+44ALZvn27vPLKePLAF154QZ544gn5+Mc/LiIir732mnR3d8vq1asL+7S0tMjKlStlx44dEx6zq6tLWlpaCj9Lly6d6rkAAGaRVHdAN9xwg/T398uyZcskm81KPp+Xm2++WdauXSsiIt3d4xmiW1tbnf1aW1sL67TNmzfLpk2bCsv9/f10QgAwB6TqgH74wx/KvffeK/fdd5+cccYZsmvXLtm4caO0t7fLunXrptSAXC4nuVwu8roZG3Ofaym8rp4hmKm0K0sWO8sZq235d95VG8e0KeWzFvYYvi5JUDFJY+cJ6fVj18Vci0iJBTu1ShnPpCSmSYqLx6QovR75LmqnnFj8/YXfqH0n/1lWstzH6KrTC7/XvuOWXzC/ejmmEUnfibj0QOqzVM8f2WUgYktClCGorVNtcr+nsZ8HsaVpk6oD+tKXviQ33HBDIZZz1llnyeuvvy5dXV2ybt06aWsbz+PU09MjS5YsKezX09MjZ599duVaDQCY9VLFgI4ePSoZNcMom81K+H4Syc7OTmlra5Pt27cX1vf398vTTz8tq1atqkBzAQDHilR3QH/2Z38mN998s3R0dMgZZ5whv/zlL+Wb3/ymfOYznxGR8dQjGzdulK997Wty6qmnFqZht7e3y+WXXz4d7Z8+6jZ79Ph5znLdG2Mlt511ktpfwUzIMyJp7nGlzifhOGampoOnENYW2xQMlzGkW8bnPLZovrPsDHBN03BX4nBp7M5V8J0+RqXqgL797W/Lli1b5Atf+IIcPHhQ2tvb5e/+7u/kxhtvLGxz/fXXy8DAgFx99dXS29srF110kTzyyCOpngECABz7AjNTxVMmqb+/X1paWuQjcpnUBLXRDTwFBPMf+bCzXLfrd8V1vX16c0xFxgpOlzEJIahx/181bZM3ktpxzhnFNvzy117aoI1+bEXh9/rXe511+T2/nZE2mAv+l7McPPmCtTBNf98VfCgdycbMqPxcfiR9fX3S3NxccrvqGyMAAMwJ1VuOoZRK/k8l404Hdec8u+9T+9RL7raN1ji2Pk45ae1niH2XoO8YdKXMSNob+46ikv9jrdB103c8+vyCuuKU3KCp0VmX7zlYel9dviBhenT2rUOF3/3cg0Xluo8Ufg8b1eMPcSl01GeTXdDiLKcZBcg8494NOt+YaboTqek40X1BfcfHunucZadUhf7c9TT/WfD3Xq24AwIAeEEHBADwgg4IAODF7IsBJUma7WJvGimpbPfHCc+SWGPIkePElY9IMlOzcexxbfW8SnI57JjS3/q6pTmfSpVT1zG5rI7RWW3UM+RSfH+StjWjZTx7UimqjZl3rGzzNe51GVOfpR0HMfrZqjDm80lM7TTz/+81AyrFjy7zEPmOl25jRUuZz3HcAQEAvKADAgB4cewNwaWo1Gj0MEJMChd76q6IiNQWL130OCmGJ3yxqo/q1urzCTIpsl+XY7qGH1WlVWMNoQR6Sq3OeG1di0AShlrUvsE8K32Tzpg+U1SbwkXWQ4H6u7hPZYi2T1dflxTVa7WgVj0oPDpSYssZpLNjW6cXiBpyK+Pc4eIOCADgBR0QAMALOiAAgBfHXgwoTmQ6aBnHqlQs51hLglgN5xOJS2Un3Gx8Vcy6xPepgnNNy/reTlu5iKTrElYwbjhZ+nPW07DhBXdAAAAv6IAAAF7QAQEAvJhbMaA09PMTg0POstNzp3kWpgrjBpECbvrZGM0+h2p5rsmmr7Ea/3dSqRx/nLvte++VPq6OIyQUujNHBmLX+xActUpIzHPLMehvppOKR6WbCepVKYeByZ+rLvcxE4J5De4LtarYZW+vu72Tqsr9jgdGP09YbuvmLu6AAABe0AEBALygAwIAeEEMyBYTn4mMedfVTrzhLOHke0uIZcx6OneXHbd6tzd2V6c0RcocYMF8KxdcXGxpBuWPK5aSNzWq/ILaNi7nmRmKL0cex0s5Ax13Svgs7XOP5IKLK0WBVLgDAgB4QQcEAPCCIThbXEVOXd3SHoLT1RNN9adrt4eW9JBIdMjqGPt/il2OQZfZiNk2Imka9tBQ7Hofsr1WZVBVETWvK6Ja3xE91ViXVEjDSzmDnPqc1WMV+jvu/n3ov+8wbhEpHGP/sgAAZgs6IACAF3RAAAAvZl8MSKd+Kaf8ddzgrdo3s6AloWGTlNimmZni6UwlHRkpvU4mKMltn0MkPqS2TXM+cTG4NDIJ02aHi1OI84fi08LEpiVK+C4GzU3FhUPvxL7PTDH1VuxSf/1jylLrc9WpqRwJ1yW7cIGznJ+BaxO+nfAecece2ZRp2JXCHRAAwAs6IACAF3RAAAAvZmEMKMUzN+XEEdQ49vCyDzjLdbtfLy6kSSVSLeUYrDYnPccQWW9fG33u5ZRnsD/bSj5LpdqY6rmNuM824VxNQy52vQ+mrvgnnz3U76wLU3w3zXBMKp6E44ydeqKzHNgxIBW/i3xYU/z7CQeOxm8Qc1zjK1VVOfHuavl3JgF3QAAAL+iAAABezLohuCAzQ9UI1S1sZtR9o3xCFuVpMU3DE6nFvc9Up12LSMZKbxQOTX0Ibsa+I0kOHKzMcSo4vBLsKQ4d5wcHp3yccrx75jxnedHTxe91zUnu8Fx48JC7bE//TjP0PRMZtycQ1LopgMzoSIkto7Itzc6yfe6RIdBZmA5MhDsgAIAndEAAAC/ogAAAXlRvDCgIimPf1ph3JA3GDE0/zP7qd85yOF2BhZipvZmGemdZTw91xoWnM+VPmpQ5cdvqceuMtaziXZG4jk7p72PaacK4e3j4cOl94+J5OqVPTW3JbRNLG6hjOdORPQXHWh9901kOre91OL/BWZdpaix5nEg6IE9xnjgZdT75vpjUT7okxMLj3GMdHige55AbG6uauGdK3AEBALygAwIAeEEHBADwonpjQNVGl+SeqZiDPS6cUWPEKs5jykmDk0Y1pPmYBc89VGXa/phY07TR38u4Ehf6v8S6JHpm7vyf2ehS4LZIGZTZ6dg4CwDArEMHBADwYvYNwXmaahkOxVSAnEb29MqgXk3D1sOC7o7ucjVMUdVTj5VwyJpGrueRZt2UJkHgDh8Ze2ryTGUvTprral9zNQwVnVYupbdVQzHGWN+JSJPUsJoempyuYbcUU/PH3jzgLGdPOKF4mHfV1HU1zdz+GwhUKiH7ukymHTOiRv0Ta/1d6u9AdFhZf5bF71ukquw77061hV5xBwQA8IIOCADgBR0QAMCL6o0BBRlrTDR0X7fFjcNXcgxYxy+mGlNJmiodN71Sxzb0FNWY8WUjFWq/SMpUPKXPJzIGbp2P0VN1U0xpDtS4+7TFhJKmgqeZFm8fS1dwTfE9rsqULOo6ZXJuaiFjxVeDGvd7GvnsRqy4Z+S7VQVxTi2f4gNQH1Yw6JZcsK+FTkOk0zWlKfvgE3dAAAAv6IAAAF7QAQEAvKjeGFCYLxE/mKEy1Gr8Pnva/3DXd79d+DX/7ntTb1Nkrr8a/7dON/+eep+4w6YZ+08qaREXy0iKc8TEmiJttOM8Og6SJmaVlKYkTQwrTsJFzp52auH3/EuvuLumiEtFyi+XITj3rMLvmSNuHCH/m70pDpTimRv12QWdp5R83/DIkckfNwUdF9RiP48ySr7ke3tL7pv0N6qfl4o1U2m4Kow7IACAF3RAAAAv6IAAAF5UbwyoREnuQD37MlOlmfNNOWc585o1fl5OzEeLG28up8x23HGT2hQXE0oTL0poU1Bb/DqakYQB8rg26+eL0uybRsJzQPo5DndlirhC3DVNeS7Zd4q51kxff6p9y3lf2/CSZme55uXi+WUa3BLWZsR9niW2BHlMm8p6Fqys8vVTLxsS1Lr5D51z1zHRWVCeZCLcAQEAvKADAgB4Ub1DcMaISPTWd+ZS7bvvPdroprrITde0x7jb/bKmDFdwaLJSbVTbVmq6cSWnLcdKmBoeNjWUXlnJYdsU3risvfD7kp/Pd1fOUEr/+lffdpbHrPMLjx6dkTbMmDJSXqVKp1MN5VamgDsgAIAXdEAAAC+qbgjuD5l/x2R0ohE4b8bG3KfGs6Z4exyamMqkmLMy+eJQYLV8R/LDxe/xWF5lW56pNobu+45VybVB5YzJ+GealMk9MGlyvc+AN954Q5YuXeq7GQCAMu3fv19OPPHEkuurrgMKw1AOHDggxhjp6OiQ/fv3S3Nzc/KOc1R/f78sXbqU65SA6zQ5XKfJ4TrFM8bI4cOHpb29XTKZ0pGeqhuCy2QycuKJJ0p///hDcs3NzXzAk8B1mhyu0+RwnSaH61RaS0tL4jZMQgAAeEEHBADwomo7oFwuJ//4j/8ouVwueeM5jOs0OVynyeE6TQ7XqTKqbhICAGBuqNo7IADAsY0OCADgBR0QAMALOiAAgBd0QAAAL6q2A9q6daucfPLJUl9fLytXrpRnnnnGd5O86erqknPPPVeamppk8eLFcvnll8uePXucbYaGhmT9+vWyaNEiaWxslDVr1khPT4+nFleHW2+9VYIgkI0bNxZe4zqNe/PNN+VTn/qULFq0SBoaGuSss86S5557rrDeGCM33nijLFmyRBoaGmT16tWyd+9ejy2eefl8XrZs2SKdnZ3S0NAgH/zgB+WrX/2qk2CT61QmU4Xuv/9+U1dXZ/7t3/7N/PrXvzZ/+7d/axYsWGB6enp8N82LSy65xGzbts3s3r3b7Nq1y/zpn/6p6ejoMEeOHCls87nPfc4sXbrUbN++3Tz33HPm/PPPNxdccIHHVvv1zDPPmJNPPtl86EMfMtdee23hda6TMe+++6456aSTzKc//Wnz9NNPm1dffdX89Kc/Nb/97W8L29x6662mpaXFPPTQQ+aFF14wf/7nf246OzvN4OCgx5bPrJtvvtksWrTIPPzww+a1114zDzzwgGlsbDT/8i//UtiG61SequyAzjvvPLN+/frCcj6fN+3t7aarq8tjq6rHwYMHjYiYxx9/3BhjTG9vr6mtrTUPPPBAYZvf/OY3RkTMjh07fDXTm8OHD5tTTz3VPProo+aP//iPCx0Q12ncl7/8ZXPRRReVXB+GoWlrazP//M//XHitt7fX5HI584Mf/GAmmlgVLr30UvOZz3zGee2KK64wa9euNcZwnSqh6obgRkZGZOfOnbJ69erCa5lMRlavXi07duzw2LLq0dfXJyIiCxcuFBGRnTt3yujoqHPNli1bJh0dHXPymq1fv14uvfRS53qIcJ3+4Mc//rGsWLFCPvnJT8rixYvlnHPOkbvvvruw/rXXXpPu7m7nOrW0tMjKlSvn1HW64IILZPv27fLKK6+IiMgLL7wgTzzxhHz84x8XEa5TJVRdNuxDhw5JPp+X1tZW5/XW1lZ5+eWXPbWqeoRhKBs3bpQLL7xQzjzzTBER6e7ulrq6OlmwYIGzbWtrq3R3d3topT/333+/PP/88/Lss89G1nGdxr366qtyxx13yKZNm+Tv//7v5dlnn5VrrrlG6urqZN26dYVrMdHf4Fy6TjfccIP09/fLsmXLJJvNSj6fl5tvvlnWrl0rIsJ1qoCq64AQb/369bJ792554oknfDel6uzfv1+uvfZaefTRR6W+vt53c6pWGIayYsUKueWWW0RE5JxzzpHdu3fLnXfeKevWrfPcuurxwx/+UO69916577775IwzzpBdu3bJxo0bpb29netUIVU3BHf88cdLNpuNzEzq6emRtrY2T62qDhs2bJCHH35YfvaznzlVBtva2mRkZER6e3ud7efaNdu5c6ccPHhQPvzhD0tNTY3U1NTI448/LrfffrvU1NRIa2sr10lElixZIqeffrrz2mmnnSb79u0TESlci7n+N/ilL31JbrjhBrnyyivlrLPOkr/+67+W6667Trq6ukSE61QJVdcB1dXVyfLly2X79u2F18IwlO3bt8uqVas8tswfY4xs2LBBHnzwQXnssceks7PTWb98+XKpra11rtmePXtk3759c+qaXXzxxfLiiy/Krl27Cj8rVqyQtWvXFn7nOolceOGFkWn8r7zyipx00kkiItLZ2SltbW3Oderv75enn356Tl2no0ePRqp5ZrNZCcNQRLhOFeF7FsRE7r//fpPL5cz3vvc989JLL5mrr77aLFiwwHR3d/tumhef//znTUtLi/n5z39u3nrrrcLP0aNHC9t87nOfMx0dHeaxxx4zzz33nFm1apVZtWqVx1ZXB3sWnDFcJ2PGp6jX1NSYm2++2ezdu9fce++9Zt68eebf//3fC9vceuutZsGCBeZHP/qR+dWvfmUuu+yyOTe9eN26deYDH/hAYRr2f/7nf5rjjz/eXH/99YVtuE7lqcoOyBhjvv3tb5uOjg5TV1dnzjvvPPPUU0/5bpI3IjLhz7Zt2wrbDA4Omi984QvmuOOOM/PmzTN/8Rd/Yd566y1/ja4SugPiOo37yU9+Ys4880yTy+XMsmXLzF133eWsD8PQbNmyxbS2tppcLmcuvvhis2fPHk+t9aO/v99ce+21pqOjw9TX15tTTjnF/MM//IMZHh4ubMN1Kg/1gAAAXlRdDAgAMDfQAQEAvKADAgB4QQcEAPCCDggA4AUdEADACzogAIAXdEAAAC/ogAAAXtABAQC8oAMCAHjxfwGPvqSy/i4c3wAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if type(model.layers[1]) == SimpleHandmadeTFLayer:\n",
    "    x = model.embedding(in_data.to('cuda:0')) * (model.embedding.embedding_dim ** 0.5)\n",
    "            \n",
    "    if model.positional_encoding == \"sine\":\n",
    "        x = x + model.pos_encoder(x) # For the sinusoidal positional encoding class\n",
    "    if model.positional_encoding == \"learned\":\n",
    "        x = x + model.pos_encoder # For the learned positional encodings\n",
    "\n",
    "    x = model.layers[0](x, mask)\n",
    "    \n",
    "    # xx = x.permute(1, 0, 2)\n",
    "    xx = x\n",
    "    \n",
    "    x_norm = model.layers[1].transformer_encoder.norm1(xx)\n",
    "    \n",
    "    mha = model.layers[1].transformer_encoder.mha\n",
    "    B, T, _ = x_norm.size()\n",
    "    q = mha.q_proj(x_norm).view(B, T, mha.num_heads, mha.head_dim).transpose(1, 2)\n",
    "    k = mha.k_proj(x_norm).view(B, T, mha.num_heads, mha.head_dim).transpose(1, 2)\n",
    "    v = mha.v_proj(x_norm).view(B, T, mha.num_heads, mha.head_dim).transpose(1, 2)\n",
    "    \n",
    "    attn_output, attn_weights = scaled_dot_product_attention(q, k, v, mask)\n",
    "\n",
    "    head_number = 0\n",
    "    plt.imshow(attn_weights[0,head_number].to('cpu').detach())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "c88fa0db-2a1a-41d0-b1b4-1f0ac5667b3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_mask = (out_data != args.vocab_size-1)\n",
    "acc = torch.sum(loss_mask.to(\"cuda:0\") & ((torch.argmax(model(in_data.to(\"cuda:0\"), mask.to(\"cuda:0\")), dim=-1) - out_data.to(\"cuda:0\")) == 0)).item()\n",
    "acc /= loss_mask.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "4ebd177f-0218-4613-88a6-c15d6c907cb2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.1383)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a42964f-6872-4b8f-ab37-e30c9f79e54e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "hybrid",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}