Upload 4 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+bert_domain_advertising_classifier filter=lfs diff=lfs merge=lfs -text

bert_domain_advertising_classifier

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70dc19f538e361f7f28848b9aea577b20e9cc941e3eaec4841113d9b2f04f129
+size 8187846

bert_domain_advertising_classifier.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9c9c83fc410ef6a07638af1e5fb5183d2686d7ccfe4b89d678ab6a5c8f62ec6
+size 8207749

bert_domain_advertising_classifier.onnx.ipynb

@@ -0,0 +1,2572 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "source": [
+        "%pip install onnx"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "yE8Z_9M87Mth",
+        "outputId": "ca9c9cdd-54c2-4527-dad9-5947dc8b7345"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Requirement already satisfied: onnx in /usr/local/lib/python3.10/dist-packages (1.15.0)\n",
+            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from onnx) (1.23.5)\n",
+            "Requirement already satisfied: protobuf>=3.20.2 in /usr/local/lib/python3.10/dist-packages (from onnx) (3.20.3)\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "V2O5yNWuifv3"
+      },
+      "outputs": [],
+      "source": [
+        "import ast\n",
+        "import torch\n",
+        "import pandas as pd\n",
+        "import torch.nn as nn\n",
+        "from tqdm import tqdm\n",
+        "from torch.utils.data import DataLoader, TensorDataset\n",
+        "from sklearn.model_selection import train_test_split\n",
+        "from sklearn.preprocessing import MultiLabelBinarizer\n",
+        "from sklearn.metrics import accuracy_score\n",
+        "from transformers import BertTokenizer, AdamW, BertForSequenceClassification"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "G4PfTgErIXIj"
+      },
+      "outputs": [],
+      "source": [
+        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TSKX7sHE6Bnr"
+      },
+      "outputs": [],
+      "source": [
+        "df = pd.read_csv('dataset.csv')\n",
+        "df['classes'] = df['classes'].apply(ast.literal_eval)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QTwsZltwhPTt"
+      },
+      "outputs": [],
+      "source": [
+        "classes_count = [0 for i in range(630)]\n",
+        "\n",
+        "for classes in df['classes']:\n",
+        "  for c in classes:\n",
+        "    classes_count[c] +=1\n",
+        "\n",
+        "classes_min = min(classes_count)\n",
+        "classes_max = max(classes_count)\n",
+        "\n",
+        "pos_weights = torch.tensor([0.3 + 0.7 * (1 - (c - classes_min)  / (classes_max - classes_min)) for c in classes_count]).to(device)  # Adjust weights for each class"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZRao8_5mitBa",
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 201,
+          "referenced_widgets": [
+            "e3ebca197a0a426babfb75caf8f81799",
+            "548e251c68d044368ecb68898037bb01",
+            "7be5e1bf58f34d359bc1726c88c2006f",
+            "ab4a6233ff044573958ef204775509c5",
+            "887871dfa5414422b63285477cda1930",
+            "2af63a8b833a4582babaec7bfc219a36",
+            "5b046c94863140dd858051e1b30cf02a",
+            "7ae4e4874f704c7d95479a435d3f5a18",
+            "78b5111866184eb096cdfb432a4d2c49",
+            "2e91d31351fe4c06b0d7827e12686607",
+            "44dfbae45a8848a1bc84e12ebd9e93b6",
+            "d0afdae8789b42d4985b0c4fda3c2564",
+            "7c960dc2e1264d6d813ea2b9c878ffa2",
+            "bd09a7a58ff14190b3a683c0e2e4ff79",
+            "4b832750eab0451997a0f892d38db423",
+            "1bbf5fae9a2c4e32b3d3d162db1c9b5f",
+            "7ec362f6554f416384b1a03a3ea41eb9",
+            "594493b81e6f45cc94d9d44c2804d042",
+            "141ba72718e043edb793f7869b2fedb5",
+            "494f55304a184a518e4e35edf39dd7a2",
+            "48d903cf22d84ecb8cd5af5874567536",
+            "c97fb6b54c474f11ab1fee9aff91cf47",
+            "03f2530b9b444d48931daa2ca900bb9e",
+            "ded0e4189e594d31b190dbca2efd9cd4",
+            "1bed16416cb945828b250748028e4be9",
+            "75b649dfd7304bebad995e992d379fa5",
+            "a20d444fee7b4919ab47e5bcd69e853a",
+            "c330eadc3acc4ddc95f061bf8e841dad",
+            "777ec9ccae5240048cdd3212a40ce632",
+            "1a61aa0c0b6047f58eff4feb7aa32959",
+            "17a3d48a545b4876849fe4ffc764752c",
+            "681827839e92433e8dd25faa888c0680",
+            "514ab933abbb44eda2cbfd7b6d5ff47a",
+            "338ca79c5d294a2e967166437813b668",
+            "841711bda59a4625b9a8bae26caed181",
+            "3f04bf02bc2149c189bbf00c36ac7107",
+            "21e9b97580f440a38a484e136ddadfb2",
+            "d1e66db731e949cb8fb16c0bb35729ae",
+            "725592c335604e9b95eaaf9852fd588e",
+            "8c25c42889db4936af2626284c8c2bfc",
+            "d0c60871f88d4405881aabac6e732de9",
+            "c7f502323c1e45b696d1c79db8b93ba7",
+            "ef643c62650f419ca01d4266049f6c30",
+            "e3c9de4c383e489cb1d4e39ab9bd3683"
+          ]
+        },
+        "outputId": "0e9bba12-bd74-4ece-e7d6-aeaae0fa0da9"
+      },
+      "outputs": [
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "tokenizer_config.json:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
+            ],
+            "application/vnd.jupyter.widget-view+json": {
+              "version_major": 2,
+              "version_minor": 0,
+              "model_id": "e3ebca197a0a426babfb75caf8f81799"
+            }
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
+            ],
+            "application/vnd.jupyter.widget-view+json": {
+              "version_major": 2,
+              "version_minor": 0,
+              "model_id": "d0afdae8789b42d4985b0c4fda3c2564"
+            }
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]"
+            ],
+            "application/vnd.jupyter.widget-view+json": {
+              "version_major": 2,
+              "version_minor": 0,
+              "model_id": "03f2530b9b444d48931daa2ca900bb9e"
+            }
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
+            ],
+            "application/vnd.jupyter.widget-view+json": {
+              "version_major": 2,
+              "version_minor": 0,
+              "model_id": "338ca79c5d294a2e967166437813b668"
+            }
+          },
+          "metadata": {}
+        }
+      ],
+      "source": [
+        "mlb = MultiLabelBinarizer()\n",
+        "mlb.fit([range(len(classes_count))])\n",
+        "train_df, val_df = train_test_split(df, test_size=0.2, random_state=42)\n",
+        "\n",
+        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
+        "\n",
+        "def tokenize_data(data, max_length=16):\n",
+        "    input_ids = []\n",
+        "    attention_masks = []\n",
+        "    labels = []\n",
+        "\n",
+        "    for _, row in data.iterrows():\n",
+        "        text = row['domain'].replace('.', ' ')\n",
+        "        classes = row['classes']\n",
+        "\n",
+        "        encoding = tokenizer.encode_plus(\n",
+        "            text,\n",
+        "            max_length=max_length,\n",
+        "            padding='max_length',\n",
+        "            truncation=True,\n",
+        "            return_tensors='pt'\n",
+        "        )\n",
+        "\n",
+        "        input_ids.append(encoding['input_ids'])\n",
+        "        attention_masks.append(encoding['attention_mask'])\n",
+        "        labels.append(torch.tensor(mlb.transform([classes])))\n",
+        "\n",
+        "    input_ids = torch.cat(input_ids, dim=0)\n",
+        "    attention_masks = torch.cat(attention_masks, dim=0)\n",
+        "    labels = torch.cat(labels, dim=0)\n",
+        "\n",
+        "    return TensorDataset(input_ids, attention_masks, labels)\n",
+        "\n",
+        "\n",
+        "train_dataset = tokenize_data(train_df)\n",
+        "val_dataset = tokenize_data(val_df)\n",
+        "\n",
+        "train_dataloader = DataLoader(train_dataset, batch_size=32, shuffle=True)\n",
+        "val_dataloader = DataLoader(val_dataset, batch_size=32, shuffle=False)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ScZSPDtsFST9"
+      },
+      "outputs": [],
+      "source": [
+        "class SmallBERT(nn.Module):\n",
+        "    def __init__(self, hidden_size=256, num_layers=2, num_attention_heads=4, num_classes=2, vocab_size=30522):\n",
+        "        super(SmallBERT, self).__init__()\n",
+        "\n",
+        "        self.embedding = nn.Embedding(vocab_size, hidden_size)\n",
+        "\n",
+        "        self.transformer = nn.TransformerEncoder(\n",
+        "            encoder_layer=nn.TransformerEncoderLayer(\n",
+        "                d_model=hidden_size,\n",
+        "                nhead=num_attention_heads,\n",
+        "                dim_feedforward=hidden_size * 4\n",
+        "            ),\n",
+        "            num_layers=num_layers\n",
+        "        )\n",
+        "\n",
+        "        self.dropout = nn.Dropout(0.1)\n",
+        "        self.classifier = nn.Linear(hidden_size, num_classes)\n",
+        "\n",
+        "    def forward(self, input_ids, attention_mask):\n",
+        "        embedded = self.embedding(input_ids)\n",
+        "        transformer_output = self.transformer(embedded)\n",
+        "        pooled_output = transformer_output.mean(dim=1)\n",
+        "        pooled_output = self.dropout(pooled_output)\n",
+        "        logits = self.classifier(pooled_output)\n",
+        "\n",
+        "        return logits"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "uRGMQeqxygB-",
+        "outputId": "2ff0b6bf-bd9a-4edd-dd3c-eaf4af9ed972"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/transformer.py:282: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance)\n",
+            "  warnings.warn(f\"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}\")\n"
+          ]
+        },
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "SmallBERT(\n",
+              "  (embedding): Embedding(30522, 64)\n",
+              "  (transformer): TransformerEncoder(\n",
+              "    (layers): ModuleList(\n",
+              "      (0): TransformerEncoderLayer(\n",
+              "        (self_attn): MultiheadAttention(\n",
+              "          (out_proj): NonDynamicallyQuantizableLinear(in_features=64, out_features=64, bias=True)\n",
+              "        )\n",
+              "        (linear1): Linear(in_features=64, out_features=256, bias=True)\n",
+              "        (dropout): Dropout(p=0.1, inplace=False)\n",
+              "        (linear2): Linear(in_features=256, out_features=64, bias=True)\n",
+              "        (norm1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
+              "        (norm2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
+              "        (dropout1): Dropout(p=0.1, inplace=False)\n",
+              "        (dropout2): Dropout(p=0.1, inplace=False)\n",
+              "      )\n",
+              "    )\n",
+              "  )\n",
+              "  (dropout): Dropout(p=0.1, inplace=False)\n",
+              "  (classifier): Linear(in_features=64, out_features=630, bias=True)\n",
+              ")"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 9
+        }
+      ],
+      "source": [
+        "model = SmallBERT(hidden_size=64, num_layers=1, num_attention_heads=2, num_classes=len(mlb.classes_), vocab_size=30522)\n",
+        "model.to(device)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "I6SmEEwfyu02",
+        "outputId": "ef09ac44-2346-4940-9cb4-a5cbf955f8d9"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
+            "  warnings.warn(\n",
+            "Epoch 1: 100%|██████████| 17333/17333 [01:34<00:00, 183.05it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.01455639958662022\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 2: 100%|██████████| 17333/17333 [01:32<00:00, 187.41it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.009372596806357445\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 3: 100%|██████████| 17333/17333 [01:33<00:00, 185.43it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.00895946430422158\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 4: 100%|██████████| 17333/17333 [01:33<00:00, 184.48it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.008654748138614369\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 5: 100%|██████████| 17333/17333 [01:37<00:00, 178.17it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.008430107505195222\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 6: 100%|██████████| 17333/17333 [01:35<00:00, 181.19it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.00825132777584388\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 7: 100%|██████████| 17333/17333 [01:35<00:00, 181.75it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.008098585383865228\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 8: 100%|██████████| 17333/17333 [01:36<00:00, 178.89it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.007973489129822018\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 9: 100%|██████████| 17333/17333 [01:34<00:00, 182.66it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.007862447003308942\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 10: 100%|██████████| 17333/17333 [01:34<00:00, 184.37it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.007767102444360781\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 11: 100%|██████████| 17333/17333 [01:32<00:00, 187.08it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.007681683442265059\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 12: 100%|██████████| 17333/17333 [01:34<00:00, 183.56it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.0076040456217698555\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 13: 100%|██████████| 17333/17333 [01:33<00:00, 185.58it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.007536577820711911\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 14: 100%|██████████| 17333/17333 [01:33<00:00, 185.95it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.007473697705334162\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 15: 100%|██████████| 17333/17333 [01:35<00:00, 182.32it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.0074216005095400365\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 16: 100%|██████████| 17333/17333 [01:34<00:00, 183.65it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.007366839836092643\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 17: 100%|██████████| 17333/17333 [01:35<00:00, 182.22it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.007323595992545943\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 18: 100%|██████████| 17333/17333 [01:35<00:00, 182.39it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.007281322365534597\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 19: 100%|██████████| 17333/17333 [01:34<00:00, 184.08it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.007237244195990475\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 20: 100%|██████████| 17333/17333 [01:32<00:00, 186.76it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.007204567027232412\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 21: 100%|██████████| 17333/17333 [01:34<00:00, 182.81it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.007169707122033225\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 22: 100%|██████████| 17333/17333 [01:34<00:00, 183.90it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.0071396396113856736\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 23: 100%|██████████| 17333/17333 [01:34<00:00, 184.27it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.0071061778510986525\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 24: 100%|██████████| 17333/17333 [01:32<00:00, 187.58it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.007079116968562793\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 25: 100%|██████████| 17333/17333 [01:35<00:00, 182.40it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.007053613749052303\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 26: 100%|██████████| 17333/17333 [01:33<00:00, 184.62it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.007027668537672933\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 27: 100%|██████████| 17333/17333 [01:32<00:00, 187.06it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.007002266663781792\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 28: 100%|██████████| 17333/17333 [01:33<00:00, 185.74it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006983264061160072\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 29: 100%|██████████| 17333/17333 [01:32<00:00, 186.62it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006960326002187834\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 30: 100%|██████████| 17333/17333 [01:33<00:00, 185.09it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006937417333392953\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 31: 100%|██████████| 17333/17333 [01:34<00:00, 183.75it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006922033323461952\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 32: 100%|██████████| 17333/17333 [01:33<00:00, 184.46it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006898388598138539\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 33: 100%|██████████| 17333/17333 [01:33<00:00, 185.13it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006879616245666734\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 34: 100%|██████████| 17333/17333 [01:33<00:00, 184.60it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006862940901992164\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 35: 100%|██████████| 17333/17333 [01:42<00:00, 169.91it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006847698244776712\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 36: 100%|██████████| 17333/17333 [01:34<00:00, 183.32it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006830427775150129\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 37: 100%|██████████| 17333/17333 [01:34<00:00, 183.19it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006812137309707921\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 38: 100%|██████████| 17333/17333 [01:33<00:00, 184.78it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.0067945668246664325\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 39: 100%|██████████| 17333/17333 [01:35<00:00, 181.25it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.0067820949417340725\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 40: 100%|██████████| 17333/17333 [01:35<00:00, 182.22it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006767237709435089\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 41: 100%|██████████| 17333/17333 [01:33<00:00, 186.12it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006754826359595461\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 42: 100%|██████████| 17333/17333 [01:34<00:00, 184.04it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006737771247406829\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 43: 100%|██████████| 17333/17333 [01:33<00:00, 185.51it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006729339890418221\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 44: 100%|██████████| 17333/17333 [01:34<00:00, 183.86it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006713867076464513\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 45: 100%|██████████| 17333/17333 [01:36<00:00, 180.05it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006700394152881958\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 46: 100%|██████████| 17333/17333 [01:34<00:00, 183.63it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006688476914265044\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 47: 100%|██████████| 17333/17333 [01:33<00:00, 185.67it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006676017345109116\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 48: 100%|██████████| 17333/17333 [01:33<00:00, 184.72it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006663939434730227\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 49: 100%|██████████| 17333/17333 [01:33<00:00, 185.32it/s]\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006649965298220924\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Epoch 50: 100%|██████████| 17333/17333 [01:33<00:00, 185.48it/s]"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Training Loss: 0.006640829532034061\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "\n"
+          ]
+        }
+      ],
+      "source": [
+        "model.train()\n",
+        "\n",
+        "optimizer = AdamW(model.parameters(), lr=2e-4)\n",
+        "criterion = nn.BCEWithLogitsLoss(pos_weight=pos_weights, reduction='mean')\n",
+        "\n",
+        "epochs = 50\n",
+        "for epoch in range(epochs):\n",
+        "  total_loss = 0\n",
+        "\n",
+        "  for batch in tqdm(train_dataloader, desc=f'Epoch {epoch + 1}'):\n",
+        "    input_ids = batch[0].to(device)\n",
+        "    attention_mask = batch[1].to(device)\n",
+        "    labels = batch[2].to(device, dtype=torch.float)\n",
+        "\n",
+        "    optimizer.zero_grad()\n",
+        "    logits = model(input_ids=input_ids, attention_mask=attention_mask)\n",
+        "\n",
+        "    loss = criterion(logits, labels)\n",
+        "    total_loss += loss.item()\n",
+        "\n",
+        "    loss.backward()\n",
+        "    optimizer.step()\n",
+        "\n",
+        "  average_loss = total_loss / len(train_dataloader)\n",
+        "  print(f'Training Loss: {average_loss}')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GO0fi0qQ7oKb",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "outputId": "2863272e-3956-4e9a-b7ba-fd8b850ec406"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Validation: 100%|██████████| 4334/4334 [00:07<00:00, 563.03it/s]\n",
+            "<ipython-input-11-49aedd21c825>:16: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:261.)\n",
+            "  predicted_labels = mlb.inverse_transform(torch.tensor(predicted_logits))\n"
+          ]
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Predicted labels [(), (), (332,), (183, 194, 215), (215,), (23,), (534,), (), (140,), (183,), (229,), (239,), (), (227, 542), (536,), (140,), (1,), (140, 299), (), (250,), (173,), (227, 542), (239,), (363, 365), (), (215,), (243,), (1, 183, 363), (332, 340), (215,), (289,), (), (96, 289, 572), (), (), (), (), (160,), (215,), (1, 104, 183), (140, 227, 439), (), (289,), (408, 412, 601), (), (140, 215, 250, 528), (332,), (), (183,), (289,), (183, 194), (183,), (), (48, 363), (), (215,), (104,), (), (239, 423), (215, 250), (), (183,), (1,), (444,), (215,), (243, 245), (299, 325, 363, 364, 365), (104, 533), (239,), (534,), (183, 533), (400,), (332,), (), (250,), (215,), (103, 215, 439), (), (), (1, 183, 186, 215), (), (), (103, 126, 289), (363,), (172,), (215,), (140, 215, 439), (183,), (), (23,), (351,), (227, 229, 542), (1, 23), (), (), (), (183,), (215,), (183, 351), (325,)]\n",
+            "True inputs ['usw1.green.ops.kargo.com', '6002359.global.siteimproveanalytics.io', 'didiglobal.com.dob.sibl.support-intelligence.net', 'ns500248.ns500202.ns500248.ns500242.ns500197.ns500219.sweetchicksclub.com', 'ms.email.nextdoor.com', 'd.hello.plowandhearth.com', 'p45-contentws.icloud.com', 'pakasak.com', 'vm3-proxy-hisgeneral-scus-sat02p-6.connector.his.msappproxy.net', 'gahu.hit.gemius.pl', 'ep.wakehealth.edu', 'fi-prj-poc-slim.frontsrv.com', 'v3.pebblepad.co.uk', 'osceolak12.schoology.com', 'mydnspt.net', 'default-download.splashtop.com', 'flow.performgroup.io', 'instagram.feoh11-1.fna.fbcdn.net', 'arin.authdns.ripe.net', 'lee.ns.cloudflare.com', 'goodhoodstore.com', 'gw01mem01-eastus.classroom.cloud', 'www.sos.state.mn.us', 'security.netflix.net', 'prod.lb.mlsmatrix.com', 'apps.paris.fr', 'rc.samsungweather.com', 'pull-flv-f6.douyincdn.com.hdlvcloud.ks-cdn.com', 'visitutah.com', 'ogs.google.co.kr', 'daraz-sg.alibaba.com.gds.alibabadns.com', 'stor.g2.ph.dell.com', 'zalando.es', 'perfumist.net', 'lifesparking.com', 'signon.hoovers.dnb.com', 'appdl2drcndbankcdn.cache.qcloudcdn.qq.com', 'api.my.healthequity.com', 'db3pcor006-meta.fe.1drv.com', 'cdn.pubble.io', 'jminsure-my.sharepoint.com', 'aws.upstart.com', 'shop.rewe-static.de', 'anonymous.print.avery.com', 'mygts.dhl.com', 'app.chat.global.xiaomi.net', 'citifxvelocity.com', 'core.gss-service.com', 'kdev.msap.io', 'i-subscriptions.thx.lazparking.com', 'ns500245.ns500249.ns500197.hostmaster.sweetxladies.com', 's7.tengu0.xyz', 'link.entrata.com', 'r5.sn-q4flrnsl.c.2mdn.net', 'static.one.network', 'expurgate.de', 'a2467.casalemedia.com', 'squadup.com', 'licensemanager.kaspersky-labs.com', 'doc-0k-7c-docstext.googleusercontent.com', 'perficient.com', 'r4---sn-vgqsrnzz.gvt1.com', 'blockworks-com.gallerycdn.vsassets.io', 'prod-s6-piv.nubank.com.br', 'mailchi.mp.dob.sibl.support-intelligence.net', 'www.steelexpress.co.uk', 'arlostreaming19261-z2-prod.ar.arlo.com', 'ic3-calling-enterpriseproxy.brazilsouth-prod.cosmic.office.net', 'eservices.dor.nc.gov', 'nevacloud.io', 'fceb4-1.fna.fbcdn.net', 'api.cosmopolitan.com.hk', 'map3.viamichelin.com', 'api.foreflight.com', 'o1276079.ingest.sentry.io', 'up.cm.ksmobile.com', 'triconenergy.sharepoint.com', 'mb.cdn.srv-hub.org', 'serv.ad-adapex.io', 'cname.mail.cname.cname.mail.cname.cname.cname.cname.cname.cdn.wan02.com', 'ssb-eu-secure-6.smartadserver.com', 'cap.co.uk', 'ppc-oauth.wac.trafficmanager.net.wac-0003.wac-dc-msedge.net.wac-0003.wac-msedge.net', 'nvidia.tt.omtrdc.net', 'euc.vision.meraki.com', 'ginkgobioworks.zoom.us', 'capanoinc-my.sharepoint.com', 'r2---sn-vgqskned.gvt1.com', 'beplb01.portal.hewitt.com', 'external-media.grailed.com', 'api-senso-cloud-anbaathcgfgmhqhw.z01.azurefd.net', 'classroom.emeritus.org', 'assets.omny.fm', 'r5---sn-nx5s7nel.c.2mdn.net', 'api.kfdealeraccess.com', 'puntown.com', 'st-v3-univ-srs-linux-3040.api.splashtop.com', 'planetoftheapps.com', 'ekbhgdvznq.dbysdbuylr.net', 'fcs.myforcura.com']\n",
+            "True labels [[238], [183, 164], [332], [215, 183], [215], [351], [1], [1], [140], [578], [229], [363, 351, 365], [250, 439], [227, 542], [250], [1, 363], [103], [299], [140], [137, 534], [298], [227], [215, 140], [423], [570], [215], [243], [1, 183, 186], [332], [215], [1, 289], [103], [289, 572, 96], [94], [250], [183, 194], [103], [158, 160], [215], [1], [140, 227, 439], [103, 533], [289], [601], [103], [215], [310], [224, 439], [215, 140], [100], [1, 183, 194], [243], [1], [1, 363], [533], [243, 245], [245], [1, 183], [423], [250], [436, 439], [183], [263], [444], [1], [103, 164], [363, 325, 364], [104, 533], [239], [1, 183, 250], [183, 533], [254, 400], [332], [1], [1], [126, 215, 243, 183], [439], [250], [140, 227], [215, 183, 186], [183], [1], [215], [140], [172, 414], [215, 140], [439], [183], [151], [23], [243], [227, 542], [23], [1], [436], [1], [140, 363], [215, 250], [243], [243, 299, 325]]\n",
+            "Validation Accuracy: 0.18336410315582993\n"
+          ]
+        }
+      ],
+      "source": [
+        "model.eval()\n",
+        "\n",
+        "predicted_logits = []\n",
+        "with torch.no_grad():\n",
+        "    for batch in tqdm(val_dataloader, desc='Validation'):\n",
+        "      input_ids = batch[0].to(device)\n",
+        "      attention_mask = batch[1].to(device)\n",
+        "      labels = batch[2].to(device, dtype=torch.float)\n",
+        "\n",
+        "      logits = model(input_ids=input_ids, attention_mask=attention_mask)\n",
+        "      probabilities = torch.sigmoid(logits)\n",
+        "\n",
+        "      preds = (probabilities > 0.2).cpu().numpy().astype(int)\n",
+        "      predicted_logits.extend(preds)\n",
+        "\n",
+        "predicted_labels = mlb.inverse_transform(torch.tensor(predicted_logits))\n",
+        "print('Predicted labels', predicted_labels[1000:1100])\n",
+        "\n",
+        "true_inputs = val_df['domain'].tolist()\n",
+        "print('True inputs', true_inputs[1000:1100])\n",
+        "\n",
+        "true_labels = val_df['classes'].tolist()\n",
+        "print('True labels', true_labels[1000:1100])\n",
+        "\n",
+        "true_logits = mlb.transform(true_labels)\n",
+        "accuracy = accuracy_score(true_logits, predicted_logits)\n",
+        "print(f'Validation Accuracy: {accuracy}')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "torch.save(model, \"bert_domain_classifier\")\n",
+        "torch.save(model.state_dict(), \"bert_domain_classifier.pth\")"
+      ],
+      "metadata": {
+        "id": "jSo-SvyK8Nan"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "dummy_input_ids = torch.zeros((1, 16), dtype=torch.long).to(device)\n",
+        "dummy_attention_mask = torch.zeros((1, 16), dtype=torch.long).to(device)\n",
+        "input_names = ['input_ids', 'attention_mask']\n",
+        "output_names = ['logits']\n",
+        "dynamic_axes = {'input_ids': {0: 'batch_size'}, 'attention_mask': {0: 'batch_size'},\n",
+        "                'logits': {0: 'batch_size'}}\n",
+        "\n",
+        "torch.onnx.export(model, (dummy_input_ids, dummy_attention_mask),\n",
+        "                  \"bert_domain_classifier.onnx\", opset_version=14,\n",
+        "                  input_names=input_names,\n",
+        "                  output_names=output_names,\n",
+        "                  dynamic_axes=dynamic_axes)"
+      ],
+      "metadata": {
+        "id": "pqanQz3i8DBx"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "provenance": [],
+      "gpuType": "V100"
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "widgets": {
+      "application/vnd.jupyter.widget-state+json": {
+        "e3ebca197a0a426babfb75caf8f81799": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HBoxModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_548e251c68d044368ecb68898037bb01",
+              "IPY_MODEL_7be5e1bf58f34d359bc1726c88c2006f",
+              "IPY_MODEL_ab4a6233ff044573958ef204775509c5"
+            ],
+            "layout": "IPY_MODEL_887871dfa5414422b63285477cda1930"
+          }
+        },
+        "548e251c68d044368ecb68898037bb01": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_2af63a8b833a4582babaec7bfc219a36",
+            "placeholder": "​",
+            "style": "IPY_MODEL_5b046c94863140dd858051e1b30cf02a",
+            "value": "tokenizer_config.json: 100%"
+          }
+        },
+        "7be5e1bf58f34d359bc1726c88c2006f": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "FloatProgressModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_7ae4e4874f704c7d95479a435d3f5a18",
+            "max": 28,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_78b5111866184eb096cdfb432a4d2c49",
+            "value": 28
+          }
+        },
+        "ab4a6233ff044573958ef204775509c5": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_2e91d31351fe4c06b0d7827e12686607",
+            "placeholder": "​",
+            "style": "IPY_MODEL_44dfbae45a8848a1bc84e12ebd9e93b6",
+            "value": " 28.0/28.0 [00:00&lt;00:00, 1.54kB/s]"
+          }
+        },
+        "887871dfa5414422b63285477cda1930": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "2af63a8b833a4582babaec7bfc219a36": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "5b046c94863140dd858051e1b30cf02a": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "7ae4e4874f704c7d95479a435d3f5a18": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "78b5111866184eb096cdfb432a4d2c49": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "ProgressStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "2e91d31351fe4c06b0d7827e12686607": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "44dfbae45a8848a1bc84e12ebd9e93b6": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "d0afdae8789b42d4985b0c4fda3c2564": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HBoxModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_7c960dc2e1264d6d813ea2b9c878ffa2",
+              "IPY_MODEL_bd09a7a58ff14190b3a683c0e2e4ff79",
+              "IPY_MODEL_4b832750eab0451997a0f892d38db423"
+            ],
+            "layout": "IPY_MODEL_1bbf5fae9a2c4e32b3d3d162db1c9b5f"
+          }
+        },
+        "7c960dc2e1264d6d813ea2b9c878ffa2": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_7ec362f6554f416384b1a03a3ea41eb9",
+            "placeholder": "​",
+            "style": "IPY_MODEL_594493b81e6f45cc94d9d44c2804d042",
+            "value": "vocab.txt: 100%"
+          }
+        },
+        "bd09a7a58ff14190b3a683c0e2e4ff79": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "FloatProgressModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_141ba72718e043edb793f7869b2fedb5",
+            "max": 231508,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_494f55304a184a518e4e35edf39dd7a2",
+            "value": 231508
+          }
+        },
+        "4b832750eab0451997a0f892d38db423": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_48d903cf22d84ecb8cd5af5874567536",
+            "placeholder": "​",
+            "style": "IPY_MODEL_c97fb6b54c474f11ab1fee9aff91cf47",
+            "value": " 232k/232k [00:00&lt;00:00, 2.28MB/s]"
+          }
+        },
+        "1bbf5fae9a2c4e32b3d3d162db1c9b5f": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "7ec362f6554f416384b1a03a3ea41eb9": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "594493b81e6f45cc94d9d44c2804d042": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "141ba72718e043edb793f7869b2fedb5": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "494f55304a184a518e4e35edf39dd7a2": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "ProgressStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "48d903cf22d84ecb8cd5af5874567536": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "c97fb6b54c474f11ab1fee9aff91cf47": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "03f2530b9b444d48931daa2ca900bb9e": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HBoxModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_ded0e4189e594d31b190dbca2efd9cd4",
+              "IPY_MODEL_1bed16416cb945828b250748028e4be9",
+              "IPY_MODEL_75b649dfd7304bebad995e992d379fa5"
+            ],
+            "layout": "IPY_MODEL_a20d444fee7b4919ab47e5bcd69e853a"
+          }
+        },
+        "ded0e4189e594d31b190dbca2efd9cd4": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_c330eadc3acc4ddc95f061bf8e841dad",
+            "placeholder": "​",
+            "style": "IPY_MODEL_777ec9ccae5240048cdd3212a40ce632",
+            "value": "tokenizer.json: 100%"
+          }
+        },
+        "1bed16416cb945828b250748028e4be9": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "FloatProgressModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_1a61aa0c0b6047f58eff4feb7aa32959",
+            "max": 466062,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_17a3d48a545b4876849fe4ffc764752c",
+            "value": 466062
+          }
+        },
+        "75b649dfd7304bebad995e992d379fa5": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_681827839e92433e8dd25faa888c0680",
+            "placeholder": "​",
+            "style": "IPY_MODEL_514ab933abbb44eda2cbfd7b6d5ff47a",
+            "value": " 466k/466k [00:00&lt;00:00, 9.05MB/s]"
+          }
+        },
+        "a20d444fee7b4919ab47e5bcd69e853a": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "c330eadc3acc4ddc95f061bf8e841dad": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "777ec9ccae5240048cdd3212a40ce632": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "1a61aa0c0b6047f58eff4feb7aa32959": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "17a3d48a545b4876849fe4ffc764752c": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "ProgressStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "681827839e92433e8dd25faa888c0680": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "514ab933abbb44eda2cbfd7b6d5ff47a": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "338ca79c5d294a2e967166437813b668": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HBoxModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_841711bda59a4625b9a8bae26caed181",
+              "IPY_MODEL_3f04bf02bc2149c189bbf00c36ac7107",
+              "IPY_MODEL_21e9b97580f440a38a484e136ddadfb2"
+            ],
+            "layout": "IPY_MODEL_d1e66db731e949cb8fb16c0bb35729ae"
+          }
+        },
+        "841711bda59a4625b9a8bae26caed181": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_725592c335604e9b95eaaf9852fd588e",
+            "placeholder": "​",
+            "style": "IPY_MODEL_8c25c42889db4936af2626284c8c2bfc",
+            "value": "config.json: 100%"
+          }
+        },
+        "3f04bf02bc2149c189bbf00c36ac7107": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "FloatProgressModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_d0c60871f88d4405881aabac6e732de9",
+            "max": 570,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_c7f502323c1e45b696d1c79db8b93ba7",
+            "value": 570
+          }
+        },
+        "21e9b97580f440a38a484e136ddadfb2": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_ef643c62650f419ca01d4266049f6c30",
+            "placeholder": "​",
+            "style": "IPY_MODEL_e3c9de4c383e489cb1d4e39ab9bd3683",
+            "value": " 570/570 [00:00&lt;00:00, 17.8kB/s]"
+          }
+        },
+        "d1e66db731e949cb8fb16c0bb35729ae": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "725592c335604e9b95eaaf9852fd588e": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "8c25c42889db4936af2626284c8c2bfc": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "d0c60871f88d4405881aabac6e732de9": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "c7f502323c1e45b696d1c79db8b93ba7": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "ProgressStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "ef643c62650f419ca01d4266049f6c30": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "e3c9de4c383e489cb1d4e39ab9bd3683": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        }
+      }
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

bert_domain_advertising_classifier.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:830e351871999ea5d394e80a709bd2faebb38c552cd060993e00f8f746702195
+size 8183622