{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "9e544464", "metadata": {}, "outputs": [], "source": [ "import sys\n", "import os\n", "sys.path.append(os.path.abspath('..'))" ] }, { "cell_type": "code", "execution_count": 2, "id": "785c1981", "metadata": {}, "outputs": [], "source": [ "import torch\n", "from tensor_transformers.nv_bert import modeling_tensor" ] }, { "cell_type": "code", "execution_count": 3, "id": "13cd16dc", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "dict_keys(['cola', 'mnli', 'mrpc', 'sst-2'])\n" ] }, { "data": { "text/plain": [ "2" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from processors.glue import PROCESSORS\n", "print(PROCESSORS.keys())\n", "# GLU dataset processor\n", "processor = PROCESSORS['cola']()\n", "num_labels = len(processor.get_labels())\n", "num_labels" ] }, { "cell_type": "code", "execution_count": 4, "id": "824c6f62", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Total number of parameters in the model: 123790338\n" ] } ], "source": [ "# config = modeling_tensor.TensorBertConfig(\"tensor_bert/config.json\")\n", "config = modeling_tensor.TensorBertConfig.from_pretrained(\"tensor_bert\") # -> bug in build_rank not set (tmp fix to modify pretrained config)\n", "model = modeling_tensor.TensorBertForSequenceClassification(\n", " config,\n", " num_labels=num_labels,\n", " )\n", "\n", "total_params = sum(p.numel() for p in model.parameters())\n", "print(f\"Total number of parameters in the model: {total_params}\")" ] }, { "cell_type": "code", "execution_count": 5, "id": "91c67e4b", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "_IncompatibleKeys(missing_keys=['classifier.weight', 'classifier.bias'], unexpected_keys=['cls.predictions.bias', 'cls.predictions.transform.dense_act.linear.weight', 'cls.predictions.transform.dense_act.linear.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias'])" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "checkpoint = torch.load(\"tensor_bert/model.pt\", map_location=\"cpu\")\n", "checkpoint = checkpoint[\"model\"] if \"model\" in checkpoint.keys() else checkpoint\n", "checkpoint.keys()\n", "\n", "model.load_state_dict(checkpoint, strict=False)" ] }, { "cell_type": "code", "execution_count": 6, "id": "25e110ee", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from tokenization import BertTokenizer\n", "vocab_file = \"../vocab\"\n", "tokenizer = BertTokenizer(\n", " vocab_file = vocab_file,\n", " do_lower_case=True,\n", " max_len=512,\n", " ) \n", "tokenizer" ] }, { "cell_type": "code", "execution_count": 7, "id": "32e2c801", "metadata": {}, "outputs": [], "source": [ "from processors.glue import PROCESSORS, convert_examples_to_features\n", "import pickle\n", "import logging\n", "from utils import is_main_process, mkdir_by_main_process, format_step, get_world_size\n", "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset\n", "\n", "\n", "logger = logging.getLogger(__name__)\n", "\n", "def get_train_features(\n", " data_dir,\n", " bert_model,\n", " max_seq_length,\n", " do_lower_case,\n", " local_rank,\n", " train_batch_size,\n", " gradient_accumulation_steps,\n", " num_train_epochs,\n", " tokenizer,\n", " processor,\n", "):\n", " cached_train_features_file = os.path.join(\n", " data_dir,\n", " \"{0}_{1}_{2}\".format(\n", " list(filter(None, bert_model.split(\"/\"))).pop(),\n", " str(max_seq_length),\n", " str(do_lower_case),\n", " ),\n", " )\n", " train_features = None\n", " try:\n", " with open(cached_train_features_file, \"rb\") as reader:\n", " train_features = pickle.load(reader)\n", " logger.info(\n", " \"Loaded pre-processed features from {}\".format(cached_train_features_file)\n", " )\n", " except:\n", " logger.info(\n", " \"Did not find pre-processed features from {}\".format(\n", " cached_train_features_file\n", " )\n", " )\n", " train_examples = processor.get_train_examples(data_dir)\n", " train_features, _ = convert_examples_to_features(\n", " train_examples,\n", " processor.get_labels(),\n", " max_seq_length,\n", " tokenizer,\n", " )\n", " if is_main_process():\n", " logger.info(\n", " \" Saving train features into cached file %s\",\n", " cached_train_features_file,\n", " )\n", " with open(cached_train_features_file, \"wb\") as writer:\n", " pickle.dump(train_features, writer)\n", " return train_features\n", "\n", "\n", "\n", "def gen_tensor_dataset(features):\n", " all_input_ids = torch.tensor(\n", " [f.input_ids for f in features],\n", " dtype=torch.long,\n", " )\n", " all_input_mask = torch.tensor(\n", " [f.input_mask for f in features],\n", " dtype=torch.long,\n", " )\n", " all_segment_ids = torch.tensor(\n", " [f.segment_ids for f in features],\n", " dtype=torch.long,\n", " )\n", " all_label_ids = torch.tensor(\n", " [f.label_id for f in features],\n", " dtype=torch.long,\n", " )\n", " return TensorDataset(\n", " all_input_ids,\n", " all_input_mask,\n", " all_segment_ids,\n", " all_label_ids,\n", " )\n" ] }, { "cell_type": "code", "execution_count": 8, "id": "743b50bd", "metadata": {}, "outputs": [], "source": [ "data_dir = \"../glue_data/CoLA\"\n", "bert_model = \"tensor-bert\"\n", "max_seq_length = max_pos_embeddings = 512\n", "do_lower_case = True\n", "local_rank = 0\n", "train_batch_size = 2\n", "gradient_accumulation_steps = 1\n", "num_train_epochs = 3\n", "\n", "tokenizer = tokenizer\n", "processor = processor\n", "\n", "train_features = get_train_features(\n", " data_dir,\n", " bert_model,\n", " max_seq_length,\n", " do_lower_case,\n", " local_rank,\n", " train_batch_size,\n", " gradient_accumulation_steps,\n", " num_train_epochs,\n", " tokenizer,\n", " processor,\n", " )\n", "train_data = gen_tensor_dataset(train_features)\n", "train_sampler = RandomSampler(train_data)\n", "train_dataloader = DataLoader(\n", " train_data,\n", " sampler=train_sampler,\n", " batch_size=train_batch_size,\n", " )" ] }, { "cell_type": "markdown", "id": "b97b356d", "metadata": {}, "source": [ "# Explore Model a bit" ] }, { "cell_type": "code", "execution_count": 17, "id": "42117cd5", "metadata": {}, "outputs": [], "source": [ "encoderLayers = model.bert.encoder.layer\n", "block = encoderLayers[0]" ] }, { "cell_type": "code", "execution_count": 22, "id": "2b912f29", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "odict_keys(['attention.self.query.bias', 'attention.self.query.tensor.factors.0', 'attention.self.query.tensor.factors.1', 'attention.self.query.tensor.factors.2', 'attention.self.query.tensor.factors.3', 'attention.self.query.tensor.factors.4', 'attention.self.query.tensor.factors.5', 'attention.self.key.bias', 'attention.self.key.tensor.factors.0', 'attention.self.key.tensor.factors.1', 'attention.self.key.tensor.factors.2', 'attention.self.key.tensor.factors.3', 'attention.self.key.tensor.factors.4', 'attention.self.key.tensor.factors.5', 'attention.self.value.bias', 'attention.self.value.tensor.factors.0', 'attention.self.value.tensor.factors.1', 'attention.self.value.tensor.factors.2', 'attention.self.value.tensor.factors.3', 'attention.self.value.tensor.factors.4', 'attention.self.value.tensor.factors.5', 'attention.output.dense.bias', 'attention.output.dense.tensor.factors.0', 'attention.output.dense.tensor.factors.1', 'attention.output.dense.tensor.factors.2', 'attention.output.dense.tensor.factors.3', 'attention.output.dense.tensor.factors.4', 'attention.output.dense.tensor.factors.5', 'attention.output.LayerNorm.weight', 'attention.output.LayerNorm.bias', 'intermediate.dense_act.linear.bias', 'intermediate.dense_act.linear.tensor.factors.0', 'intermediate.dense_act.linear.tensor.factors.1', 'intermediate.dense_act.linear.tensor.factors.2', 'intermediate.dense_act.linear.tensor.factors.3', 'intermediate.dense_act.linear.tensor.factors.4', 'intermediate.dense_act.linear.tensor.factors.5', 'output.dense.bias', 'output.dense.tensor.factors.0', 'output.dense.tensor.factors.1', 'output.dense.tensor.factors.2', 'output.dense.tensor.factors.3', 'output.dense.tensor.factors.4', 'output.dense.tensor.factors.5', 'output.LayerNorm.weight', 'output.LayerNorm.bias'])" ] }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "block.state_dict().keys()" ] }, { "cell_type": "code", "execution_count": 28, "id": "98292093", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "TensorizedLinear(\n", " (tensor): TensorTrain(\n", " (factors): ParameterList(\n", " (0): Parameter containing: [torch.float32 of size 1x16x12]\n", " (1): Parameter containing: [torch.float32 of size 12x16x144]\n", " (2): Parameter containing: [torch.float32 of size 144x16x300]\n", " (3): Parameter containing: [torch.float32 of size 300x8x96]\n", " (4): Parameter containing: [torch.float32 of size 96x8x12]\n", " (5): Parameter containing: [torch.float32 of size 12x16x1]\n", " )\n", " )\n", ")" ] }, "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tensoredizedLinear = block.output.dense\n", "tensoredizedLinear" ] }, { "cell_type": "code", "execution_count": 33, "id": "98ca9cf5", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(4096, 1024, [16, 16, 16, 8, 8, 16])" ] }, "execution_count": 33, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tensoredizedLinear.in_features, tensoredizedLinear.out_features, tensoredizedLinear.shape" ] }, { "cell_type": "code", "execution_count": 35, "id": "23ab333d", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(4096, 1024)" ] }, "execution_count": 35, "metadata": {}, "output_type": "execute_result" } ], "source": [ "16*16*16, 8*8*16" ] }, { "cell_type": "code", "execution_count": 36, "id": "83786f7f", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "TensorBertLayer(\n", " (attention): TensorBertAttention(\n", " (self): TensorBertSelfAttention(\n", " (query): TensorizedLinear(\n", " (tensor): TensorTrain(\n", " (factors): ParameterList(\n", " (0): Parameter containing: [torch.float32 of size 1x16x12]\n", " (1): Parameter containing: [torch.float32 of size 12x8x72]\n", " (2): Parameter containing: [torch.float32 of size 72x8x300]\n", " (3): Parameter containing: [torch.float32 of size 300x8x96]\n", " (4): Parameter containing: [torch.float32 of size 96x8x12]\n", " (5): Parameter containing: [torch.float32 of size 12x16x1]\n", " )\n", " )\n", " )\n", " (key): TensorizedLinear(\n", " (tensor): TensorTrain(\n", " (factors): ParameterList(\n", " (0): Parameter containing: [torch.float32 of size 1x16x12]\n", " (1): Parameter containing: [torch.float32 of size 12x8x72]\n", " (2): Parameter containing: [torch.float32 of size 72x8x300]\n", " (3): Parameter containing: [torch.float32 of size 300x8x96]\n", " (4): Parameter containing: [torch.float32 of size 96x8x12]\n", " (5): Parameter containing: [torch.float32 of size 12x16x1]\n", " )\n", " )\n", " )\n", " (value): TensorizedLinear(\n", " (tensor): TensorTrain(\n", " (factors): ParameterList(\n", " (0): Parameter containing: [torch.float32 of size 1x16x12]\n", " (1): Parameter containing: [torch.float32 of size 12x8x72]\n", " (2): Parameter containing: [torch.float32 of size 72x8x300]\n", " (3): Parameter containing: [torch.float32 of size 300x8x96]\n", " (4): Parameter containing: [torch.float32 of size 96x8x12]\n", " (5): Parameter containing: [torch.float32 of size 12x16x1]\n", " )\n", " )\n", " )\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): TensorBertSelfOutput(\n", " (dense): TensorizedLinear(\n", " (tensor): TensorTrain(\n", " (factors): ParameterList(\n", " (0): Parameter containing: [torch.float32 of size 1x16x12]\n", " (1): Parameter containing: [torch.float32 of size 12x8x72]\n", " (2): Parameter containing: [torch.float32 of size 72x8x300]\n", " (3): Parameter containing: [torch.float32 of size 300x8x96]\n", " (4): Parameter containing: [torch.float32 of size 96x8x12]\n", " (5): Parameter containing: [torch.float32 of size 12x16x1]\n", " )\n", " )\n", " )\n", " (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): TensorBertIntermediate(\n", " (dense_act): TensorLinearActivation(\n", " (act_fn): GELU(approximate='none')\n", " (linear): TensorizedLinear(\n", " (tensor): TensorTrain(\n", " (factors): ParameterList(\n", " (0): Parameter containing: [torch.float32 of size 1x16x12]\n", " (1): Parameter containing: [torch.float32 of size 12x8x72]\n", " (2): Parameter containing: [torch.float32 of size 72x8x300]\n", " (3): Parameter containing: [torch.float32 of size 300x16x192]\n", " (4): Parameter containing: [torch.float32 of size 192x16x12]\n", " (5): Parameter containing: [torch.float32 of size 12x16x1]\n", " )\n", " )\n", " )\n", " )\n", " )\n", " (output): TensorBertOutput(\n", " (dense): TensorizedLinear(\n", " (tensor): TensorTrain(\n", " (factors): ParameterList(\n", " (0): Parameter containing: [torch.float32 of size 1x16x12]\n", " (1): Parameter containing: [torch.float32 of size 12x16x144]\n", " (2): Parameter containing: [torch.float32 of size 144x16x300]\n", " (3): Parameter containing: [torch.float32 of size 300x8x96]\n", " (4): Parameter containing: [torch.float32 of size 96x8x12]\n", " (5): Parameter containing: [torch.float32 of size 12x16x1]\n", " )\n", " )\n", " )\n", " (LayerNorm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", ")" ] }, "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ "block" ] }, { "cell_type": "markdown", "id": "673b12ed", "metadata": {}, "source": [ "# Check Backward pass works" ] }, { "cell_type": "code", "execution_count": null, "id": "fc558bbd", "metadata": {}, "outputs": [], "source": [ "batch = next(iter(train_dataloader))\n", "input_ids, input_mask, segment_ids, label_ids = batch\n", "# input_ids.shape, input_mask.shape, segment_ids.shape, label_ids.shape" ] }, { "cell_type": "code", "execution_count": null, "id": "b5d6cf09", "metadata": {}, "outputs": [], "source": [ "device = 'cuda'\n", "model = model.to(device)\n", "input_ids, input_mask, segment_ids, label_ids = [x.to(device) for x in batch]" ] }, { "cell_type": "code", "execution_count": null, "id": "b83b5c32", "metadata": {}, "outputs": [], "source": [ "# loss_fct = torch.nn.CrossEntropyLoss()" ] }, { "cell_type": "code", "execution_count": null, "id": "25f1764f", "metadata": {}, "outputs": [], "source": [ "# model.train()\n", "# out = model(input_ids, segment_ids, input_mask)\n", "# loss = loss_fct(out.view(-1, num_labels), label_ids.view(-1))\n", "# loss.backward()\n", "\n", "# for name, param in model.named_parameters():\n", "# print(name,param.shape)\n", "# # if param.grad is not None:\n", "# # print(f\"{name}: {param.grad.shape}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "9c6ae700", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "bert.embeddings.word_embeddings.weight torch.Size([30528, 1024])\n", "bert.embeddings.position_embeddings.weight torch.Size([512, 1024])\n", "bert.embeddings.token_type_embeddings.weight torch.Size([2, 1024])\n", "bert.embeddings.LayerNorm.weight torch.Size([1024])\n", "bert.embeddings.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.0.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.0.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.0.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.0.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.0.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.0.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.0.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.0.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.0.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.0.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.0.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.0.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.0.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.0.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.0.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.0.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.0.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.0.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.0.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.0.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.0.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.0.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.0.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.0.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.0.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.0.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.0.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.0.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.0.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.0.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.0.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.0.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.0.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.0.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.0.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.0.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.0.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.0.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.0.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.0.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.0.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.0.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.0.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.0.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.0.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.0.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.1.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.1.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.1.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.1.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.1.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.1.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.1.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.1.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.1.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.1.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.1.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.1.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.1.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.1.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.1.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.1.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.1.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.1.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.1.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.1.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.1.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.1.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.1.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.1.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.1.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.1.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.1.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.1.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.1.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.1.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.1.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.1.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.1.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.1.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.1.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.1.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.1.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.1.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.1.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.1.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.1.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.1.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.1.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.1.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.1.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.1.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.2.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.2.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.2.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.2.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.2.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.2.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.2.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.2.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.2.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.2.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.2.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.2.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.2.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.2.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.2.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.2.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.2.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.2.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.2.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.2.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.2.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.2.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.2.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.2.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.2.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.2.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.2.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.2.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.2.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.2.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.2.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.2.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.2.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.2.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.2.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.2.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.2.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.2.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.2.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.2.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.2.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.2.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.2.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.2.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.2.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.2.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.3.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.3.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.3.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.3.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.3.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.3.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.3.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.3.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.3.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.3.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.3.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.3.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.3.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.3.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.3.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.3.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.3.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.3.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.3.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.3.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.3.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.3.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.3.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.3.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.3.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.3.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.3.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.3.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.3.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.3.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.3.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.3.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.3.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.3.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.3.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.3.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.3.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.3.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.3.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.3.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.3.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.3.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.3.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.3.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.3.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.3.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.4.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.4.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.4.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.4.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.4.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.4.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.4.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.4.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.4.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.4.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.4.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.4.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.4.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.4.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.4.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.4.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.4.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.4.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.4.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.4.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.4.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.4.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.4.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.4.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.4.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.4.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.4.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.4.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.4.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.4.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.4.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.4.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.4.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.4.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.4.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.4.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.4.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.4.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.4.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.4.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.4.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.4.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.4.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.4.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.4.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.4.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.5.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.5.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.5.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.5.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.5.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.5.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.5.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.5.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.5.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.5.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.5.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.5.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.5.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.5.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.5.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.5.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.5.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.5.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.5.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.5.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.5.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.5.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.5.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.5.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.5.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.5.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.5.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.5.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.5.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.5.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.5.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.5.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.5.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.5.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.5.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.5.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.5.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.5.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.5.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.5.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.5.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.5.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.5.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.5.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.5.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.5.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.6.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.6.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.6.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.6.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.6.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.6.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.6.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.6.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.6.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.6.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.6.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.6.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.6.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.6.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.6.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.6.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.6.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.6.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.6.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.6.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.6.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.6.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.6.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.6.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.6.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.6.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.6.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.6.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.6.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.6.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.6.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.6.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.6.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.6.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.6.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.6.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.6.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.6.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.6.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.6.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.6.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.6.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.6.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.6.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.6.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.6.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.7.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.7.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.7.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.7.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.7.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.7.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.7.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.7.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.7.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.7.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.7.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.7.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.7.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.7.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.7.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.7.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.7.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.7.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.7.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.7.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.7.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.7.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.7.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.7.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.7.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.7.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.7.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.7.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.7.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.7.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.7.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.7.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.7.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.7.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.7.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.7.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.7.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.7.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.7.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.7.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.7.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.7.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.7.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.7.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.7.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.7.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.8.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.8.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.8.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.8.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.8.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.8.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.8.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.8.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.8.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.8.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.8.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.8.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.8.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.8.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.8.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.8.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.8.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.8.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.8.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.8.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.8.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.8.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.8.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.8.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.8.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.8.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.8.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.8.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.8.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.8.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.8.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.8.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.8.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.8.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.8.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.8.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.8.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.8.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.8.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.8.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.8.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.8.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.8.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.8.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.8.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.8.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.9.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.9.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.9.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.9.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.9.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.9.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.9.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.9.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.9.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.9.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.9.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.9.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.9.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.9.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.9.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.9.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.9.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.9.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.9.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.9.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.9.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.9.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.9.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.9.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.9.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.9.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.9.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.9.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.9.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.9.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.9.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.9.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.9.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.9.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.9.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.9.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.9.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.9.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.9.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.9.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.9.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.9.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.9.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.9.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.9.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.9.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.10.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.10.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.10.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.10.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.10.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.10.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.10.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.10.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.10.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.10.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.10.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.10.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.10.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.10.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.10.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.10.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.10.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.10.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.10.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.10.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.10.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.10.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.10.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.10.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.10.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.10.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.10.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.10.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.10.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.10.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.10.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.10.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.10.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.10.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.10.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.10.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.10.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.10.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.10.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.10.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.10.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.10.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.10.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.10.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.10.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.10.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.11.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.11.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.11.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.11.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.11.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.11.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.11.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.11.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.11.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.11.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.11.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.11.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.11.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.11.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.11.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.11.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.11.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.11.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.11.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.11.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.11.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.11.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.11.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.11.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.11.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.11.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.11.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.11.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.11.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.11.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.11.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.11.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.11.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.11.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.11.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.11.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.11.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.11.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.11.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.11.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.11.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.11.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.11.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.11.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.11.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.11.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.12.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.12.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.12.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.12.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.12.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.12.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.12.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.12.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.12.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.12.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.12.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.12.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.12.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.12.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.12.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.12.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.12.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.12.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.12.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.12.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.12.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.12.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.12.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.12.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.12.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.12.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.12.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.12.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.12.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.12.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.12.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.12.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.12.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.12.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.12.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.12.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.12.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.12.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.12.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.12.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.12.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.12.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.12.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.12.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.12.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.12.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.13.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.13.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.13.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.13.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.13.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.13.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.13.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.13.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.13.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.13.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.13.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.13.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.13.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.13.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.13.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.13.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.13.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.13.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.13.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.13.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.13.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.13.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.13.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.13.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.13.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.13.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.13.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.13.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.13.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.13.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.13.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.13.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.13.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.13.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.13.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.13.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.13.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.13.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.13.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.13.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.13.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.13.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.13.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.13.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.13.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.13.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.14.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.14.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.14.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.14.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.14.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.14.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.14.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.14.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.14.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.14.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.14.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.14.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.14.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.14.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.14.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.14.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.14.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.14.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.14.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.14.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.14.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.14.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.14.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.14.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.14.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.14.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.14.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.14.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.14.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.14.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.14.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.14.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.14.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.14.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.14.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.14.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.14.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.14.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.14.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.14.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.14.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.14.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.14.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.14.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.14.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.14.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.15.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.15.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.15.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.15.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.15.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.15.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.15.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.15.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.15.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.15.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.15.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.15.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.15.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.15.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.15.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.15.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.15.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.15.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.15.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.15.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.15.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.15.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.15.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.15.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.15.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.15.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.15.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.15.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.15.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.15.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.15.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.15.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.15.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.15.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.15.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.15.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.15.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.15.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.15.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.15.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.15.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.15.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.15.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.15.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.15.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.15.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.16.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.16.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.16.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.16.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.16.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.16.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.16.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.16.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.16.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.16.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.16.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.16.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.16.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.16.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.16.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.16.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.16.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.16.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.16.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.16.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.16.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.16.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.16.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.16.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.16.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.16.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.16.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.16.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.16.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.16.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.16.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.16.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.16.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.16.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.16.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.16.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.16.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.16.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.16.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.16.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.16.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.16.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.16.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.16.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.16.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.16.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.17.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.17.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.17.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.17.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.17.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.17.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.17.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.17.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.17.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.17.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.17.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.17.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.17.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.17.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.17.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.17.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.17.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.17.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.17.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.17.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.17.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.17.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.17.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.17.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.17.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.17.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.17.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.17.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.17.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.17.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.17.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.17.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.17.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.17.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.17.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.17.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.17.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.17.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.17.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.17.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.17.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.17.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.17.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.17.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.17.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.17.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.18.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.18.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.18.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.18.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.18.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.18.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.18.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.18.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.18.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.18.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.18.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.18.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.18.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.18.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.18.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.18.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.18.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.18.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.18.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.18.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.18.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.18.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.18.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.18.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.18.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.18.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.18.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.18.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.18.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.18.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.18.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.18.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.18.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.18.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.18.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.18.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.18.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.18.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.18.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.18.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.18.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.18.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.18.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.18.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.18.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.18.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.19.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.19.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.19.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.19.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.19.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.19.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.19.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.19.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.19.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.19.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.19.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.19.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.19.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.19.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.19.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.19.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.19.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.19.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.19.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.19.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.19.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.19.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.19.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.19.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.19.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.19.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.19.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.19.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.19.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.19.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.19.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.19.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.19.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.19.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.19.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.19.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.19.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.19.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.19.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.19.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.19.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.19.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.19.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.19.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.19.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.19.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.20.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.20.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.20.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.20.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.20.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.20.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.20.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.20.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.20.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.20.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.20.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.20.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.20.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.20.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.20.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.20.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.20.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.20.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.20.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.20.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.20.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.20.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.20.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.20.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.20.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.20.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.20.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.20.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.20.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.20.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.20.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.20.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.20.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.20.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.20.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.20.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.20.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.20.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.20.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.20.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.20.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.20.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.20.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.20.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.20.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.20.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.21.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.21.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.21.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.21.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.21.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.21.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.21.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.21.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.21.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.21.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.21.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.21.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.21.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.21.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.21.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.21.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.21.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.21.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.21.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.21.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.21.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.21.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.21.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.21.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.21.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.21.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.21.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.21.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.21.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.21.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.21.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.21.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.21.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.21.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.21.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.21.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.21.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.21.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.21.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.21.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.21.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.21.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.21.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.21.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.21.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.21.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.22.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.22.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.22.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.22.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.22.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.22.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.22.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.22.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.22.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.22.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.22.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.22.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.22.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.22.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.22.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.22.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.22.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.22.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.22.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.22.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.22.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.22.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.22.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.22.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.22.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.22.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.22.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.22.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.22.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.22.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.22.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.22.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.22.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.22.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.22.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.22.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.22.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.22.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.22.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.22.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.22.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.22.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.22.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.22.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.22.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.22.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.23.attention.self.query.bias torch.Size([1024])\n", "bert.encoder.layer.23.attention.self.query.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.23.attention.self.query.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.23.attention.self.query.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.23.attention.self.query.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.23.attention.self.query.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.23.attention.self.query.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.23.attention.self.key.bias torch.Size([1024])\n", "bert.encoder.layer.23.attention.self.key.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.23.attention.self.key.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.23.attention.self.key.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.23.attention.self.key.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.23.attention.self.key.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.23.attention.self.key.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.23.attention.self.value.bias torch.Size([1024])\n", "bert.encoder.layer.23.attention.self.value.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.23.attention.self.value.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.23.attention.self.value.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.23.attention.self.value.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.23.attention.self.value.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.23.attention.self.value.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.23.attention.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.23.attention.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.23.attention.output.dense.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.23.attention.output.dense.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.23.attention.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.23.attention.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.23.attention.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.23.attention.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.23.attention.output.LayerNorm.bias torch.Size([1024])\n", "bert.encoder.layer.23.intermediate.dense_act.linear.bias torch.Size([4096])\n", "bert.encoder.layer.23.intermediate.dense_act.linear.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.23.intermediate.dense_act.linear.tensor.factors.1 torch.Size([12, 8, 72])\n", "bert.encoder.layer.23.intermediate.dense_act.linear.tensor.factors.2 torch.Size([72, 8, 300])\n", "bert.encoder.layer.23.intermediate.dense_act.linear.tensor.factors.3 torch.Size([300, 16, 192])\n", "bert.encoder.layer.23.intermediate.dense_act.linear.tensor.factors.4 torch.Size([192, 16, 12])\n", "bert.encoder.layer.23.intermediate.dense_act.linear.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.23.output.dense.bias torch.Size([1024])\n", "bert.encoder.layer.23.output.dense.tensor.factors.0 torch.Size([1, 16, 12])\n", "bert.encoder.layer.23.output.dense.tensor.factors.1 torch.Size([12, 16, 144])\n", "bert.encoder.layer.23.output.dense.tensor.factors.2 torch.Size([144, 16, 300])\n", "bert.encoder.layer.23.output.dense.tensor.factors.3 torch.Size([300, 8, 96])\n", "bert.encoder.layer.23.output.dense.tensor.factors.4 torch.Size([96, 8, 12])\n", "bert.encoder.layer.23.output.dense.tensor.factors.5 torch.Size([12, 16, 1])\n", "bert.encoder.layer.23.output.LayerNorm.weight torch.Size([1024])\n", "bert.encoder.layer.23.output.LayerNorm.bias torch.Size([1024])\n", "bert.pooler.dense_act.linear.weight torch.Size([1024, 1024])\n", "bert.pooler.dense_act.linear.bias torch.Size([1024])\n", "classifier.weight torch.Size([2, 1024])\n", "classifier.bias torch.Size([2])\n" ] } ], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f340e458", "metadata": {}, "outputs": [], "source": [ "# import json\n", "\n", "# def save_state_dict_structure(state_dict, file_path):\n", "# def serialize_state_dict(state_dict):\n", "# serialized = {}\n", "# for key, value in state_dict.items():\n", "# if isinstance(value, torch.Tensor):\n", "# serialized[key] = f\"Tensor with shape {list(value.shape)}\"\n", "# elif isinstance(value, (int, float, str)):\n", "# serialized[key] = value\n", "# elif isinstance(value, dict):\n", "# serialized[key] = serialize_state_dict(value)\n", "# else:\n", "# serialized[key] = str(type(value))\n", "# return serialized\n", "\n", "# with open(file_path, 'w') as f:\n", "# json.dump(serialize_state_dict(state_dict), f, indent=4)\n", "\n", "# save_state_dict_structure(stateDict, \"state_dict_structure.json\")" ] }, { "cell_type": "code", "execution_count": null, "id": "e4a54a77", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9b6c70fd", "metadata": {}, "outputs": [], "source": [ "\t" ] } ], "metadata": { "kernelspec": { "display_name": "colaBert", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.13" } }, "nbformat": 4, "nbformat_minor": 5 }