Upload 12 files

deberta-mnli/checkpoint-24544/config.json

@@ -0,0 +1,42 @@
+{
+  "_name_or_path": "/ibex/user/anb/Model_merging/EMR_Merging-main/merge_lm/ckpts/deberta/deberta-base",
+  "architectures": [
+    "DebertaForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": [
+    "c2p",
+    "p2c"
+  ],
+  "position_biased_input": false,
+  "relative_attention": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.45.1",
+  "type_vocab_size": 0,
+  "vocab_size": 50265
+}

deberta-mnli/checkpoint-24544/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:298d07f2f91fc467c0d00f92328adebef199c2f8b33b2e25f858d8dc0239edc5
+size 556802644

deberta-mnli/checkpoint-24544/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b17778356ce7a420024428acc45bda191ba15abe5d8e2469445e95f9d1d6170b
+size 1113724282

deberta-mnli/checkpoint-24544/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36a2571db718d633322af805463b0b2faf2e475ff1ca9e040162e39bf99ac6dc
+size 14244

deberta-mnli/checkpoint-24544/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60e55f3aecf8b8786448fdb441e26658f605e89258d669fdd92bd1383ac19ec2
+size 1064

deberta-mnli/checkpoint-24544/special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

deberta-mnli/checkpoint-24544/tokenizer_config.json

@@ -0,0 +1,59 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50264": {
+      "content": "[MASK]",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "[CLS]",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "errors": "replace",
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "DebertaTokenizer",
+  "unk_token": "[UNK]",
+  "vocab_type": "gpt2"
+}

deberta-mnli/checkpoint-24544/trainer_state.json

@@ -0,0 +1,412 @@
+{
+  "best_metric": 0.8852776362710137,
+  "best_model_checkpoint": "./deberta/deberta-mnli/checkpoint-24544",
+  "epoch": 4.0,
+  "eval_steps": 500,
+  "global_step": 24544,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.08148631029986962,
+      "grad_norm": 9.7207612991333,
+      "learning_rate": 1.9738391105297582e-05,
+      "loss": 0.7226,
+      "step": 500
+    },
+    {
+      "epoch": 0.16297262059973924,
+      "grad_norm": 6.806437015533447,
+      "learning_rate": 1.9411379986919555e-05,
+      "loss": 0.4742,
+      "step": 1000
+    },
+    {
+      "epoch": 0.24445893089960888,
+      "grad_norm": 6.399604320526123,
+      "learning_rate": 1.9084368868541532e-05,
+      "loss": 0.4375,
+      "step": 1500
+    },
+    {
+      "epoch": 0.3259452411994785,
+      "grad_norm": 6.315491676330566,
+      "learning_rate": 1.8757357750163508e-05,
+      "loss": 0.4139,
+      "step": 2000
+    },
+    {
+      "epoch": 0.4074315514993481,
+      "grad_norm": 5.844660758972168,
+      "learning_rate": 1.843034663178548e-05,
+      "loss": 0.4046,
+      "step": 2500
+    },
+    {
+      "epoch": 0.48891786179921776,
+      "grad_norm": 3.927537679672241,
+      "learning_rate": 1.8103335513407458e-05,
+      "loss": 0.3975,
+      "step": 3000
+    },
+    {
+      "epoch": 0.5704041720990873,
+      "grad_norm": 3.775937557220459,
+      "learning_rate": 1.777632439502943e-05,
+      "loss": 0.3827,
+      "step": 3500
+    },
+    {
+      "epoch": 0.651890482398957,
+      "grad_norm": 3.8927953243255615,
+      "learning_rate": 1.7449313276651408e-05,
+      "loss": 0.3781,
+      "step": 4000
+    },
+    {
+      "epoch": 0.7333767926988266,
+      "grad_norm": 6.637417316436768,
+      "learning_rate": 1.7122302158273384e-05,
+      "loss": 0.3783,
+      "step": 4500
+    },
+    {
+      "epoch": 0.8148631029986962,
+      "grad_norm": 6.947172164916992,
+      "learning_rate": 1.6795291039895357e-05,
+      "loss": 0.3649,
+      "step": 5000
+    },
+    {
+      "epoch": 0.8963494132985659,
+      "grad_norm": 7.5690531730651855,
+      "learning_rate": 1.6468279921517334e-05,
+      "loss": 0.3669,
+      "step": 5500
+    },
+    {
+      "epoch": 0.9778357235984355,
+      "grad_norm": 6.342957496643066,
+      "learning_rate": 1.6141268803139307e-05,
+      "loss": 0.3627,
+      "step": 6000
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.8798777381558839,
+      "eval_loss": 0.32735681533813477,
+      "eval_runtime": 11.0908,
+      "eval_samples_per_second": 884.971,
+      "eval_steps_per_second": 13.885,
+      "step": 6136
+    },
+    {
+      "epoch": 1.0593220338983051,
+      "grad_norm": 4.847009658813477,
+      "learning_rate": 1.5814257684761283e-05,
+      "loss": 0.2928,
+      "step": 6500
+    },
+    {
+      "epoch": 1.1408083441981747,
+      "grad_norm": 7.34955358505249,
+      "learning_rate": 1.548724656638326e-05,
+      "loss": 0.2716,
+      "step": 7000
+    },
+    {
+      "epoch": 1.2222946544980444,
+      "grad_norm": 3.077202796936035,
+      "learning_rate": 1.5160235448005235e-05,
+      "loss": 0.2784,
+      "step": 7500
+    },
+    {
+      "epoch": 1.303780964797914,
+      "grad_norm": 4.981752872467041,
+      "learning_rate": 1.483322432962721e-05,
+      "loss": 0.2747,
+      "step": 8000
+    },
+    {
+      "epoch": 1.3852672750977835,
+      "grad_norm": 4.702407360076904,
+      "learning_rate": 1.4506213211249183e-05,
+      "loss": 0.2714,
+      "step": 8500
+    },
+    {
+      "epoch": 1.4667535853976532,
+      "grad_norm": 5.884511947631836,
+      "learning_rate": 1.4179202092871159e-05,
+      "loss": 0.2741,
+      "step": 9000
+    },
+    {
+      "epoch": 1.548239895697523,
+      "grad_norm": 7.298000335693359,
+      "learning_rate": 1.3852190974493134e-05,
+      "loss": 0.2727,
+      "step": 9500
+    },
+    {
+      "epoch": 1.6297262059973925,
+      "grad_norm": 5.3628339767456055,
+      "learning_rate": 1.3525179856115109e-05,
+      "loss": 0.2711,
+      "step": 10000
+    },
+    {
+      "epoch": 1.711212516297262,
+      "grad_norm": 5.2779765129089355,
+      "learning_rate": 1.3198168737737085e-05,
+      "loss": 0.2709,
+      "step": 10500
+    },
+    {
+      "epoch": 1.7926988265971318,
+      "grad_norm": 8.934818267822266,
+      "learning_rate": 1.2871157619359058e-05,
+      "loss": 0.2713,
+      "step": 11000
+    },
+    {
+      "epoch": 1.8741851368970013,
+      "grad_norm": 4.539167881011963,
+      "learning_rate": 1.2544146500981033e-05,
+      "loss": 0.2696,
+      "step": 11500
+    },
+    {
+      "epoch": 1.9556714471968708,
+      "grad_norm": 4.068707466125488,
+      "learning_rate": 1.221713538260301e-05,
+      "loss": 0.2698,
+      "step": 12000
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.8799796230259807,
+      "eval_loss": 0.34052515029907227,
+      "eval_runtime": 11.0931,
+      "eval_samples_per_second": 884.781,
+      "eval_steps_per_second": 13.882,
+      "step": 12272
+    },
+    {
+      "epoch": 2.0371577574967406,
+      "grad_norm": 5.666093826293945,
+      "learning_rate": 1.1890124264224984e-05,
+      "loss": 0.2327,
+      "step": 12500
+    },
+    {
+      "epoch": 2.1186440677966103,
+      "grad_norm": 5.032351970672607,
+      "learning_rate": 1.1563113145846961e-05,
+      "loss": 0.1798,
+      "step": 13000
+    },
+    {
+      "epoch": 2.2001303780964796,
+      "grad_norm": 4.264206886291504,
+      "learning_rate": 1.1236102027468936e-05,
+      "loss": 0.1905,
+      "step": 13500
+    },
+    {
+      "epoch": 2.2816166883963493,
+      "grad_norm": 7.420534133911133,
+      "learning_rate": 1.0909090909090909e-05,
+      "loss": 0.1908,
+      "step": 14000
+    },
+    {
+      "epoch": 2.363102998696219,
+      "grad_norm": 7.359429836273193,
+      "learning_rate": 1.0582079790712885e-05,
+      "loss": 0.1843,
+      "step": 14500
+    },
+    {
+      "epoch": 2.444589308996089,
+      "grad_norm": 7.396298408508301,
+      "learning_rate": 1.025506867233486e-05,
+      "loss": 0.1848,
+      "step": 15000
+    },
+    {
+      "epoch": 2.526075619295958,
+      "grad_norm": 8.213353157043457,
+      "learning_rate": 9.928057553956835e-06,
+      "loss": 0.1885,
+      "step": 15500
+    },
+    {
+      "epoch": 2.607561929595828,
+      "grad_norm": 3.7509984970092773,
+      "learning_rate": 9.60104643557881e-06,
+      "loss": 0.1869,
+      "step": 16000
+    },
+    {
+      "epoch": 2.6890482398956976,
+      "grad_norm": 5.412872791290283,
+      "learning_rate": 9.274035317200786e-06,
+      "loss": 0.1834,
+      "step": 16500
+    },
+    {
+      "epoch": 2.770534550195567,
+      "grad_norm": 6.171872615814209,
+      "learning_rate": 8.94702419882276e-06,
+      "loss": 0.1932,
+      "step": 17000
+    },
+    {
+      "epoch": 2.8520208604954367,
+      "grad_norm": 4.4419050216674805,
+      "learning_rate": 8.620013080444736e-06,
+      "loss": 0.1796,
+      "step": 17500
+    },
+    {
+      "epoch": 2.9335071707953064,
+      "grad_norm": 4.847419738769531,
+      "learning_rate": 8.29300196206671e-06,
+      "loss": 0.1857,
+      "step": 18000
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.8828323993886907,
+      "eval_loss": 0.3725295662879944,
+      "eval_runtime": 11.0431,
+      "eval_samples_per_second": 888.792,
+      "eval_steps_per_second": 13.945,
+      "step": 18408
+    },
+    {
+      "epoch": 3.014993481095176,
+      "grad_norm": 5.3139729499816895,
+      "learning_rate": 7.965990843688686e-06,
+      "loss": 0.1749,
+      "step": 18500
+    },
+    {
+      "epoch": 3.0964797913950455,
+      "grad_norm": 8.973730087280273,
+      "learning_rate": 7.63897972531066e-06,
+      "loss": 0.1251,
+      "step": 19000
+    },
+    {
+      "epoch": 3.1779661016949152,
+      "grad_norm": 5.263895511627197,
+      "learning_rate": 7.311968606932637e-06,
+      "loss": 0.1311,
+      "step": 19500
+    },
+    {
+      "epoch": 3.259452411994785,
+      "grad_norm": 7.2389397621154785,
+      "learning_rate": 6.984957488554611e-06,
+      "loss": 0.1254,
+      "step": 20000
+    },
+    {
+      "epoch": 3.3409387222946547,
+      "grad_norm": 5.466920375823975,
+      "learning_rate": 6.6579463701765866e-06,
+      "loss": 0.131,
+      "step": 20500
+    },
+    {
+      "epoch": 3.422425032594524,
+      "grad_norm": 3.5215988159179688,
+      "learning_rate": 6.330935251798561e-06,
+      "loss": 0.1289,
+      "step": 21000
+    },
+    {
+      "epoch": 3.5039113428943938,
+      "grad_norm": 5.995067119598389,
+      "learning_rate": 6.003924133420537e-06,
+      "loss": 0.1348,
+      "step": 21500
+    },
+    {
+      "epoch": 3.5853976531942635,
+      "grad_norm": 6.122405529022217,
+      "learning_rate": 5.676913015042512e-06,
+      "loss": 0.135,
+      "step": 22000
+    },
+    {
+      "epoch": 3.666883963494133,
+      "grad_norm": 7.12537145614624,
+      "learning_rate": 5.349901896664487e-06,
+      "loss": 0.1351,
+      "step": 22500
+    },
+    {
+      "epoch": 3.7483702737940026,
+      "grad_norm": 2.369924783706665,
+      "learning_rate": 5.022890778286462e-06,
+      "loss": 0.1261,
+      "step": 23000
+    },
+    {
+      "epoch": 3.8298565840938723,
+      "grad_norm": 9.514494895935059,
+      "learning_rate": 4.695879659908437e-06,
+      "loss": 0.1342,
+      "step": 23500
+    },
+    {
+      "epoch": 3.9113428943937416,
+      "grad_norm": 3.0919198989868164,
+      "learning_rate": 4.368868541530412e-06,
+      "loss": 0.1362,
+      "step": 24000
+    },
+    {
+      "epoch": 3.9928292046936114,
+      "grad_norm": 4.291722774505615,
+      "learning_rate": 4.041857423152388e-06,
+      "loss": 0.126,
+      "step": 24500
+    },
+    {
+      "epoch": 4.0,
+      "eval_accuracy": 0.8852776362710137,
+      "eval_loss": 0.43032392859458923,
+      "eval_runtime": 10.8684,
+      "eval_samples_per_second": 903.081,
+      "eval_steps_per_second": 14.17,
+      "step": 24544
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 30680,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 9.890772120374488e+16,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

deberta-mnli/checkpoint-24544/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c99385eb30c86c40ef5ece4146e57520c621b89437abcdf5139cfd693d43451
+size 5176