final: UltraFeedback(openbmb) baseline fixed1k + sampled synth (T=5, B=2000, lr=2e-05, gas=16)

config.json

@@ -0,0 +1,43 @@
+{
+  "architectures": [
+    "DebertaV2ForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "dtype": "float32",
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_eps": 1e-07,
+  "legacy": true,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "problem_type": "regression",
+  "relative_attention": true,
+  "share_att_key": true,
+  "transformers_version": "4.57.3",
+  "type_vocab_size": 0,
+  "use_cache": false,
+  "vocab_size": 128001
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89f958f84b0a28cf45ab0769e7cb1ba749ceb77e95536fc312b3f80f68e02f2b
+size 393219224

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:768f110c5a023fe66bcc40c10f1c644a24c8fba1cde9336ddc8f3719379de846
+size 198480360

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05655d8f6072f8b95fce237c4c9dceb11ab4d54808734532aae252e4f8ad0fb2
+size 145712228

model.safetensors.index.json

@@ -0,0 +1,210 @@
+{
+  "metadata": {
+    "total_parameters": 184346881,
+    "total_size": 737387524
+  },
+  "weight_map": {
+    "classifier.bias": "model-00003-of-00003.safetensors",
+    "classifier.weight": "model-00003-of-00003.safetensors",
+    "deberta.embeddings.LayerNorm.bias": "model-00002-of-00003.safetensors",
+    "deberta.embeddings.LayerNorm.weight": "model-00002-of-00003.safetensors",
+    "deberta.embeddings.word_embeddings.weight": "model-00001-of-00003.safetensors",
+    "deberta.encoder.LayerNorm.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.LayerNorm.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.0.attention.output.LayerNorm.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.0.attention.output.LayerNorm.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.0.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.0.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.0.attention.self.key_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.0.attention.self.key_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.0.attention.self.query_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.0.attention.self.query_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.0.attention.self.value_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.0.attention.self.value_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.0.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.0.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.0.output.LayerNorm.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.0.output.LayerNorm.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.0.output.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.0.output.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.1.attention.output.LayerNorm.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.1.attention.output.LayerNorm.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.1.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.1.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.1.attention.self.key_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.1.attention.self.key_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.1.attention.self.query_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.1.attention.self.query_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.1.attention.self.value_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.1.attention.self.value_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.1.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.1.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.1.output.LayerNorm.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.1.output.LayerNorm.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.1.output.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.1.output.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.10.attention.output.LayerNorm.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.10.attention.output.LayerNorm.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.10.attention.output.dense.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.10.attention.output.dense.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.10.attention.self.key_proj.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.10.attention.self.key_proj.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.10.attention.self.query_proj.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.10.attention.self.query_proj.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.10.attention.self.value_proj.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.10.attention.self.value_proj.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.10.intermediate.dense.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.10.intermediate.dense.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.10.output.LayerNorm.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.10.output.LayerNorm.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.10.output.dense.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.10.output.dense.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.11.attention.output.LayerNorm.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.11.attention.output.LayerNorm.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.11.attention.output.dense.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.11.attention.output.dense.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.11.attention.self.key_proj.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.11.attention.self.key_proj.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.11.attention.self.query_proj.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.11.attention.self.query_proj.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.11.attention.self.value_proj.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.11.attention.self.value_proj.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.11.intermediate.dense.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.11.intermediate.dense.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.11.output.LayerNorm.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.11.output.LayerNorm.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.11.output.dense.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.11.output.dense.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.2.attention.output.LayerNorm.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.2.attention.output.LayerNorm.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.2.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.2.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.2.attention.self.key_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.2.attention.self.key_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.2.attention.self.query_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.2.attention.self.query_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.2.attention.self.value_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.2.attention.self.value_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.2.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.2.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.2.output.LayerNorm.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.2.output.LayerNorm.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.2.output.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.2.output.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.3.attention.output.LayerNorm.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.3.attention.output.LayerNorm.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.3.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.3.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.3.attention.self.key_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.3.attention.self.key_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.3.attention.self.query_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.3.attention.self.query_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.3.attention.self.value_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.3.attention.self.value_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.3.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.3.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.3.output.LayerNorm.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.3.output.LayerNorm.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.3.output.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.3.output.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.4.attention.output.LayerNorm.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.4.attention.output.LayerNorm.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.4.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.4.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.4.attention.self.key_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.4.attention.self.key_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.4.attention.self.query_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.4.attention.self.query_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.4.attention.self.value_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.4.attention.self.value_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.4.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.4.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.4.output.LayerNorm.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.4.output.LayerNorm.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.4.output.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.4.output.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.5.attention.output.LayerNorm.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.5.attention.output.LayerNorm.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.5.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.5.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.5.attention.self.key_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.5.attention.self.key_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.5.attention.self.query_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.5.attention.self.query_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.5.attention.self.value_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.5.attention.self.value_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.5.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.5.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.5.output.LayerNorm.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.5.output.LayerNorm.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.5.output.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.5.output.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.6.attention.output.LayerNorm.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.6.attention.output.LayerNorm.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.6.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.6.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.6.attention.self.key_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.6.attention.self.key_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.6.attention.self.query_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.6.attention.self.query_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.6.attention.self.value_proj.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.6.attention.self.value_proj.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.6.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.6.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.6.output.LayerNorm.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.6.output.LayerNorm.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.6.output.dense.bias": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.6.output.dense.weight": "model-00002-of-00003.safetensors",
+    "deberta.encoder.layer.7.attention.output.LayerNorm.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.7.attention.output.LayerNorm.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.7.attention.output.dense.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.7.attention.output.dense.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.7.attention.self.key_proj.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.7.attention.self.key_proj.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.7.attention.self.query_proj.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.7.attention.self.query_proj.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.7.attention.self.value_proj.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.7.attention.self.value_proj.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.7.intermediate.dense.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.7.intermediate.dense.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.7.output.LayerNorm.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.7.output.LayerNorm.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.7.output.dense.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.7.output.dense.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.8.attention.output.LayerNorm.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.8.attention.output.LayerNorm.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.8.attention.output.dense.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.8.attention.output.dense.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.8.attention.self.key_proj.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.8.attention.self.key_proj.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.8.attention.self.query_proj.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.8.attention.self.query_proj.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.8.attention.self.value_proj.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.8.attention.self.value_proj.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.8.intermediate.dense.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.8.intermediate.dense.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.8.output.LayerNorm.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.8.output.LayerNorm.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.8.output.dense.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.8.output.dense.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.9.attention.output.LayerNorm.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.9.attention.output.LayerNorm.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.9.attention.output.dense.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.9.attention.output.dense.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.9.attention.self.key_proj.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.9.attention.self.key_proj.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.9.attention.self.query_proj.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.9.attention.self.query_proj.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.9.attention.self.value_proj.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.9.attention.self.value_proj.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.9.intermediate.dense.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.9.intermediate.dense.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.9.output.LayerNorm.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.9.output.LayerNorm.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.9.output.dense.bias": "model-00003-of-00003.safetensors",
+    "deberta.encoder.layer.9.output.dense.weight": "model-00003-of-00003.safetensors",
+    "deberta.encoder.rel_embeddings.weight": "model-00003-of-00003.safetensors",
+    "pooler.dense.bias": "model-00003-of-00003.safetensors",
+    "pooler.dense.weight": "model-00003-of-00003.safetensors"
+  }
+}