Model save

README.md

@@ -0,0 +1,52 @@
+---
+tags:
+- generated_from_trainer
+model-index:
+- name: online_dpo_llmjudge
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/costa-huang/huggingface/runs/e52d1ezu)
+# online_dpo_llmjudge
+
+This model was trained from scratch on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 3e-06
+- train_batch_size: 16
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 512
+- total_eval_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 3.0
+
+### Framework versions
+
+- Transformers 4.42.3
+- Pytorch 2.3.0+cu121
+- Datasets 2.19.1
+- Tokenizers 0.19.1

config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "cleanrl/EleutherAI_pythia-1b-deduped__sft__tldr",
+  "architectures": [
+    "GPTNeoXForCausalLM"
+  ],
+  "attention_bias": true,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "classifier_dropout": 0.1,
+  "eos_token_id": 0,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.0,
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2048,
+  "model_type": "gpt_neox",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 16,
+  "rope_scaling": null,
+  "rotary_emb_base": 10000,
+  "rotary_pct": 0.25,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.42.3",
+  "use_cache": true,
+  "use_parallel_residual": true,
+  "vocab_size": 50304
+}

generation_config.json

@@ -0,0 +1,5 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "transformers_version": "4.42.3"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ce66e12687cf780b2e4178d85ee4a709512eb63206f442f8d1e466e8c4e6940
+size 2023586384

runs/Jul16_19-28-05_ip-26-0-160-192/events.out.tfevents.1721158090.ip-26-0-160-192.486807.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dea1a8854d668d03d3f135c2e13ce7a22378e85ea1cee36282f9d04551b15bad
+size 6312

runs/Jul16_19-28-55_ip-26-0-160-192/events.out.tfevents.1721158138.ip-26-0-160-192.489756.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa4e6920560cad5eeed46b29b089e75172659f37dad6303b23f943ae3b58950e
+size 6312

runs/Jul16_19-29-28_ip-26-0-160-192/events.out.tfevents.1721158172.ip-26-0-160-192.491088.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc56aac544deea4c2aeef37c23770bf2e5acd7ca114b0456b29972a0d0dc8eda
+size 6312

runs/Jul16_19-30-31_ip-26-0-160-100/events.out.tfevents.1721158236.ip-26-0-160-100.2233327.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4aa962531745f25707df83572eebf93fb85a8429deaf38335faf52d7506c3838
+size 6312

runs/Jul16_19-33-40_ip-26-0-160-100/events.out.tfevents.1721158423.ip-26-0-160-100.3402169.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdc5adc930f13cbc64d8d2fc63dbcc0e4af0d96a8899f97e14bfab8337369d39
+size 6312

runs/Jul16_19-36-11_ip-26-0-160-100/events.out.tfevents.1721158574.ip-26-0-160-100.3889662.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0f041c98914ea36a06b4eb448deb9119293891e5b7860b47fb8d5c0596096f3
+size 6312

runs/Jul16_19-38-13_ip-26-0-160-100/events.out.tfevents.1721158696.ip-26-0-160-100.82708.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0fbc5ed9b50b3665df6e320213b14e9690e614465d81f7010a3131a4749c2fb
+size 6312

runs/Jul16_19-42-12_ip-26-0-160-192/events.out.tfevents.1721158939.ip-26-0-160-192.496384.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd097d781e9863ffdee76ebdb4a0cfe536308aa49d687e802d8d4c76ceb28d0
+size 6312

runs/Jul16_19-51-28_ip-26-0-160-192/events.out.tfevents.1721159558.ip-26-0-160-192.501459.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67e838f0f284357f14d47e33a22335f806b8c3e42de7c2d5bb1a2357abe48527
+size 7443

runs/Jul16_19-58-21_ip-26-0-160-192/events.out.tfevents.1721159935.ip-26-0-160-192.503495.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92e016eb684dc75251ce329e68264b0fda964c4bde23d3f35b42ec0592f0c9c4
+size 7443

runs/Jul16_20-03-09_ip-26-0-160-192/events.out.tfevents.1721160223.ip-26-0-160-192.505748.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de929c185dd28705f9e95f5b6cad189cc754641f362f1ee740c222feb1714370
+size 12673

runs/Jul16_20-25-10_ip-26-0-167-51/events.out.tfevents.1721161576.ip-26-0-167-51.2192748.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953603241172c931c22496a0fd3c0347cbf91275876b09264300b417063aa677
+size 6398

runs/Jul16_20-28-49_ip-26-0-167-51/events.out.tfevents.1721161782.ip-26-0-167-51.2195066.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3f57e02279f95b2d20d983c84fa793bf0b6502ba8c8c2f3f32da7ab062eb033
+size 6398

runs/Jul16_20-35-23_ip-26-0-175-241/events.out.tfevents.1721162187.ip-26-0-175-241.1729472.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfdb6bb8e0bb6c901b52e77c219c90817f9ee1675212f7e423b8b298381ba8ee
+size 6399

runs/Jul16_20-37-37_ip-26-0-175-241/events.out.tfevents.1721162313.ip-26-0-175-241.1731639.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9140614c6832180bbead231a493e32b2b241557b86e1f99dafe345f3f25169b
+size 6399

runs/Jul16_20-40-54_ip-26-0-175-241/events.out.tfevents.1721162505.ip-26-0-175-241.1734374.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb6deca754b2120d828de19df2bd10645ca0a1a5269890fff073b2b92abbbd74
+size 28365

runs/Jul16_22-16-02_ip-26-0-171-168/events.out.tfevents.1721168229.ip-26-0-171-168.1704982.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:015fe15ea884f48d3f4016732450c5072a7a697bcf9be37b897c70d81a965de5
+size 2082105

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,224 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|padding|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50254": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50255": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50256": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50257": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50258": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50259": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50260": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50261": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50262": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50263": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50264": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50265": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50266": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50267": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50268": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50269": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50270": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50271": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50272": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50273": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50274": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50275": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50276": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50277": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "chat_template": "{% for message in messages %}{{' ' + message['content']}}{% endfor %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "padding_side": "left",
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "unk_token": "<|endoftext|>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a577af5edc5f1f0519202485d410fcfb73210fc0b3b97f23cd93c31d2c67676
+size 7480