Upload folder using huggingface_hub

README.md

@@ -0,0 +1,209 @@
+---
+base_model: microsoft/Phi-tiny-MoE-instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:microsoft/Phi-tiny-MoE-instruct
+- grpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1

adapter_config.json

@@ -0,0 +1,56 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {
+    ".*\\.gate_up_proj": 64
+  },
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "microsoft/Phi-tiny-MoE-instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {
+    ".*\\.gate_up_proj": 32
+  },
+  "revision": null,
+  "target_modules": [
+    "up_proj",
+    "o_proj",
+    "gate_proj",
+    "q_proj",
+    "v_proj",
+    "down_proj",
+    "k_proj"
+  ],
+  "target_parameters": [
+    "experts.gate_up_proj",
+    "experts.down_proj",
+    "router.weight"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17aa2abe397bfab9f67c460c7c95a8e569abf536f3b3fae82d6cd58b376b2ecf
+size 528575792

chat_template.jinja

@@ -0,0 +1,4 @@
+{% for message in messages %}{{'<|' + message['role'] + '|>' + '
+' + message['content'] + '<|end|>
+' }}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>
+' }}{% else %}{{ eos_token }}{% endif %}

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd3233fc482e9d53ef5479126ff97a1df0450d0592c122c43a33260842858b71
+size 1057414883

rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03bf306d9773e53636d8214fbbf9aaa92373718df13d50d694415af3b80b7035
+size 15429

rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0e9298eee0e5369b66ad2d5ad8d4130090b303c32421f6ed2284ebb89a77422
+size 15365

rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee27f3a291d5be071038ec5f08019d0423cdfca9c7bd05ecda4e8b2c7b49a39f
+size 15429

rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9da1ab1e243accf371b1b4af8245d21aa78bf7185b684b0159f6af7d5e9c1389
+size 15429

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3336668d1a4abea674cdf2f97d84dbc38557a073c80d8fc2c06d15f6cae19c4c
+size 1465

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "is_local": false,
+  "legacy": false,
+  "local_files_only": true,
+  "model_max_length": 4096,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

trainer_state.json

@@ -0,0 +1,614 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.5037593984962406,
+  "eval_steps": 500,
+  "global_step": 201,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "cache_hit_rate": 0.2987321466207504,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.8671875,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 809.9,
+      "completions/mean_length": 961.46875,
+      "completions/mean_terminated_length": 563.5086151123047,
+      "completions/min_length": 327.9,
+      "completions/min_terminated_length": 327.9,
+      "entropy": 0.7783569753170013,
+      "epoch": 0.02506265664160401,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.19229525327682495,
+      "kl": 0.025735817663371564,
+      "learning_rate": 9e-05,
+      "loss": 0.03023299276828766,
+      "num_tokens": 773884.0,
+      "reward": 0.2986815273761749,
+      "reward_std": 0.021126844733953477,
+      "rewards/cache_reward/mean": 0.2986815273761749,
+      "rewards/cache_reward/std": 0.021126844733953477,
+      "step": 10,
+      "step_time": 90.0082708384376
+    },
+    {
+      "cache_hit_rate": 0.3069839864969254,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.8828125,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 821.6,
+      "completions/mean_length": 971.9640625,
+      "completions/mean_terminated_length": 570.3135467529297,
+      "completions/min_length": 287.1,
+      "completions/min_terminated_length": 287.1,
+      "entropy": 0.754186350107193,
+      "epoch": 0.05012531328320802,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 1.0587414503097534,
+      "kl": 0.028826666437089442,
+      "learning_rate": 0.0001,
+      "loss": -0.013258826732635499,
+      "num_tokens": 1581589.0,
+      "reward": 0.306694495677948,
+      "reward_std": 0.021151169389486312,
+      "rewards/cache_reward/mean": 0.306694495677948,
+      "rewards/cache_reward/std": 0.021151169389486312,
+      "step": 20,
+      "step_time": 90.82258757026867
+    },
+    {
+      "cache_hit_rate": 0.3231631815433502,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.9515625,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 664.1,
+      "completions/mean_length": 1000.9328125,
+      "completions/mean_terminated_length": 459.4166717529297,
+      "completions/min_length": 412.7,
+      "completions/min_terminated_length": 310.3,
+      "entropy": 0.5696786761283874,
+      "epoch": 0.07518796992481203,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1489810347557068,
+      "kl": 0.5070431146770715,
+      "learning_rate": 0.0001,
+      "loss": 0.052439266443252565,
+      "num_tokens": 2390842.0,
+      "reward": 0.3228506237268448,
+      "reward_std": 0.02563070598989725,
+      "rewards/cache_reward/mean": 0.3228506237268448,
+      "rewards/cache_reward/std": 0.02563070598989725,
+      "step": 30,
+      "step_time": 90.69367619913537
+    },
+    {
+      "cache_hit_rate": 0.3287877798080444,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.9921875,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 150.8,
+      "completions/mean_length": 1018.8375,
+      "completions/mean_terminated_length": 93.1,
+      "completions/min_length": 752.2,
+      "completions/min_terminated_length": 35.4,
+      "entropy": 0.5493248522281646,
+      "epoch": 0.10025062656641603,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.15555262565612793,
+      "kl": 0.04194357730448246,
+      "learning_rate": 0.0001,
+      "loss": 0.004945866391062736,
+      "num_tokens": 3234082.0,
+      "reward": 0.3287609279155731,
+      "reward_std": 0.024521841667592525,
+      "rewards/cache_reward/mean": 0.3287609279155731,
+      "rewards/cache_reward/std": 0.024521841667592525,
+      "step": 40,
+      "step_time": 90.52190033134539
+    },
+    {
+      "cache_hit_rate": 0.3247293561697006,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.9921875,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 361.6,
+      "completions/mean_length": 1021.65,
+      "completions/mean_terminated_length": 361.6,
+      "completions/min_length": 873.6,
+      "completions/min_terminated_length": 361.6,
+      "entropy": 0.6632817029953003,
+      "epoch": 0.12531328320802004,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.18938110768795013,
+      "kl": 0.054390712082386015,
+      "learning_rate": 0.0001,
+      "loss": 0.008095483481884002,
+      "num_tokens": 4060218.0,
+      "reward": 0.32472453713417054,
+      "reward_std": 0.025068807974457742,
+      "rewards/cache_reward/mean": 0.32472453713417054,
+      "rewards/cache_reward/std": 0.025068807974457742,
+      "step": 50,
+      "step_time": 90.76908840958495
+    },
+    {
+      "cache_hit_rate": 0.33069254755973815,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.98125,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 455.7,
+      "completions/mean_length": 1014.6015625,
+      "completions/mean_terminated_length": 384.475,
+      "completions/min_length": 629.5,
+      "completions/min_terminated_length": 322.3,
+      "entropy": 0.6398827135562897,
+      "epoch": 0.15037593984962405,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1729903519153595,
+      "kl": 0.04443241953849793,
+      "learning_rate": 0.0001,
+      "loss": -1.8284097313880922e-05,
+      "num_tokens": 4852611.0,
+      "reward": 0.3305449992418289,
+      "reward_std": 0.02162333866581321,
+      "rewards/cache_reward/mean": 0.3305449992418289,
+      "rewards/cache_reward/std": 0.02162333866581321,
+      "step": 60,
+      "step_time": 90.52109750902746
+    },
+    {
+      "cache_hit_rate": 0.3291244089603424,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.9953125,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 118.3,
+      "completions/mean_length": 1021.0484375,
+      "completions/mean_terminated_length": 118.3,
+      "completions/min_length": 835.1,
+      "completions/min_terminated_length": 118.3,
+      "entropy": 0.6163755863904953,
+      "epoch": 0.17543859649122806,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.6231634020805359,
+      "kl": 0.04733915701508522,
+      "learning_rate": 0.0001,
+      "loss": -0.0013297975063323974,
+      "num_tokens": 5682834.0,
+      "reward": 0.3290934026241302,
+      "reward_std": 0.025694792345166206,
+      "rewards/cache_reward/mean": 0.3290934026241302,
+      "rewards/cache_reward/std": 0.025694792345166206,
+      "step": 70,
+      "step_time": 91.40459936279804
+    },
+    {
+      "cache_hit_rate": 0.32914042472839355,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.9859375,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 280.2,
+      "completions/mean_length": 1016.7546875,
+      "completions/mean_terminated_length": 271.95,
+      "completions/min_length": 673.3,
+      "completions/min_terminated_length": 263.7,
+      "entropy": 0.6250631839036942,
+      "epoch": 0.20050125313283207,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.14358767867088318,
+      "kl": 0.04605277068912983,
+      "learning_rate": 0.0001,
+      "loss": 0.0016001952812075616,
+      "num_tokens": 6469845.0,
+      "reward": 0.3290808916091919,
+      "reward_std": 0.027931400947272776,
+      "rewards/cache_reward/mean": 0.3290808916091919,
+      "rewards/cache_reward/std": 0.027931400947272776,
+      "step": 80,
+      "step_time": 90.27690456421115
+    },
+    {
+      "cache_hit_rate": 0.3384521096944809,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.990625,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 164.9,
+      "completions/mean_length": 1017.1234375,
+      "completions/mean_terminated_length": 164.3,
+      "completions/min_length": 675.7,
+      "completions/min_terminated_length": 163.7,
+      "entropy": 0.44799884855747224,
+      "epoch": 0.22556390977443608,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.19581231474876404,
+      "kl": 0.042757127434015274,
+      "learning_rate": 0.0001,
+      "loss": -0.004632307216525078,
+      "num_tokens": 7271556.0,
+      "reward": 0.3383351147174835,
+      "reward_std": 0.023683687672019005,
+      "rewards/cache_reward/mean": 0.3383351147174835,
+      "rewards/cache_reward/std": 0.023683687672019005,
+      "step": 90,
+      "step_time": 91.19093074775301
+    },
+    {
+      "cache_hit_rate": 0.3332447409629822,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.9921875,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 279.5,
+      "completions/mean_length": 1020.3671875,
+      "completions/mean_terminated_length": 279.5,
+      "completions/min_length": 791.5,
+      "completions/min_terminated_length": 279.5,
+      "entropy": 0.5914906859397888,
+      "epoch": 0.2506265664160401,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.22322462499141693,
+      "kl": 0.04192206896841526,
+      "learning_rate": 0.0001,
+      "loss": 0.004385053366422653,
+      "num_tokens": 8067287.0,
+      "reward": 0.33321733176708224,
+      "reward_std": 0.02411998054012656,
+      "rewards/cache_reward/mean": 0.33321733176708224,
+      "rewards/cache_reward/std": 0.02411998054012656,
+      "step": 100,
+      "step_time": 90.90292567885481
+    },
+    {
+      "cache_hit_rate": 0.33464895784854887,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.98125,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 373.3,
+      "completions/mean_length": 1013.3484375,
+      "completions/mean_terminated_length": 336.025,
+      "completions/min_length": 503.6,
+      "completions/min_terminated_length": 298.8,
+      "entropy": 0.4564924329519272,
+      "epoch": 0.2756892230576441,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1496705561876297,
+      "kl": 0.0379826420918107,
+      "learning_rate": 0.0001,
+      "loss": 0.0033605631440877916,
+      "num_tokens": 8892334.0,
+      "reward": 0.33460802435874937,
+      "reward_std": 0.024270175583660604,
+      "rewards/cache_reward/mean": 0.33460802435874937,
+      "rewards/cache_reward/std": 0.024270175583660604,
+      "step": 110,
+      "step_time": 91.2041310617933
+    },
+    {
+      "cache_hit_rate": 0.3374886929988861,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.9828125,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 459.2,
+      "completions/mean_length": 1015.209375,
+      "completions/mean_terminated_length": 380.6,
+      "completions/min_length": 506.8,
+      "completions/min_terminated_length": 302.0,
+      "entropy": 0.5316164076328278,
+      "epoch": 0.3007518796992481,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.16703373193740845,
+      "kl": 0.0434085251763463,
+      "learning_rate": 0.0001,
+      "loss": 0.008729323744773865,
+      "num_tokens": 9680356.0,
+      "reward": 0.33764258623123167,
+      "reward_std": 0.024684826284646987,
+      "rewards/cache_reward/mean": 0.33764258623123167,
+      "rewards/cache_reward/std": 0.024684826284646987,
+      "step": 120,
+      "step_time": 91.07375006200746
+    },
+    {
+      "cache_hit_rate": 0.33408362567424776,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.9859375,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 328.1,
+      "completions/mean_length": 1015.221875,
+      "completions/mean_terminated_length": 301.4,
+      "completions/min_length": 581.9,
+      "completions/min_terminated_length": 274.7,
+      "entropy": 0.5473819822072983,
+      "epoch": 0.3258145363408521,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.17018599808216095,
+      "kl": 0.08177528716623783,
+      "learning_rate": 0.0001,
+      "loss": 0.008963721990585326,
+      "num_tokens": 10495674.0,
+      "reward": 0.3341580301523209,
+      "reward_std": 0.024980502761900426,
+      "rewards/cache_reward/mean": 0.3341580301523209,
+      "rewards/cache_reward/std": 0.024980502761900426,
+      "step": 130,
+      "step_time": 90.68363874959759
+    },
+    {
+      "cache_hit_rate": 0.33405864536762236,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.9859375,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 253.5,
+      "completions/mean_length": 1015.403125,
+      "completions/mean_terminated_length": 207.71666870117187,
+      "completions/min_length": 675.1,
+      "completions/min_terminated_length": 163.1,
+      "entropy": 0.5471912950277329,
+      "epoch": 0.3508771929824561,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.20733150839805603,
+      "kl": 0.35445162318646906,
+      "learning_rate": 0.0001,
+      "loss": 0.04615514874458313,
+      "num_tokens": 11295188.0,
+      "reward": 0.3339251846075058,
+      "reward_std": 0.02682190975174308,
+      "rewards/cache_reward/mean": 0.3339251846075058,
+      "rewards/cache_reward/std": 0.02682190975174308,
+      "step": 140,
+      "step_time": 91.03833641626406
+    },
+    {
+      "cache_hit_rate": 0.33871919512748716,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.978125,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 519.8,
+      "completions/mean_length": 1013.0640625,
+      "completions/mean_terminated_length": 432.925,
+      "completions/min_length": 537.9,
+      "completions/min_terminated_length": 333.1,
+      "entropy": 0.49129725992679596,
+      "epoch": 0.37593984962406013,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1506304144859314,
+      "kl": 0.045265543088316915,
+      "learning_rate": 0.0001,
+      "loss": -0.0007558207958936692,
+      "num_tokens": 12104301.0,
+      "reward": 0.3386825680732727,
+      "reward_std": 0.025650268793106078,
+      "rewards/cache_reward/mean": 0.3386825680732727,
+      "rewards/cache_reward/std": 0.025650268793106078,
+      "step": 150,
+      "step_time": 91.25925274270121
+    },
+    {
+      "cache_hit_rate": 0.33684642016887667,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.98125,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 453.5,
+      "completions/mean_length": 1014.05,
+      "completions/mean_terminated_length": 370.8833374023437,
+      "completions/min_length": 471.5,
+      "completions/min_terminated_length": 266.7,
+      "entropy": 0.5017873764038085,
+      "epoch": 0.40100250626566414,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 2.4631996154785156,
+      "kl": 24.154931166768073,
+      "learning_rate": 0.0001,
+      "loss": 3.8478744506835936,
+      "num_tokens": 12925005.0,
+      "reward": 0.3366888463497162,
+      "reward_std": 0.02669309824705124,
+      "rewards/cache_reward/mean": 0.3366888463497162,
+      "rewards/cache_reward/std": 0.02669309824705124,
+      "step": 160,
+      "step_time": 90.7307717730524
+    },
+    {
+      "cache_hit_rate": 0.3352246254682541,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.984375,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 195.4,
+      "completions/mean_length": 1014.4828125,
+      "completions/mean_terminated_length": 127.05000305175781,
+      "completions/min_length": 784.3,
+      "completions/min_terminated_length": 67.5,
+      "entropy": 0.5599089354276657,
+      "epoch": 0.42606516290726815,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.6771759390830994,
+      "kl": 0.057728367298841475,
+      "learning_rate": 0.0001,
+      "loss": -0.005303345993161202,
+      "num_tokens": 13762762.0,
+      "reward": 0.33506683707237245,
+      "reward_std": 0.022994110081344842,
+      "rewards/cache_reward/mean": 0.33506683707237245,
+      "rewards/cache_reward/std": 0.022994110081344842,
+      "step": 170,
+      "step_time": 91.37958506643773
+    },
+    {
+      "cache_hit_rate": 0.33833766877651217,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.99375,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 346.8,
+      "completions/mean_length": 1023.01875,
+      "completions/mean_terminated_length": 346.8,
+      "completions/min_length": 961.2,
+      "completions/min_terminated_length": 346.8,
+      "entropy": 0.5654248386621475,
+      "epoch": 0.45112781954887216,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.14284838736057281,
+      "kl": 2.4538527205586433,
+      "learning_rate": 0.0001,
+      "loss": 0.39122982025146485,
+      "num_tokens": 14591590.0,
+      "reward": 0.33830906450748444,
+      "reward_std": 0.02580958418548107,
+      "rewards/cache_reward/mean": 0.33830906450748444,
+      "rewards/cache_reward/std": 0.02580958418548107,
+      "step": 180,
+      "step_time": 91.57028467489872
+    },
+    {
+      "cache_hit_rate": 0.3408436506986618,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.9890625,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 202.6,
+      "completions/mean_length": 1016.6109375,
+      "completions/mean_terminated_length": 183.8,
+      "completions/min_length": 677.0,
+      "completions/min_terminated_length": 165.0,
+      "entropy": 0.5419578611850738,
+      "epoch": 0.47619047619047616,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.13886591792106628,
+      "kl": 0.07005284689366817,
+      "learning_rate": 0.0001,
+      "loss": 0.008208285272121429,
+      "num_tokens": 15394349.0,
+      "reward": 0.3408004969358444,
+      "reward_std": 0.025611967965960502,
+      "rewards/cache_reward/mean": 0.3408004969358444,
+      "rewards/cache_reward/std": 0.025611967965960502,
+      "step": 190,
+      "step_time": 91.14261374084745
+    },
+    {
+      "cache_hit_rate": 0.3391114443540573,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.9921875,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 179.6,
+      "completions/mean_length": 1019.0921875,
+      "completions/mean_terminated_length": 179.0,
+      "completions/min_length": 792.8,
+      "completions/min_terminated_length": 178.4,
+      "entropy": 0.5290060758590698,
+      "epoch": 0.5012531328320802,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 0.1409655660390854,
+      "kl": 0.0499697394669056,
+      "learning_rate": 0.0001,
+      "loss": 0.004792223125696183,
+      "num_tokens": 16176376.0,
+      "reward": 0.33905126750469206,
+      "reward_std": 0.026329115405678748,
+      "rewards/cache_reward/mean": 0.33905126750469206,
+      "rewards/cache_reward/std": 0.026329115405678748,
+      "step": 200,
+      "step_time": 90.93274403410032
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 200,
+  "num_input_tokens_seen": 16261463,
+  "num_train_epochs": 1,
+  "save_steps": 50,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fa0515887e3471c404fa0917b4bd4f0dff257d719ef4ce4a6dadc0daac7ba8c
+size 7825