Training in progress, step 10000

README.md

@@ -0,0 +1,111 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+datasets:
+- craa/100M
+metrics:
+- accuracy
+model-index:
+- name: 100M_6910
+  results:
+  - task:
+      name: Causal Language Modeling
+      type: text-generation
+    dataset:
+      name: craa/100M
+      type: craa/100M
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 0.39686979717370413
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# 100M_6910
+
+This model is a fine-tuned version of [](https://huggingface.co/) on the craa/100M dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.2710
+- Accuracy: 0.3969
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 32
+- eval_batch_size: 16
+- seed: 6910
+- gradient_accumulation_steps: 5
+- total_train_batch_size: 160
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 20.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch   | Step  | Validation Loss | Accuracy |
+|:-------------:|:-------:|:-----:|:---------------:|:--------:|
+| 21.4198       | 0.5391  | 1000  | 4.2180          | 0.2974   |
+| 19.135        | 1.0782  | 2000  | 3.7959          | 0.3386   |
+| 18.3835       | 1.6173  | 3000  | 3.6398          | 0.3533   |
+| 17.6178       | 2.1563  | 4000  | 3.5501          | 0.3623   |
+| 17.4994       | 2.6954  | 5000  | 3.4885          | 0.3689   |
+| 16.9783       | 3.2345  | 6000  | 3.4492          | 0.3732   |
+| 16.9938       | 3.7736  | 7000  | 3.4134          | 0.3765   |
+| 16.5676       | 4.3127  | 8000  | 3.3927          | 0.3794   |
+| 16.623        | 4.8518  | 9000  | 3.3670          | 0.3820   |
+| 16.268        | 5.3908  | 10000 | 3.3580          | 0.3835   |
+| 16.3595       | 5.9299  | 11000 | 3.3376          | 0.3858   |
+| 16.0848       | 6.4690  | 12000 | 3.3341          | 0.3867   |
+| 15.9683       | 7.0081  | 13000 | 3.3240          | 0.3878   |
+| 15.9139       | 7.5472  | 14000 | 3.3172          | 0.3890   |
+| 15.5172       | 8.0863  | 15000 | 3.3149          | 0.3893   |
+| 15.7165       | 8.6253  | 16000 | 3.3008          | 0.3906   |
+| 15.4386       | 9.1644  | 17000 | 3.3030          | 0.3910   |
+| 15.6082       | 9.7035  | 18000 | 3.2893          | 0.3922   |
+| 15.3025       | 10.2426 | 19000 | 3.2937          | 0.3925   |
+| 15.4821       | 10.7817 | 20000 | 3.2815          | 0.3933   |
+| 15.1951       | 11.3208 | 21000 | 3.2858          | 0.3935   |
+| 15.3396       | 11.8598 | 22000 | 3.2750          | 0.3947   |
+| 15.155        | 12.3989 | 23000 | 3.2803          | 0.3945   |
+| 15.2463       | 12.9380 | 24000 | 3.2670          | 0.3956   |
+| 15.0772       | 13.4771 | 25000 | 3.2737          | 0.3955   |
+| 14.8798       | 14.0162 | 26000 | 3.2740          | 0.3958   |
+| 14.9491       | 14.5553 | 27000 | 3.2703          | 0.3961   |
+| 14.7585       | 15.0943 | 28000 | 3.2742          | 0.3963   |
+| 14.8196       | 15.6334 | 29000 | 3.2677          | 0.3969   |
+| 14.6783       | 16.1725 | 30000 | 3.2710          | 0.3969   |
+| 14.7716       | 16.7116 | 31000 | 3.2666          | 0.3974   |
+| 14.6427       | 17.2507 | 32000 | 3.2697          | 0.3972   |
+| 14.6409       | 17.7898 | 33000 | 3.2631          | 0.3980   |
+| 14.5329       | 18.3288 | 34000 | 3.2663          | 0.3980   |
+| 14.5702       | 18.8679 | 35000 | 3.2617          | 0.3984   |
+| 14.4615       | 19.4070 | 36000 | 3.2644          | 0.3984   |
+| 14.4647       | 19.9461 | 37000 | 3.2621          | 0.3985   |
+
+
+### Framework versions
+
+- Transformers 4.47.0.dev0
+- Pytorch 2.5.0+cu124
+- Datasets 3.0.2
+- Tokenizers 0.20.1

all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 20.0,
+    "eval_accuracy": 0.39686979717370413,
+    "eval_loss": 3.2710256576538086,
+    "eval_runtime": 180.4566,
+    "eval_samples": 18012,
+    "eval_samples_per_second": 99.813,
+    "eval_steps_per_second": 6.24,
+    "perplexity": 26.33833961399547,
+    "total_flos": 1.55087795257344e+18,
+    "train_loss": 16.0859583358559,
+    "train_runtime": 127733.4325,
+    "train_samples": 296771,
+    "train_samples_per_second": 46.467,
+    "train_steps_per_second": 0.29
+}

config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.0.dev0",
+  "use_cache": true,
+  "vocab_size": 52000
+}

eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 20.0,
+    "eval_accuracy": 0.39686979717370413,
+    "eval_loss": 3.2710256576538086,
+    "eval_runtime": 180.4566,
+    "eval_samples": 18012,
+    "eval_samples_per_second": 99.813,
+    "eval_steps_per_second": 6.24,
+    "perplexity": 26.33833961399547
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.47.0.dev0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fccc9d15f81a3d2550f3189e5101a64358903254e2b9013215aafa4bbfef0fd
+size 503128704

special_tokens_map.json

@@ -0,0 +1 @@
+{}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 20.0,
+    "total_flos": 1.55087795257344e+18,
+    "train_loss": 16.0859583358559,
+    "train_runtime": 127733.4325,
+    "train_samples": 296771,
+    "train_samples_per_second": 46.467,
+    "train_steps_per_second": 0.29
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfe144d512ffe8a80f31c3c4898102180c338015ceb78579bf68b3086243e742
+size 5304