Training in progress, step 10000

README.md

@@ -0,0 +1,111 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+datasets:
+- craa/100M
+metrics:
+- accuracy
+model-index:
+- name: 100M_8397
+  results:
+  - task:
+      name: Causal Language Modeling
+      type: text-generation
+    dataset:
+      name: craa/100M
+      type: craa/100M
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 0.39711088454620164
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# 100M_8397
+
+This model is a fine-tuned version of [](https://huggingface.co/) on the craa/100M dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.2698
+- Accuracy: 0.3971
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 32
+- eval_batch_size: 16
+- seed: 8397
+- gradient_accumulation_steps: 5
+- total_train_batch_size: 160
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 20.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch   | Step  | Accuracy | Validation Loss |
+|:-------------:|:-------:|:-----:|:--------:|:---------------:|
+| 21.3858       | 0.5391  | 1000  | 0.3000   | 4.1883          |
+| 19.0764       | 1.0782  | 2000  | 0.3387   | 3.7939          |
+| 18.4093       | 1.6173  | 3000  | 0.3536   | 3.6363          |
+| 17.6272       | 2.1563  | 4000  | 0.3624   | 3.5476          |
+| 17.4682       | 2.6954  | 5000  | 0.3687   | 3.4876          |
+| 16.9702       | 3.2345  | 6000  | 0.3729   | 3.4503          |
+| 16.9854       | 3.7736  | 7000  | 0.3767   | 3.4158          |
+| 16.6055       | 4.3127  | 8000  | 0.3792   | 3.3926          |
+| 16.5975       | 4.8518  | 9000  | 0.3819   | 3.3694          |
+| 16.2438       | 5.3908  | 10000 | 0.3834   | 3.3585          |
+| 16.3082       | 5.9299  | 11000 | 3.3380   | 0.3854          |
+| 16.0444       | 6.4690  | 12000 | 3.3349   | 0.3863          |
+| 15.9733       | 7.0081  | 13000 | 3.3216   | 0.3876          |
+| 15.8821       | 7.5472  | 14000 | 3.3151   | 0.3887          |
+| 15.5282       | 8.0863  | 15000 | 3.3097   | 0.3900          |
+| 15.7186       | 8.6253  | 16000 | 3.2999   | 0.3909          |
+| 15.4334       | 9.1644  | 17000 | 3.3002   | 0.3910          |
+| 15.6017       | 9.7035  | 18000 | 3.2890   | 0.3923          |
+| 15.3661       | 10.2426 | 19000 | 3.2917   | 0.3925          |
+| 15.4528       | 10.7817 | 20000 | 3.2816   | 0.3936          |
+| 15.2093       | 11.3208 | 21000 | 3.2851   | 0.3936          |
+| 15.3565       | 11.8598 | 22000 | 3.2755   | 0.3947          |
+| 15.1753       | 12.3989 | 23000 | 3.2789   | 0.3949          |
+| 15.2719       | 12.9380 | 24000 | 3.2695   | 0.3954          |
+| 15.0743       | 13.4771 | 25000 | 3.2756   | 0.3956          |
+| 14.9025       | 14.0162 | 26000 | 3.2737   | 0.3958          |
+| 14.8969       | 14.5553 | 27000 | 3.2681   | 0.3964          |
+| 14.7568       | 15.0943 | 28000 | 3.2735   | 0.3963          |
+| 14.8698       | 15.6334 | 29000 | 3.2674   | 0.3968          |
+| 14.6981       | 16.1725 | 30000 | 3.2698   | 0.3971          |
+| 14.7843       | 16.7116 | 31000 | 3.2641   | 0.3975          |
+| 14.6185       | 17.2507 | 32000 | 3.2667   | 0.3976          |
+| 14.6404       | 17.7898 | 33000 | 3.2624   | 0.3981          |
+| 14.5449       | 18.3288 | 34000 | 3.2648   | 0.3981          |
+| 14.5924       | 18.8679 | 35000 | 3.2608   | 0.3984          |
+| 14.4439       | 19.4070 | 36000 | 3.2623   | 0.3985          |
+| 14.484        | 19.9461 | 37000 | 3.2608   | 0.3987          |
+
+
+### Framework versions
+
+- Transformers 4.47.0.dev0
+- Pytorch 2.5.0+cu124
+- Datasets 3.0.2
+- Tokenizers 0.20.1

all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 20.0,
+    "eval_accuracy": 0.39711088454620164,
+    "eval_loss": 3.2697906494140625,
+    "eval_runtime": 181.9633,
+    "eval_samples": 18012,
+    "eval_samples_per_second": 98.987,
+    "eval_steps_per_second": 6.188,
+    "perplexity": 26.305831625499668,
+    "total_flos": 1.55087795257344e+18,
+    "train_loss": 11.102719344887129,
+    "train_runtime": 93781.9567,
+    "train_samples": 296771,
+    "train_samples_per_second": 63.29,
+    "train_steps_per_second": 0.396
+}

config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.0.dev0",
+  "use_cache": true,
+  "vocab_size": 52000
+}

eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 20.0,
+    "eval_accuracy": 0.39711088454620164,
+    "eval_loss": 3.2697906494140625,
+    "eval_runtime": 181.9633,
+    "eval_samples": 18012,
+    "eval_samples_per_second": 98.987,
+    "eval_steps_per_second": 6.188,
+    "perplexity": 26.305831625499668
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.47.0.dev0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ec302ecd369b52dd09d862f2670f9b9bb168926d0e86791c661a9e174fb7d29
+size 503128704

special_tokens_map.json

@@ -0,0 +1 @@
+{}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 20.0,
+    "total_flos": 1.55087795257344e+18,
+    "train_loss": 11.102719344887129,
+    "train_runtime": 93781.9567,
+    "train_samples": 296771,
+    "train_samples_per_second": 63.29,
+    "train_steps_per_second": 0.396
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de806f958cda4fd28e35b926e99af957f1556e2a4665381fb905a2ed66ce40ee
+size 5304