Final models for exp1b, exp2a, exp3 (Leong & Linzen exceptions project)

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

exp1b/100M_1001/README.md

@@ -0,0 +1,150 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: exceptions_exp1_100M_1001
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/craaaa/exceptions_exp1/runs/cfjz2isg)
+# exceptions_exp1_100M_1001
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.5623
+- Accuracy: 0.3689
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 1001
+- gradient_accumulation_steps: 5
+- total_train_batch_size: 80
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch   | Step  | Accuracy | Validation Loss |
+|:-------------:|:-------:|:-----:|:--------:|:---------------:|
+| 4.8847        | 0.2951  | 1000  | 0.2453   | 4.8093          |
+| 4.3746        | 0.5902  | 2000  | 0.2922   | 4.3241          |
+| 4.1863        | 0.8853  | 3000  | 0.3085   | 4.1315          |
+| 4.0369        | 1.1803  | 4000  | 0.3186   | 4.0273          |
+| 3.9654        | 1.4754  | 5000  | 0.3253   | 3.9474          |
+| 3.9249        | 1.7705  | 6000  | 0.3304   | 3.8899          |
+| 3.786         | 2.0655  | 7000  | 0.3349   | 3.8473          |
+| 3.7917        | 2.3606  | 8000  | 0.3378   | 3.8159          |
+| 3.7684        | 2.6557  | 9000  | 0.3407   | 3.7837          |
+| 3.7542        | 2.9508  | 10000 | 0.3432   | 3.7577          |
+| 3.6746        | 3.2458  | 11000 | 0.3453   | 3.7447          |
+| 3.6654        | 3.5409  | 12000 | 0.3469   | 3.7254          |
+| 3.6774        | 3.8360  | 13000 | 0.3488   | 3.7077          |
+| 3.5885        | 4.1310  | 14000 | 0.3498   | 3.7021          |
+| 3.6017        | 4.4261  | 15000 | 0.3508   | 3.6899          |
+| 3.5987        | 4.7212  | 16000 | 0.3526   | 3.6744          |
+| 3.4887        | 5.0162  | 17000 | 0.3533   | 3.6721          |
+| 3.5186        | 5.3113  | 18000 | 0.3542   | 3.6642          |
+| 3.5419        | 5.6064  | 19000 | 0.3553   | 3.6526          |
+| 3.5426        | 5.9015  | 20000 | 0.3562   | 3.6411          |
+| 3.4803        | 6.1965  | 21000 | 0.3563   | 3.6450          |
+| 3.4819        | 6.4916  | 22000 | 0.3573   | 3.6376          |
+| 3.4882        | 6.7867  | 23000 | 0.3580   | 3.6279          |
+| 3.4116        | 7.0817  | 24000 | 0.3583   | 3.6321          |
+| 3.4392        | 7.3768  | 25000 | 0.3591   | 3.6237          |
+| 3.4731        | 7.6719  | 26000 | 0.3596   | 3.6154          |
+| 3.4655        | 7.9670  | 27000 | 0.3608   | 3.6046          |
+| 3.4087        | 8.2620  | 28000 | 0.3606   | 3.6178          |
+| 3.4331        | 8.5571  | 29000 | 0.3612   | 3.6057          |
+| 3.4359        | 8.8522  | 30000 | 0.3617   | 3.5982          |
+| 3.3483        | 9.1472  | 31000 | 0.3616   | 3.6064          |
+| 3.375         | 9.4423  | 32000 | 0.3618   | 3.6010          |
+| 3.3966        | 9.7374  | 33000 | 0.3629   | 3.5916          |
+| 3.2985        | 10.0325 | 34000 | 0.3626   | 3.5990          |
+| 3.3549        | 10.3275 | 35000 | 0.3631   | 3.5967          |
+| 3.3627        | 10.6226 | 36000 | 0.3636   | 3.5898          |
+| 3.38          | 10.9177 | 37000 | 0.3645   | 3.5812          |
+| 3.3015        | 11.2128 | 38000 | 0.3639   | 3.5934          |
+| 3.3476        | 11.5078 | 39000 | 0.3645   | 3.5861          |
+| 3.3484        | 11.8029 | 40000 | 0.3651   | 3.5774          |
+| 3.2683        | 12.0980 | 41000 | 0.3649   | 3.5889          |
+| 3.297         | 12.3931 | 42000 | 0.3651   | 3.5842          |
+| 3.3315        | 12.6881 | 43000 | 0.3658   | 3.5759          |
+| 3.3232        | 12.9832 | 44000 | 0.3664   | 3.5708          |
+| 3.2725        | 13.2783 | 45000 | 0.3657   | 3.5821          |
+| 3.2729        | 13.5734 | 46000 | 0.3657   | 3.5750          |
+| 3.2951        | 13.8684 | 47000 | 0.3667   | 3.5700          |
+| 3.236         | 14.1635 | 48000 | 0.3664   | 3.5793          |
+| 3.2491        | 14.4586 | 49000 | 0.3666   | 3.5752          |
+| 3.2657        | 14.7537 | 50000 | 0.3673   | 3.5662          |
+| 3.1916        | 15.0487 | 51000 | 0.3670   | 3.5746          |
+| 3.2148        | 15.3438 | 52000 | 0.3672   | 3.5727          |
+| 3.247         | 15.6389 | 53000 | 0.3676   | 3.5686          |
+| 3.2419        | 15.9340 | 54000 | 0.3682   | 3.5630          |
+| 3.2048        | 16.2290 | 55000 | 0.3679   | 3.5720          |
+| 3.2104        | 16.5241 | 56000 | 0.3681   | 3.5673          |
+| 3.207         | 16.8192 | 57000 | 0.3686   | 3.5625          |
+| 3.1696        | 17.1142 | 58000 | 0.3683   | 3.5687          |
+| 3.1813        | 17.4093 | 59000 | 0.3686   | 3.5648          |
+| 3.1985        | 17.7044 | 60000 | 0.3689   | 3.5623          |
+| 3.2021        | 17.9995 | 61000 | 0.3693   | 3.5579          |
+| 3.1442        | 18.2945 | 62000 | 0.3691   | 3.5645          |
+| 3.1619        | 18.5896 | 63000 | 0.3694   | 3.5608          |
+| 3.1752        | 18.8847 | 64000 | 0.3697   | 3.5585          |
+| 3.1365        | 19.1797 | 65000 | 0.3696   | 3.5616          |
+| 3.1577        | 19.4748 | 66000 | 0.3698   | 3.5602          |
+| 3.1488        | 19.7699 | 67000 | 0.3699   | 3.5584          |
+| 3.2429        | 20.0649 | 68000 | 3.6295   | 0.3624          |
+| 3.3271        | 20.3600 | 69000 | 3.6218   | 0.3621          |
+| 3.3381        | 20.6551 | 70000 | 3.6095   | 0.3627          |
+| 3.3746        | 20.9502 | 71000 | 3.5961   | 0.3634          |
+| 3.2922        | 21.2452 | 72000 | 3.6116   | 0.3625          |
+| 3.3342        | 21.5403 | 73000 | 3.6010   | 0.3633          |
+| 3.3622        | 21.8354 | 74000 | 3.5974   | 0.3637          |
+| 3.2714        | 22.1304 | 75000 | 3.6103   | 0.3632          |
+| 3.3055        | 22.4255 | 76000 | 3.6010   | 0.3637          |
+| 3.3373        | 22.7206 | 77000 | 3.5948   | 0.3641          |
+| 3.2286        | 23.0156 | 78000 | 3.6035   | 0.3635          |
+| 3.2831        | 23.3107 | 79000 | 3.6030   | 0.3641          |
+| 3.3058        | 23.6058 | 80000 | 3.5908   | 0.3649          |
+| 3.3188        | 23.9009 | 81000 | 3.5826   | 0.3656          |
+| 3.2553        | 24.1959 | 82000 | 3.6001   | 0.3646          |
+| 3.2926        | 24.4910 | 83000 | 3.5937   | 0.3645          |
+| 3.2975        | 24.7861 | 84000 | 3.5818   | 0.3652          |
+| 3.2157        | 25.0811 | 85000 | 3.5992   | 0.3648          |
+| 3.2564        | 25.3762 | 86000 | 3.5932   | 0.3651          |
+| 3.2929        | 25.6713 | 87000 | 3.5883   | 0.3654          |
+
+
+### Framework versions
+
+- Transformers 4.55.2
+- Pytorch 2.8.0+cu128
+- Datasets 4.0.0
+- Tokenizers 0.21.4

exp1b/100M_1001/all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 25.67132908404155,
+    "eval_accuracy": 0.3689158231470823,
+    "eval_loss": 3.5623369216918945,
+    "eval_runtime": 174.6896,
+    "eval_samples": 16421,
+    "eval_samples_per_second": 94.001,
+    "eval_steps_per_second": 5.879,
+    "perplexity": 35.24546687478834,
+    "total_flos": 1.8184096382976e+18,
+    "train_loss": 0.7290977372355845,
+    "train_runtime": 38154.3334,
+    "train_samples": 271092,
+    "train_samples_per_second": 355.257,
+    "train_steps_per_second": 4.441
+}

exp1b/100M_1001/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.55.2",
+  "use_cache": true,
+  "vocab_size": 50257
+}

exp1b/100M_1001/eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 25.67132908404155,
+    "eval_accuracy": 0.3689158231470823,
+    "eval_loss": 3.5623369216918945,
+    "eval_runtime": 174.6896,
+    "eval_samples": 16421,
+    "eval_samples_per_second": 94.001,
+    "eval_steps_per_second": 5.879,
+    "perplexity": 35.24546687478834
+}

exp1b/100M_1001/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.55.2"
+}

exp1b/100M_1001/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:272f9b2af35f94bade086896a229775a9e7803d4b005bbc3e072ad9935daa20c
+size 497774208

exp1b/100M_1001/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

exp1b/100M_1001/tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

exp1b/100M_1001/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 25.67132908404155,
+    "total_flos": 1.8184096382976e+18,
+    "train_loss": 0.7290977372355845,
+    "train_runtime": 38154.3334,
+    "train_samples": 271092,
+    "train_samples_per_second": 355.257,
+    "train_steps_per_second": 4.441
+}

exp1b/100M_1001/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9ffc6e453bf69df3d46451195234a969aa04635647addaae8ecaebfec8832cd
+size 5905

exp1b/100M_2128/README.md

@@ -0,0 +1,150 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: exceptions_exp1_100M_2128
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/craaaa/exceptions_exp1/runs/0c403z3m)
+# exceptions_exp1_100M_2128
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.5807
+- Accuracy: 0.3646
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 2128
+- gradient_accumulation_steps: 5
+- total_train_batch_size: 80
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch   | Step  | Accuracy | Validation Loss |
+|:-------------:|:-------:|:-----:|:--------:|:---------------:|
+| 4.8689        | 0.2951  | 1000  | 0.2471   | 4.7995          |
+| 4.383         | 0.5902  | 2000  | 0.2918   | 4.3266          |
+| 4.1846        | 0.8853  | 3000  | 0.3081   | 4.1402          |
+| 4.0325        | 1.1803  | 4000  | 0.3177   | 4.0315          |
+| 3.974         | 1.4754  | 5000  | 0.3244   | 3.9547          |
+| 3.9318        | 1.7705  | 6000  | 0.3301   | 3.8942          |
+| 3.7918        | 2.0655  | 7000  | 0.3342   | 3.8535          |
+| 3.7845        | 2.3606  | 8000  | 0.3376   | 3.8197          |
+| 3.7738        | 2.6557  | 9000  | 0.3406   | 3.7893          |
+| 3.7483        | 2.9508  | 10000 | 0.3430   | 3.7616          |
+| 3.6683        | 3.2458  | 11000 | 0.3447   | 3.7483          |
+| 3.6712        | 3.5409  | 12000 | 0.3469   | 3.7315          |
+| 3.6689        | 3.8360  | 13000 | 0.3481   | 3.7095          |
+| 3.5763        | 4.1310  | 14000 | 0.3498   | 3.7044          |
+| 3.5891        | 4.4261  | 15000 | 0.3508   | 3.6915          |
+| 3.6057        | 4.7212  | 16000 | 0.3522   | 3.6779          |
+| 3.4892        | 5.0162  | 17000 | 0.3534   | 3.6712          |
+| 3.5369        | 5.3113  | 18000 | 0.3540   | 3.6680          |
+| 3.5446        | 5.6064  | 19000 | 0.3550   | 3.6545          |
+| 3.5616        | 5.9015  | 20000 | 0.3561   | 3.6425          |
+| 3.4672        | 6.1965  | 21000 | 0.3564   | 3.6489          |
+| 3.5086        | 6.4916  | 22000 | 0.3572   | 3.6392          |
+| 3.5005        | 6.7867  | 23000 | 0.3580   | 3.6297          |
+| 3.4054        | 7.0817  | 24000 | 0.3581   | 3.6337          |
+| 3.4465        | 7.3768  | 25000 | 0.3590   | 3.6270          |
+| 3.4615        | 7.6719  | 26000 | 0.3596   | 3.6174          |
+| 3.4791        | 7.9670  | 27000 | 0.3605   | 3.6082          |
+| 3.4062        | 8.2620  | 28000 | 0.3601   | 3.6178          |
+| 3.4112        | 8.5571  | 29000 | 0.3610   | 3.6094          |
+| 3.4401        | 8.8522  | 30000 | 0.3615   | 3.6015          |
+| 3.3538        | 9.1472  | 31000 | 0.3614   | 3.6083          |
+| 3.384         | 9.4423  | 32000 | 0.3617   | 3.6056          |
+| 3.4035        | 9.7374  | 33000 | 0.3626   | 3.5939          |
+| 3.3157        | 10.0325 | 34000 | 0.3625   | 3.6021          |
+| 3.3505        | 10.3275 | 35000 | 0.3627   | 3.6010          |
+| 3.3722        | 10.6226 | 36000 | 0.3635   | 3.5930          |
+| 3.3869        | 10.9177 | 37000 | 0.3641   | 3.5854          |
+| 3.3191        | 11.2128 | 38000 | 0.3636   | 3.5964          |
+| 3.3325        | 11.5078 | 39000 | 0.3645   | 3.5886          |
+| 3.3426        | 11.8029 | 40000 | 0.3646   | 3.5807          |
+| 3.2761        | 12.0980 | 41000 | 0.3647   | 3.5898          |
+| 3.2964        | 12.3931 | 42000 | 0.3648   | 3.5878          |
+| 3.3183        | 12.6881 | 43000 | 0.3653   | 3.5789          |
+| 3.322         | 12.9832 | 44000 | 0.3660   | 3.5718          |
+| 3.2651        | 13.2783 | 45000 | 0.3656   | 3.5837          |
+| 3.2814        | 13.5734 | 46000 | 0.3660   | 3.5784          |
+| 3.298         | 13.8684 | 47000 | 0.3665   | 3.5733          |
+| 3.2335        | 14.1635 | 48000 | 0.3661   | 3.5808          |
+| 3.2496        | 14.4586 | 49000 | 0.3665   | 3.5792          |
+| 3.2703        | 14.7537 | 50000 | 0.3669   | 3.5723          |
+| 3.2032        | 15.0487 | 51000 | 0.3669   | 3.5775          |
+| 3.2325        | 15.3438 | 52000 | 0.3669   | 3.5759          |
+| 3.2393        | 15.6389 | 53000 | 0.3675   | 3.5718          |
+| 3.2526        | 15.9340 | 54000 | 0.3679   | 3.5645          |
+| 3.1932        | 16.2290 | 55000 | 0.3674   | 3.5745          |
+| 3.2053        | 16.5241 | 56000 | 0.3680   | 3.5692          |
+| 3.2298        | 16.8192 | 57000 | 0.3684   | 3.5633          |
+| 3.1751        | 17.1142 | 58000 | 0.3682   | 3.5708          |
+| 3.1901        | 17.4093 | 59000 | 0.3683   | 3.5686          |
+| 3.1929        | 17.7044 | 60000 | 0.3687   | 3.5642          |
+| 3.1961        | 17.9995 | 61000 | 0.3689   | 3.5612          |
+| 3.1735        | 18.2945 | 62000 | 0.3688   | 3.5651          |
+| 3.178         | 18.5896 | 63000 | 0.3691   | 3.5637          |
+| 3.1571        | 18.8847 | 64000 | 0.3693   | 3.5608          |
+| 3.1466        | 19.1797 | 65000 | 0.3693   | 3.5632          |
+| 3.159         | 19.4748 | 66000 | 0.3695   | 3.5623          |
+| 3.1559        | 19.7699 | 67000 | 0.3696   | 3.5610          |
+| 3.2497        | 20.0649 | 68000 | 3.6288   | 0.3621          |
+| 3.33          | 20.3600 | 69000 | 3.6255   | 0.3617          |
+| 3.3575        | 20.6551 | 70000 | 3.6107   | 0.3626          |
+| 3.3782        | 20.9502 | 71000 | 3.6027   | 0.3630          |
+| 3.322         | 21.2452 | 72000 | 3.6153   | 0.3626          |
+| 3.338         | 21.5403 | 73000 | 3.6063   | 0.3628          |
+| 3.3459        | 21.8354 | 74000 | 3.5981   | 0.3635          |
+| 3.2705        | 22.1304 | 75000 | 3.6113   | 0.3630          |
+| 3.3096        | 22.4255 | 76000 | 3.6040   | 0.3636          |
+| 3.3383        | 22.7206 | 77000 | 3.5934   | 0.3641          |
+| 3.2273        | 23.0156 | 78000 | 3.6063   | 0.3638          |
+| 3.2916        | 23.3107 | 79000 | 3.6030   | 0.3636          |
+| 3.3047        | 23.6058 | 80000 | 3.5957   | 0.3640          |
+| 3.3125        | 23.9009 | 81000 | 3.5867   | 0.3648          |
+| 3.2465        | 24.1959 | 82000 | 3.6044   | 0.3642          |
+| 3.2973        | 24.4910 | 83000 | 3.5942   | 0.3645          |
+| 3.309         | 24.7861 | 84000 | 3.5862   | 0.3652          |
+| 3.2202        | 25.0811 | 85000 | 3.5991   | 0.3647          |
+| 3.2581        | 25.3762 | 86000 | 3.5943   | 0.3648          |
+| 3.2807        | 25.6713 | 87000 | 3.5899   | 0.3651          |
+
+
+### Framework versions
+
+- Transformers 4.55.2
+- Pytorch 2.8.0+cu128
+- Datasets 4.0.0
+- Tokenizers 0.21.4

exp1b/100M_2128/all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 25.67132908404155,
+    "eval_accuracy": 0.3645632513662342,
+    "eval_loss": 3.580665111541748,
+    "eval_runtime": 178.3177,
+    "eval_samples": 16421,
+    "eval_samples_per_second": 92.088,
+    "eval_steps_per_second": 5.759,
+    "perplexity": 35.89740868963242,
+    "total_flos": 1.8184096382976e+18,
+    "train_loss": 0.729940791952199,
+    "train_runtime": 38426.6729,
+    "train_samples": 271092,
+    "train_samples_per_second": 352.739,
+    "train_steps_per_second": 4.41
+}

exp1b/100M_2128/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.55.2",
+  "use_cache": true,
+  "vocab_size": 50257
+}

exp1b/100M_2128/eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 25.67132908404155,
+    "eval_accuracy": 0.3645632513662342,
+    "eval_loss": 3.580665111541748,
+    "eval_runtime": 178.3177,
+    "eval_samples": 16421,
+    "eval_samples_per_second": 92.088,
+    "eval_steps_per_second": 5.759,
+    "perplexity": 35.89740868963242
+}

exp1b/100M_2128/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.55.2"
+}

exp1b/100M_2128/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a87ba3e2b616761aca15095ce3df8fa3e4bebd37e516e2347f1b18a789c26b26
+size 497774208

exp1b/100M_2128/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

exp1b/100M_2128/tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

exp1b/100M_2128/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 25.67132908404155,
+    "total_flos": 1.8184096382976e+18,
+    "train_loss": 0.729940791952199,
+    "train_runtime": 38426.6729,
+    "train_samples": 271092,
+    "train_samples_per_second": 352.739,
+    "train_steps_per_second": 4.41
+}

exp1b/100M_2128/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b09db6950905f6a6c2749f212af299e103d6fc457c2aab0131f6d60d54fe86c7
+size 5905

exp1b/100M_3591/README.md

@@ -0,0 +1,150 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: exceptions_exp1_100M_3591
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/craaaa/exceptions_exp1/runs/jcjz7uof)
+# exceptions_exp1_100M_3591
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.5675
+- Accuracy: 0.3670
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 3591
+- gradient_accumulation_steps: 5
+- total_train_batch_size: 80
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch   | Step  | Accuracy | Validation Loss |
+|:-------------:|:-------:|:-----:|:--------:|:---------------:|
+| 4.8678        | 0.2951  | 1000  | 0.2467   | 4.8027          |
+| 4.3918        | 0.5902  | 2000  | 0.2917   | 4.3312          |
+| 4.1903        | 0.8853  | 3000  | 0.3089   | 4.1325          |
+| 4.0326        | 1.1803  | 4000  | 0.3179   | 4.0291          |
+| 3.9643        | 1.4754  | 5000  | 0.3250   | 3.9517          |
+| 3.9098        | 1.7705  | 6000  | 0.3301   | 3.8925          |
+| 3.7929        | 2.0655  | 7000  | 0.3346   | 3.8521          |
+| 3.7859        | 2.3606  | 8000  | 0.3373   | 3.8186          |
+| 3.7806        | 2.6557  | 9000  | 0.3407   | 3.7877          |
+| 3.7599        | 2.9508  | 10000 | 0.3429   | 3.7605          |
+| 3.6705        | 3.2458  | 11000 | 0.3446   | 3.7493          |
+| 3.6714        | 3.5409  | 12000 | 0.3469   | 3.7277          |
+| 3.6649        | 3.8360  | 13000 | 0.3486   | 3.7104          |
+| 3.5667        | 4.1310  | 14000 | 0.3497   | 3.7045          |
+| 3.6008        | 4.4261  | 15000 | 0.3507   | 3.6930          |
+| 3.6091        | 4.7212  | 16000 | 0.3523   | 3.6793          |
+| 3.4967        | 5.0162  | 17000 | 0.3533   | 3.6696          |
+| 3.541         | 5.3113  | 18000 | 0.3539   | 3.6655          |
+| 3.5478        | 5.6064  | 19000 | 0.3552   | 3.6535          |
+| 3.5467        | 5.9015  | 20000 | 0.3562   | 3.6420          |
+| 3.476         | 6.1965  | 21000 | 0.3563   | 3.6473          |
+| 3.4914        | 6.4916  | 22000 | 0.3573   | 3.6370          |
+| 3.5007        | 6.7867  | 23000 | 0.3579   | 3.6268          |
+| 3.4102        | 7.0817  | 24000 | 0.3583   | 3.6301          |
+| 3.4483        | 7.3768  | 25000 | 0.3586   | 3.6274          |
+| 3.4581        | 7.6719  | 26000 | 0.3597   | 3.6162          |
+| 3.4618        | 7.9670  | 27000 | 0.3604   | 3.6058          |
+| 3.397         | 8.2620  | 28000 | 0.3604   | 3.6156          |
+| 3.4103        | 8.5571  | 29000 | 0.3610   | 3.6107          |
+| 3.4341        | 8.8522  | 30000 | 0.3616   | 3.6001          |
+| 3.355         | 9.1472  | 31000 | 0.3614   | 3.6075          |
+| 3.3785        | 9.4423  | 32000 | 0.3620   | 3.6000          |
+| 3.3876        | 9.7374  | 33000 | 0.3628   | 3.5929          |
+| 3.3069        | 10.0325 | 34000 | 0.3625   | 3.6004          |
+| 3.3535        | 10.3275 | 35000 | 0.3628   | 3.6001          |
+| 3.3628        | 10.6226 | 36000 | 0.3636   | 3.5911          |
+| 3.3778        | 10.9177 | 37000 | 0.3639   | 3.5846          |
+| 3.3051        | 11.2128 | 38000 | 0.3635   | 3.5938          |
+| 3.3301        | 11.5078 | 39000 | 0.3642   | 3.5866          |
+| 3.3608        | 11.8029 | 40000 | 0.3650   | 3.5788          |
+| 3.2706        | 12.0980 | 41000 | 0.3648   | 3.5897          |
+| 3.2911        | 12.3931 | 42000 | 0.3648   | 3.5859          |
+| 3.3135        | 12.6881 | 43000 | 0.3655   | 3.5767          |
+| 3.3211        | 12.9832 | 44000 | 0.3658   | 3.5711          |
+| 3.2658        | 13.2783 | 45000 | 0.3659   | 3.5840          |
+| 3.2762        | 13.5734 | 46000 | 0.3661   | 3.5766          |
+| 3.2925        | 13.8684 | 47000 | 0.3665   | 3.5712          |
+| 3.2331        | 14.1635 | 48000 | 0.3662   | 3.5805          |
+| 3.2413        | 14.4586 | 49000 | 0.3667   | 3.5738          |
+| 3.2652        | 14.7537 | 50000 | 0.3670   | 3.5675          |
+| 3.2115        | 15.0487 | 51000 | 0.3668   | 3.5755          |
+| 3.224         | 15.3438 | 52000 | 0.3671   | 3.5755          |
+| 3.2369        | 15.6389 | 53000 | 0.3675   | 3.5698          |
+| 3.2507        | 15.9340 | 54000 | 0.3680   | 3.5615          |
+| 3.1948        | 16.2290 | 55000 | 0.3676   | 3.5719          |
+| 3.2114        | 16.5241 | 56000 | 0.3680   | 3.5662          |
+| 3.2077        | 16.8192 | 57000 | 0.3684   | 3.5619          |
+| 3.1628        | 17.1142 | 58000 | 0.3681   | 3.5689          |
+| 3.1858        | 17.4093 | 59000 | 0.3683   | 3.5655          |
+| 3.1918        | 17.7044 | 60000 | 0.3689   | 3.5627          |
+| 3.1859        | 17.9995 | 61000 | 0.3693   | 3.5571          |
+| 3.1706        | 18.2945 | 62000 | 0.3689   | 3.5644          |
+| 3.1673        | 18.5896 | 63000 | 0.3690   | 3.5620          |
+| 3.1629        | 18.8847 | 64000 | 0.3695   | 3.5591          |
+| 3.1322        | 19.1797 | 65000 | 0.3693   | 3.5622          |
+| 3.1415        | 19.4748 | 66000 | 0.3695   | 3.5605          |
+| 3.1359        | 19.7699 | 67000 | 0.3697   | 3.5590          |
+| 3.2424        | 20.0649 | 68000 | 3.6273   | 0.3623          |
+| 3.3252        | 20.3600 | 69000 | 3.6223   | 0.3621          |
+| 3.344         | 20.6551 | 70000 | 3.6115   | 0.3622          |
+| 3.3684        | 20.9502 | 71000 | 3.5968   | 0.3632          |
+| 3.3116        | 21.2452 | 72000 | 3.6109   | 0.3626          |
+| 3.3365        | 21.5403 | 73000 | 3.6038   | 0.3631          |
+| 3.3741        | 21.8354 | 74000 | 3.5934   | 0.3639          |
+| 3.2761        | 22.1304 | 75000 | 3.6076   | 0.3630          |
+| 3.3           | 22.4255 | 76000 | 3.6009   | 0.3635          |
+| 3.326         | 22.7206 | 77000 | 3.5952   | 0.3640          |
+| 3.2265        | 23.0156 | 78000 | 3.6051   | 0.3635          |
+| 3.2837        | 23.3107 | 79000 | 3.6029   | 0.3640          |
+| 3.311         | 23.6058 | 80000 | 3.5903   | 0.3646          |
+| 3.3287        | 23.9009 | 81000 | 3.5842   | 0.3648          |
+| 3.2544        | 24.1959 | 82000 | 3.6015   | 0.3642          |
+| 3.2882        | 24.4910 | 83000 | 3.5936   | 0.3646          |
+| 3.2988        | 24.7861 | 84000 | 3.5864   | 0.3652          |
+| 3.2204        | 25.0811 | 85000 | 3.6007   | 0.3647          |
+| 3.2633        | 25.3762 | 86000 | 3.5927   | 0.3649          |
+| 3.2874        | 25.6713 | 87000 | 3.5875   | 0.3654          |
+
+
+### Framework versions
+
+- Transformers 4.55.2
+- Pytorch 2.8.0+cu128
+- Datasets 4.0.0
+- Tokenizers 0.21.4

exp1b/100M_3591/all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 25.67132908404155,
+    "eval_accuracy": 0.36698962273381264,
+    "eval_loss": 3.567456007003784,
+    "eval_runtime": 174.0059,
+    "eval_samples": 16421,
+    "eval_samples_per_second": 94.37,
+    "eval_steps_per_second": 5.902,
+    "perplexity": 35.426354019929015,
+    "total_flos": 1.8184096382976e+18,
+    "train_loss": 0.7292301019252032,
+    "train_runtime": 38261.3792,
+    "train_samples": 271092,
+    "train_samples_per_second": 354.263,
+    "train_steps_per_second": 4.429
+}

exp1b/100M_3591/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.55.2",
+  "use_cache": true,
+  "vocab_size": 50257
+}

exp1b/100M_3591/eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 25.67132908404155,
+    "eval_accuracy": 0.36698962273381264,
+    "eval_loss": 3.567456007003784,
+    "eval_runtime": 174.0059,
+    "eval_samples": 16421,
+    "eval_samples_per_second": 94.37,
+    "eval_steps_per_second": 5.902,
+    "perplexity": 35.426354019929015
+}

exp1b/100M_3591/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.55.2"
+}

exp1b/100M_3591/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5daca3b8e1eb4599e58275946e47f73f97ad834314fbb859530829284c4ef4bd
+size 497774208

exp1b/100M_3591/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

exp1b/100M_3591/tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

exp1b/100M_3591/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 25.67132908404155,
+    "total_flos": 1.8184096382976e+18,
+    "train_loss": 0.7292301019252032,
+    "train_runtime": 38261.3792,
+    "train_samples": 271092,
+    "train_samples_per_second": 354.263,
+    "train_steps_per_second": 4.429
+}

exp1b/100M_3591/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6446d0d1a0eaad24b8e3f63f068c8db98c4613e49616fc2d0e980db623354f64
+size 5905

exp1b/100M_40817/README.md

@@ -0,0 +1,150 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: exceptions_exp1_100M_40817
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/craaaa/exceptions_exp1/runs/na2q91k3)
+# exceptions_exp1_100M_40817
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.5703
+- Accuracy: 0.3671
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0006
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 40817
+- gradient_accumulation_steps: 5
+- total_train_batch_size: 80
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.98) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 50.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch   | Step  | Accuracy | Validation Loss |
+|:-------------:|:-------:|:-----:|:--------:|:---------------:|
+| 4.8725        | 0.2951  | 1000  | 0.2474   | 4.7947          |
+| 4.3801        | 0.5902  | 2000  | 0.2920   | 4.3249          |
+| 4.1873        | 0.8853  | 3000  | 0.3078   | 4.1363          |
+| 4.032         | 1.1803  | 4000  | 0.3171   | 4.0304          |
+| 3.9652        | 1.4754  | 5000  | 0.3246   | 3.9570          |
+| 3.91          | 1.7705  | 6000  | 0.3301   | 3.8920          |
+| 3.7859        | 2.0655  | 7000  | 0.3346   | 3.8509          |
+| 3.7943        | 2.3606  | 8000  | 0.3377   | 3.8192          |
+| 3.7767        | 2.6557  | 9000  | 0.3403   | 3.7890          |
+| 3.7617        | 2.9508  | 10000 | 0.3429   | 3.7622          |
+| 3.6815        | 3.2458  | 11000 | 0.3448   | 3.7493          |
+| 3.6721        | 3.5409  | 12000 | 0.3465   | 3.7310          |
+| 3.6757        | 3.8360  | 13000 | 0.3484   | 3.7141          |
+| 3.5722        | 4.1310  | 14000 | 0.3496   | 3.7061          |
+| 3.6101        | 4.4261  | 15000 | 0.3507   | 3.6925          |
+| 3.6088        | 4.7212  | 16000 | 0.3521   | 3.6758          |
+| 3.498         | 5.0162  | 17000 | 0.3532   | 3.6710          |
+| 3.5171        | 5.3113  | 18000 | 0.3537   | 3.6691          |
+| 3.5412        | 5.6064  | 19000 | 0.3549   | 3.6557          |
+| 3.5401        | 5.9015  | 20000 | 0.3560   | 3.6433          |
+| 3.4626        | 6.1965  | 21000 | 0.3564   | 3.6500          |
+| 3.496         | 6.4916  | 22000 | 0.3569   | 3.6400          |
+| 3.5145        | 6.7867  | 23000 | 0.3579   | 3.6295          |
+| 3.4109        | 7.0817  | 24000 | 0.3583   | 3.6361          |
+| 3.4462        | 7.3768  | 25000 | 0.3589   | 3.6272          |
+| 3.468         | 7.6719  | 26000 | 0.3597   | 3.6187          |
+| 3.4715        | 7.9670  | 27000 | 0.3602   | 3.6087          |
+| 3.3947        | 8.2620  | 28000 | 0.3596   | 3.6230          |
+| 3.4164        | 8.5571  | 29000 | 0.3605   | 3.6127          |
+| 3.4344        | 8.8522  | 30000 | 0.3616   | 3.6028          |
+| 3.3535        | 9.1472  | 31000 | 0.3610   | 3.6133          |
+| 3.3903        | 9.4423  | 32000 | 0.3617   | 3.6059          |
+| 3.4025        | 9.7374  | 33000 | 0.3622   | 3.5982          |
+| 3.3045        | 10.0325 | 34000 | 0.3623   | 3.6054          |
+| 3.3549        | 10.3275 | 35000 | 0.3626   | 3.6024          |
+| 3.3632        | 10.6226 | 36000 | 0.3636   | 3.5924          |
+| 3.3807        | 10.9177 | 37000 | 0.3638   | 3.5868          |
+| 3.317         | 11.2128 | 38000 | 0.3635   | 3.5983          |
+| 3.3479        | 11.5078 | 39000 | 0.3641   | 3.5922          |
+| 3.3453        | 11.8029 | 40000 | 0.3647   | 3.5839          |
+| 3.2688        | 12.0980 | 41000 | 0.3646   | 3.5886          |
+| 3.2927        | 12.3931 | 42000 | 0.3645   | 3.5906          |
+| 3.3313        | 12.6881 | 43000 | 0.3653   | 3.5813          |
+| 3.3281        | 12.9832 | 44000 | 0.3658   | 3.5743          |
+| 3.2765        | 13.2783 | 45000 | 0.3652   | 3.5870          |
+| 3.2919        | 13.5734 | 46000 | 0.3656   | 3.5816          |
+| 3.2944        | 13.8684 | 47000 | 0.3661   | 3.5754          |
+| 3.2396        | 14.1635 | 48000 | 0.3658   | 3.5825          |
+| 3.2595        | 14.4586 | 49000 | 0.3663   | 3.5805          |
+| 3.2607        | 14.7537 | 50000 | 0.3671   | 3.5703          |
+| 3.1915        | 15.0487 | 51000 | 0.3665   | 3.5820          |
+| 3.2424        | 15.3438 | 52000 | 0.3670   | 3.5767          |
+| 3.2413        | 15.6389 | 53000 | 0.3672   | 3.5730          |
+| 3.2607        | 15.9340 | 54000 | 0.3676   | 3.5679          |
+| 3.2079        | 16.2290 | 55000 | 0.3674   | 3.5760          |
+| 3.2199        | 16.5241 | 56000 | 0.3677   | 3.5726          |
+| 3.2205        | 16.8192 | 57000 | 0.3681   | 3.5679          |
+| 3.177         | 17.1142 | 58000 | 0.3677   | 3.5774          |
+| 3.1933        | 17.4093 | 59000 | 0.3682   | 3.5734          |
+| 3.1919        | 17.7044 | 60000 | 0.3682   | 3.5713          |
+| 3.1911        | 17.9995 | 61000 | 0.3688   | 3.5647          |
+| 3.1667        | 18.2945 | 62000 | 0.3684   | 3.5713          |
+| 3.1737        | 18.5896 | 63000 | 0.3687   | 3.5695          |
+| 3.1813        | 18.8847 | 64000 | 0.3690   | 3.5655          |
+| 3.1399        | 19.1797 | 65000 | 0.3690   | 3.5684          |
+| 3.1481        | 19.4748 | 66000 | 0.3692   | 3.5671          |
+| 3.1488        | 19.7699 | 67000 | 0.3692   | 3.5656          |
+| 3.2411        | 20.0649 | 68000 | 3.6358   | 0.3624          |
+| 3.3289        | 20.3600 | 69000 | 3.6231   | 0.3618          |
+| 3.3478        | 20.6551 | 70000 | 3.6162   | 0.3622          |
+| 3.3725        | 20.9502 | 71000 | 3.6033   | 0.3627          |
+| 3.3007        | 21.2452 | 72000 | 3.6182   | 0.3622          |
+| 3.3419        | 21.5403 | 73000 | 3.6106   | 0.3627          |
+| 3.3618        | 21.8354 | 74000 | 3.6021   | 0.3633          |
+| 3.2813        | 22.1304 | 75000 | 3.6149   | 0.3629          |
+| 3.3123        | 22.4255 | 76000 | 3.6065   | 0.3634          |
+| 3.3259        | 22.7206 | 77000 | 3.5958   | 0.3639          |
+| 3.228         | 23.0156 | 78000 | 3.6096   | 0.3632          |
+| 3.2912        | 23.3107 | 79000 | 3.6097   | 0.3633          |
+| 3.3226        | 23.6058 | 80000 | 3.5983   | 0.3639          |
+| 3.3181        | 23.9009 | 81000 | 3.5953   | 0.3639          |
+| 3.2561        | 24.1959 | 82000 | 3.6054   | 0.3641          |
+| 3.2976        | 24.4910 | 83000 | 3.6010   | 0.3642          |
+| 3.3102        | 24.7861 | 84000 | 3.5936   | 0.3650          |
+| 3.2345        | 25.0811 | 85000 | 3.6085   | 0.3645          |
+| 3.2577        | 25.3762 | 86000 | 3.6016   | 0.3643          |
+| 3.2873        | 25.6713 | 87000 | 3.5933   | 0.3648          |
+
+
+### Framework versions
+
+- Transformers 4.55.2
+- Pytorch 2.8.0+cu128
+- Datasets 4.0.0
+- Tokenizers 0.21.4

exp1b/100M_40817/all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 25.67132908404155,
+    "eval_accuracy": 0.36708937090840316,
+    "eval_loss": 3.570321559906006,
+    "eval_runtime": 179.5812,
+    "eval_samples": 16421,
+    "eval_samples_per_second": 91.441,
+    "eval_steps_per_second": 5.719,
+    "perplexity": 35.52801570040239,
+    "total_flos": 1.8184096382976e+18,
+    "train_loss": 0.7298843362742456,
+    "train_runtime": 38440.3857,
+    "train_samples": 271092,
+    "train_samples_per_second": 352.614,
+    "train_steps_per_second": 4.408
+}

exp1b/100M_40817/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.55.2",
+  "use_cache": true,
+  "vocab_size": 50257
+}

exp1b/100M_40817/eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 25.67132908404155,
+    "eval_accuracy": 0.36708937090840316,
+    "eval_loss": 3.570321559906006,
+    "eval_runtime": 179.5812,
+    "eval_samples": 16421,
+    "eval_samples_per_second": 91.441,
+    "eval_steps_per_second": 5.719,
+    "perplexity": 35.52801570040239
+}

exp1b/100M_40817/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.55.2"
+}

exp1b/100M_40817/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fef642a9a34d69ab2e47bea91ff909c2d140b668a681815b6a156ed7803ed473
+size 497774208