Upload folder using huggingface_hub

beeper_final.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:053d16f81c5f0091fedc994f6901c890291df9c5ce8e0f102fdf73fd7609150b
+size 110234924

beeper_final.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4148b1b7b7646abec817f34d1da0349068540de25b0efa75979049b067a5691
+size 110221080

config.json

@@ -0,0 +1,47 @@
+{
+  "name": "Beeper",
+  "dataset_name": "roneneldan/TinyStories",
+  "dataset_split": "train[:10%]",
+  "context": 512,
+  "vocab_size": 8192,
+  "add_bos_eos": true,
+  "val_ratio": 0.01,
+  "test_ratio": 0.01,
+  "dim": 512,
+  "n_layers": 6,
+  "n_heads": 8,
+  "mlp_ratio": 4.0,
+  "dropout": 0.0,
+  "resid_dropout": 0.1,
+  "grad_checkpoint": false,
+  "compile_model": true,
+  "batch_size": 32,
+  "grad_accum_steps": 1,
+  "epochs": 3,
+  "lr": 0.0003,
+  "betas": [
+    0.9,
+    0.95
+  ],
+  "weight_decay": 0.1,
+  "warmup_steps": 500,
+  "max_steps": null,
+  "clip_grad": 1.0,
+  "min_lr": 1e-06,
+  "label_smoothing": 0.0,
+  "mixed_precision": "bf16",
+  "log_dir": "./runs/beeper",
+  "log_interval": 50,
+  "ckpt_dir": "./beeper_checkpoints",
+  "export_dir": "./beeper_export",
+  "temperature": 0.9,
+  "top_k": 40,
+  "top_p": 0.9,
+  "repetition_penalty": 1.1,
+  "presence_penalty": 0.6,
+  "frequency_penalty": 0.0,
+  "upload_to_hub": true,
+  "hf_repo": "AbstractPhil/beeper-tinystories-6l-512d-ctx512",
+  "tokenizer_path": "beeper.tokenizer.json",
+  "seed": 1337
+}