Upload 3 files

12-layers/data_config.json

@@ -0,0 +1,16 @@
+{
+  "num_registers": 4,
+  "modulus": 16,
+  "max_program_len": 16,
+  "max_exec_steps": 20,
+  "num_samples": 1000000,
+  "max_loops": 0,
+  "max_nesting_depth": 0,
+  "mode": "direct_to_final",
+  "include_initial_state": true,
+  "enable_add_sub": true,
+  "enable_cp": true,
+  "enable_load": true,
+  "enable_loops": false,
+  "seed": 42
+}

12-layers/model_config.json

@@ -0,0 +1,10 @@
+{
+  "vocab_size": -1,
+  "d_model": 768,
+  "n_heads": 12,
+  "n_layers": 12,
+  "max_len": 512,
+  "rope_base": 1000.0,
+  "dropout_rate": 0.0,
+  "pad_token_id": -1
+}

12-layers/train_config.json

@@ -0,0 +1,30 @@
+{
+  "device": "cuda",
+  "batch_size": 64,
+  "max_iters": 50000,
+  "learning_rate": 0.0003,
+  "weight_decay": 0.01,
+  "beta1": 0.9,
+  "beta2": 0.999,
+  "grad_clip": 1.0,
+  "decay_lr": true,
+  "warmup_iters": 1000,
+  "lr_decay_iters": 15000,
+  "min_lr": 3e-05,
+  "log_interval": 50,
+  "eval_interval": 500,
+  "eval_iters": 100,
+  "seed": 42,
+  "train_split": 0.8,
+  "use_wandb": true,
+  "wandb_project": "assembly-gpt",
+  "wandb_entity": null,
+  "wandb_run_name": null,
+  "wandb_tags": [],
+  "wandb_notes": null,
+  "base_exp_dir": "/user/jalal.naghiyev/u13581/disk_space/looped_exp",
+  "exp_name": null,
+  "save_every": 1000,
+  "keep_last_n_checkpoints": 5,
+  "save_path": null
+}