Upload training_config.yaml with huggingface_hub

training_config.yaml

@@ -0,0 +1,62 @@
+model:
+  n_layer: 4
+  d_model: 4096
+  n_ctx: 512
+  d_head: 16
+  d_mlp: 16384
+  vocab_size: 4096
+  use_rms_norm: true
+  tie_embeddings: false
+  use_positional_embeddings: false
+  use_bigram_table: false
+  use_attention_sinks: true
+  activation: gelu
+  dropout: 0.0
+  use_bias: true
+  use_flash_attention: true
+sparsity:
+  enable_weight_sparsity: true
+  target_l0_fraction: 0.0039
+  sparsity_anneal_start_fraction: 0.01
+  sparsity_anneal_end_fraction: 0.5
+  min_weights_per_neuron: 4
+  enable_activation_sparsity: true
+  activation_topk_fraction: 0.25
+  activation_sparsity_locations: attn_in,attn_out,mlp_in,mlp_out,mlp_neuron,attn_v,attn_k,attn_q
+optimizer:
+  learning_rate: 0.001
+  beta1: 0.9
+  beta2: 0.95
+  weight_decay: 0.1
+  eps: 0.1
+  enable_grad_clip: true
+  grad_clip_rms: 1.0
+  warmup_fraction: 0.01
+  enable_lr_decay: true
+  use_sharkfin_schedule: false
+training:
+  dataset_name: SimpleStories/SimpleStories
+  dataset_split: train
+  text_column: story
+  tokenizer_name: SimpleStories/SimpleStories-1.25M
+  total_tokens: 2000000000
+  batch_size: 128
+  gradient_accumulation_steps: 1
+  mixed_precision: bf16
+  checkpoint_dir: checkpoints
+  checkpoint_every_n_steps: 1000
+  keep_n_checkpoints: 5
+  log_every_n_steps: 10
+  log_gradients_every_n_steps: 100
+  log_weights_every_n_steps: 100
+  log_sparsity_every_n_steps: 100
+  eval_every_n_steps: 20
+  val_split: test
+  val_holdout_fraction: 0.01
+  val_max_batches: 20
+  wandb_project: my_sparsity
+  wandb_run_name: d4096_f0.0039
+  wandb_entity: null
+  use_wandb: true
+  seed: 0
+  hf_repo: jacobcd52/ss_d4096_f0.0039