pixel_training_config.json

{
  "size": "100m",
  "data_path": "data\\bootstrap\\demo_corpus.txt",
  "output_dir": "checkpoints/pixel_100m",
  "batch_size": 1,
  "grad_accumulation_steps": 2,
  "learning_rate": 0.0003,
  "min_learning_rate": 3e-05,
  "warmup_steps": 4,
  "total_steps": 10,
  "weight_decay": 0.1,
  "max_grad_norm": 1.0,
  "eval_every": 10,
  "save_every": 10,
  "seed": 42,
  "gradient_checkpointing": false,
  "mode": "pretrain",
  "sequence_length": 32,
  "num_workers": 0
}