config.json

{
  "aux_loss_alpha": 0.02,
  "betas": [
    0.9,
    0.95
  ],
  "ckpt_dir": "artifacts/checkpoint",
  "config_filename": "config.json",
  "d_model": 512,
  "dataset_config": null,
  "dataset_name": "HuggingFaceFW/fineweb",
  "dataset_split": "train",
  "expert_hidden": 768,
  "grad_accum_steps": 16,
  "grad_clip": 1.0,
  "latest_bundle_dir": "artifacts/latest_bundle",
  "log_every_steps": 20,
  "lr": 0.00025,
  "max_seq_len": 256,
  "max_steps_this_run": 4000,
  "micro_batch_size": 1,
  "min_text_chars": 128,
  "model_filename": "model.safetensors",
  "moe_noise_std": 0.02,
  "n_heads": 8,
  "n_layers": 6,
  "optimizer_filename": "optimizer.pt",
  "quality_fields": [
    "quality_score",
    "score",
    "quality",
    "fw_quality",
    "document_quality"
  ],
  "readme_filename": "README.md",
  "repo_id": "wop/MoE-model-test-kaggle",
  "repo_type": "model",
  "rng_filename": "rng.pt",
  "router_grad_clip": 0.5,
  "router_lr_scale": 1.0,
  "run_rows_target": 10000,
  "save_every_steps": 200,
  "scaler_filename": "scaler.pt",
  "scan_rows_per_run": 25000,
  "scheduler_filename": "scheduler.pt",
  "shared_num_experts": 100,
  "state_filename": "state.json",
  "text_fields": [
    "text",
    "content",
    "raw_content",
    "markdown",
    "body"
  ],
  "top_k_experts": 12,
  "use_amp": true,
  "vocab_size": 32000,
  "warmup_steps": 100,
  "weight_decay": 0.1,
  "work_dir": "artifacts"
}