models/full-128k-bpe-nfc-plus2-balanced-mathcode-scratch/conversion_metadata.json

{
  "run": "full-128k-bpe-nfc-plus2-balanced-mathcode-scratch",
  "source_checkpoint": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-bpe-nfc-plus2-balanced-mathcode-scratch",
  "source_model_file": "model_019073.pt",
  "source_step": 19073,
  "source_meta": {
    "step": 19073,
    "val_bpb": 0.3384368951749275,
    "model_config": {
      "sequence_len": 2048,
      "vocab_size": 128256,
      "n_layer": 24,
      "n_head": 12,
      "n_kv_head": 12,
      "n_embd": 1536,
      "window_pattern": "SSSL",
      "ve_dim": 128,
      "logit_softcap": 15.0
    },
    "tokenizer_path": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/tokenizers/bpe-nfc-plus2-balanced",
    "training_config": {
      "name": "full-128k-mathcode-scratch",
      "model": {
        "n_layer": 24,
        "n_embd": 1536,
        "n_kv_head": 12,
        "sequence_len": 2048,
        "window_pattern": "SSSL",
        "ve_dim": 128
      },
      "training": {
        "target_param_data_ratio": -1,
        "max_tokens": 20000000000,
        "num_iterations": -1,
        "total_batch_size": 1048576,
        "device_batch_size": 16,
        "warmup_steps": 40,
        "warmdown_ratio": 0.65,
        "final_lr_frac": 0.05,
        "eval_tokens": 41943040,
        "compile": true
      },
      "optimizer": {
        "matrix_lr": 0.02,
        "embedding_lr": 0.3,
        "unembedding_lr": 0.008,
        "scalar_lr": 0.5,
        "weight_decay": 0.28,
        "width_lr_exponent": -1.0
      },
      "data": {
        "data_dir": null
      },
      "logging": {
        "eval_every": 500,
        "save_every": 500,
        "max_checkpoints": 0
      },
      "wandb": {
        "enabled": true,
        "project": "tokenizer-lm-experiments",
        "run_name": null
      },
      "output_dir": "/capstor/scratch/cscs/${USER}/tokenizer-lm/checkpoints",
      "seed": 42
    },
    "total_batch_size": 1048576,
    "device_batch_size": 16,
    "max_seq_len": 2048,
    "dataloader_state_dict": {
      "pq_idx": 208,
      "rg_idx": 40,
      "epoch": 2
    },
    "loop_state": {
      "min_val_bpb": 0.3384368951749275,
      "smooth_train_loss": 0.7209605803493717,
      "total_training_time": 92953.48900318146,
      "total_bytes_consumed": 151753084755
    }
  },
  "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)",
  "weight_dtype_counts": {
    "torch.float32": 174,
    "torch.bfloat16": 13
  },
  "total_parameters": 1272841994,
  "logit_softcap_source": "meta",
  "bos_token": "<s>",
  "bos_token_id": 0,
  "bos_resolved_from": "packaged tokenizer",
  "tokenizer_slug": "bpe-nfc-plus2-balanced",
  "tokenizer_redistributed": true,
  "tokenizer_hub_id": null,
  "weight_key_mapping": "strip _orig_mod. and transformer. prefixes",
  "converted_by": "scripts/hf_release/convert_for_release.py",
  "repo_commit": "1a9a718",
  "nanochat_commit": "f919449",
  "nanochat_branch": "tokenizer-lm-patches"
}