Hugging Face's logo

cmeister
/
tokenizer-lm-ablations

Text Generation
Safetensors
tokenization
tokenizer-ablation
research
multilingual
code
Model card Files
xet
 Community
tokenizer-lm-ablations
File size: 2,943 Bytes
6275c4d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
{
  "run": "full-128k-apertus-mathcode-scratch",
  "source_checkpoint": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-apertus-mathcode-scratch",
  "source_model_file": "model_019073.pt",
  "source_step": 19073,
  "source_meta": {
    "step": 19073,
    "val_bpb": 0.3368332422686206,
    "model_config": {
      "sequence_len": 2048,
      "vocab_size": 131072,
      "n_layer": 24,
      "n_head": 12,
      "n_kv_head": 12,
      "n_embd": 1536,
      "window_pattern": "SSSL",
      "ve_dim": 128,
      "logit_softcap": 15.0
    },
    "tokenizer_path": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/tokenizers/apertus",
    "training_config": {
      "name": "full-128k-mathcode-scratch",
      "model": {
        "n_layer": 24,
        "n_embd": 1536,
        "n_kv_head": 12,
        "sequence_len": 2048,
        "window_pattern": "SSSL",
        "ve_dim": 128
      },
      "training": {
        "target_param_data_ratio": -1,
        "max_tokens": 20000000000,
        "num_iterations": -1,
        "total_batch_size": 1048576,
        "device_batch_size": 16,
        "warmup_steps": 40,
        "warmdown_ratio": 0.65,
        "final_lr_frac": 0.05,
        "eval_tokens": 41943040,
        "compile": true
      },
      "optimizer": {
        "matrix_lr": 0.02,
        "embedding_lr": 0.3,
        "unembedding_lr": 0.008,
        "scalar_lr": 0.5,
        "weight_decay": 0.28,
        "width_lr_exponent": -1.0
      },
      "data": {
        "data_dir": null
      },
      "logging": {
        "eval_every": 500,
        "save_every": 500,
        "max_checkpoints": 0
      },
      "wandb": {
        "enabled": true,
        "project": "tokenizer-lm-experiments",
        "run_name": null
      },
      "output_dir": "/capstor/scratch/cscs/${USER}/tokenizer-lm/checkpoints",
      "seed": 42
    },
    "total_batch_size": 1048576,
    "device_batch_size": 16,
    "max_seq_len": 2048,
    "dataloader_state_dict": {
      "pq_idx": 245,
      "rg_idx": 24,
      "epoch": 2
    },
    "loop_state": {
      "min_val_bpb": 0.3368332422686206,
      "smooth_train_loss": 0.9292727896131538,
      "total_training_time": 94543.93335485458,
      "total_bytes_consumed": 133241384072
    }
  },
  "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)",
  "weight_dtype_counts": {
    "torch.float32": 174,
    "torch.bfloat16": 13
  },
  "total_parameters": 1285818122,
  "logit_softcap_source": "meta",
  "bos_token": "<s>",
  "bos_token_id": 1,
  "bos_resolved_from": "local mirror apertus (not shipped)",
  "tokenizer_slug": "apertus",
  "tokenizer_redistributed": false,
  "tokenizer_hub_id": "swiss-ai/Apertus-70B-2509",
  "weight_key_mapping": "strip _orig_mod. and transformer. prefixes",
  "converted_by": "scripts/hf_release/convert_for_release.py",
  "repo_commit": "1a9a718",
  "nanochat_commit": "f919449",
  "nanochat_branch": "tokenizer-lm-patches"
}