| { |
| "run": "full-128k-apertus-mathcode-scratch", |
| "source_checkpoint": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/checkpoints/full-128k-apertus-mathcode-scratch", |
| "source_model_file": "model_019073.pt", |
| "source_step": 19073, |
| "source_meta": { |
| "step": 19073, |
| "val_bpb": 0.3368332422686206, |
| "model_config": { |
| "sequence_len": 2048, |
| "vocab_size": 131072, |
| "n_layer": 24, |
| "n_head": 12, |
| "n_kv_head": 12, |
| "n_embd": 1536, |
| "window_pattern": "SSSL", |
| "ve_dim": 128, |
| "logit_softcap": 15.0 |
| }, |
| "tokenizer_path": "/capstor/store/cscs/swissai/a139/cmeister/tokenizer-lm/tokenizers/apertus", |
| "training_config": { |
| "name": "full-128k-mathcode-scratch", |
| "model": { |
| "n_layer": 24, |
| "n_embd": 1536, |
| "n_kv_head": 12, |
| "sequence_len": 2048, |
| "window_pattern": "SSSL", |
| "ve_dim": 128 |
| }, |
| "training": { |
| "target_param_data_ratio": -1, |
| "max_tokens": 20000000000, |
| "num_iterations": -1, |
| "total_batch_size": 1048576, |
| "device_batch_size": 16, |
| "warmup_steps": 40, |
| "warmdown_ratio": 0.65, |
| "final_lr_frac": 0.05, |
| "eval_tokens": 41943040, |
| "compile": true |
| }, |
| "optimizer": { |
| "matrix_lr": 0.02, |
| "embedding_lr": 0.3, |
| "unembedding_lr": 0.008, |
| "scalar_lr": 0.5, |
| "weight_decay": 0.28, |
| "width_lr_exponent": -1.0 |
| }, |
| "data": { |
| "data_dir": null |
| }, |
| "logging": { |
| "eval_every": 500, |
| "save_every": 500, |
| "max_checkpoints": 0 |
| }, |
| "wandb": { |
| "enabled": true, |
| "project": "tokenizer-lm-experiments", |
| "run_name": null |
| }, |
| "output_dir": "/capstor/scratch/cscs/${USER}/tokenizer-lm/checkpoints", |
| "seed": 42 |
| }, |
| "total_batch_size": 1048576, |
| "device_batch_size": 16, |
| "max_seq_len": 2048, |
| "dataloader_state_dict": { |
| "pq_idx": 245, |
| "rg_idx": 24, |
| "epoch": 2 |
| }, |
| "loop_state": { |
| "min_val_bpb": 0.3368332422686206, |
| "smooth_train_loss": 0.9292727896131538, |
| "total_training_time": 94543.93335485458, |
| "total_bytes_consumed": 133241384072 |
| } |
| }, |
| "weight_dtype": "as trained (float32 matrices, bfloat16 embeddings)", |
| "weight_dtype_counts": { |
| "torch.float32": 174, |
| "torch.bfloat16": 13 |
| }, |
| "total_parameters": 1285818122, |
| "logit_softcap_source": "meta", |
| "bos_token": "<s>", |
| "bos_token_id": 1, |
| "bos_resolved_from": "local mirror apertus (not shipped)", |
| "tokenizer_slug": "apertus", |
| "tokenizer_redistributed": false, |
| "tokenizer_hub_id": "swiss-ai/Apertus-70B-2509", |
| "weight_key_mapping": "strip _orig_mod. and transformer. prefixes", |
| "converted_by": "scripts/hf_release/convert_for_release.py", |
| "repo_commit": "1a9a718", |
| "nanochat_commit": "f919449", |
| "nanochat_branch": "tokenizer-lm-patches" |
| } |
|
|
