Hugging Face's logo

chtugha
/
moshi-german-lora

TensorBoard
Model card Files
xet
 Metrics Community
chtugha commited on
Commit
4917d64
·
verified ·
1 Parent(s): 6e6a9f4

Upload 10 files

Browse files
Files changed (10) hide show
  1. args.yaml +45 -0
  2. checkpoints/checkpoint_002000/consolidated/config.json +51 -0
  3. checkpoints/checkpoint_002000/consolidated/lora.safetensors +3 -0
  4. checkpoints/checkpoint_002500/consolidated/config.json +51 -0
  5. checkpoints/checkpoint_002500/consolidated/lora.safetensors +3 -0
  6. checkpoints/checkpoint_003000/consolidated/config.json +51 -0
  7. checkpoints/checkpoint_003000/consolidated/lora.safetensors +3 -0
  8. metrics.train.jsonl +0 -0
  9. tb/events.out.tfevents.1777658430.modal.6.0.train +3 -0
  10. tb/events.out.tfevents.1777658430.modal.6.1.eval +3 -0
args.yaml ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ batch_size: 8
2
+ ckpt_freq: 500
3
+ data:
4
+ eval_data: ''
5
+ shuffle: true
6
+ train_data: /data/train.jsonl
7
+ do_ckpt: true
8
+ do_eval: false
9
+ duration_sec: 80.0
10
+ eval_freq: 300
11
+ first_codebook_weight_multiplier: 100.0
12
+ full_finetuning: false
13
+ gradient_checkpointing: true
14
+ log_freq: 10
15
+ lora:
16
+ enable: true
17
+ ft_embed: false
18
+ rank: 64
19
+ scaling: 2.0
20
+ max_norm: 1.0
21
+ max_steps: 3000
22
+ moshi_paths:
23
+ config_path: null
24
+ hf_repo_id: kyutai/moshiko-pytorch-bf16
25
+ mimi_path: null
26
+ moshi_path: null
27
+ tokenizer_path: null
28
+ num_ckpt_keep: 3
29
+ num_microbatches: 1
30
+ optim:
31
+ lr: 2.0e-06
32
+ pct_start: 0.05
33
+ weight_decay: 0.1
34
+ overwrite_run_dir: true
35
+ param_dtype: bfloat16
36
+ run_dir: /checkpoints/run_001
37
+ save_adapters: true
38
+ seed: 42
39
+ text_padding_weight: 0.5
40
+ wandb:
41
+ key: null
42
+ offline: false
43
+ project: null
44
+ run_name: null
45
+ world_size: 1
checkpoints/checkpoint_002000/consolidated/config.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dim": 4096,
3
+ "text_card": 32000,
4
+ "existing_text_padding_id": 3,
5
+ "n_q": 16,
6
+ "dep_q": 8,
7
+ "card": 2048,
8
+ "num_heads": 32,
9
+ "num_layers": 32,
10
+ "hidden_scale": 4.125,
11
+ "causal": true,
12
+ "layer_scale": null,
13
+ "context": 3000,
14
+ "max_period": 10000,
15
+ "gating": "silu",
16
+ "norm": "rms_norm_f32",
17
+ "positional_embedding": "rope",
18
+ "depformer_dim": 1024,
19
+ "depformer_dim_feedforward": 4224,
20
+ "depformer_num_heads": 16,
21
+ "depformer_num_layers": 6,
22
+ "depformer_layer_scale": null,
23
+ "depformer_multi_linear": true,
24
+ "depformer_context": 8,
25
+ "depformer_max_period": 10000,
26
+ "depformer_gating": "silu",
27
+ "depformer_pos_emb": "none",
28
+ "depformer_weights_per_step": true,
29
+ "delays": [
30
+ 0,
31
+ 0,
32
+ 1,
33
+ 1,
34
+ 1,
35
+ 1,
36
+ 1,
37
+ 1,
38
+ 1,
39
+ 0,
40
+ 1,
41
+ 1,
42
+ 1,
43
+ 1,
44
+ 1,
45
+ 1,
46
+ 1
47
+ ],
48
+ "lora": true,
49
+ "lora_rank": 64,
50
+ "lora_scaling": 2.0
51
+ }
checkpoints/checkpoint_002000/consolidated/lora.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:85ad3f8dd978042689831514fda445c6b1b2297b9c29b8b15c3027ad0fb6fa6f
3
+ size 387959568
checkpoints/checkpoint_002500/consolidated/config.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dim": 4096,
3
+ "text_card": 32000,
4
+ "existing_text_padding_id": 3,
5
+ "n_q": 16,
6
+ "dep_q": 8,
7
+ "card": 2048,
8
+ "num_heads": 32,
9
+ "num_layers": 32,
10
+ "hidden_scale": 4.125,
11
+ "causal": true,
12
+ "layer_scale": null,
13
+ "context": 3000,
14
+ "max_period": 10000,
15
+ "gating": "silu",
16
+ "norm": "rms_norm_f32",
17
+ "positional_embedding": "rope",
18
+ "depformer_dim": 1024,
19
+ "depformer_dim_feedforward": 4224,
20
+ "depformer_num_heads": 16,
21
+ "depformer_num_layers": 6,
22
+ "depformer_layer_scale": null,
23
+ "depformer_multi_linear": true,
24
+ "depformer_context": 8,
25
+ "depformer_max_period": 10000,
26
+ "depformer_gating": "silu",
27
+ "depformer_pos_emb": "none",
28
+ "depformer_weights_per_step": true,
29
+ "delays": [
30
+ 0,
31
+ 0,
32
+ 1,
33
+ 1,
34
+ 1,
35
+ 1,
36
+ 1,
37
+ 1,
38
+ 1,
39
+ 0,
40
+ 1,
41
+ 1,
42
+ 1,
43
+ 1,
44
+ 1,
45
+ 1,
46
+ 1
47
+ ],
48
+ "lora": true,
49
+ "lora_rank": 64,
50
+ "lora_scaling": 2.0
51
+ }
checkpoints/checkpoint_002500/consolidated/lora.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ddaa9a803e5f43a46723890bc5aaec388c9ad0b8aa96a13713f2261f9ca92155
3
+ size 387959568
checkpoints/checkpoint_003000/consolidated/config.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dim": 4096,
3
+ "text_card": 32000,
4
+ "existing_text_padding_id": 3,
5
+ "n_q": 16,
6
+ "dep_q": 8,
7
+ "card": 2048,
8
+ "num_heads": 32,
9
+ "num_layers": 32,
10
+ "hidden_scale": 4.125,
11
+ "causal": true,
12
+ "layer_scale": null,
13
+ "context": 3000,
14
+ "max_period": 10000,
15
+ "gating": "silu",
16
+ "norm": "rms_norm_f32",
17
+ "positional_embedding": "rope",
18
+ "depformer_dim": 1024,
19
+ "depformer_dim_feedforward": 4224,
20
+ "depformer_num_heads": 16,
21
+ "depformer_num_layers": 6,
22
+ "depformer_layer_scale": null,
23
+ "depformer_multi_linear": true,
24
+ "depformer_context": 8,
25
+ "depformer_max_period": 10000,
26
+ "depformer_gating": "silu",
27
+ "depformer_pos_emb": "none",
28
+ "depformer_weights_per_step": true,
29
+ "delays": [
30
+ 0,
31
+ 0,
32
+ 1,
33
+ 1,
34
+ 1,
35
+ 1,
36
+ 1,
37
+ 1,
38
+ 1,
39
+ 0,
40
+ 1,
41
+ 1,
42
+ 1,
43
+ 1,
44
+ 1,
45
+ 1,
46
+ 1
47
+ ],
48
+ "lora": true,
49
+ "lora_rank": 64,
50
+ "lora_scaling": 2.0
51
+ }
checkpoints/checkpoint_003000/consolidated/lora.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:50ff9a571adcc9e9ff5d345ec70f437d41c06689800f3ce2612d7d8034259ca0
3
+ size 387959568
metrics.train.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
tb/events.out.tfevents.1777658430.modal.6.0.train ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5559934c7184a94b61ff6b296cc304c737c5cb5908ead322403d58d7b6e538ce
3
+ size 148180
tb/events.out.tfevents.1777658430.modal.6.1.eval ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aab83abba463ea60e01202d3c7cc265f5138f56b6d3cdec5f8ee21b7a300500c
3
+ size 88