Hugging Face's logo

rubentium
/
sparse-checkpoints

Model card Files
xet
 Community
rubentium commited on
Commit
1e1d716
·
verified ·
1 Parent(s): f72ef86

EPFL ALiBi and training provenance 1-50 of 1184

Browse files
Files changed (50) hide show
  1. epfl_final/configs_dropctl/a100_40gb_vram_probe_p28.json +67 -0
  2. epfl_final/configs_dropctl/calib_p00.json +68 -0
  3. epfl_final/configs_dropctl/calib_p05.json +68 -0
  4. epfl_final/configs_dropctl/calib_p10.json +68 -0
  5. epfl_final/configs_dropctl/calib_p20.json +68 -0
  6. epfl_final/configs_dropctl/calib_p28_verify.json +68 -0
  7. epfl_final/configs_dropctl/calib_p30_verify.json +68 -0
  8. epfl_final/configs_dropctl/diag_p28_20.json +68 -0
  9. epfl_final/configs_dropctl/full_p28/seed42.json +67 -0
  10. epfl_final/configs_dropctl/full_p28/seed43.json +67 -0
  11. epfl_final/configs_dropctl/full_p28/seed44.json +67 -0
  12. epfl_final/configs_dropctl/full_p28/seed45.json +67 -0
  13. epfl_final/configs_dropctl/full_p28/seed46.json +67 -0
  14. epfl_final/configs_dropctl/resume_p28/seed42.json +67 -0
  15. epfl_final/configs_dropctl/resume_p28/seed43.json +67 -0
  16. epfl_final/configs_dropctl/resume_p28/seed44.json +67 -0
  17. epfl_final/models/ALIBI_20260921/1p2b_alibi_s42/run_20260922_004519/checkpoint-17000/config.json +38 -0
  18. epfl_final/models/ALIBI_20260921/1p2b_alibi_s42/run_20260922_004519/checkpoint-17000/loader_gen_state_rank0.pt +3 -0
  19. epfl_final/models/ALIBI_20260921/1p2b_alibi_s42/run_20260922_004519/checkpoint-17000/loader_gen_state_rank1.pt +3 -0
  20. epfl_final/models/ALIBI_20260921/1p2b_alibi_s42/run_20260922_004519/checkpoint-17000/loader_gen_state_rank2.pt +3 -0
  21. epfl_final/models/ALIBI_20260921/1p2b_alibi_s42/run_20260922_004519/checkpoint-17000/model.pt +3 -0
  22. epfl_final/models/ALIBI_20260921/1p2b_alibi_s42/run_20260922_004519/checkpoint-17000/rng_state_rank0.pt +3 -0
  23. epfl_final/models/ALIBI_20260921/1p2b_alibi_s42/run_20260922_004519/checkpoint-17000/rng_state_rank1.pt +3 -0
  24. epfl_final/models/ALIBI_20260921/1p2b_alibi_s42/run_20260922_004519/checkpoint-17000/rng_state_rank2.pt +3 -0
  25. epfl_final/models/ALIBI_20260921/1p2b_alibi_s42/run_20260922_004519/checkpoint-17000/training_state.pt +3 -0
  26. epfl_final/models/ALIBI_20260921/365m_alibi_s42/run_20260921_142951/checkpoint-44000/config.json +38 -0
  27. epfl_final/models/ALIBI_20260921/365m_alibi_s42/run_20260921_142951/checkpoint-44000/model.pt +3 -0
  28. epfl_final/models/ALIBI_20260921/365m_alibi_s43/run_20260921_142951/checkpoint-44000/config.json +38 -0
  29. epfl_final/models/ALIBI_20260921/365m_alibi_s43/run_20260921_142951/checkpoint-44000/model.pt +3 -0
  30. epfl_final/models/ALIBI_20260921/365m_alibi_s44/run_20260921_142951/checkpoint-44000/config.json +38 -0
  31. epfl_final/models/ALIBI_20260921/365m_alibi_s44/run_20260921_142951/checkpoint-44000/model.pt +3 -0
  32. epfl_final/models/ALIBI_20260921/365m_alibi_s45/run_20260921_142951/checkpoint-44000/config.json +38 -0
  33. epfl_final/models/ALIBI_20260921/365m_alibi_s45/run_20260921_142951/checkpoint-44000/model.pt +3 -0
  34. epfl_final/models/ALIBI_20260921/365m_alibi_s46/run_20260921_142951/checkpoint-44000/config.json +38 -0
  35. epfl_final/models/ALIBI_20260921/365m_alibi_s46/run_20260921_142951/checkpoint-44000/model.pt +3 -0
  36. epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/config.json +38 -0
  37. epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/loader_gen_state_rank0.pt +3 -0
  38. epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/loader_gen_state_rank1.pt +3 -0
  39. epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/loader_gen_state_rank2.pt +3 -0
  40. epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/loader_gen_state_rank3.pt +3 -0
  41. epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/loader_gen_state_rank4.pt +3 -0
  42. epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/model.pt +3 -0
  43. epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/rng_state_rank0.pt +3 -0
  44. epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/rng_state_rank1.pt +3 -0
  45. epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/rng_state_rank2.pt +3 -0
  46. epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/rng_state_rank3.pt +3 -0
  47. epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/rng_state_rank4.pt +3 -0
  48. epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/training_state.pt +3 -0
  49. epfl_final/models/ALIBI_20260921/365m_alibi_s47_restart_20260922/run_20260922_192827/checkpoint-44000/config.json +40 -0
  50. epfl_final/models/ALIBI_20260921/365m_alibi_s47_restart_20260922/run_20260922_192827/checkpoint-44000/model.pt +3 -0
epfl_final/configs_dropctl/a100_40gb_vram_probe_p28.json ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": {
3
+ "path": "/mloscratch/homes/navasard/qwen3_06b_base",
4
+ "dtype": "bfloat16",
5
+ "init_from_scratch": true,
6
+ "hidden_size": 768,
7
+ "intermediate_size": 3072,
8
+ "num_hidden_layers": 28,
9
+ "num_attention_heads": 6,
10
+ "num_key_value_heads": 2,
11
+ "attention_dropout": 0.28
12
+ },
13
+ "data": {
14
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
15
+ "val_path": null,
16
+ "max_seq_length": 8192,
17
+ "num_workers": 8
18
+ },
19
+ "training": {
20
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/a100_40gb_vram_probe_p28",
21
+ "force_new_run": true,
22
+ "num_epochs": 1,
23
+ "max_steps": 2,
24
+ "per_device_batch_size": 2,
25
+ "gradient_accumulation_steps": 10,
26
+ "learning_rate": 0.0005,
27
+ "weight_decay": 0.01,
28
+ "warmup_steps": 750,
29
+ "lr_scheduler": "cosine",
30
+ "max_grad_norm": 1.0,
31
+ "save_steps": 999999,
32
+ "eval_steps": 999999,
33
+ "eval_batches": 50,
34
+ "logging_steps": 1,
35
+ "seed": 42,
36
+ "verbose": true,
37
+ "checkpoint_activations": false
38
+ },
39
+ "distributed": {
40
+ "strategy": "ddp",
41
+ "mixed_precision": "bf16"
42
+ },
43
+ "wandb": {
44
+ "enabled": false,
45
+ "project": "stochastic_sparsity",
46
+ "name": "l8k365m_dense_dropout_p28_seed42",
47
+ "tags": [
48
+ "qwen3",
49
+ "365m",
50
+ "8k",
51
+ "scaleladder",
52
+ "dense"
53
+ ]
54
+ },
55
+ "attention": {
56
+ "type": "sparse_stochastic",
57
+ "dense_mode": true,
58
+ "sparse_block_size": 128,
59
+ "sparse_num_random_blocks": 31,
60
+ "sparse_mask_pool_size": 1,
61
+ "sparse_val_dense_compare": false,
62
+ "dense_step_intervals": null,
63
+ "sparse_layer_indices": null,
64
+ "sparse_mask_cache_dir": null,
65
+ "sparse_mask_seed": 0
66
+ }
67
+ }
epfl_final/configs_dropctl/calib_p00.json ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": {
3
+ "path": "/mloscratch/homes/navasard/qwen3_06b_base",
4
+ "dtype": "bfloat16",
5
+ "init_from_scratch": false,
6
+ "hidden_size": 768,
7
+ "intermediate_size": 3072,
8
+ "num_hidden_layers": 28,
9
+ "num_attention_heads": 6,
10
+ "num_key_value_heads": 2,
11
+ "attention_dropout": 0.0
12
+ },
13
+ "data": {
14
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
15
+ "val_path": "/mloscratch/homes/navasard/data/fineweb_val_seed42",
16
+ "max_seq_length": 8192,
17
+ "num_workers": 8
18
+ },
19
+ "training": {
20
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/dropctl_calib_p00",
21
+ "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-44000",
22
+ "force_new_run": true,
23
+ "num_epochs": 1,
24
+ "max_steps": 44300,
25
+ "per_device_batch_size": 2,
26
+ "gradient_accumulation_steps": 10,
27
+ "learning_rate": 5e-05,
28
+ "weight_decay": 0.01,
29
+ "warmup_steps": 0,
30
+ "lr_scheduler": "constant",
31
+ "max_grad_norm": 1.0,
32
+ "save_steps": 300,
33
+ "eval_steps": 75,
34
+ "eval_batches": 50,
35
+ "logging_steps": 10,
36
+ "seed": 42,
37
+ "verbose": true,
38
+ "checkpoint_activations": false
39
+ },
40
+ "distributed": {
41
+ "strategy": "ddp",
42
+ "mixed_precision": "bf16"
43
+ },
44
+ "wandb": {
45
+ "enabled": false,
46
+ "project": "stochastic_sparsity",
47
+ "name": "dropctl_calib_p00",
48
+ "tags": [
49
+ "qwen3",
50
+ "365m",
51
+ "8k",
52
+ "scaleladder",
53
+ "dense"
54
+ ]
55
+ },
56
+ "attention": {
57
+ "type": "sparse_stochastic",
58
+ "dense_mode": true,
59
+ "sparse_block_size": 128,
60
+ "sparse_num_random_blocks": 31,
61
+ "sparse_mask_pool_size": 1,
62
+ "sparse_val_dense_compare": false,
63
+ "dense_step_intervals": null,
64
+ "sparse_layer_indices": null,
65
+ "sparse_mask_cache_dir": null,
66
+ "sparse_mask_seed": 0
67
+ }
68
+ }
epfl_final/configs_dropctl/calib_p05.json ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": {
3
+ "path": "/mloscratch/homes/navasard/qwen3_06b_base",
4
+ "dtype": "bfloat16",
5
+ "init_from_scratch": false,
6
+ "hidden_size": 768,
7
+ "intermediate_size": 3072,
8
+ "num_hidden_layers": 28,
9
+ "num_attention_heads": 6,
10
+ "num_key_value_heads": 2,
11
+ "attention_dropout": 0.05
12
+ },
13
+ "data": {
14
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
15
+ "val_path": "/mloscratch/homes/navasard/data/fineweb_val_seed42",
16
+ "max_seq_length": 8192,
17
+ "num_workers": 8
18
+ },
19
+ "training": {
20
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/dropctl_calib_p05",
21
+ "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-44000",
22
+ "force_new_run": true,
23
+ "num_epochs": 1,
24
+ "max_steps": 44300,
25
+ "per_device_batch_size": 2,
26
+ "gradient_accumulation_steps": 10,
27
+ "learning_rate": 5e-05,
28
+ "weight_decay": 0.01,
29
+ "warmup_steps": 0,
30
+ "lr_scheduler": "constant",
31
+ "max_grad_norm": 1.0,
32
+ "save_steps": 300,
33
+ "eval_steps": 75,
34
+ "eval_batches": 50,
35
+ "logging_steps": 10,
36
+ "seed": 42,
37
+ "verbose": true,
38
+ "checkpoint_activations": false
39
+ },
40
+ "distributed": {
41
+ "strategy": "ddp",
42
+ "mixed_precision": "bf16"
43
+ },
44
+ "wandb": {
45
+ "enabled": false,
46
+ "project": "stochastic_sparsity",
47
+ "name": "dropctl_calib_p05",
48
+ "tags": [
49
+ "qwen3",
50
+ "365m",
51
+ "8k",
52
+ "scaleladder",
53
+ "dense"
54
+ ]
55
+ },
56
+ "attention": {
57
+ "type": "sparse_stochastic",
58
+ "dense_mode": true,
59
+ "sparse_block_size": 128,
60
+ "sparse_num_random_blocks": 31,
61
+ "sparse_mask_pool_size": 1,
62
+ "sparse_val_dense_compare": false,
63
+ "dense_step_intervals": null,
64
+ "sparse_layer_indices": null,
65
+ "sparse_mask_cache_dir": null,
66
+ "sparse_mask_seed": 0
67
+ }
68
+ }
epfl_final/configs_dropctl/calib_p10.json ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": {
3
+ "path": "/mloscratch/homes/navasard/qwen3_06b_base",
4
+ "dtype": "bfloat16",
5
+ "init_from_scratch": false,
6
+ "hidden_size": 768,
7
+ "intermediate_size": 3072,
8
+ "num_hidden_layers": 28,
9
+ "num_attention_heads": 6,
10
+ "num_key_value_heads": 2,
11
+ "attention_dropout": 0.1
12
+ },
13
+ "data": {
14
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
15
+ "val_path": "/mloscratch/homes/navasard/data/fineweb_val_seed42",
16
+ "max_seq_length": 8192,
17
+ "num_workers": 8
18
+ },
19
+ "training": {
20
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/dropctl_calib_p10",
21
+ "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-44000",
22
+ "force_new_run": true,
23
+ "num_epochs": 1,
24
+ "max_steps": 44300,
25
+ "per_device_batch_size": 2,
26
+ "gradient_accumulation_steps": 10,
27
+ "learning_rate": 5e-05,
28
+ "weight_decay": 0.01,
29
+ "warmup_steps": 0,
30
+ "lr_scheduler": "constant",
31
+ "max_grad_norm": 1.0,
32
+ "save_steps": 300,
33
+ "eval_steps": 75,
34
+ "eval_batches": 50,
35
+ "logging_steps": 10,
36
+ "seed": 42,
37
+ "verbose": true,
38
+ "checkpoint_activations": false
39
+ },
40
+ "distributed": {
41
+ "strategy": "ddp",
42
+ "mixed_precision": "bf16"
43
+ },
44
+ "wandb": {
45
+ "enabled": false,
46
+ "project": "stochastic_sparsity",
47
+ "name": "dropctl_calib_p10",
48
+ "tags": [
49
+ "qwen3",
50
+ "365m",
51
+ "8k",
52
+ "scaleladder",
53
+ "dense"
54
+ ]
55
+ },
56
+ "attention": {
57
+ "type": "sparse_stochastic",
58
+ "dense_mode": true,
59
+ "sparse_block_size": 128,
60
+ "sparse_num_random_blocks": 31,
61
+ "sparse_mask_pool_size": 1,
62
+ "sparse_val_dense_compare": false,
63
+ "dense_step_intervals": null,
64
+ "sparse_layer_indices": null,
65
+ "sparse_mask_cache_dir": null,
66
+ "sparse_mask_seed": 0
67
+ }
68
+ }
epfl_final/configs_dropctl/calib_p20.json ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": {
3
+ "path": "/mloscratch/homes/navasard/qwen3_06b_base",
4
+ "dtype": "bfloat16",
5
+ "init_from_scratch": false,
6
+ "hidden_size": 768,
7
+ "intermediate_size": 3072,
8
+ "num_hidden_layers": 28,
9
+ "num_attention_heads": 6,
10
+ "num_key_value_heads": 2,
11
+ "attention_dropout": 0.2
12
+ },
13
+ "data": {
14
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
15
+ "val_path": "/mloscratch/homes/navasard/data/fineweb_val_seed42",
16
+ "max_seq_length": 8192,
17
+ "num_workers": 8
18
+ },
19
+ "training": {
20
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/dropctl_calib_p20",
21
+ "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-44000",
22
+ "force_new_run": true,
23
+ "num_epochs": 1,
24
+ "max_steps": 44300,
25
+ "per_device_batch_size": 2,
26
+ "gradient_accumulation_steps": 10,
27
+ "learning_rate": 5e-05,
28
+ "weight_decay": 0.01,
29
+ "warmup_steps": 0,
30
+ "lr_scheduler": "constant",
31
+ "max_grad_norm": 1.0,
32
+ "save_steps": 300,
33
+ "eval_steps": 75,
34
+ "eval_batches": 50,
35
+ "logging_steps": 10,
36
+ "seed": 42,
37
+ "verbose": true,
38
+ "checkpoint_activations": false
39
+ },
40
+ "distributed": {
41
+ "strategy": "ddp",
42
+ "mixed_precision": "bf16"
43
+ },
44
+ "wandb": {
45
+ "enabled": false,
46
+ "project": "stochastic_sparsity",
47
+ "name": "dropctl_calib_p20",
48
+ "tags": [
49
+ "qwen3",
50
+ "365m",
51
+ "8k",
52
+ "scaleladder",
53
+ "dense"
54
+ ]
55
+ },
56
+ "attention": {
57
+ "type": "sparse_stochastic",
58
+ "dense_mode": true,
59
+ "sparse_block_size": 128,
60
+ "sparse_num_random_blocks": 31,
61
+ "sparse_mask_pool_size": 1,
62
+ "sparse_val_dense_compare": false,
63
+ "dense_step_intervals": null,
64
+ "sparse_layer_indices": null,
65
+ "sparse_mask_cache_dir": null,
66
+ "sparse_mask_seed": 0
67
+ }
68
+ }
epfl_final/configs_dropctl/calib_p28_verify.json ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": {
3
+ "path": "/mloscratch/homes/navasard/qwen3_06b_base",
4
+ "dtype": "bfloat16",
5
+ "init_from_scratch": false,
6
+ "hidden_size": 768,
7
+ "intermediate_size": 3072,
8
+ "num_hidden_layers": 28,
9
+ "num_attention_heads": 6,
10
+ "num_key_value_heads": 2,
11
+ "attention_dropout": 0.28
12
+ },
13
+ "data": {
14
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
15
+ "val_path": "/mloscratch/homes/navasard/data/fineweb_val_seed42",
16
+ "max_seq_length": 8192,
17
+ "num_workers": 8
18
+ },
19
+ "training": {
20
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/dropctl_calib_p28_verify",
21
+ "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-44000",
22
+ "force_new_run": true,
23
+ "num_epochs": 1,
24
+ "max_steps": 44300,
25
+ "per_device_batch_size": 2,
26
+ "gradient_accumulation_steps": 10,
27
+ "learning_rate": 5e-05,
28
+ "weight_decay": 0.01,
29
+ "warmup_steps": 0,
30
+ "lr_scheduler": "constant",
31
+ "max_grad_norm": 1.0,
32
+ "save_steps": 300,
33
+ "eval_steps": 75,
34
+ "eval_batches": 50,
35
+ "logging_steps": 10,
36
+ "seed": 42,
37
+ "verbose": true,
38
+ "checkpoint_activations": false
39
+ },
40
+ "distributed": {
41
+ "strategy": "ddp",
42
+ "mixed_precision": "bf16"
43
+ },
44
+ "wandb": {
45
+ "enabled": false,
46
+ "project": "stochastic_sparsity",
47
+ "name": "dropctl_calib_p28_verify",
48
+ "tags": [
49
+ "qwen3",
50
+ "365m",
51
+ "8k",
52
+ "scaleladder",
53
+ "dense"
54
+ ]
55
+ },
56
+ "attention": {
57
+ "type": "sparse_stochastic",
58
+ "dense_mode": true,
59
+ "sparse_block_size": 128,
60
+ "sparse_num_random_blocks": 31,
61
+ "sparse_mask_pool_size": 1,
62
+ "sparse_val_dense_compare": false,
63
+ "dense_step_intervals": null,
64
+ "sparse_layer_indices": null,
65
+ "sparse_mask_cache_dir": null,
66
+ "sparse_mask_seed": 0
67
+ }
68
+ }
epfl_final/configs_dropctl/calib_p30_verify.json ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": {
3
+ "path": "/mloscratch/homes/navasard/qwen3_06b_base",
4
+ "dtype": "bfloat16",
5
+ "init_from_scratch": false,
6
+ "hidden_size": 768,
7
+ "intermediate_size": 3072,
8
+ "num_hidden_layers": 28,
9
+ "num_attention_heads": 6,
10
+ "num_key_value_heads": 2,
11
+ "attention_dropout": 0.3
12
+ },
13
+ "data": {
14
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
15
+ "val_path": "/mloscratch/homes/navasard/data/fineweb_val_seed42",
16
+ "max_seq_length": 8192,
17
+ "num_workers": 8
18
+ },
19
+ "training": {
20
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/dropctl_calib_p30_verify",
21
+ "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-44000",
22
+ "force_new_run": true,
23
+ "num_epochs": 1,
24
+ "max_steps": 44300,
25
+ "per_device_batch_size": 2,
26
+ "gradient_accumulation_steps": 10,
27
+ "learning_rate": 5e-05,
28
+ "weight_decay": 0.01,
29
+ "warmup_steps": 0,
30
+ "lr_scheduler": "constant",
31
+ "max_grad_norm": 1.0,
32
+ "save_steps": 300,
33
+ "eval_steps": 75,
34
+ "eval_batches": 50,
35
+ "logging_steps": 10,
36
+ "seed": 42,
37
+ "verbose": true,
38
+ "checkpoint_activations": false
39
+ },
40
+ "distributed": {
41
+ "strategy": "ddp",
42
+ "mixed_precision": "bf16"
43
+ },
44
+ "wandb": {
45
+ "enabled": false,
46
+ "project": "stochastic_sparsity",
47
+ "name": "dropctl_calib_p30_verify",
48
+ "tags": [
49
+ "qwen3",
50
+ "365m",
51
+ "8k",
52
+ "scaleladder",
53
+ "dense"
54
+ ]
55
+ },
56
+ "attention": {
57
+ "type": "sparse_stochastic",
58
+ "dense_mode": true,
59
+ "sparse_block_size": 128,
60
+ "sparse_num_random_blocks": 31,
61
+ "sparse_mask_pool_size": 1,
62
+ "sparse_val_dense_compare": false,
63
+ "dense_step_intervals": null,
64
+ "sparse_layer_indices": null,
65
+ "sparse_mask_cache_dir": null,
66
+ "sparse_mask_seed": 0
67
+ }
68
+ }
epfl_final/configs_dropctl/diag_p28_20.json ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": {
3
+ "path": "/mloscratch/homes/navasard/qwen3_06b_base",
4
+ "dtype": "bfloat16",
5
+ "init_from_scratch": false,
6
+ "hidden_size": 768,
7
+ "intermediate_size": 3072,
8
+ "num_hidden_layers": 28,
9
+ "num_attention_heads": 6,
10
+ "num_key_value_heads": 2,
11
+ "attention_dropout": 0.28
12
+ },
13
+ "data": {
14
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
15
+ "val_path": "/mloscratch/homes/navasard/data/fineweb_val_seed42",
16
+ "max_seq_length": 8192,
17
+ "num_workers": 8
18
+ },
19
+ "training": {
20
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/dropctl_diag_p28_20",
21
+ "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-44000",
22
+ "force_new_run": true,
23
+ "num_epochs": 1,
24
+ "max_steps": 44020,
25
+ "per_device_batch_size": 2,
26
+ "gradient_accumulation_steps": 10,
27
+ "learning_rate": 5e-05,
28
+ "weight_decay": 0.01,
29
+ "warmup_steps": 0,
30
+ "lr_scheduler": "constant",
31
+ "max_grad_norm": 1.0,
32
+ "save_steps": 20,
33
+ "eval_steps": 25,
34
+ "eval_batches": 50,
35
+ "logging_steps": 10,
36
+ "seed": 42,
37
+ "verbose": true,
38
+ "checkpoint_activations": false
39
+ },
40
+ "distributed": {
41
+ "strategy": "ddp",
42
+ "mixed_precision": "bf16"
43
+ },
44
+ "wandb": {
45
+ "enabled": false,
46
+ "project": "stochastic_sparsity",
47
+ "name": "dropctl_diag_p28_20",
48
+ "tags": [
49
+ "qwen3",
50
+ "365m",
51
+ "8k",
52
+ "scaleladder",
53
+ "dense"
54
+ ]
55
+ },
56
+ "attention": {
57
+ "type": "sparse_stochastic",
58
+ "dense_mode": true,
59
+ "sparse_block_size": 128,
60
+ "sparse_num_random_blocks": 31,
61
+ "sparse_mask_pool_size": 1,
62
+ "sparse_val_dense_compare": false,
63
+ "dense_step_intervals": null,
64
+ "sparse_layer_indices": null,
65
+ "sparse_mask_cache_dir": null,
66
+ "sparse_mask_seed": 0
67
+ }
68
+ }
epfl_final/configs_dropctl/full_p28/seed42.json ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": {
3
+ "path": "/mloscratch/homes/navasard/qwen3_06b_base",
4
+ "dtype": "bfloat16",
5
+ "init_from_scratch": true,
6
+ "hidden_size": 768,
7
+ "intermediate_size": 3072,
8
+ "num_hidden_layers": 28,
9
+ "num_attention_heads": 6,
10
+ "num_key_value_heads": 2,
11
+ "attention_dropout": 0.28
12
+ },
13
+ "data": {
14
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
15
+ "val_path": null,
16
+ "max_seq_length": 8192,
17
+ "num_workers": 8
18
+ },
19
+ "training": {
20
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_dropout_p28_seed42",
21
+ "force_new_run": true,
22
+ "num_epochs": 1,
23
+ "max_steps": 44000,
24
+ "per_device_batch_size": 2,
25
+ "gradient_accumulation_steps": 10,
26
+ "learning_rate": 0.0005,
27
+ "weight_decay": 0.01,
28
+ "warmup_steps": 750,
29
+ "lr_scheduler": "cosine",
30
+ "max_grad_norm": 1.0,
31
+ "save_steps": 1000,
32
+ "eval_steps": 1000,
33
+ "eval_batches": 50,
34
+ "logging_steps": 10,
35
+ "seed": 42,
36
+ "verbose": true,
37
+ "checkpoint_activations": false
38
+ },
39
+ "distributed": {
40
+ "strategy": "ddp",
41
+ "mixed_precision": "bf16"
42
+ },
43
+ "wandb": {
44
+ "enabled": false,
45
+ "project": "stochastic_sparsity",
46
+ "name": "l8k365m_dense_dropout_p28_seed42",
47
+ "tags": [
48
+ "qwen3",
49
+ "365m",
50
+ "8k",
51
+ "scaleladder",
52
+ "dense"
53
+ ]
54
+ },
55
+ "attention": {
56
+ "type": "sparse_stochastic",
57
+ "dense_mode": true,
58
+ "sparse_block_size": 128,
59
+ "sparse_num_random_blocks": 31,
60
+ "sparse_mask_pool_size": 1,
61
+ "sparse_val_dense_compare": false,
62
+ "dense_step_intervals": null,
63
+ "sparse_layer_indices": null,
64
+ "sparse_mask_cache_dir": null,
65
+ "sparse_mask_seed": 0
66
+ }
67
+ }
epfl_final/configs_dropctl/full_p28/seed43.json ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": {
3
+ "path": "/mloscratch/homes/navasard/qwen3_06b_base",
4
+ "dtype": "bfloat16",
5
+ "init_from_scratch": true,
6
+ "hidden_size": 768,
7
+ "intermediate_size": 3072,
8
+ "num_hidden_layers": 28,
9
+ "num_attention_heads": 6,
10
+ "num_key_value_heads": 2,
11
+ "attention_dropout": 0.28
12
+ },
13
+ "data": {
14
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
15
+ "val_path": null,
16
+ "max_seq_length": 8192,
17
+ "num_workers": 8
18
+ },
19
+ "training": {
20
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_dropout_p28_seed43",
21
+ "force_new_run": true,
22
+ "num_epochs": 1,
23
+ "max_steps": 44000,
24
+ "per_device_batch_size": 2,
25
+ "gradient_accumulation_steps": 10,
26
+ "learning_rate": 0.0005,
27
+ "weight_decay": 0.01,
28
+ "warmup_steps": 750,
29
+ "lr_scheduler": "cosine",
30
+ "max_grad_norm": 1.0,
31
+ "save_steps": 1000,
32
+ "eval_steps": 1000,
33
+ "eval_batches": 50,
34
+ "logging_steps": 10,
35
+ "seed": 43,
36
+ "verbose": true,
37
+ "checkpoint_activations": false
38
+ },
39
+ "distributed": {
40
+ "strategy": "ddp",
41
+ "mixed_precision": "bf16"
42
+ },
43
+ "wandb": {
44
+ "enabled": false,
45
+ "project": "stochastic_sparsity",
46
+ "name": "l8k365m_dense_dropout_p28_seed43",
47
+ "tags": [
48
+ "qwen3",
49
+ "365m",
50
+ "8k",
51
+ "scaleladder",
52
+ "dense"
53
+ ]
54
+ },
55
+ "attention": {
56
+ "type": "sparse_stochastic",
57
+ "dense_mode": true,
58
+ "sparse_block_size": 128,
59
+ "sparse_num_random_blocks": 31,
60
+ "sparse_mask_pool_size": 1,
61
+ "sparse_val_dense_compare": false,
62
+ "dense_step_intervals": null,
63
+ "sparse_layer_indices": null,
64
+ "sparse_mask_cache_dir": null,
65
+ "sparse_mask_seed": 0
66
+ }
67
+ }
epfl_final/configs_dropctl/full_p28/seed44.json ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": {
3
+ "path": "/mloscratch/homes/navasard/qwen3_06b_base",
4
+ "dtype": "bfloat16",
5
+ "init_from_scratch": true,
6
+ "hidden_size": 768,
7
+ "intermediate_size": 3072,
8
+ "num_hidden_layers": 28,
9
+ "num_attention_heads": 6,
10
+ "num_key_value_heads": 2,
11
+ "attention_dropout": 0.28
12
+ },
13
+ "data": {
14
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
15
+ "val_path": null,
16
+ "max_seq_length": 8192,
17
+ "num_workers": 8
18
+ },
19
+ "training": {
20
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_dropout_p28_seed44",
21
+ "force_new_run": true,
22
+ "num_epochs": 1,
23
+ "max_steps": 44000,
24
+ "per_device_batch_size": 2,
25
+ "gradient_accumulation_steps": 10,
26
+ "learning_rate": 0.0005,
27
+ "weight_decay": 0.01,
28
+ "warmup_steps": 750,
29
+ "lr_scheduler": "cosine",
30
+ "max_grad_norm": 1.0,
31
+ "save_steps": 1000,
32
+ "eval_steps": 1000,
33
+ "eval_batches": 50,
34
+ "logging_steps": 10,
35
+ "seed": 44,
36
+ "verbose": true,
37
+ "checkpoint_activations": false
38
+ },
39
+ "distributed": {
40
+ "strategy": "ddp",
41
+ "mixed_precision": "bf16"
42
+ },
43
+ "wandb": {
44
+ "enabled": false,
45
+ "project": "stochastic_sparsity",
46
+ "name": "l8k365m_dense_dropout_p28_seed44",
47
+ "tags": [
48
+ "qwen3",
49
+ "365m",
50
+ "8k",
51
+ "scaleladder",
52
+ "dense"
53
+ ]
54
+ },
55
+ "attention": {
56
+ "type": "sparse_stochastic",
57
+ "dense_mode": true,
58
+ "sparse_block_size": 128,
59
+ "sparse_num_random_blocks": 31,
60
+ "sparse_mask_pool_size": 1,
61
+ "sparse_val_dense_compare": false,
62
+ "dense_step_intervals": null,
63
+ "sparse_layer_indices": null,
64
+ "sparse_mask_cache_dir": null,
65
+ "sparse_mask_seed": 0
66
+ }
67
+ }
epfl_final/configs_dropctl/full_p28/seed45.json ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": {
3
+ "path": "/mloscratch/homes/navasard/qwen3_06b_base",
4
+ "dtype": "bfloat16",
5
+ "init_from_scratch": true,
6
+ "hidden_size": 768,
7
+ "intermediate_size": 3072,
8
+ "num_hidden_layers": 28,
9
+ "num_attention_heads": 6,
10
+ "num_key_value_heads": 2,
11
+ "attention_dropout": 0.28
12
+ },
13
+ "data": {
14
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
15
+ "val_path": null,
16
+ "max_seq_length": 8192,
17
+ "num_workers": 8
18
+ },
19
+ "training": {
20
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_dropout_p28_seed45",
21
+ "force_new_run": true,
22
+ "num_epochs": 1,
23
+ "max_steps": 44000,
24
+ "per_device_batch_size": 2,
25
+ "gradient_accumulation_steps": 10,
26
+ "learning_rate": 0.0005,
27
+ "weight_decay": 0.01,
28
+ "warmup_steps": 750,
29
+ "lr_scheduler": "cosine",
30
+ "max_grad_norm": 1.0,
31
+ "save_steps": 1000,
32
+ "eval_steps": 1000,
33
+ "eval_batches": 50,
34
+ "logging_steps": 10,
35
+ "seed": 45,
36
+ "verbose": true,
37
+ "checkpoint_activations": false
38
+ },
39
+ "distributed": {
40
+ "strategy": "ddp",
41
+ "mixed_precision": "bf16"
42
+ },
43
+ "wandb": {
44
+ "enabled": false,
45
+ "project": "stochastic_sparsity",
46
+ "name": "l8k365m_dense_dropout_p28_seed45",
47
+ "tags": [
48
+ "qwen3",
49
+ "365m",
50
+ "8k",
51
+ "scaleladder",
52
+ "dense"
53
+ ]
54
+ },
55
+ "attention": {
56
+ "type": "sparse_stochastic",
57
+ "dense_mode": true,
58
+ "sparse_block_size": 128,
59
+ "sparse_num_random_blocks": 31,
60
+ "sparse_mask_pool_size": 1,
61
+ "sparse_val_dense_compare": false,
62
+ "dense_step_intervals": null,
63
+ "sparse_layer_indices": null,
64
+ "sparse_mask_cache_dir": null,
65
+ "sparse_mask_seed": 0
66
+ }
67
+ }
epfl_final/configs_dropctl/full_p28/seed46.json ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": {
3
+ "path": "/mloscratch/homes/navasard/qwen3_06b_base",
4
+ "dtype": "bfloat16",
5
+ "init_from_scratch": true,
6
+ "hidden_size": 768,
7
+ "intermediate_size": 3072,
8
+ "num_hidden_layers": 28,
9
+ "num_attention_heads": 6,
10
+ "num_key_value_heads": 2,
11
+ "attention_dropout": 0.28
12
+ },
13
+ "data": {
14
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
15
+ "val_path": null,
16
+ "max_seq_length": 8192,
17
+ "num_workers": 8
18
+ },
19
+ "training": {
20
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_dropout_p28_seed46",
21
+ "force_new_run": true,
22
+ "num_epochs": 1,
23
+ "max_steps": 44000,
24
+ "per_device_batch_size": 2,
25
+ "gradient_accumulation_steps": 10,
26
+ "learning_rate": 0.0005,
27
+ "weight_decay": 0.01,
28
+ "warmup_steps": 750,
29
+ "lr_scheduler": "cosine",
30
+ "max_grad_norm": 1.0,
31
+ "save_steps": 1000,
32
+ "eval_steps": 1000,
33
+ "eval_batches": 50,
34
+ "logging_steps": 10,
35
+ "seed": 46,
36
+ "verbose": true,
37
+ "checkpoint_activations": false
38
+ },
39
+ "distributed": {
40
+ "strategy": "ddp",
41
+ "mixed_precision": "bf16"
42
+ },
43
+ "wandb": {
44
+ "enabled": false,
45
+ "project": "stochastic_sparsity",
46
+ "name": "l8k365m_dense_dropout_p28_seed46",
47
+ "tags": [
48
+ "qwen3",
49
+ "365m",
50
+ "8k",
51
+ "scaleladder",
52
+ "dense"
53
+ ]
54
+ },
55
+ "attention": {
56
+ "type": "sparse_stochastic",
57
+ "dense_mode": true,
58
+ "sparse_block_size": 128,
59
+ "sparse_num_random_blocks": 31,
60
+ "sparse_mask_pool_size": 1,
61
+ "sparse_val_dense_compare": false,
62
+ "dense_step_intervals": null,
63
+ "sparse_layer_indices": null,
64
+ "sparse_mask_cache_dir": null,
65
+ "sparse_mask_seed": 0
66
+ }
67
+ }
epfl_final/configs_dropctl/resume_p28/seed42.json ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": {
3
+ "path": "/mloscratch/homes/navasard/qwen3_06b_base",
4
+ "dtype": "bfloat16",
5
+ "init_from_scratch": true,
6
+ "hidden_size": 768,
7
+ "intermediate_size": 3072,
8
+ "num_hidden_layers": 28,
9
+ "num_attention_heads": 6,
10
+ "num_key_value_heads": 2,
11
+ "attention_dropout": 0.28
12
+ },
13
+ "data": {
14
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
15
+ "val_path": null,
16
+ "max_seq_length": 8192,
17
+ "num_workers": 8
18
+ },
19
+ "training": {
20
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_dropout_p28_seed42",
21
+ "force_new_run": false,
22
+ "num_epochs": 1,
23
+ "max_steps": 44000,
24
+ "per_device_batch_size": 2,
25
+ "gradient_accumulation_steps": 10,
26
+ "learning_rate": 0.0005,
27
+ "weight_decay": 0.01,
28
+ "warmup_steps": 750,
29
+ "lr_scheduler": "cosine",
30
+ "max_grad_norm": 1.0,
31
+ "save_steps": 1000,
32
+ "eval_steps": 1000,
33
+ "eval_batches": 50,
34
+ "logging_steps": 10,
35
+ "seed": 42,
36
+ "verbose": true,
37
+ "checkpoint_activations": false
38
+ },
39
+ "distributed": {
40
+ "strategy": "ddp",
41
+ "mixed_precision": "bf16"
42
+ },
43
+ "wandb": {
44
+ "enabled": false,
45
+ "project": "stochastic_sparsity",
46
+ "name": "l8k365m_dense_dropout_p28_seed42",
47
+ "tags": [
48
+ "qwen3",
49
+ "365m",
50
+ "8k",
51
+ "scaleladder",
52
+ "dense"
53
+ ]
54
+ },
55
+ "attention": {
56
+ "type": "sparse_stochastic",
57
+ "dense_mode": true,
58
+ "sparse_block_size": 128,
59
+ "sparse_num_random_blocks": 31,
60
+ "sparse_mask_pool_size": 1,
61
+ "sparse_val_dense_compare": false,
62
+ "dense_step_intervals": null,
63
+ "sparse_layer_indices": null,
64
+ "sparse_mask_cache_dir": null,
65
+ "sparse_mask_seed": 0
66
+ }
67
+ }
epfl_final/configs_dropctl/resume_p28/seed43.json ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": {
3
+ "path": "/mloscratch/homes/navasard/qwen3_06b_base",
4
+ "dtype": "bfloat16",
5
+ "init_from_scratch": true,
6
+ "hidden_size": 768,
7
+ "intermediate_size": 3072,
8
+ "num_hidden_layers": 28,
9
+ "num_attention_heads": 6,
10
+ "num_key_value_heads": 2,
11
+ "attention_dropout": 0.28
12
+ },
13
+ "data": {
14
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
15
+ "val_path": null,
16
+ "max_seq_length": 8192,
17
+ "num_workers": 8
18
+ },
19
+ "training": {
20
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_dropout_p28_seed43",
21
+ "force_new_run": false,
22
+ "num_epochs": 1,
23
+ "max_steps": 44000,
24
+ "per_device_batch_size": 2,
25
+ "gradient_accumulation_steps": 10,
26
+ "learning_rate": 0.0005,
27
+ "weight_decay": 0.01,
28
+ "warmup_steps": 750,
29
+ "lr_scheduler": "cosine",
30
+ "max_grad_norm": 1.0,
31
+ "save_steps": 1000,
32
+ "eval_steps": 1000,
33
+ "eval_batches": 50,
34
+ "logging_steps": 10,
35
+ "seed": 43,
36
+ "verbose": true,
37
+ "checkpoint_activations": false
38
+ },
39
+ "distributed": {
40
+ "strategy": "ddp",
41
+ "mixed_precision": "bf16"
42
+ },
43
+ "wandb": {
44
+ "enabled": false,
45
+ "project": "stochastic_sparsity",
46
+ "name": "l8k365m_dense_dropout_p28_seed43",
47
+ "tags": [
48
+ "qwen3",
49
+ "365m",
50
+ "8k",
51
+ "scaleladder",
52
+ "dense"
53
+ ]
54
+ },
55
+ "attention": {
56
+ "type": "sparse_stochastic",
57
+ "dense_mode": true,
58
+ "sparse_block_size": 128,
59
+ "sparse_num_random_blocks": 31,
60
+ "sparse_mask_pool_size": 1,
61
+ "sparse_val_dense_compare": false,
62
+ "dense_step_intervals": null,
63
+ "sparse_layer_indices": null,
64
+ "sparse_mask_cache_dir": null,
65
+ "sparse_mask_seed": 0
66
+ }
67
+ }
epfl_final/configs_dropctl/resume_p28/seed44.json ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": {
3
+ "path": "/mloscratch/homes/navasard/qwen3_06b_base",
4
+ "dtype": "bfloat16",
5
+ "init_from_scratch": true,
6
+ "hidden_size": 768,
7
+ "intermediate_size": 3072,
8
+ "num_hidden_layers": 28,
9
+ "num_attention_heads": 6,
10
+ "num_key_value_heads": 2,
11
+ "attention_dropout": 0.28
12
+ },
13
+ "data": {
14
+ "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
15
+ "val_path": null,
16
+ "max_seq_length": 8192,
17
+ "num_workers": 8
18
+ },
19
+ "training": {
20
+ "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_dropout_p28_seed44",
21
+ "force_new_run": false,
22
+ "num_epochs": 1,
23
+ "max_steps": 44000,
24
+ "per_device_batch_size": 2,
25
+ "gradient_accumulation_steps": 10,
26
+ "learning_rate": 0.0005,
27
+ "weight_decay": 0.01,
28
+ "warmup_steps": 750,
29
+ "lr_scheduler": "cosine",
30
+ "max_grad_norm": 1.0,
31
+ "save_steps": 1000,
32
+ "eval_steps": 1000,
33
+ "eval_batches": 50,
34
+ "logging_steps": 10,
35
+ "seed": 44,
36
+ "verbose": true,
37
+ "checkpoint_activations": false
38
+ },
39
+ "distributed": {
40
+ "strategy": "ddp",
41
+ "mixed_precision": "bf16"
42
+ },
43
+ "wandb": {
44
+ "enabled": false,
45
+ "project": "stochastic_sparsity",
46
+ "name": "l8k365m_dense_dropout_p28_seed44",
47
+ "tags": [
48
+ "qwen3",
49
+ "365m",
50
+ "8k",
51
+ "scaleladder",
52
+ "dense"
53
+ ]
54
+ },
55
+ "attention": {
56
+ "type": "sparse_stochastic",
57
+ "dense_mode": true,
58
+ "sparse_block_size": 128,
59
+ "sparse_num_random_blocks": 31,
60
+ "sparse_mask_pool_size": 1,
61
+ "sparse_val_dense_compare": false,
62
+ "dense_step_intervals": null,
63
+ "sparse_layer_indices": null,
64
+ "sparse_mask_cache_dir": null,
65
+ "sparse_mask_seed": 0
66
+ }
67
+ }
epfl_final/models/ALIBI_20260921/1p2b_alibi_s42/run_20260922_004519/checkpoint-17000/config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 1536,
4
+ "intermediate_size": 6144,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 12,
7
+ "num_key_value_heads": 4,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "positional": "alibi",
18
+ "rope_train_position_scale_set": null,
19
+ "attention_bias": false,
20
+ "attention_dropout": 0.0,
21
+ "bos_token_id": 151643,
22
+ "eos_token_id": 151643,
23
+ "checkpoint_activations": false,
24
+ "attention_type": "dense",
25
+ "sparse_block_size": 128,
26
+ "sparse_num_random_blocks": 31,
27
+ "sparse_mask_pool_size": 1,
28
+ "sparse_max_seq_length": 8192,
29
+ "sparse_layer_indices": null,
30
+ "sparse_mask_cache_dir": null,
31
+ "sparse_mask_seed": 42,
32
+ "sparse_mixed_nrb_set": null,
33
+ "moba_block_size": 128,
34
+ "moba_top_k": 32,
35
+ "asentmax_alpha": 1.5,
36
+ "asentmax_learn_temp": true,
37
+ "asentmax_init_temp": 1.0
38
+ }
epfl_final/models/ALIBI_20260921/1p2b_alibi_s42/run_20260922_004519/checkpoint-17000/loader_gen_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3f59a1416dd062038a7e2dbac8c330163f45326660faeb92a199f62460447539
3
+ size 6802
epfl_final/models/ALIBI_20260921/1p2b_alibi_s42/run_20260922_004519/checkpoint-17000/loader_gen_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9907050ee7aa7455a32aae231218aac0666dd0bc7b78294eeb2cb3adc77e63c8
3
+ size 6802
epfl_final/models/ALIBI_20260921/1p2b_alibi_s42/run_20260922_004519/checkpoint-17000/loader_gen_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3adc40f4c25919716a49dee06d626601e0abca351145ddf1ed45e0bbfca3ea0c
3
+ size 6802
epfl_final/models/ALIBI_20260921/1p2b_alibi_s42/run_20260922_004519/checkpoint-17000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:567b3e5012b140ee3aad39ae7ae32268e47dcb18c21e69153e6a7091b56cf061
3
+ size 2404819483
epfl_final/models/ALIBI_20260921/1p2b_alibi_s42/run_20260922_004519/checkpoint-17000/rng_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:963d30d3ff8493b5336f2df34c8a29a71e39e205b74384a8736ec19f8b03b67b
3
+ size 14757
epfl_final/models/ALIBI_20260921/1p2b_alibi_s42/run_20260922_004519/checkpoint-17000/rng_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d9a620dedf93d9b9c78f8c5fafd2bbfb558c5195e01add35b488158d528449ec
3
+ size 14821
epfl_final/models/ALIBI_20260921/1p2b_alibi_s42/run_20260922_004519/checkpoint-17000/rng_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c208126c5358db5672b13dd0a333e87dc7238caceb878b15b81888b1f3e4bd15
3
+ size 14821
epfl_final/models/ALIBI_20260921/1p2b_alibi_s42/run_20260922_004519/checkpoint-17000/training_state.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:511e04312719b6691ded2541570561dee69cea59d1a9660c447189719000f11a
3
+ size 4809681879
epfl_final/models/ALIBI_20260921/365m_alibi_s42/run_20260921_142951/checkpoint-44000/config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "positional": "alibi",
18
+ "rope_train_position_scale_set": null,
19
+ "attention_bias": false,
20
+ "attention_dropout": 0.0,
21
+ "bos_token_id": 151643,
22
+ "eos_token_id": 151643,
23
+ "checkpoint_activations": false,
24
+ "attention_type": "dense",
25
+ "sparse_block_size": 128,
26
+ "sparse_num_random_blocks": 31,
27
+ "sparse_mask_pool_size": 1,
28
+ "sparse_max_seq_length": 8192,
29
+ "sparse_layer_indices": null,
30
+ "sparse_mask_cache_dir": null,
31
+ "sparse_mask_seed": 0,
32
+ "sparse_mixed_nrb_set": null,
33
+ "moba_block_size": 128,
34
+ "moba_top_k": 32,
35
+ "asentmax_alpha": 1.5,
36
+ "asentmax_learn_temp": true,
37
+ "asentmax_init_temp": 1.0
38
+ }
epfl_final/models/ALIBI_20260921/365m_alibi_s42/run_20260921_142951/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:709b863b6b2679ca92dafb2096661784c959e4fb46332bad30b93b0cc327e7ec
3
+ size 718031899
epfl_final/models/ALIBI_20260921/365m_alibi_s43/run_20260921_142951/checkpoint-44000/config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "positional": "alibi",
18
+ "rope_train_position_scale_set": null,
19
+ "attention_bias": false,
20
+ "attention_dropout": 0.0,
21
+ "bos_token_id": 151643,
22
+ "eos_token_id": 151643,
23
+ "checkpoint_activations": false,
24
+ "attention_type": "dense",
25
+ "sparse_block_size": 128,
26
+ "sparse_num_random_blocks": 31,
27
+ "sparse_mask_pool_size": 1,
28
+ "sparse_max_seq_length": 8192,
29
+ "sparse_layer_indices": null,
30
+ "sparse_mask_cache_dir": null,
31
+ "sparse_mask_seed": 0,
32
+ "sparse_mixed_nrb_set": null,
33
+ "moba_block_size": 128,
34
+ "moba_top_k": 32,
35
+ "asentmax_alpha": 1.5,
36
+ "asentmax_learn_temp": true,
37
+ "asentmax_init_temp": 1.0
38
+ }
epfl_final/models/ALIBI_20260921/365m_alibi_s43/run_20260921_142951/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9f68f62d83e867ae292ffac9b6fba3bf19ed722cd1132d6f790df56b65da7cd0
3
+ size 718031899
epfl_final/models/ALIBI_20260921/365m_alibi_s44/run_20260921_142951/checkpoint-44000/config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "positional": "alibi",
18
+ "rope_train_position_scale_set": null,
19
+ "attention_bias": false,
20
+ "attention_dropout": 0.0,
21
+ "bos_token_id": 151643,
22
+ "eos_token_id": 151643,
23
+ "checkpoint_activations": false,
24
+ "attention_type": "dense",
25
+ "sparse_block_size": 128,
26
+ "sparse_num_random_blocks": 31,
27
+ "sparse_mask_pool_size": 1,
28
+ "sparse_max_seq_length": 8192,
29
+ "sparse_layer_indices": null,
30
+ "sparse_mask_cache_dir": null,
31
+ "sparse_mask_seed": 0,
32
+ "sparse_mixed_nrb_set": null,
33
+ "moba_block_size": 128,
34
+ "moba_top_k": 32,
35
+ "asentmax_alpha": 1.5,
36
+ "asentmax_learn_temp": true,
37
+ "asentmax_init_temp": 1.0
38
+ }
epfl_final/models/ALIBI_20260921/365m_alibi_s44/run_20260921_142951/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c22c3d486ca9b939816aef278457d1e734e3379d64069e89eb96c29809ff44ac
3
+ size 718031899
epfl_final/models/ALIBI_20260921/365m_alibi_s45/run_20260921_142951/checkpoint-44000/config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "positional": "alibi",
18
+ "rope_train_position_scale_set": null,
19
+ "attention_bias": false,
20
+ "attention_dropout": 0.0,
21
+ "bos_token_id": 151643,
22
+ "eos_token_id": 151643,
23
+ "checkpoint_activations": false,
24
+ "attention_type": "dense",
25
+ "sparse_block_size": 128,
26
+ "sparse_num_random_blocks": 31,
27
+ "sparse_mask_pool_size": 1,
28
+ "sparse_max_seq_length": 8192,
29
+ "sparse_layer_indices": null,
30
+ "sparse_mask_cache_dir": null,
31
+ "sparse_mask_seed": 0,
32
+ "sparse_mixed_nrb_set": null,
33
+ "moba_block_size": 128,
34
+ "moba_top_k": 32,
35
+ "asentmax_alpha": 1.5,
36
+ "asentmax_learn_temp": true,
37
+ "asentmax_init_temp": 1.0
38
+ }
epfl_final/models/ALIBI_20260921/365m_alibi_s45/run_20260921_142951/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4580777302800e123942e234e26893ccbbd526acbc17c62666d6d892d9ac0284
3
+ size 718031899
epfl_final/models/ALIBI_20260921/365m_alibi_s46/run_20260921_142951/checkpoint-44000/config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "positional": "alibi",
18
+ "rope_train_position_scale_set": null,
19
+ "attention_bias": false,
20
+ "attention_dropout": 0.0,
21
+ "bos_token_id": 151643,
22
+ "eos_token_id": 151643,
23
+ "checkpoint_activations": false,
24
+ "attention_type": "dense",
25
+ "sparse_block_size": 128,
26
+ "sparse_num_random_blocks": 31,
27
+ "sparse_mask_pool_size": 1,
28
+ "sparse_max_seq_length": 8192,
29
+ "sparse_layer_indices": null,
30
+ "sparse_mask_cache_dir": null,
31
+ "sparse_mask_seed": 0,
32
+ "sparse_mixed_nrb_set": null,
33
+ "moba_block_size": 128,
34
+ "moba_top_k": 32,
35
+ "asentmax_alpha": 1.5,
36
+ "asentmax_learn_temp": true,
37
+ "asentmax_init_temp": 1.0
38
+ }
epfl_final/models/ALIBI_20260921/365m_alibi_s46/run_20260921_142951/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c01a5b323b3d9ae62c2d80e12a10bc931127de5f51af4c7c07e2c5f0c01c8fb0
3
+ size 718031899
epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "positional": "alibi",
18
+ "rope_train_position_scale_set": null,
19
+ "attention_bias": false,
20
+ "attention_dropout": 0.0,
21
+ "bos_token_id": 151643,
22
+ "eos_token_id": 151643,
23
+ "checkpoint_activations": false,
24
+ "attention_type": "dense",
25
+ "sparse_block_size": 128,
26
+ "sparse_num_random_blocks": 31,
27
+ "sparse_mask_pool_size": 1,
28
+ "sparse_max_seq_length": 8192,
29
+ "sparse_layer_indices": null,
30
+ "sparse_mask_cache_dir": null,
31
+ "sparse_mask_seed": 0,
32
+ "sparse_mixed_nrb_set": null,
33
+ "moba_block_size": 128,
34
+ "moba_top_k": 32,
35
+ "asentmax_alpha": 1.5,
36
+ "asentmax_learn_temp": true,
37
+ "asentmax_init_temp": 1.0
38
+ }
epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/loader_gen_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5e6a62e475ff11a8f3cfb4b53f3efed4418c34b86eeb62304ecf30637c1ca785
3
+ size 6802
epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/loader_gen_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d6cbcc77de4742713528531e0ef8cdd89a1973c9e431acb1771b42ca22d7bd1f
3
+ size 6802
epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/loader_gen_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9db5ff6feccc6d1a9e53c22c2c117265883048de542cfdb5f1d72684ce40e97f
3
+ size 6802
epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/loader_gen_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2448f0fdf688f2e5aca811502f056dcbab182fad72e69178143577736b6f0c8e
3
+ size 6802
epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/loader_gen_state_rank4.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e5371f5b80aa7358ef9290ec1864b1c0669c69fb1e0f3f24dfe821940c3a22a
3
+ size 6802
epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6909bb176903b0699e6276aafce1b0bd5548516a5d2f819c9d225c888bc60762
3
+ size 718031899
epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/rng_state_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eee21359592b9a9aa8168aa900104f682271dd423ff278de532a3e569a0bd911
3
+ size 14757
epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/rng_state_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:45f083b7bebcc8721a119e9dd3b542e74bb5602ebfa4c960db83c7cfc45ec7b6
3
+ size 14757
epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/rng_state_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f44114486a5293a2ced45e33b791a4eb943c859a871385e74830b2597b3c2b73
3
+ size 14821
epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/rng_state_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8b066a127581f9b4b01b4debc7761b551492449f5e8c03656f0ef1a697c3dccb
3
+ size 14757
epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/rng_state_rank4.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0e99d25b66102a8c87a83a05e42573d39d70034404540658dc05f7d05d3fe24f
3
+ size 14821
epfl_final/models/ALIBI_20260921/365m_alibi_s47/run_20260921_142951/checkpoint-5000/training_state.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a7bed2cbb589dad9c6a0d1975480bfd49e7d60896664dd8620f208bda130913c
3
+ size 1436105299
epfl_final/models/ALIBI_20260921/365m_alibi_s47_restart_20260922/run_20260922_192827/checkpoint-44000/config.json ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab_size": 151936,
3
+ "hidden_size": 768,
4
+ "intermediate_size": 3072,
5
+ "num_hidden_layers": 28,
6
+ "num_attention_heads": 6,
7
+ "num_key_value_heads": 2,
8
+ "head_dim": 128,
9
+ "hidden_act": "silu",
10
+ "max_position_embeddings": 32768,
11
+ "initializer_range": 0.02,
12
+ "rms_norm_eps": 1e-06,
13
+ "use_cache": true,
14
+ "tie_word_embeddings": true,
15
+ "rope_theta": 1000000,
16
+ "rope_scaling": null,
17
+ "positional": "alibi",
18
+ "qk_norm": "on",
19
+ "qk_gain_fp32": false,
20
+ "rope_train_position_scale_set": null,
21
+ "attention_bias": false,
22
+ "attention_dropout": 0.0,
23
+ "bos_token_id": 151643,
24
+ "eos_token_id": 151643,
25
+ "checkpoint_activations": false,
26
+ "attention_type": "dense",
27
+ "sparse_block_size": 128,
28
+ "sparse_num_random_blocks": 31,
29
+ "sparse_mask_pool_size": 1,
30
+ "sparse_max_seq_length": 8192,
31
+ "sparse_layer_indices": null,
32
+ "sparse_mask_cache_dir": null,
33
+ "sparse_mask_seed": 0,
34
+ "sparse_mixed_nrb_set": null,
35
+ "moba_block_size": 128,
36
+ "moba_top_k": 32,
37
+ "asentmax_alpha": 1.5,
38
+ "asentmax_learn_temp": true,
39
+ "asentmax_init_temp": 1.0
40
+ }
epfl_final/models/ALIBI_20260921/365m_alibi_s47_restart_20260922/run_20260922_192827/checkpoint-44000/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b678f76518f9d2c8ee5e14caac9ba4d1107b8ca3281fa78f313c2cfb8466eb62
3
+ size 718031899