Hugging Face's logo

ffurfaro
/
kairos

Text Generation
Transformers
PyTorch
TensorBoard
English
kairos
diffusion
multimodal
Mixture of Experts
trust_remote_code
custom_code
Model card Files
xet
 Metrics Community
ffurfaro commited on
Commit
b9ec341
·
verified ·
1 Parent(s): 5ad13ba

Upload training_config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. training_config.json +130 -0
training_config.json ADDED
@@ -0,0 +1,130 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_config": {
3
+ "transformers_version": "5.15.0",
4
+ "architectures": null,
5
+ "output_hidden_states": false,
6
+ "return_dict": true,
7
+ "dtype": null,
8
+ "chunk_size_feed_forward": 0,
9
+ "is_encoder_decoder": false,
10
+ "id2label": {
11
+ "0": "LABEL_0",
12
+ "1": "LABEL_1"
13
+ },
14
+ "label2id": {
15
+ "LABEL_0": 0,
16
+ "LABEL_1": 1
17
+ },
18
+ "problem_type": null,
19
+ "_name_or_path": "",
20
+ "num_modalities": 8,
21
+ "num_scales": 4,
22
+ "modality_scales": {
23
+ "0": [
24
+ 0,
25
+ 1
26
+ ],
27
+ "5": [
28
+ 0
29
+ ],
30
+ "6": [
31
+ 0
32
+ ],
33
+ "1": [
34
+ 1,
35
+ 2
36
+ ],
37
+ "4": [
38
+ 1
39
+ ],
40
+ "3": [
41
+ 2
42
+ ],
43
+ "2": [
44
+ 2,
45
+ 3
46
+ ],
47
+ "7": [
48
+ 3
49
+ ]
50
+ },
51
+ "moe_intermediate_size": 544,
52
+ "num_local_experts": 7,
53
+ "num_experts_per_tok": 1,
54
+ "n_shared_experts": 1,
55
+ "use_moe": true,
56
+ "attnres_block_size": 4,
57
+ "use_memory_gate": true,
58
+ "hidden_size": 64,
59
+ "num_attention_heads": 4,
60
+ "num_hidden_layers": 4,
61
+ "vocab_size": 291,
62
+ "text_modality_id": 0,
63
+ "stride": 3,
64
+ "sliding_window_size": 128,
65
+ "num_key_value_heads": 4,
66
+ "head_dim": 16,
67
+ "attention_dropout": 0.0,
68
+ "rope_theta": 10000.0,
69
+ "max_position_embeddings": 4096,
70
+ "linear_num_value_heads": 4,
71
+ "linear_num_key_heads": 4,
72
+ "linear_key_head_dim": 16,
73
+ "linear_value_head_dim": 16,
74
+ "linear_conv_kernel_dim": 4,
75
+ "hidden_act": "silu",
76
+ "rms_norm_eps": 1e-06,
77
+ "time_step_min": 0.001,
78
+ "time_step_max": 0.1,
79
+ "time_step_floor": 0.0001,
80
+ "A_init_range": [
81
+ 1.0,
82
+ 16.0
83
+ ],
84
+ "initializer_range": 0.02,
85
+ "intermediate_size": 544,
86
+ "routed_scaling_factor": 1.0,
87
+ "n_group": 1,
88
+ "topk_group": 1,
89
+ "norm_topk_prob": false,
90
+ "layers_config": [
91
+ "ld",
92
+ "ld",
93
+ "ld",
94
+ "ld"
95
+ ],
96
+ "slw_wsize": -1,
97
+ "use_cache": false,
98
+ "auto_map": {
99
+ "AutoConfig": "modeling.KairosConfig"
100
+ },
101
+ "model_type": "kairos",
102
+ "output_attentions": false
103
+ },
104
+ "train_config": {
105
+ "lr": 0.0003,
106
+ "epochs": 3,
107
+ "save_every": 200,
108
+ "last_ckpt_every": 20,
109
+ "grad_clip": 1.0,
110
+ "max_consecutive_nan": 50,
111
+ "run_dir": "checkpoints/kairos-multimodal/run_01",
112
+ "device": null,
113
+ "report_to": [],
114
+ "hub_repo_id": null,
115
+ "hub_push_every_ckpt": false,
116
+ "hub_private": false,
117
+ "hub_subfolder": null
118
+ },
119
+ "data_config": {
120
+ "multimodal_examples": "<17235 examples, omitted>",
121
+ "multimodal_path": null,
122
+ "text_examples": "<0 examples, omitted>",
123
+ "max_len": 1024,
124
+ "stride": 3,
125
+ "batch_size": 32,
126
+ "shuffle": true,
127
+ "drop_last": true,
128
+ "pack": true
129
+ }
130
+ }