strata-engine commited on
Commit
9010168
·
verified ·
1 Parent(s): 75446fe

Add config.json for Inkling-2-Mini tokenizer/template support

Browse files
Files changed (1) hide show
  1. config.json +122 -0
config.json ADDED
@@ -0,0 +1,122 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dtype": "bfloat16",
3
+ "architectures": [
4
+ "Inkling2LoopForConditionalGeneration"
5
+ ],
6
+ "model_type": "inkling2_loop",
7
+ "eos_token_id": 200006,
8
+ "text_config": {
9
+ "model_type": "inkling2_loop_text",
10
+ "model_max_length": 1048576,
11
+ "torch_dtype": "bfloat16",
12
+ "hidden_size": 2304,
13
+ "num_hidden_layers": 24,
14
+ "vocab_size": 201024,
15
+ "num_attention_heads": 24,
16
+ "num_key_value_heads": 8,
17
+ "head_dim": 128,
18
+ "d_rel": 16,
19
+ "rel_extent": 1024,
20
+ "q_bias": false,
21
+ "o_bias": false,
22
+ "log_scaling_n_floor": 8192,
23
+ "log_scaling_alpha": 0.1,
24
+ "rms_norm_eps": 1e-06,
25
+ "use_value_norm_no_gamma": true,
26
+ "qk_l2_clip_threshold": 22.627416997969522,
27
+ "rel_l2_clip_threshold": 8.0,
28
+ "local_layer_ids": [
29
+ 0,
30
+ 1,
31
+ 2,
32
+ 3,
33
+ 4,
34
+ 6,
35
+ 7,
36
+ 8,
37
+ 9,
38
+ 10,
39
+ 12,
40
+ 13,
41
+ 14,
42
+ 15,
43
+ 16,
44
+ 18,
45
+ 19,
46
+ 20,
47
+ 21,
48
+ 22
49
+ ],
50
+ "sconv_kernel_size": 4,
51
+ "unpadded_vocab_size": 200058,
52
+ "final_logit_softcapping": 30.0,
53
+ "swa_head_dim": 128,
54
+ "swa_num_attention_heads": 24,
55
+ "swa_num_key_value_heads": 8,
56
+ "sliding_window_size": 1024,
57
+ "n_layers_per_attn_res_block": 6,
58
+ "attn_res_normalize_values": true,
59
+ "n_routed_experts": 128,
60
+ "num_experts_per_tok": 8,
61
+ "n_shared_experts": 0,
62
+ "intermediate_size": 1152,
63
+ "mlp_gate_softcap": 4.0,
64
+ "mlp_up_softcap": 25.0,
65
+ "route_scale": 8.0,
66
+ "use_gate_bias": true,
67
+ "loop_config": {
68
+ "core_start": 6,
69
+ "core_end": 18,
70
+ "num_passes": 3,
71
+ "gate_rank": 64,
72
+ "is_varloop": false
73
+ }
74
+ },
75
+ "vision_config": {
76
+ "model_type": "inkling2_vision",
77
+ "text_hidden_size": 2304,
78
+ "temporal_patch_size": 2,
79
+ "native_patch_size": 20,
80
+ "adapter_merge_size": 2,
81
+ "projection_hidden_dim": 4096,
82
+ "projector_bias": false,
83
+ "hidden_size": 1024,
84
+ "num_hidden_layers": 23,
85
+ "num_attention_heads": 16,
86
+ "mlp_ratio": 4.0,
87
+ "posemb_grid_size": 32,
88
+ "layer_norm_eps": 1e-05
89
+ },
90
+ "mtp_config": {
91
+ "hidden_size": 2304,
92
+ "num_hidden_layers": 3,
93
+ "num_attention_heads": 24,
94
+ "num_key_value_heads": 8,
95
+ "head_dim": 128,
96
+ "intermediate_size": 9216,
97
+ "max_position_embeddings": 1048576,
98
+ "torch_dtype": "bfloat16",
99
+ "q_bias": false,
100
+ "o_bias": false,
101
+ "rms_norm_eps": 1e-06,
102
+ "rope_theta": 1000000.0,
103
+ "local_layer_ids": [
104
+ 0,
105
+ 2
106
+ ],
107
+ "sliding_window_size": 1024,
108
+ "block_size": 8,
109
+ "markov_rank": 256,
110
+ "markov_head_type": "rnn_hidden",
111
+ "target_layer_ids": [
112
+ 5,
113
+ 23,
114
+ 41,
115
+ 48
116
+ ]
117
+ },
118
+ "quantization_config": {
119
+ "quant_method": "modelopt",
120
+ "quant_algo": "NVFP4"
121
+ }
122
+ }