bralynn commited on
Commit
fef127e
·
verified ·
1 Parent(s): 0c2673a

Trained with Unsloth - config

Browse files
Files changed (1) hide show
  1. config.json +67 -0
config.json ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Lfm2ForCausalLM"
4
+ ],
5
+ "block__name_mlp": "parallel_mlp_merged",
6
+ "block_auto_adjust_ff_dim": false,
7
+ "block_dim": 1024,
8
+ "block_ff_dim": 2560,
9
+ "block_ffn_dim_multiplier": 1.0,
10
+ "block_ffn_te_autocast": false,
11
+ "block_ffn_use_quantized_params": false,
12
+ "block_mlp_init_scale": 1.0,
13
+ "block_multiple_of": 256,
14
+ "block_norm_eps": 1e-05,
15
+ "block_out_init_scale": 1.0,
16
+ "block_sequence_parallel_norm_across_tp": false,
17
+ "block_use_swiglu": true,
18
+ "block_use_xavier_init": true,
19
+ "bos_token_id": 1,
20
+ "conv_L_cache": 3,
21
+ "conv_bias": false,
22
+ "conv_dim": 1024,
23
+ "conv_use_xavier_init": true,
24
+ "torch_dtype": "bfloat16",
25
+ "eos_token_id": 7,
26
+ "ffn_te_autocast": false,
27
+ "ffn_use_quantized_params": false,
28
+ "hidden_size": 1024,
29
+ "initializer_range": 0.02,
30
+ "intermediate_size": 2560,
31
+ "layer_types": [
32
+ "conv",
33
+ "conv",
34
+ "full_attention",
35
+ "conv",
36
+ "full_attention",
37
+ "conv",
38
+ "full_attention",
39
+ "conv",
40
+ "full_attention",
41
+ "conv",
42
+ "full_attention",
43
+ "conv",
44
+ "full_attention",
45
+ "conv"
46
+ ],
47
+ "max_position_embeddings": 128000,
48
+ "model_name": "LiquidAI/LFM2.5-230M",
49
+ "model_type": "lfm2",
50
+ "norm_eps": 1e-05,
51
+ "num_attention_heads": 16,
52
+ "num_heads": 16,
53
+ "num_hidden_layers": 14,
54
+ "num_key_value_heads": 8,
55
+ "pad_token_id": 0,
56
+ "rope_parameters": {
57
+ "rope_theta": 1000000.0,
58
+ "rope_type": "default"
59
+ },
60
+ "rope_theta": 1000000.0,
61
+ "sequence_parallel_norm_across_tp": false,
62
+ "tie_embedding": true,
63
+ "unsloth_version": "2026.7.3",
64
+ "use_cache": false,
65
+ "use_pos_enc": true,
66
+ "vocab_size": 65536
67
+ }