Hugging Face's logo

yujiepan
/
lfm3-tiny-random

Text Generation
Transformers
Safetensors
lfm3
conversational
custom_code
Model card Files
xet
 Community
yujiepan commited on
Commit
92d119a
·
verified ·
1 Parent(s): 2afc1a1

Sync redesigned LFM3 from tiny-random/lfm3

Browse files
Files changed (5) hide show
  1. .meta/create_model.py +10 -2
  2. .meta/model.log +110 -110
  3. README.md +8 -6
  4. config.json +5 -5
  5. model.safetensors +2 -2
.meta/create_model.py CHANGED
@@ -25,8 +25,16 @@ def create_model(source_model_id: str, source_revision: str, save_folder: str) -
25
 
26
  config = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text())
27
  # Keep the two-conv prefix and two complete full/conv/sliding/conv cycles.
28
- # Hidden width 8 is the user's floor; keep the source 16:4 Q/KV grouping.
29
- config.update(hidden_size=8, intermediate_size=32, num_hidden_layers=10, head_dim=32, max_position_embeddings=4096)
 
 
 
 
 
 
 
 
30
  config["layer_types"] = config["layer_types"][:10]
31
  (folder / "config.json").write_text(json.dumps(config, indent=2) + "\n")
32
 
 
25
 
26
  config = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text())
27
  # Keep the two-conv prefix and two complete full/conv/sliding/conv cycles.
28
+ # Match tiny-random/lfm2's residual/MLP widths; retain LFM3's 4:1 Q/KV ratio.
29
+ config.update(
30
+ hidden_size=64,
31
+ intermediate_size=128,
32
+ num_hidden_layers=10,
33
+ num_attention_heads=4,
34
+ num_key_value_heads=1,
35
+ head_dim=32,
36
+ max_position_embeddings=128000,
37
+ )
38
  config["layer_types"] = config["layer_types"][:10]
39
  (folder / "config.json").write_text(json.dumps(config, indent=2) + "\n")
40
 
.meta/model.log CHANGED
@@ -1,170 +1,170 @@
1
  Lfm3ForCausalLM(
2
  (model): Lfm3Model(
3
- (embed_tokens): Embedding(128000, 8, padding_idx=124893)
4
  (layers): ModuleList(
5
  (0-1): 2 x Lfm3DecoderLayer(
6
  (conv): Lfm3ShortConv(
7
- (conv): Conv1d(8, 8, kernel_size=(7,), stride=(1,), padding=(6,), groups=8, bias=False)
8
- (in_proj): Linear(in_features=8, out_features=24, bias=False)
9
- (out_proj): Linear(in_features=8, out_features=8, bias=False)
10
  )
11
  (feed_forward): Lfm3MLP(
12
- (w1): Linear(in_features=8, out_features=32, bias=False)
13
- (w3): Linear(in_features=8, out_features=32, bias=False)
14
- (w2): Linear(in_features=32, out_features=8, bias=False)
15
- )
16
- (operator_norm): Lfm3RMSNorm((8,), eps=1e-05)
17
- (ffn_norm): Lfm3RMSNorm((8,), eps=1e-05)
18
- (post_operator_norm): Lfm3RMSNorm((8,), eps=1e-05)
19
- (post_ffn_norm): Lfm3RMSNorm((8,), eps=1e-05)
20
  )
21
  (2): Lfm3DecoderLayer(
22
  (self_attn): Lfm3Attention(
23
- (q_proj): Linear(in_features=8, out_features=512, bias=False)
24
- (k_proj): Linear(in_features=8, out_features=128, bias=False)
25
- (v_proj): Linear(in_features=8, out_features=128, bias=False)
26
- (o_proj): Linear(in_features=512, out_features=8, bias=False)
27
  (q_norm): Lfm3RMSNorm((32,), eps=1e-05)
28
  (k_norm): Lfm3RMSNorm((32,), eps=1e-05)
29
- (gate_proj): Linear(in_features=8, out_features=16, bias=False)
30
  )
31
  (feed_forward): Lfm3MLP(
32
- (w1): Linear(in_features=8, out_features=32, bias=False)
33
- (w3): Linear(in_features=8, out_features=32, bias=False)
34
- (w2): Linear(in_features=32, out_features=8, bias=False)
35
- )
36
- (operator_norm): Lfm3RMSNorm((8,), eps=1e-05)
37
- (ffn_norm): Lfm3RMSNorm((8,), eps=1e-05)
38
- (post_operator_norm): Lfm3RMSNorm((8,), eps=1e-05)
39
- (post_ffn_norm): Lfm3RMSNorm((8,), eps=1e-05)
40
  )
41
  (3): Lfm3DecoderLayer(
42
  (conv): Lfm3ShortConv(
43
- (conv): Conv1d(8, 8, kernel_size=(7,), stride=(1,), padding=(6,), groups=8, bias=False)
44
- (in_proj): Linear(in_features=8, out_features=24, bias=False)
45
- (out_proj): Linear(in_features=8, out_features=8, bias=False)
46
  )
47
  (feed_forward): Lfm3MLP(
48
- (w1): Linear(in_features=8, out_features=32, bias=False)
49
- (w3): Linear(in_features=8, out_features=32, bias=False)
50
- (w2): Linear(in_features=32, out_features=8, bias=False)
51
- )
52
- (operator_norm): Lfm3RMSNorm((8,), eps=1e-05)
53
- (ffn_norm): Lfm3RMSNorm((8,), eps=1e-05)
54
- (post_operator_norm): Lfm3RMSNorm((8,), eps=1e-05)
55
- (post_ffn_norm): Lfm3RMSNorm((8,), eps=1e-05)
56
  )
57
  (4): Lfm3DecoderLayer(
58
  (self_attn): Lfm3Attention(
59
- (q_proj): Linear(in_features=8, out_features=512, bias=False)
60
- (k_proj): Linear(in_features=8, out_features=128, bias=False)
61
- (v_proj): Linear(in_features=8, out_features=128, bias=False)
62
- (o_proj): Linear(in_features=512, out_features=8, bias=False)
63
  (q_norm): Lfm3RMSNorm((32,), eps=1e-05)
64
  (k_norm): Lfm3RMSNorm((32,), eps=1e-05)
65
- (gate_proj): Linear(in_features=8, out_features=16, bias=False)
66
  )
67
  (feed_forward): Lfm3MLP(
68
- (w1): Linear(in_features=8, out_features=32, bias=False)
69
- (w3): Linear(in_features=8, out_features=32, bias=False)
70
- (w2): Linear(in_features=32, out_features=8, bias=False)
71
- )
72
- (operator_norm): Lfm3RMSNorm((8,), eps=1e-05)
73
- (ffn_norm): Lfm3RMSNorm((8,), eps=1e-05)
74
- (post_operator_norm): Lfm3RMSNorm((8,), eps=1e-05)
75
- (post_ffn_norm): Lfm3RMSNorm((8,), eps=1e-05)
76
  )
77
  (5): Lfm3DecoderLayer(
78
  (conv): Lfm3ShortConv(
79
- (conv): Conv1d(8, 8, kernel_size=(7,), stride=(1,), padding=(6,), groups=8, bias=False)
80
- (in_proj): Linear(in_features=8, out_features=24, bias=False)
81
- (out_proj): Linear(in_features=8, out_features=8, bias=False)
82
  )
83
  (feed_forward): Lfm3MLP(
84
- (w1): Linear(in_features=8, out_features=32, bias=False)
85
- (w3): Linear(in_features=8, out_features=32, bias=False)
86
- (w2): Linear(in_features=32, out_features=8, bias=False)
87
- )
88
- (operator_norm): Lfm3RMSNorm((8,), eps=1e-05)
89
- (ffn_norm): Lfm3RMSNorm((8,), eps=1e-05)
90
- (post_operator_norm): Lfm3RMSNorm((8,), eps=1e-05)
91
- (post_ffn_norm): Lfm3RMSNorm((8,), eps=1e-05)
92
  )
93
  (6): Lfm3DecoderLayer(
94
  (self_attn): Lfm3Attention(
95
- (q_proj): Linear(in_features=8, out_features=512, bias=False)
96
- (k_proj): Linear(in_features=8, out_features=128, bias=False)
97
- (v_proj): Linear(in_features=8, out_features=128, bias=False)
98
- (o_proj): Linear(in_features=512, out_features=8, bias=False)
99
  (q_norm): Lfm3RMSNorm((32,), eps=1e-05)
100
  (k_norm): Lfm3RMSNorm((32,), eps=1e-05)
101
- (gate_proj): Linear(in_features=8, out_features=16, bias=False)
102
  )
103
  (feed_forward): Lfm3MLP(
104
- (w1): Linear(in_features=8, out_features=32, bias=False)
105
- (w3): Linear(in_features=8, out_features=32, bias=False)
106
- (w2): Linear(in_features=32, out_features=8, bias=False)
107
- )
108
- (operator_norm): Lfm3RMSNorm((8,), eps=1e-05)
109
- (ffn_norm): Lfm3RMSNorm((8,), eps=1e-05)
110
- (post_operator_norm): Lfm3RMSNorm((8,), eps=1e-05)
111
- (post_ffn_norm): Lfm3RMSNorm((8,), eps=1e-05)
112
  )
113
  (7): Lfm3DecoderLayer(
114
  (conv): Lfm3ShortConv(
115
- (conv): Conv1d(8, 8, kernel_size=(7,), stride=(1,), padding=(6,), groups=8, bias=False)
116
- (in_proj): Linear(in_features=8, out_features=24, bias=False)
117
- (out_proj): Linear(in_features=8, out_features=8, bias=False)
118
  )
119
  (feed_forward): Lfm3MLP(
120
- (w1): Linear(in_features=8, out_features=32, bias=False)
121
- (w3): Linear(in_features=8, out_features=32, bias=False)
122
- (w2): Linear(in_features=32, out_features=8, bias=False)
123
- )
124
- (operator_norm): Lfm3RMSNorm((8,), eps=1e-05)
125
- (ffn_norm): Lfm3RMSNorm((8,), eps=1e-05)
126
- (post_operator_norm): Lfm3RMSNorm((8,), eps=1e-05)
127
- (post_ffn_norm): Lfm3RMSNorm((8,), eps=1e-05)
128
  )
129
  (8): Lfm3DecoderLayer(
130
  (self_attn): Lfm3Attention(
131
- (q_proj): Linear(in_features=8, out_features=512, bias=False)
132
- (k_proj): Linear(in_features=8, out_features=128, bias=False)
133
- (v_proj): Linear(in_features=8, out_features=128, bias=False)
134
- (o_proj): Linear(in_features=512, out_features=8, bias=False)
135
  (q_norm): Lfm3RMSNorm((32,), eps=1e-05)
136
  (k_norm): Lfm3RMSNorm((32,), eps=1e-05)
137
- (gate_proj): Linear(in_features=8, out_features=16, bias=False)
138
  )
139
  (feed_forward): Lfm3MLP(
140
- (w1): Linear(in_features=8, out_features=32, bias=False)
141
- (w3): Linear(in_features=8, out_features=32, bias=False)
142
- (w2): Linear(in_features=32, out_features=8, bias=False)
143
- )
144
- (operator_norm): Lfm3RMSNorm((8,), eps=1e-05)
145
- (ffn_norm): Lfm3RMSNorm((8,), eps=1e-05)
146
- (post_operator_norm): Lfm3RMSNorm((8,), eps=1e-05)
147
- (post_ffn_norm): Lfm3RMSNorm((8,), eps=1e-05)
148
  )
149
  (9): Lfm3DecoderLayer(
150
  (conv): Lfm3ShortConv(
151
- (conv): Conv1d(8, 8, kernel_size=(7,), stride=(1,), padding=(6,), groups=8, bias=False)
152
- (in_proj): Linear(in_features=8, out_features=24, bias=False)
153
- (out_proj): Linear(in_features=8, out_features=8, bias=False)
154
  )
155
  (feed_forward): Lfm3MLP(
156
- (w1): Linear(in_features=8, out_features=32, bias=False)
157
- (w3): Linear(in_features=8, out_features=32, bias=False)
158
- (w2): Linear(in_features=32, out_features=8, bias=False)
159
- )
160
- (operator_norm): Lfm3RMSNorm((8,), eps=1e-05)
161
- (ffn_norm): Lfm3RMSNorm((8,), eps=1e-05)
162
- (post_operator_norm): Lfm3RMSNorm((8,), eps=1e-05)
163
- (post_ffn_norm): Lfm3RMSNorm((8,), eps=1e-05)
164
  )
165
  )
166
  (rotary_emb): Lfm3RotaryEmbedding()
167
- (embedding_norm): Lfm3RMSNorm((8,), eps=1e-05)
168
  )
169
- (lm_head): Linear(in_features=8, out_features=128000, bias=False)
170
  )
 
1
  Lfm3ForCausalLM(
2
  (model): Lfm3Model(
3
+ (embed_tokens): Embedding(128000, 64, padding_idx=124893)
4
  (layers): ModuleList(
5
  (0-1): 2 x Lfm3DecoderLayer(
6
  (conv): Lfm3ShortConv(
7
+ (conv): Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(6,), groups=64, bias=False)
8
+ (in_proj): Linear(in_features=64, out_features=192, bias=False)
9
+ (out_proj): Linear(in_features=64, out_features=64, bias=False)
10
  )
11
  (feed_forward): Lfm3MLP(
12
+ (w1): Linear(in_features=64, out_features=128, bias=False)
13
+ (w3): Linear(in_features=64, out_features=128, bias=False)
14
+ (w2): Linear(in_features=128, out_features=64, bias=False)
15
+ )
16
+ (operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
17
+ (ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
18
+ (post_operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
19
+ (post_ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
20
  )
21
  (2): Lfm3DecoderLayer(
22
  (self_attn): Lfm3Attention(
23
+ (q_proj): Linear(in_features=64, out_features=128, bias=False)
24
+ (k_proj): Linear(in_features=64, out_features=32, bias=False)
25
+ (v_proj): Linear(in_features=64, out_features=32, bias=False)
26
+ (o_proj): Linear(in_features=128, out_features=64, bias=False)
27
  (q_norm): Lfm3RMSNorm((32,), eps=1e-05)
28
  (k_norm): Lfm3RMSNorm((32,), eps=1e-05)
29
+ (gate_proj): Linear(in_features=64, out_features=4, bias=False)
30
  )
31
  (feed_forward): Lfm3MLP(
32
+ (w1): Linear(in_features=64, out_features=128, bias=False)
33
+ (w3): Linear(in_features=64, out_features=128, bias=False)
34
+ (w2): Linear(in_features=128, out_features=64, bias=False)
35
+ )
36
+ (operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
37
+ (ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
38
+ (post_operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
39
+ (post_ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
40
  )
41
  (3): Lfm3DecoderLayer(
42
  (conv): Lfm3ShortConv(
43
+ (conv): Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(6,), groups=64, bias=False)
44
+ (in_proj): Linear(in_features=64, out_features=192, bias=False)
45
+ (out_proj): Linear(in_features=64, out_features=64, bias=False)
46
  )
47
  (feed_forward): Lfm3MLP(
48
+ (w1): Linear(in_features=64, out_features=128, bias=False)
49
+ (w3): Linear(in_features=64, out_features=128, bias=False)
50
+ (w2): Linear(in_features=128, out_features=64, bias=False)
51
+ )
52
+ (operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
53
+ (ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
54
+ (post_operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
55
+ (post_ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
56
  )
57
  (4): Lfm3DecoderLayer(
58
  (self_attn): Lfm3Attention(
59
+ (q_proj): Linear(in_features=64, out_features=128, bias=False)
60
+ (k_proj): Linear(in_features=64, out_features=32, bias=False)
61
+ (v_proj): Linear(in_features=64, out_features=32, bias=False)
62
+ (o_proj): Linear(in_features=128, out_features=64, bias=False)
63
  (q_norm): Lfm3RMSNorm((32,), eps=1e-05)
64
  (k_norm): Lfm3RMSNorm((32,), eps=1e-05)
65
+ (gate_proj): Linear(in_features=64, out_features=4, bias=False)
66
  )
67
  (feed_forward): Lfm3MLP(
68
+ (w1): Linear(in_features=64, out_features=128, bias=False)
69
+ (w3): Linear(in_features=64, out_features=128, bias=False)
70
+ (w2): Linear(in_features=128, out_features=64, bias=False)
71
+ )
72
+ (operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
73
+ (ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
74
+ (post_operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
75
+ (post_ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
76
  )
77
  (5): Lfm3DecoderLayer(
78
  (conv): Lfm3ShortConv(
79
+ (conv): Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(6,), groups=64, bias=False)
80
+ (in_proj): Linear(in_features=64, out_features=192, bias=False)
81
+ (out_proj): Linear(in_features=64, out_features=64, bias=False)
82
  )
83
  (feed_forward): Lfm3MLP(
84
+ (w1): Linear(in_features=64, out_features=128, bias=False)
85
+ (w3): Linear(in_features=64, out_features=128, bias=False)
86
+ (w2): Linear(in_features=128, out_features=64, bias=False)
87
+ )
88
+ (operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
89
+ (ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
90
+ (post_operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
91
+ (post_ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
92
  )
93
  (6): Lfm3DecoderLayer(
94
  (self_attn): Lfm3Attention(
95
+ (q_proj): Linear(in_features=64, out_features=128, bias=False)
96
+ (k_proj): Linear(in_features=64, out_features=32, bias=False)
97
+ (v_proj): Linear(in_features=64, out_features=32, bias=False)
98
+ (o_proj): Linear(in_features=128, out_features=64, bias=False)
99
  (q_norm): Lfm3RMSNorm((32,), eps=1e-05)
100
  (k_norm): Lfm3RMSNorm((32,), eps=1e-05)
101
+ (gate_proj): Linear(in_features=64, out_features=4, bias=False)
102
  )
103
  (feed_forward): Lfm3MLP(
104
+ (w1): Linear(in_features=64, out_features=128, bias=False)
105
+ (w3): Linear(in_features=64, out_features=128, bias=False)
106
+ (w2): Linear(in_features=128, out_features=64, bias=False)
107
+ )
108
+ (operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
109
+ (ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
110
+ (post_operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
111
+ (post_ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
112
  )
113
  (7): Lfm3DecoderLayer(
114
  (conv): Lfm3ShortConv(
115
+ (conv): Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(6,), groups=64, bias=False)
116
+ (in_proj): Linear(in_features=64, out_features=192, bias=False)
117
+ (out_proj): Linear(in_features=64, out_features=64, bias=False)
118
  )
119
  (feed_forward): Lfm3MLP(
120
+ (w1): Linear(in_features=64, out_features=128, bias=False)
121
+ (w3): Linear(in_features=64, out_features=128, bias=False)
122
+ (w2): Linear(in_features=128, out_features=64, bias=False)
123
+ )
124
+ (operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
125
+ (ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
126
+ (post_operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
127
+ (post_ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
128
  )
129
  (8): Lfm3DecoderLayer(
130
  (self_attn): Lfm3Attention(
131
+ (q_proj): Linear(in_features=64, out_features=128, bias=False)
132
+ (k_proj): Linear(in_features=64, out_features=32, bias=False)
133
+ (v_proj): Linear(in_features=64, out_features=32, bias=False)
134
+ (o_proj): Linear(in_features=128, out_features=64, bias=False)
135
  (q_norm): Lfm3RMSNorm((32,), eps=1e-05)
136
  (k_norm): Lfm3RMSNorm((32,), eps=1e-05)
137
+ (gate_proj): Linear(in_features=64, out_features=4, bias=False)
138
  )
139
  (feed_forward): Lfm3MLP(
140
+ (w1): Linear(in_features=64, out_features=128, bias=False)
141
+ (w3): Linear(in_features=64, out_features=128, bias=False)
142
+ (w2): Linear(in_features=128, out_features=64, bias=False)
143
+ )
144
+ (operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
145
+ (ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
146
+ (post_operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
147
+ (post_ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
148
  )
149
  (9): Lfm3DecoderLayer(
150
  (conv): Lfm3ShortConv(
151
+ (conv): Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(6,), groups=64, bias=False)
152
+ (in_proj): Linear(in_features=64, out_features=192, bias=False)
153
+ (out_proj): Linear(in_features=64, out_features=64, bias=False)
154
  )
155
  (feed_forward): Lfm3MLP(
156
+ (w1): Linear(in_features=64, out_features=128, bias=False)
157
+ (w3): Linear(in_features=64, out_features=128, bias=False)
158
+ (w2): Linear(in_features=128, out_features=64, bias=False)
159
+ )
160
+ (operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
161
+ (ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
162
+ (post_operator_norm): Lfm3RMSNorm((64,), eps=1e-05)
163
+ (post_ffn_norm): Lfm3RMSNorm((64,), eps=1e-05)
164
  )
165
  )
166
  (rotary_emb): Lfm3RotaryEmbedding()
167
+ (embedding_norm): Lfm3RMSNorm((64,), eps=1e-05)
168
  )
169
+ (lm_head): Linear(in_features=64, out_features=128000, bias=False)
170
  )
README.md CHANGED
@@ -9,22 +9,23 @@ This is a randomly initialized tiny model based on [LiquidAI/LFM3-1.7B](https://
9
 
10
  | Feature | Source | This repo |
11
  |---|---|---|
12
- | Checkpoint tensor elements | 1.73B BF16 | 1.08M BF16 |
13
- | Safetensors weight files | 3.46 GB (model.safetensors) | 2.16 MB (model.safetensors) |
14
  | Layers / layer-type cycle | 24 / two-conv prefix, then full/conv/sliding/conv | 10 / same prefix and two complete cycles |
15
- | Hidden width / MLP width | 2,048 / 7,680 | 8 / 32 |
16
- | Q heads / KV heads / head_dim | 16 / 4 / 128 | 16 / 4 / 32 |
17
  | Short-conv kernel / cache | 7 / 7 | Same |
18
  | RoPE | full 0.5 partial, theta 1e6; sliding theta 1e4 | Same |
19
  | Sandwich norms / QK RMSNorm / sigmoid head gate | present | Same |
20
  | Vocabulary rows / embedding tying | 128,000 / tied | Same |
 
21
  | Stored precision | BF16 | Same |
22
 
23
- Hidden width 8 follows the requested minimum. The pinned upstream implementation is included for Transformers loading. This repo provides a Transformers example; no serving-backend validation is claimed.
24
 
25
  | File path | Size |
26
  |------|------|
27
- | model.safetensors | 2.2MB |
28
 
29
  ### Example usage
30
 
@@ -73,5 +74,6 @@ if __name__ == "__main__":
73
  ### Change log
74
 
75
  - 2026-10-30: Initial version.
 
76
 
77
  *This repo is generated by GPT-6.1-Sol.*
 
9
 
10
  | Feature | Source | This repo |
11
  |---|---|---|
12
+ | Checkpoint tensor elements | 1.73B BF16 | 8.62M BF16 |
13
+ | Safetensors weight files | 3.46 GB (model.safetensors) | 17.26 MB (model.safetensors) |
14
  | Layers / layer-type cycle | 24 / two-conv prefix, then full/conv/sliding/conv | 10 / same prefix and two complete cycles |
15
+ | Hidden width / MLP width | 2,048 / 7,680 | 64 / 128 |
16
+ | Q heads / KV heads / head_dim | 16 / 4 / 128 | 4 / 1 / 32 |
17
  | Short-conv kernel / cache | 7 / 7 | Same |
18
  | RoPE | full 0.5 partial, theta 1e6; sliding theta 1e4 | Same |
19
  | Sandwich norms / QK RMSNorm / sigmoid head gate | present | Same |
20
  | Vocabulary rows / embedding tying | 128,000 / tied | Same |
21
+ | Maximum context length | 262,144 | 128,000 |
22
  | Stored precision | BF16 | Same |
23
 
24
+ Residual width 64 and MLP width 128 follow [tiny-random/lfm2](https://huggingface.co/tiny-random/lfm2). The pinned upstream implementation is included for Transformers loading. This repo provides a Transformers example; no serving-backend validation is claimed.
25
 
26
  | File path | Size |
27
  |------|------|
28
+ | model.safetensors | 17.3MB |
29
 
30
  ### Example usage
31
 
 
74
  ### Change log
75
 
76
  - 2026-10-30: Initial version.
77
+ - 2026-10-31: Resize to hidden width 64 and MLP width 128, following tiny-random/lfm2; use four Q heads and one KV head while preserving the LFM3 layer cadence.
78
 
79
  *This repo is generated by GPT-6.1-Sol.*
config.json CHANGED
@@ -13,8 +13,8 @@
13
  "dtype": "bfloat16",
14
  "eos_token_id": 124900,
15
  "head_dim": 32,
16
- "hidden_size": 8,
17
- "intermediate_size": 32,
18
  "layer_types": [
19
  "conv",
20
  "conv",
@@ -27,12 +27,12 @@
27
  "sliding_attention",
28
  "conv"
29
  ],
30
- "max_position_embeddings": 4096,
31
  "model_type": "lfm3",
32
  "norm_eps": 1e-05,
33
- "num_attention_heads": 16,
34
  "num_hidden_layers": 10,
35
- "num_key_value_heads": 4,
36
  "pad_token_id": 124893,
37
  "rope_parameters": {
38
  "full_attention": {
 
13
  "dtype": "bfloat16",
14
  "eos_token_id": 124900,
15
  "head_dim": 32,
16
+ "hidden_size": 64,
17
+ "intermediate_size": 128,
18
  "layer_types": [
19
  "conv",
20
  "conv",
 
27
  "sliding_attention",
28
  "conv"
29
  ],
30
+ "max_position_embeddings": 128000,
31
  "model_type": "lfm3",
32
  "norm_eps": 1e-05,
33
+ "num_attention_heads": 4,
34
  "num_hidden_layers": 10,
35
+ "num_key_value_heads": 1,
36
  "pad_token_id": 124893,
37
  "rope_parameters": {
38
  "full_attention": {
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e220eee4fbdc3991d6f0679469ae70fb685e1c81c9c97e136b866bee497f69ae
3
- size 2163400
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c2c8b4d4fde884ed076ad2e6ad1ba42744f997a124105b5be2d6ef5de1fdcf9
3
+ size 17261704