CMSManhattan
/

JiRackNative_3b

ternarytransformer

Model card Files Files and versions

kgrabko commited on Jul 9

Commit

80e3270

·

verified ·

1 Parent(s): a6f00ef

Update JiRackNative_3b.py

Files changed (1) hide show

JiRackNative_3b.py +2 -2

JiRackNative_3b.py CHANGED Viewed

@@ -33,8 +33,8 @@ NUM_HEADS = 24
 NUM_KV_HEADS = 8
 # NOTE: with INTERMEDIATE_SIZE = 4096 the model is ~2.05B params, not 3B.
 # Restore 8192 for a true ~2.8-3B model (wider SwiGLU FFN). Your call.
-#INTERMEDIATE_SIZE = 8192
-INTERMEDIATE_SIZE = 4096
 MAX_SEQ_LEN = 4096
 RMS_EPS = 1e-6
 STABILITY_EPS = 1e-9

 NUM_KV_HEADS = 8
 # NOTE: with INTERMEDIATE_SIZE = 4096 the model is ~2.05B params, not 3B.
 # Restore 8192 for a true ~2.8-3B model (wider SwiGLU FFN). Your call.
+INTERMEDIATE_SIZE = 8192
+#INTERMEDIATE_SIZE = 4096
 MAX_SEQ_LEN = 4096
 RMS_EPS = 1e-6
 STABILITY_EPS = 1e-9