barakplasma commited on
Commit
7f4c0e9
·
verified ·
1 Parent(s): e468f0d

Upload scripts/convert_translategemma_android.py with huggingface_hub

Browse files
scripts/convert_translategemma_android.py CHANGED
@@ -142,6 +142,7 @@ def build_translategemma_4b(checkpoint_path: str):
142
  num_heads=8, head_dim=256, num_query_groups=4,
143
  rotary_base=1_000_000 if (idx + 1) % 6 == 0 else 10_000,
144
  rotary_percentage=1.0, qkv_transpose_before_split=True,
 
145
  query_norm_config=norm, key_norm_config=norm, logit_softcap=None,
146
  sliding_window_size=1024,
147
  attn_type=cfg_mod.AttentionType.GLOBAL if (idx + 1) % 6 == 0
 
142
  num_heads=8, head_dim=256, num_query_groups=4,
143
  rotary_base=1_000_000 if (idx + 1) % 6 == 0 else 10_000,
144
  rotary_percentage=1.0, qkv_transpose_before_split=True,
145
+ qkv_fused_interleaved=False,
146
  query_norm_config=norm, key_norm_config=norm, logit_softcap=None,
147
  sliding_window_size=1024,
148
  attn_type=cfg_mod.AttentionType.GLOBAL if (idx + 1) % 6 == 0