Upload folder using huggingface_hub

README.md

@@ -158,6 +158,15 @@ config = AutoConfig.from_pretrained(
 print(config)
 torch.set_default_dtype(torch.bfloat16)
 model = Qwen3_5MoeForConditionalGeneration(config)
+with torch.no_grad():
+    for i in range(3):
+        attn = model.model.language_model.layers[i].linear_attn
+        attn.A_log = torch.nn.Parameter(attn.A_log.float())
+        attn.norm.float()
+
+print(model.state_dict()['model.language_model.layers.0.linear_attn.A_log'].dtype)
+print(model.state_dict()['model.language_model.layers.0.linear_attn.norm.weight'].dtype)
+
 model.mtp = torch.nn.ModuleDict({
     "pre_fc_norm_embedding": torch.nn.RMSNorm(config.text_config.hidden_size),
     "fc": torch.nn.Linear(config.text_config.hidden_size * 2, config.text_config.hidden_size, bias=False),

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a68280ca252dadbb9875aa7455a9def12207fc29aa7bf34f10f578a4e4102cb
-size 10057952
+oid sha256:4e568307fe7ff224d8af7fca904d2c196fbf8231257434b99ce9b9b01cbc31b0
+size 10058136