Upload MLX converted model with quantization settings

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,44 @@
+---
+language:
+- en
+- zh
+library_name: transformers
+license: mit
+pipeline_tag: text-generation
+base_model: zai-org/GLM-4.6
+tags:
+- mlx
+---
+
+# cs2764/GLM-4.6-mlx-6Bit
+
+The Model [cs2764/GLM-4.6-mlx-6Bit](https://huggingface.co/cs2764/GLM-4.6-mlx-6Bit) was converted to MLX format from [zai-org/GLM-4.6](https://huggingface.co/zai-org/GLM-4.6) using mlx-lm version **0.28.0**.
+
+## Quantization Details
+
+This model was converted with the following quantization settings:
+
+- **Quantization Strategy**: 6-bit quantization
+- **Average bits per weight**: 6.502
+
+## Use with mlx
+
+```bash
+pip install mlx-lm
+```
+
+```python
+from mlx_lm import load, generate
+
+model, tokenizer = load("cs2764/GLM-4.6-mlx-6Bit")
+
+prompt="hello"
+
+if hasattr(tokenizer, "apply_chat_template") and tokenizer.chat_template is not None:
+    messages = [{"role": "user", "content": prompt}]
+    prompt = tokenizer.apply_chat_template(
+        messages, tokenize=False, add_generation_prompt=True
+    )
+
+response = generate(model, tokenizer, prompt=prompt, verbose=True)
+```

chat_template.jinja

@@ -0,0 +1,103 @@
+[gMASK]<sop>
+{%- if tools -%}
+<|system|>
+# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{% for tool in tools %}
+{{ tool | tojson(ensure_ascii=False) }}
+{% endfor %}
+</tools>
+
+For each function call, output the function name and arguments within the following XML format:
+<tool_call>{function-name}
+<arg_key>{arg-key-1}</arg_key>
+<arg_value>{arg-value-1}</arg_value>
+<arg_key>{arg-key-2}</arg_key>
+<arg_value>{arg-value-2}</arg_value>
+...
+</tool_call>{%- endif -%}
+{%- macro visible_text(content) -%}
+    {%- if content is string -%}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping -%}
+        {%- for item in content -%}
+            {%- if item is mapping and item.type == 'text' -%}
+                {{- item.text }}
+            {%- elif item is string -%}
+                {{- item }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{- content }}
+    {%- endif -%}
+{%- endmacro -%}
+{%- set ns = namespace(last_user_index=-1) %}
+{%- for m in messages %}
+    {%- if m.role == 'user' %}
+        {% set ns.last_user_index = loop.index0 -%}
+    {%- endif %}
+{%- endfor %}
+{% for m in messages %}
+{%- if m.role == 'user' -%}<|user|>
+{{ visible_text(m.content) }}
+{{- '/nothink' if (enable_thinking is defined and not enable_thinking and not visible_text(m.content).endswith("/nothink")) else '' -}}
+{%- elif m.role == 'assistant' -%}
+<|assistant|>
+{%- set reasoning_content = '' %}
+{%- set content = visible_text(m.content) %}
+{%- if m.reasoning_content is string %}
+    {%- set reasoning_content = m.reasoning_content %}
+{%- else %}
+    {%- if '</think>' in content %}
+        {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+        {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+    {%- endif %}
+{%- endif %}
+{%- if loop.index0 > ns.last_user_index and reasoning_content -%}
+{{ '\n<think>' + reasoning_content.strip() +  '</think>'}}
+{%- else -%}
+{{ '\n<think></think>' }}
+{%- endif -%}
+{%- if content.strip() -%}
+{{ '\n' + content.strip() }}
+{%- endif -%}
+{% if m.tool_calls %}
+{% for tc in m.tool_calls %}
+{%- if tc.function %}
+    {%- set tc = tc.function %}
+{%- endif %}
+{{ '\n<tool_call>' + tc.name }}
+{% set _args = tc.arguments %}
+{% for k, v in _args.items() %}
+<arg_key>{{ k }}</arg_key>
+<arg_value>{{ v | tojson(ensure_ascii=False) if v is not string else v }}</arg_value>
+{% endfor %}
+</tool_call>{% endfor %}
+{% endif %}
+{%- elif m.role == 'tool' -%}
+{%- if m.content is string -%}
+{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+    {{- '<|observation|>' }}
+{%- endif %}
+{{- '\n<tool_response>\n' }}
+{{- m.content }}
+{{- '\n</tool_response>' }}
+{%- else -%}
+<|observation|>{% for tr in m.content %}
+
+<tool_response>
+{{ tr.output if tr.output is defined else tr }}
+</tool_response>{% endfor -%}
+{% endif -%}
+{%- elif m.role == 'system' -%}
+<|system|>
+{{ visible_text(m.content) }}
+{%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    <|assistant|>{{- '\n<think></think>' if (enable_thinking is defined and not enable_thinking) else '' -}}
+{%- endif -%}

config.json

@@ -0,0 +1,53 @@
+{
+    "architectures": [
+        "Glm4MoeForCausalLM"
+    ],
+    "attention_bias": true,
+    "attention_dropout": 0.0,
+    "eos_token_id": [
+        151329,
+        151336,
+        151338
+    ],
+    "first_k_dense_replace": 3,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 5120,
+    "initializer_range": 0.02,
+    "intermediate_size": 12288,
+    "max_position_embeddings": 202752,
+    "model_type": "glm4_moe",
+    "moe_intermediate_size": 1536,
+    "n_group": 1,
+    "n_routed_experts": 160,
+    "n_shared_experts": 1,
+    "norm_topk_prob": true,
+    "num_attention_heads": 96,
+    "num_experts_per_tok": 8,
+    "num_hidden_layers": 92,
+    "num_key_value_heads": 8,
+    "num_nextn_predict_layers": 1,
+    "pad_token_id": 151329,
+    "partial_rotary_factor": 0.5,
+    "quantization": {
+        "group_size": 64,
+        "bits": 6,
+        "mode": "affine"
+    },
+    "quantization_config": {
+        "group_size": 64,
+        "bits": 6,
+        "mode": "affine"
+    },
+    "rms_norm_eps": 1e-05,
+    "rope_scaling": null,
+    "rope_theta": 1000000,
+    "routed_scaling_factor": 2.5,
+    "tie_word_embeddings": false,
+    "topk_group": 1,
+    "torch_dtype": "bfloat16",
+    "transformers_version": "4.54.0",
+    "use_cache": true,
+    "use_qk_norm": true,
+    "vocab_size": 151552
+}

generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": [
+    151329,
+    151336,
+    151338
+  ],
+  "pad_token_id": 151329,
+  "temperature": 1.0,
+  "transformers_version": "4.56.2"
+}

model-00001-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:651de89c10bbc509155b13df3f0b9c4e1974376e367e3eee1dec9267c118ee7f
+size 4732435585

model-00002-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:663969396a528243b56d9ca5f5308fa480d973b680badc4298af23a95a144803
+size 5243427265

model-00003-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5294979e3f3e3be0bf7ced5f83ff2a9ecd44f64014b1aa2f8f727139f9094727
+size 5335722898

model-00004-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78c1b13cddbc08cbb94651b25e076b9565d53dd46e743651bc2b3ce8553680c8
+size 5363247106

model-00005-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fcb3e6fef5f12a7062ae2694154645a55d1313c1582ece752656e532121fb3e
+size 5294546458

model-00006-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c006861df314f0d5c03cee80d62d0b22c4c0c509747a9a4b2c75de0f7ec8a772
+size 5335722946

model-00007-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:554ca4ebb1d0057f528b4f5814165894da24fe7a256565dca5e51bd8148a7640
+size 5363247185

model-00008-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d43fdfb0f384f6c46ac5a25ca06ac717eed7b6cab28cef40e890c7a4540222a7
+size 5294546451

model-00009-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:610bd0de3537e203ce02c19c76691eaf5f2f7ac76cae7c7681a20cd6dc3ac84c
+size 5335722966

model-00010-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e653dbd679a7471a3b8e86e5ba49f981ad7bc7e007d3fa40d051e431707938d
+size 5363247163

model-00011-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdad75f6c388efe0106b30d17ae087d62b9edae24b22e0e8dcd457065f3b3eca
+size 5294546461

model-00012-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3b3e0a6ce477c58b8717e96cade9485d59606b7075f0c5231541941c9ee9906
+size 5335722940

model-00013-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f3301fc2d137d1303a5d57e544959ecc4f7f0a1f72e4e91fcb8d78f1f6a9695
+size 5363247197

model-00014-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f60aadb4160f8f9490872aa70c422ebac940d1c1130a3aa693481851bd47183
+size 5294546487

model-00015-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3863fead247704010f027c2945e77c193c24f353bc3ff1a9ffa8c16c09ef4a53
+size 5335722976

model-00016-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2c7abcebb3cc21dd3176418d36ad10f64f3404dca21ae1a43fdde542448a4e9
+size 5363247143

model-00017-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c00246155e177e0be2afcbcda99d56e89b8e601a77ea05eb5bf18bff66a85a5b
+size 5294546487

model-00018-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35bcc543c57a17e1f57b5e68cdb5036aad970580ba0367f96eb5ed7bc709e028
+size 5335722962

model-00019-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a9e3cf786ccf08795d453dff901a1a74771f9283009bf63f1a34d7654bb8e1d
+size 5363247181

model-00020-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7223cc2f3abe72b3bf3da7e76577d30b5dfb2203c1f46ab2389064eed8a6f152
+size 5294546489

model-00021-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23a0a63141efd0a6b4118612dc633f810abac22e3d0d8efdd0cf71a8722ceede
+size 5335722984

model-00022-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d3a295840275c5b0cf0071d2184985e742ed40eaa7d7c119254636658d8f016
+size 5363247173

model-00023-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:364864110ee87ebe121cf13d61cb2a6a372eaf547e49af12c7d4b7d26dc45ddc
+size 5294546481

model-00024-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e58c3d73ed5dff2df23c2e92f52c83c26a6c17c3afbe4dc9f283098f8b4b7207
+size 5335722908

model-00025-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6e495fb5a80895cb8b81b7560b39b94a6c38752d18cb52c1d80afb421634e47
+size 5363247193

model-00026-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa654f20c7168b9c2f50b4e039f9f069ff7b73511620ecc6f244dcfe560ab4e3
+size 5294546487

model-00027-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02489e05d453416bacdb5a9ec258ed49dfc6d7e846acffd5d22a8d533676b358
+size 5335722916

model-00028-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fc12fbcb7ba1a6239acbeac737488b801957a9a0509106e8be2f45a415cbf95
+size 5363247169

model-00029-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65d54fa57b37bd685ebf4c3d5724ad184b401688b2c803dd67594d5618164a47
+size 5294546487

model-00030-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c70f38843c83f55b4ce516ee2a2c490917dde226143c3ff421f6b3e153c6f532
+size 5335722970

model-00031-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:266742861fdceb9f54d46810c5c0fc5d60c230989ad74e7323f70f78536de6d7
+size 5363247195

model-00032-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d93c98569bbfb4dd8c1af9f4d283adfd5c139bff3c88a4bfebffd48cb928528
+size 5294546489

model-00033-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9a3738cd42ce5f03b1624ab100f870a2af04870a4c90b96d08367e4c7a46b4a
+size 5335722968

model-00034-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6acd6c99b4d2c25c1b7e109b26309e6b5fc83fc6e266ad27a719fdd505825fa7
+size 5363247173

model-00035-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:929febcaa26abe331aa4d1e3aaf0e636781486d25efb870a1bd253a39a45df15
+size 5294546489

model-00036-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:946a521f24b5489b0c21c204228a9940139c08792c73406c64d59c85f81f0b98
+size 5335722910

model-00037-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd0a07f2340f8a2644f9871fd85ddb2914de56bd703a7c4f336cc1ea983dbb11
+size 5363247177

model-00038-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd9aaa71c0e3011ee5e1153774ff71208f475bbde3f79839fae073fbbd54ebe9
+size 5294546491

model-00039-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de76a40b1738565371c056365305dd04f654923d13a9912adcdece0f3fa83e0e
+size 5335722982

model-00040-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb7b631b903d630b79f73ccb2d764a7830a8969f5ccb768711f826a6ceb12ea8
+size 5363247177

model-00041-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55322ea59aef6410b27807253ad611732e369d2f90f322aeda8db66ecbb8cbf8
+size 5294546479

model-00042-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2271b2cb7dc4f8e26df910d1ff06b0187853861b6bbd8e2c8ea536a91aa8533
+size 5335722966

model-00043-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f28997f45c43ec45f34b30c8c1e41a0a7e5547500be9a6e9d55a31bbcd4b5fec
+size 5363247181

model-00044-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:353d1aab366d8693a214d9a7380c0f30d5b8d73df3cc0ad9df7dd3cc2a004138
+size 5294546447

model-00045-of-00054.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cd4c5e00e11a4804b7aeffe52bf3b6d51ad1ab4df44ca951c7fff06624876f1
+size 5335722978