Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

chat_template.jinja

@@ -0,0 +1,86 @@
+[gMASK]<sop>
+{%- if tools -%}
+<|system|>
+# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{% for tool in tools %}
+{{ tool | tojson(ensure_ascii=False) }}
+{% endfor %}
+</tools>
+
+For each function call, output the function name and arguments within the following XML format:
+<tool_call>{function-name}<arg_key>{arg-key-1}</arg_key><arg_value>{arg-value-1}</arg_value><arg_key>{arg-key-2}</arg_key><arg_value>{arg-value-2}</arg_value>...</tool_call>{%- endif -%}
+{%- macro visible_text(content) -%}
+    {%- if content is string -%}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping -%}
+        {%- for item in content -%}
+            {%- if item is mapping and item.type == 'text' -%}
+                {{- item.text }}
+            {%- elif item is string -%}
+                {{- item }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{- content }}
+    {%- endif -%}
+{%- endmacro -%}
+{%- set ns = namespace(last_user_index=-1) %}
+{%- for m in messages %}
+    {%- if m.role == 'user' %}
+        {% set ns.last_user_index = loop.index0 -%}
+    {%- endif %}
+{%- endfor %}
+{% for m in messages %}
+{%- if m.role == 'user' -%}<|user|>{{ visible_text(m.content) }}
+{%- elif m.role == 'assistant' -%}
+<|assistant|>
+{%- set reasoning_content = '' %}
+{%- set content = visible_text(m.content) %}
+{%- if m.reasoning_content is string %}
+    {%- set reasoning_content = m.reasoning_content %}
+{%- else %}
+    {%- if '</think>' in content %}
+        {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+        {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+    {%- endif %}
+{%- endif %}
+{%- if ((clear_thinking is defined and not clear_thinking) or loop.index0 > ns.last_user_index) and reasoning_content -%}
+{{ '<think>' + reasoning_content.strip() +  '</think>'}}
+{%- else -%}
+{{ '</think>' }}
+{%- endif -%}
+{%- if content.strip() -%}
+{{ content.strip() }}
+{%- endif -%}
+{% if m.tool_calls %}
+{% for tc in m.tool_calls %}
+{%- if tc.function %}
+    {%- set tc = tc.function %}
+{%- endif %}
+{{- '<tool_call>' + tc.name -}}
+{% set _args = tc.arguments %}{% for k, v in _args.items() %}<arg_key>{{ k }}</arg_key><arg_value>{{ v | tojson(ensure_ascii=False) if v is not string else v }}</arg_value>{% endfor %}</tool_call>{% endfor %}
+{% endif %}
+{%- elif m.role == 'tool' -%}
+{%- if m.content is string -%}
+{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+    {{- '<|observation|>' }}
+{%- endif %}
+{{- '<tool_response>' }}
+{{- m.content }}
+{{- '</tool_response>' }}
+{%- else -%}
+<|observation|>{% for tr in m.content %}
+<tool_response>{{ tr.output if tr.output is defined else tr }}</tool_response>{% endfor -%}
+{% endif -%}
+{%- elif m.role == 'system' -%}
+<|system|>{{ visible_text(m.content) }}
+{%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    <|assistant|>{{- '</think>' if (enable_thinking is defined and not enable_thinking) else '<think>' -}}
+{%- endif -%}

config.json

@@ -0,0 +1,45 @@
+{
+  "architectures": [
+    "Glm4MoeLiteForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "pad_token_id": 154820,
+  "eos_token_id": [
+    154820,
+    154827,
+    154829
+  ],
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "intermediate_size": 10240,
+  "max_position_embeddings": 202752,
+  "model_type": "glm4_moe_lite",
+  "moe_intermediate_size": 1536,
+  "topk_method": "noaux_tc",
+  "norm_topk_prob": true,
+  "num_attention_heads": 20,
+  "n_group": 1,
+  "topk_group": 1,
+  "n_routed_experts": 64,
+  "n_shared_experts": 1,
+  "routed_scaling_factor": 1.8,
+  "num_experts_per_tok": 4,
+  "first_k_dense_replace": 1,
+  "num_hidden_layers": 75,
+  "num_key_value_heads": 20,
+  "num_nextn_predict_layers": 1,
+  "partial_rotary_factor": 1.0,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 1000000,
+  "tie_word_embeddings": false,
+  "dtype": "bfloat16",
+  "transformers_version": "5.0.0rc0",
+  "q_lora_rank": 768,
+  "kv_lora_rank": 512,
+  "qk_nope_head_dim": 192,
+  "qk_rope_head_dim": 64,
+  "v_head_dim": 256,
+  "vocab_size": 154880
+}

growth_metadata.json

@@ -0,0 +1,40 @@
+{
+  "model_type": "glm4_moe_lite",
+  "src": {
+    "model_type": "glm4_moe_lite",
+    "vocab_size": 154880,
+    "hidden_size": 2048,
+    "intermediate_size": 10240,
+    "moe_intermediate_size": 1536,
+    "num_hidden_layers": 47,
+    "num_nextn_predict_layers": 1,
+    "num_attention_heads": 20,
+    "num_key_value_heads": 20,
+    "q_lora_rank": 768,
+    "kv_lora_rank": 512,
+    "qk_nope_head_dim": 192,
+    "qk_rope_head_dim": 64,
+    "v_head_dim": 256,
+    "n_routed_experts": 64,
+    "n_shared_experts": 1,
+    "num_experts_per_tok": 4,
+    "first_k_dense_replace": 1,
+    "moe_layer_freq": 1,
+    "rms_norm_eps": 1e-05,
+    "tie_word_embeddings": false
+  },
+  "tgt": {
+    "hidden_size": 2048,
+    "intermediate_size": 10240,
+    "moe_intermediate_size": 1536,
+    "num_hidden_layers": 75,
+    "rms_norm_eps": 1e-05
+  },
+  "operators": {
+    "hidden_operator": "vzero",
+    "mlp_operator": "dormant_prime",
+    "new_layer_init": "copy_base",
+    "depth_identity_operator": "inproj_zero_prime",
+    "seed": 42
+  }
+}

model-00001-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e5cb46944b08fb0e77e23134ece62e85a96463a001e8db8c2dd5c6add068c61
+size 4294014464

model-00002-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e318e1145e164cf8fe382295a53381026eb44c19580139923ba9f88e06f395d
+size 4290105192

model-00003-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8b9a20045f63c70c577c08413a2bcbe30ac38f3b6396f92b476a231fe194e50
+size 4289854464

model-00004-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbe524ee1e12f80e6f8981b4398ccc3d14d92ea0c0a9f6b5929c1c236a24c134
+size 4290105888

model-00005-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c81bf645b96d851b00ebcf757c07c8663412cdb1c785cb99abf8041cc9619ae
+size 4290105888

model-00006-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc2f76d3390ca6fbf2da1e1d343cf23dbc531d916d4f419d42a5ff84eac0a8b8
+size 4289855168

model-00007-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59ad9d71a62e2c196ea85744c0ced98c97e91454fbaa1b2a7d572e24e2259342
+size 4290105896

model-00008-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61d34b61b9b35bcb1d559b6b9924572088c969b34fa6f1e07a673b4d19a15ddf
+size 4290105888

model-00009-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa2419b3e4b3925d61e4d1eb97a0f2c7de7875b81d76daaaf3230a4ee64bb988
+size 4289855168

model-00010-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bff04303b779fbcd0172f798a0d1bf72dade9c3ea26e034cb82051ce64e1fc2
+size 4290105888

model-00011-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:785d58bdd0b250c0f1e444550cde21aa1b3cb9fa7e0ac9713d5913d1bb5bb042
+size 4289855168

model-00012-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f42713e6175c89882353b0516780c8d5620d85e1e55d295165323717c6f7efb
+size 4290105888

model-00013-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b15f671b430c58e24ab7100988b0b7849764a460e4516da6bc4920be7fca466
+size 4290105888

model-00014-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb2856cfaa79e21cd77ff537c63c75ec3cf76e4d086f5a95b642da0ae2001756
+size 4289044984

model-00015-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:122b3a9dce6b83e63ad490b28d0500b07fffbab99c55b331fcc448fa9d033ee8
+size 4293255736

model-00016-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1a2b17629734737e5992ef502a49c8b607a66512c2cb53252b9a14983e441a1
+size 4290105888

model-00017-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d6982be9f0b0acbf21f104170a8b95408b62d4602d550045ce5dc9677258c4d
+size 4289855168

model-00018-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1924d21c413261016592f95f50b206c741be4d2179ebe1b030db4b05346dc886
+size 4290105888

model-00019-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2882d1008564c81c5f2911dd027f290a0c767fcd794b96e6400f159fcbb352b7
+size 4289855168

model-00020-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb3db8742f143bdc0efb5ec3f4a932a6e91a87bf8880e56822715f0719f636b6
+size 4290105888

model-00021-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:512bc64dbe6049ed35a53db5612f952335688dc62c54f97065682d2fd872b10c
+size 4290105888

model-00022-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3cd0f585efc38fd265978d1cabf5ea016d854d10be577cfbc64d0ace82a0f83
+size 4289855168

model-00023-of-00023.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17df5ba9c14fe25ee8c4f39b53798339fde9abd2dd4030630e3abd6d41ba131a
+size 3635777240

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19e773648cb4e65de8660ea6365e10acca112d42a854923df93db4a6f333a82d
+size 20217442

tokenizer_config.json

@@ -0,0 +1,321 @@
+{
+  "added_tokens_decoder": {
+    "154820": {
+      "content": "<|endoftext|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "154821": {
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "154822": {
+      "content": "[gMASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "154823": {
+      "content": "[sMASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "154824": {
+      "content": "<sop>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "154825": {
+      "content": "<eop>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "154826": {
+      "content": "<|system|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "154827": {
+      "content": "<|user|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "154828": {
+      "content": "<|assistant|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "154829": {
+      "content": "<|observation|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "154830": {
+      "content": "<|begin_of_image|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "154831": {
+      "content": "<|end_of_image|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "154832": {
+      "content": "<|begin_of_video|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "154833": {
+      "content": "<|end_of_video|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "154834": {
+      "content": "<|begin_of_audio|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "154835": {
+      "content": "<|end_of_audio|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "154836": {
+      "content": "<|begin_of_transcription|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "154837": {
+      "content": "<|end_of_transcription|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    "154838": {
+      "content": "<|code_prefix|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "154839": {
+      "content": "<|code_middle|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "154840": {
+      "content": "<|code_suffix|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "154841": {
+      "content": "<think>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "154842": {
+      "content": "</think>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "154843": {
+      "content": "<tool_call>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "154844": {
+      "content": "</tool_call>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "154845": {
+      "content": "<tool_response>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "154846": {
+      "content": "</tool_response>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "154847": {
+      "content": "<arg_key>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "154848": {
+      "content": "</arg_key>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "154849": {
+      "content": "<arg_value>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "154850": {
+      "content": "</arg_value>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "154851": {
+      "content": "/nothink",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "154852": {
+      "content": "<|begin_of_box|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "154853": {
+      "content": "<|end_of_box|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "154854": {
+      "content": "<|image|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    "154855": {
+      "content": "<|video|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "[MASK]",
+    "[gMASK]",
+    "[sMASK]",
+    "<sop>",
+    "<eop>",
+    "<|system|>",
+    "<|user|>",
+    "<|assistant|>",
+    "<|observation|>",
+    "<|begin_of_image|>",
+    "<|end_of_image|>",
+    "<|begin_of_video|>",
+    "<|end_of_video|>",
+    "<|begin_of_audio|>",
+    "<|end_of_audio|>",
+    "<|begin_of_transcription|>",
+    "<|end_of_transcription|>"
+  ],
+  "clean_up_tokenization_spaces": false,
+  "do_lower_case": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 128000,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "remove_space": false,
+  "tokenizer_class": "PreTrainedTokenizer"
+}