Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,250 @@
+---
+library_name: transformers
+base_model:
+- zai-org/GLM-4.7-Flash
+---
+
+This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from [zai-org/GLM-4.7-Flash](https://huggingface.co/zai-org/GLM-4.7-Flash).
+
+### Example usage:
+
+- vLLM
+
+```bash
+# Multi-token prediction is supported
+model_id=yujiepan/glm-4.7-flash-tiny-random
+vllm serve $model_id \
+     --tensor-parallel-size 2 \
+     --speculative-config.method mtp \
+     --speculative-config.num_speculative_tokens 1 \
+     --tool-call-parser glm47 \
+     --reasoning-parser glm45 \
+     --enable-auto-tool-choice
+```
+
+- SGLang
+
+```bash
+# Multi-token prediction is supported
+model_id=yujiepan/glm-4.7-flash-tiny-random
+python3 -m sglang.launch_server   --model-path $model_id   --tp-size 2 \
+  --tool-call-parser glm47  \
+  --reasoning-parser glm45 \
+  --speculative-algorithm EAGLE \
+  --speculative-num-steps 3 \
+  --speculative-eagle-topk 1 \
+  --speculative-num-draft-tokens 4
+```
+
+- Transformers
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+# Load model and tokenizer
+model_id = "yujiepan/glm-4.7-flash-tiny-random"
+messages = [{"role": "user", "content": "hello"}]
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+inputs = tokenizer.apply_chat_template(
+    messages,
+    tokenize=True,
+    add_generation_prompt=True,
+    return_dict=True,
+    return_tensors="pt",
+)
+model = AutoModelForCausalLM.from_pretrained(
+    pretrained_model_name_or_path=model_id,
+    torch_dtype=torch.bfloat16,
+    device_map="cuda",
+)
+inputs = inputs.to(model.device)
+generated_ids = model.generate(
+    **inputs, max_new_tokens=32, do_sample=False)
+output_text = tokenizer.decode(
+    generated_ids[0][inputs.input_ids.shape[1]:])
+print(output_text)
+```
+
+### Codes to create this repo:
+
+```python
+import json
+from copy import deepcopy
+from pathlib import Path
+
+import accelerate
+import torch
+import torch.nn as nn
+from huggingface_hub import file_exists, hf_hub_download
+from transformers import (
+    AutoConfig,
+    AutoModelForCausalLM,
+    AutoProcessor,
+    GenerationConfig,
+    set_seed,
+)
+
+source_model_id = "zai-org/GLM-4.7-Flash"
+save_folder = "/tmp/yujiepan/glm-4.7-flash-tiny-random"
+
+processor = AutoProcessor.from_pretrained(
+    source_model_id, trust_remote_code=True)
+processor.save_pretrained(save_folder)
+
+with open(hf_hub_download(source_model_id, filename='config.json', repo_type='model'), 'r', encoding='utf-8') as f:
+    config_json = json.load(f)
+config_json.update({
+    'kv_lora_rank': 384,
+    'num_key_value_heads': 1,
+    'q_lora_rank': 32,
+    'qk_nope_head_dim': 64,
+    'qk_rope_head_dim': 192,
+    'v_head_dim': 64,
+    'num_key_value_heads': 4,
+    'num_attention_heads': 4,
+})
+config_json['hidden_size'] = 8
+config_json['intermediate_size'] = 32
+config_json['moe_intermediate_size'] = 32
+config_json['num_hidden_layers'] = 2
+config_json['tie_word_embeddings'] = False
+config_json['use_cache'] = True
+with open(f"{save_folder}/config.json", "w", encoding='utf-8') as f:
+    json.dump(config_json, f, indent=2)
+
+config = AutoConfig.from_pretrained(
+    save_folder,
+    trust_remote_code=True,
+)
+print(config)
+torch.set_default_dtype(torch.bfloat16)
+model = AutoModelForCausalLM.from_config(config)
+torch.set_default_dtype(torch.float32)
+if file_exists(filename="generation_config.json", repo_id=source_model_id, repo_type='model'):
+    model.generation_config = GenerationConfig.from_pretrained(
+        source_model_id, trust_remote_code=True,
+    )
+    model.generation_config.do_sample = True
+    print(model.generation_config)
+model = model.cpu()
+set_seed(42)
+with torch.no_grad():
+    for name, p in sorted(model.named_parameters()):
+        torch.nn.init.normal_(p, 0, 0.1)
+        print(name, p.shape)
+# MTP
+set_seed(42)
+model.model.layers.append(nn.ModuleDict(dict(
+    embed_tokens=deepcopy(model.model.embed_tokens),
+    shared_head=nn.ModuleDict(dict(
+        norm=nn.RMSNorm(config.hidden_size),
+        head=deepcopy(model.model.embed_tokens),
+    )),
+    eh_proj=nn.Linear(config.hidden_size * 2,
+                      config.hidden_size, bias=False),
+    enorm=nn.RMSNorm(config.hidden_size),
+    hnorm=nn.RMSNorm(config.hidden_size),
+    input_layernorm=nn.RMSNorm(config.hidden_size),
+    post_attention_layernorm=nn.RMSNorm(config.hidden_size),
+    self_attn=deepcopy(model.model.layers[1].self_attn),
+    mlp=deepcopy(model.model.layers[1].mlp),
+)))
+for i in range(1, len(model.model.layers)):
+    model.model.layers[i].mlp.gate.e_score_correction_bias = torch.rand_like(
+        model.model.layers[i].mlp.gate.e_score_correction_bias).float()
+model.save_pretrained(save_folder)
+print(model)
+```
+
+### Printing the model:
+
+```text
+Glm4MoeLiteForCausalLM(
+  (model): Glm4MoeLiteModel(
+    (embed_tokens): Embedding(154880, 8, padding_idx=154820)
+    (layers): ModuleList(
+      (0): Glm4MoeLiteDecoderLayer(
+        (self_attn): Glm4MoeLiteAttention(
+          (q_a_proj): Linear(in_features=8, out_features=32, bias=False)
+          (q_a_layernorm): Glm4MoeLiteRMSNorm((32,), eps=1e-06)
+          (q_b_proj): Linear(in_features=32, out_features=1024, bias=False)
+          (kv_a_proj_with_mqa): Linear(in_features=8, out_features=576, bias=False)
+          (kv_a_layernorm): Glm4MoeLiteRMSNorm((384,), eps=1e-06)
+          (kv_b_proj): Linear(in_features=384, out_features=512, bias=False)
+          (o_proj): Linear(in_features=256, out_features=8, bias=False)
+        )
+        (mlp): Glm4MoeLiteMLP(
+          (gate_proj): Linear(in_features=8, out_features=32, bias=False)
+          (up_proj): Linear(in_features=8, out_features=32, bias=False)
+          (down_proj): Linear(in_features=32, out_features=8, bias=False)
+          (act_fn): SiLUActivation()
+        )
+        (input_layernorm): Glm4MoeLiteRMSNorm((8,), eps=1e-05)
+        (post_attention_layernorm): Glm4MoeLiteRMSNorm((8,), eps=1e-05)
+      )
+      (1): Glm4MoeLiteDecoderLayer(
+        (self_attn): Glm4MoeLiteAttention(
+          (q_a_proj): Linear(in_features=8, out_features=32, bias=False)
+          (q_a_layernorm): Glm4MoeLiteRMSNorm((32,), eps=1e-06)
+          (q_b_proj): Linear(in_features=32, out_features=1024, bias=False)
+          (kv_a_proj_with_mqa): Linear(in_features=8, out_features=576, bias=False)
+          (kv_a_layernorm): Glm4MoeLiteRMSNorm((384,), eps=1e-06)
+          (kv_b_proj): Linear(in_features=384, out_features=512, bias=False)
+          (o_proj): Linear(in_features=256, out_features=8, bias=False)
+        )
+        (mlp): Glm4MoeLiteMoE(
+          (experts): Glm4MoeLiteNaiveMoe(
+            (act_fn): SiLUActivation()
+          )
+          (gate): Glm4MoeLiteTopkRouter()
+          (shared_experts): Glm4MoeLiteMLP(
+            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
+            (up_proj): Linear(in_features=8, out_features=32, bias=False)
+            (down_proj): Linear(in_features=32, out_features=8, bias=False)
+            (act_fn): SiLUActivation()
+          )
+        )
+        (input_layernorm): Glm4MoeLiteRMSNorm((8,), eps=1e-05)
+        (post_attention_layernorm): Glm4MoeLiteRMSNorm((8,), eps=1e-05)
+      )
+      (2): ModuleDict(
+        (embed_tokens): Embedding(154880, 8, padding_idx=154820)
+        (shared_head): ModuleDict(
+          (norm): RMSNorm((8,), eps=None, elementwise_affine=True)
+          (head): Embedding(154880, 8, padding_idx=154820)
+        )
+        (eh_proj): Linear(in_features=16, out_features=8, bias=False)
+        (enorm): RMSNorm((8,), eps=None, elementwise_affine=True)
+        (hnorm): RMSNorm((8,), eps=None, elementwise_affine=True)
+        (input_layernorm): RMSNorm((8,), eps=None, elementwise_affine=True)
+        (post_attention_layernorm): RMSNorm((8,), eps=None, elementwise_affine=True)
+        (self_attn): Glm4MoeLiteAttention(
+          (q_a_proj): Linear(in_features=8, out_features=32, bias=False)
+          (q_a_layernorm): Glm4MoeLiteRMSNorm((32,), eps=1e-06)
+          (q_b_proj): Linear(in_features=32, out_features=1024, bias=False)
+          (kv_a_proj_with_mqa): Linear(in_features=8, out_features=576, bias=False)
+          (kv_a_layernorm): Glm4MoeLiteRMSNorm((384,), eps=1e-06)
+          (kv_b_proj): Linear(in_features=384, out_features=512, bias=False)
+          (o_proj): Linear(in_features=256, out_features=8, bias=False)
+        )
+        (mlp): Glm4MoeLiteMoE(
+          (experts): Glm4MoeLiteNaiveMoe(
+            (act_fn): SiLUActivation()
+          )
+          (gate): Glm4MoeLiteTopkRouter()
+          (shared_experts): Glm4MoeLiteMLP(
+            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
+            (up_proj): Linear(in_features=8, out_features=32, bias=False)
+            (down_proj): Linear(in_features=32, out_features=8, bias=False)
+            (act_fn): SiLUActivation()
+          )
+        )
+      )
+    )
+    (norm): Glm4MoeLiteRMSNorm((8,), eps=1e-05)
+    (rotary_emb): Glm4MoeLiteRotaryEmbedding()
+  )
+  (lm_head): Linear(in_features=8, out_features=154880, bias=False)
+)
+```

chat_template.jinja

@@ -0,0 +1,86 @@
+[gMASK]<sop>
+{%- if tools -%}
+<|system|>
+# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{% for tool in tools %}
+{{ tool | tojson(ensure_ascii=False) }}
+{% endfor %}
+</tools>
+
+For each function call, output the function name and arguments within the following XML format:
+<tool_call>{function-name}<arg_key>{arg-key-1}</arg_key><arg_value>{arg-value-1}</arg_value><arg_key>{arg-key-2}</arg_key><arg_value>{arg-value-2}</arg_value>...</tool_call>{%- endif -%}
+{%- macro visible_text(content) -%}
+    {%- if content is string -%}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping -%}
+        {%- for item in content -%}
+            {%- if item is mapping and item.type == 'text' -%}
+                {{- item.text }}
+            {%- elif item is string -%}
+                {{- item }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{- content }}
+    {%- endif -%}
+{%- endmacro -%}
+{%- set ns = namespace(last_user_index=-1) %}
+{%- for m in messages %}
+    {%- if m.role == 'user' %}
+        {% set ns.last_user_index = loop.index0 -%}
+    {%- endif %}
+{%- endfor %}
+{% for m in messages %}
+{%- if m.role == 'user' -%}<|user|>{{ visible_text(m.content) }}
+{%- elif m.role == 'assistant' -%}
+<|assistant|>
+{%- set reasoning_content = '' %}
+{%- set content = visible_text(m.content) %}
+{%- if m.reasoning_content is string %}
+    {%- set reasoning_content = m.reasoning_content %}
+{%- else %}
+    {%- if '</think>' in content %}
+        {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+        {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+    {%- endif %}
+{%- endif %}
+{%- if ((clear_thinking is defined and not clear_thinking) or loop.index0 > ns.last_user_index) and reasoning_content -%}
+{{ '<think>' + reasoning_content.strip() +  '</think>'}}
+{%- else -%}
+{{ '</think>' }}
+{%- endif -%}
+{%- if content.strip() -%}
+{{ content.strip() }}
+{%- endif -%}
+{% if m.tool_calls %}
+{% for tc in m.tool_calls %}
+{%- if tc.function %}
+    {%- set tc = tc.function %}
+{%- endif %}
+{{- '<tool_call>' + tc.name -}}
+{% set _args = tc.arguments %}{% for k, v in _args.items() %}<arg_key>{{ k }}</arg_key><arg_value>{{ v | tojson(ensure_ascii=False) if v is not string else v }}</arg_value>{% endfor %}</tool_call>{% endfor %}
+{% endif %}
+{%- elif m.role == 'tool' -%}
+{%- if m.content is string -%}
+{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+    {{- '<|observation|>' }}
+{%- endif %}
+{{- '<tool_response>' }}
+{{- m.content }}
+{{- '</tool_response>' }}
+{%- else -%}
+<|observation|>{% for tr in m.content %}
+<tool_response>{{ tr.output if tr.output is defined else tr }}</tool_response>{% endfor -%}
+{% endif -%}
+{%- elif m.role == 'system' -%}
+<|system|>{{ visible_text(m.content) }}
+{%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    <|assistant|>{{- '</think>' if (enable_thinking is defined and not enable_thinking) else '<think>' -}}
+{%- endif -%}

config.json

@@ -0,0 +1,59 @@
+{
+  "architectures": [
+    "Glm4MoeLiteForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dtype": "bfloat16",
+  "eos_token_id": [
+    154820,
+    154827,
+    154829
+  ],
+  "first_k_dense_replace": 1,
+  "head_dim": 192,
+  "hidden_act": "silu",
+  "hidden_size": 8,
+  "initializer_range": 0.02,
+  "intermediate_size": 32,
+  "kv_lora_rank": 384,
+  "max_position_embeddings": 202752,
+  "mlp_layer_types": [
+    "dense",
+    "sparse"
+  ],
+  "model_type": "glm4_moe_lite",
+  "moe_intermediate_size": 32,
+  "n_group": 1,
+  "n_routed_experts": 64,
+  "n_shared_experts": 1,
+  "norm_topk_prob": true,
+  "num_attention_heads": 4,
+  "num_experts_per_tok": 4,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 4,
+  "num_nextn_predict_layers": 1,
+  "pad_token_id": 154820,
+  "partial_rotary_factor": 1.0,
+  "pretraining_tp": 1,
+  "q_lora_rank": 32,
+  "qk_head_dim": 256,
+  "qk_nope_head_dim": 64,
+  "qk_rope_head_dim": 192,
+  "rms_norm_eps": 1e-05,
+  "rope_interleave": true,
+  "rope_parameters": {
+    "partial_rotary_factor": 1.0,
+    "rope_theta": 1000000,
+    "rope_type": "default"
+  },
+  "routed_scaling_factor": 1.8,
+  "tie_word_embeddings": false,
+  "topk_group": 1,
+  "topk_method": "noaux_tc",
+  "transformers_version": "5.0.0.dev0",
+  "use_cache": true,
+  "v_head_dim": 64,
+  "vocab_size": 154880
+}

generation_config.json

@@ -0,0 +1,12 @@
+{
+  "_from_model_config": true,
+  "do_sample": true,
+  "eos_token_id": [
+    154820,
+    154827,
+    154829
+  ],
+  "pad_token_id": 154820,
+  "temperature": 1.0,
+  "transformers_version": "5.0.0.dev0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd035174c24e5786d4d7172564bd239feb38916442a6c30dc52762e6fed7aa4d
+size 11585784

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19e773648cb4e65de8660ea6365e10acca112d42a854923df93db4a6f333a82d
+size 20217442

tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "backend": "tokenizers",
+  "clean_up_tokenization_spaces": false,
+  "do_lower_case": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": [
+    "<|endoftext|>",
+    "[MASK]",
+    "[gMASK]",
+    "[sMASK]",
+    "<sop>",
+    "<eop>",
+    "<|system|>",
+    "<|user|>",
+    "<|assistant|>",
+    "<|observation|>",
+    "<|begin_of_image|>",
+    "<|end_of_image|>",
+    "<|begin_of_video|>",
+    "<|end_of_video|>",
+    "<|begin_of_audio|>",
+    "<|end_of_audio|>",
+    "<|begin_of_transcription|>",
+    "<|end_of_transcription|>"
+  ],
+  "is_local": false,
+  "model_max_length": 128000,
+  "model_specific_special_tokens": {},
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "remove_space": false,
+  "tokenizer_class": "TokenizersBackend"
+}