Upload folder using huggingface_hub

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 qwen3/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 Hy-MT2/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+llama3.1-8b-fp4/tokenizer.json filter=lfs diff=lfs merge=lfs -text

Hy-MT2/convert_weights_fp8.py

@@ -0,0 +1,211 @@
+"""
+convert_weights_fp8.py — 将 tencent/Hy-MT2-7B-FP8 官方量化 safetensors 转换为 cccc INIReaderBin 格式
+
+权重格式说明（compressed-tensors/AngelSlim FP8）：
+  <name>.weight       → float8_e4m3fn  (量化权重，直接存储原始字节)
+  <name>.weight_scale → float32        (per-tensor 标量 scale)
+  <name>.input_scale  → float32        (激活 scale，W8A8 用，W8A16 忽略)
+  非量化权重（embed/RMSNorm）→ bfloat16/float32
+
+用法：
+  python convert_weights_fp8.py --hf_dir <下载目录> --output hy_mt2_7b_fp8_cccc.bin
+
+依赖：
+  pip install safetensors torch numpy
+
+cccc INIReaderBin 格式（FP8 variant）：
+  data_type = fp8_e4m3
+  weight_fp8_e4m3_<name> = raw FP8 E4M3 bytes (1 byte/element)   ← 含类型标记的新格式
+  scale_<name>           = raw float32 bytes (4 bytes, per-tensor scale)
+  weight_bf16_<name>     = raw BF16 bytes (2 bytes/element, for small unquantized weights)
+"""
+
+import argparse
+import os
+import struct
+import sys
+import numpy as np
+
+try:
+    import torch
+except ImportError:
+    print("请先安装: pip install torch")
+    sys.exit(1)
+
+try:
+    from safetensors import safe_open
+except ImportError:
+    print("请先安装: pip install safetensors")
+    sys.exit(1)
+
+
+# ── 权重名称映射：HuggingFace → cccc ─────────────────────────────────────────
+# cccc 列主序 BLAS: element(in_i, out_j) 在 in_i + out_j*in_features
+# HF [out, in] C-order: 同样的内存布局（加法交换律），无需转置
+# needs_transpose=False 适用于所有 2D 权重
+def build_weight_map(num_layers: int = 32):
+    mapping = []
+    # 全局权重（非量化，存 BF16）
+    mapping.append(("model.embed_tokens.weight", "W_emb", False, False))  # (hf_name, cccc_name, needs_transpose, is_quantized)
+    mapping.append(("model.norm.weight",          "W_rms_final", False, False))
+
+    for i in range(num_layers):
+        pfx = f"model.layers.{i}"
+        # 量化大权重 (FP8 E4M3 + weight_scale)
+        mapping.extend([
+            (f"{pfx}.self_attn.q_proj.weight",         f"W_q_{i}",     False, True),
+            (f"{pfx}.self_attn.k_proj.weight",         f"W_k_{i}",     False, True),
+            (f"{pfx}.self_attn.v_proj.weight",         f"W_v_{i}",     False, True),
+            (f"{pfx}.self_attn.o_proj.weight",         f"W_o_{i}",     False, True),
+            (f"{pfx}.mlp.gate_proj.weight",            f"W_gate_{i}",  False, True),
+            (f"{pfx}.mlp.up_proj.weight",              f"W_up_{i}",    False, True),
+            (f"{pfx}.mlp.down_proj.weight",            f"W_down_{i}",  False, True),
+        ])
+        # 非量化小权重 (BF16 RMSNorm / QKNorm)
+        mapping.extend([
+            (f"{pfx}.input_layernorm.weight",           f"W_rms_attn_{i}", False, False),
+            (f"{pfx}.post_attention_layernorm.weight",  f"W_rms_ffn_{i}",  False, False),
+            (f"{pfx}.self_attn.query_layernorm.weight", f"W_qnorm_{i}",    False, False),
+            (f"{pfx}.self_attn.key_layernorm.weight",   f"W_knorm_{i}",    False, False),
+        ])
+    return mapping
+
+
+# ── INIReaderBin 序列化 ───────────────────────────────────────────────────────
+def build_inireaderbin(named_blobs: dict) -> bytes:
+    blob_parts = []
+    index_lines = []
+    offset = 0
+    for key, data in named_blobs.items():
+        blob_parts.append(data)
+        index_lines.append(f"{key} = {offset},{len(data)}\n")
+        offset += len(data)
+
+    ini_text = "".join(index_lines)
+    ini_bytes = ini_text.encode("utf-8")
+    binary_content = b"".join(blob_parts)
+
+    header = b"CFG_BIN INI" + b"\x00" * (32 - len("CFG_BIN INI"))
+    size_ini = struct.pack("<Q", len(ini_bytes))
+    return header + size_ini + ini_bytes + binary_content
+
+
+def convert(hf_dir: str, output_path: str):
+    mapping = build_weight_map(num_layers=32)
+
+    # 扫描所有 safetensors 文件
+    shard_files = sorted(
+        f for f in os.listdir(hf_dir) if f.endswith(".safetensors")
+    )
+    if not shard_files:
+        print(f"在 {hf_dir} 中未找到 .safetensors 文件")
+        sys.exit(1)
+    print(f"找到 {len(shard_files)} 个 safetensors 分片：{shard_files}")
+
+    # 打开所有分片（lazy load），建立全键名→handle 映射
+    handles: dict[str, object] = {}
+    for sf in shard_files:
+        path = os.path.join(hf_dir, sf)
+        h = safe_open(path, framework="pt", device="cpu")
+        for key in h.keys():
+            handles[key] = h
+
+    # 检查几个代表性权重的 dtype
+    sample_keys = [k for k in handles if k.endswith(".weight") and "q_proj" in k][:3]
+    for sk in sample_keys:
+        t = handles[sk].get_tensor(sk)
+        print(f"  dtype check: {sk} → {t.dtype}  shape={list(t.shape)}")
+
+    named_blobs: dict[str, bytes] = {}
+    # 文件标识头
+    named_blobs["data_type"] = b"fp8_e4m3"
+    named_blobs["named_weights"] = b"1"
+
+    missing = []
+    fp8_count = 0
+    bf16_count = 0
+
+    for hf_name, cccc_name, needs_transpose, is_quantized in mapping:
+        if hf_name not in handles:
+            missing.append(hf_name)
+            continue
+
+        tensor = handles[hf_name].get_tensor(hf_name)
+
+        if is_quantized:
+            # FP8 量化权重：直接取原始字节（无转换）
+            scale_key = hf_name + "_scale"
+            if scale_key not in handles:
+                print(f"  WARNING: missing scale for {hf_name}, using scale=1.0")
+                scale_val = 1.0
+            else:
+                scale_tensor = handles[scale_key].get_tensor(scale_key)
+                scale_val = float(scale_tensor.reshape(-1)[0].item())
+
+            # input_scale：激活量化 scale（W8A8），key = <module>.input_scale
+            # e.g. model.layers.0.self_attn.q_proj.weight → model.layers.0.self_attn.q_proj.input_scale
+            input_scale_key = hf_name.replace(".weight", ".input_scale")
+            input_scale_val = None
+            if input_scale_key in handles:
+                try:
+                    ist = handles[input_scale_key].get_tensor(input_scale_key)
+                    input_scale_val = float(ist.reshape(-1)[0].item())
+                except Exception:
+                    input_scale_val = None
+
+            # 取 FP8 原始字节（view as uint8）
+            if needs_transpose and tensor.ndim == 2:
+                tensor = tensor.T.contiguous()
+            fp8_blob = tensor.view(torch.uint8).numpy().tobytes()
+            # cccc 约定：quant_scale_ = fp8_max / absmax = 448 / absmax（正向量化 scale）
+            # HF weight_scale = absmax / 448（反量化 scale），两者互为倒数
+            # GEMM 里用 invScaleA = 1/quant_scale_ = weight_scale 做反量化 (fp8 * weight_scale)
+            scale_blob = struct.pack('<f', scale_val)
+
+            named_blobs[f"weight_fp8_e4m3_{cccc_name}"] = fp8_blob
+            named_blobs[f"scale_{cccc_name}"] = scale_blob
+            if input_scale_val is not None and input_scale_val > 0:
+                named_blobs[f"input_scale_{cccc_name}"] = struct.pack('<f', input_scale_val)
+            fp8_count += 1
+            iscale_str = f"  input_scale={input_scale_val:.6g}" if input_scale_val is not None else ""
+            print(f"  FP8  {hf_name:60s} → fp8_e4m3/{cccc_name:20s}  "
+                  f"shape={list(tensor.shape)}  scale={scale_val:.6g}{iscale_str}  {len(fp8_blob)//1024}KB")
+        else:
+            # 非量化权重：转换为 BF16 存储
+            if needs_transpose and tensor.ndim == 2:
+                tensor = tensor.T.contiguous()
+            tensor = tensor.to(torch.bfloat16)
+            bf16_blob = tensor.view(torch.int16).numpy().tobytes()
+            named_blobs[f"weight_bf16_{cccc_name}"] = bf16_blob
+            bf16_count += 1
+            print(f"  BF16 {hf_name:60s} → bf16/{cccc_name:24s}  "
+                  f"shape={list(tensor.shape)}  {len(bf16_blob)//1024}KB")
+
+    if missing:
+        print(f"\n警告：以下权重未找到：{missing}")
+
+    print(f"\nFP8 权重: {fp8_count}  BF16 权重: {bf16_count}")
+    print(f"\n正在写入 {output_path} ...")
+    content = build_inireaderbin(named_blobs)
+    with open(output_path, "wb") as f:
+        f.write(content)
+    size_gb = os.path.getsize(output_path) / 1024**3
+    print(f"完成！文件大小：{size_gb:.2f} GB")
+    print(f"\n格式说明：")
+    print(f"  data_type = fp8_e4m3   (cccc 会调用 loadFp8E4m3Weights)")
+    print(f"  weight_fp8_e4m3_<name> = FP8 E4M3 原始字节，含类型标记（新格式）")
+    print(f"  scale_<name>           = per-tensor float32 scale (官方 weight_scale)")
+    print(f"  weight_bf16_<name>     = BF16 非量化小权重 (RMSNorm/embed)")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Hy-MT2-7B-FP8 → cccc fp8 bin 转换工具")
+    parser.add_argument("--hf_dir",  required=True, help="下载的 HF 权重目录（含 *.safetensors）")
+    parser.add_argument("--output",  default="hy_mt2_7b_fp8_cccc.bin", help="输出 .bin 文件路径")
+    args = parser.parse_args()
+
+    convert(args.hf_dir, args.output)
+
+
+if __name__ == "__main__":
+    main()

Hy-MT2/extract_input_scales.py

@@ -0,0 +1,142 @@
+"""
+extract_input_scales.py — 从 Hy-MT2-7B-FP8 safetensors 提取 W8A8 激活量化 scale
+
+从 HF safetensors 文件中提取每层 input_scale（静态激活量化 scale），
+生成一个很小的 INIReaderBin 格式 sidecar 文件（通常仅约 1KB）。
+
+输出文件可与现有 hy_mt2_7b_fp8_cccc.bin 配合使用，通过 INI 中的
+  input_scale_file = hy_mt2_input_scales.bin
+选项加载，无需重新生成整个 7.48GB 权重文件。
+
+key 格式：input_scale_<cccc_name> = 4 bytes float32（小端序）
+
+用法：
+  python extract_input_scales.py --hf_dir <HF权重目录> --output hy_mt2_input_scales.bin
+
+依赖：
+  pip install safetensors
+"""
+
+import argparse
+import os
+import struct
+import sys
+
+try:
+    from safetensors import safe_open
+except ImportError:
+    print("请先安装: pip install safetensors")
+    sys.exit(1)
+
+try:
+    import torch
+    _ST_FRAMEWORK = "pt"
+except ImportError:
+    _ST_FRAMEWORK = "numpy"
+
+
+# 与 convert_weights_fp8.py 保持一致的量化权重名称映射
+def build_weight_map(num_layers: int = 32):
+    mapping = []
+    for i in range(num_layers):
+        pfx = f"model.layers.{i}"
+        mapping.extend([
+            (f"{pfx}.self_attn.q_proj.weight",  f"W_q_{i}"),
+            (f"{pfx}.self_attn.k_proj.weight",  f"W_k_{i}"),
+            (f"{pfx}.self_attn.v_proj.weight",  f"W_v_{i}"),
+            (f"{pfx}.self_attn.o_proj.weight",  f"W_o_{i}"),
+            (f"{pfx}.mlp.gate_proj.weight",     f"W_gate_{i}"),
+            (f"{pfx}.mlp.up_proj.weight",       f"W_up_{i}"),
+            (f"{pfx}.mlp.down_proj.weight",     f"W_down_{i}"),
+        ])
+    return mapping
+
+
+def build_inireaderbin(named_blobs: dict) -> bytes:
+    blob_parts = []
+    index_lines = []
+    offset = 0
+    for key, data in named_blobs.items():
+        blob_parts.append(data)
+        index_lines.append(f"{key} = {offset},{len(data)}\n")
+        offset += len(data)
+
+    ini_text = "".join(index_lines)
+    ini_bytes = ini_text.encode("utf-8")
+    binary_content = b"".join(blob_parts)
+
+    header = b"CFG_BIN INI" + b"\x00" * (32 - len("CFG_BIN INI"))
+    size_ini = struct.pack("<Q", len(ini_bytes))
+    return header + size_ini + ini_bytes + binary_content
+
+
+def extract(hf_dir: str, output_path: str):
+    mapping = build_weight_map(num_layers=32)
+
+    shard_files = sorted(f for f in os.listdir(hf_dir) if f.endswith(".safetensors"))
+    if not shard_files:
+        print(f"在 {hf_dir} 中未找到 .safetensors 文件")
+        sys.exit(1)
+    print(f"找到 {len(shard_files)} 个 safetensors 分片：{shard_files}")
+
+    handles: dict[str, object] = {}
+    for sf in shard_files:
+        path = os.path.join(hf_dir, sf)
+        h = safe_open(path, framework=_ST_FRAMEWORK, device="cpu")
+        for key in h.keys():
+            handles[key] = h
+
+    named_blobs: dict[str, bytes] = {}
+    found = 0
+    missing = []
+
+    for hf_name, cccc_name in mapping:
+        # HF input_scale key: model.layers.0.self_attn.q_proj.input_scale
+        input_scale_key = hf_name.replace(".weight", ".input_scale")
+        if input_scale_key not in handles:
+            missing.append(input_scale_key)
+            continue
+        try:
+            t = handles[input_scale_key].get_tensor(input_scale_key)
+            if _ST_FRAMEWORK == "pt":
+                val = float(t.reshape(-1)[0].item())
+            else:
+                val = float(t.reshape(-1)[0])
+        except Exception as e:
+            print(f"  WARNING: failed to read {input_scale_key}: {e}")
+            continue
+        if val <= 0:
+            print(f"  WARNING: {input_scale_key} = {val} (non-positive, skipping)")
+            continue
+        named_blobs[f"input_scale_{cccc_name}"] = struct.pack('<f', val)
+        found += 1
+        print(f"  {cccc_name:20s}  input_scale = {val:.6g}")
+
+    if missing:
+        print(f"\n未找到的 input_scale 键（共 {len(missing)} 个）：{missing[:5]}{'...' if len(missing)>5 else ''}")
+
+    if found == 0:
+        print("\n警告：未提取到任何 input_scale（HF 文件可能不含此字段）")
+        sys.exit(1)
+
+    print(f"\n共提取 {found} 个 input_scale")
+    print(f"正在写入 {output_path} ...")
+    content = build_inireaderbin(named_blobs)
+    with open(output_path, "wb") as f:
+        f.write(content)
+    size_b = os.path.getsize(output_path)
+    print(f"完成！文件大小：{size_b} bytes（{size_b // 1024} KB）")
+    print(f"\n使用方法：在 INI 配置文件中添加")
+    print(f"  input_scale_file = hy_mt2_input_scales.bin")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="提取 Hy-MT2-7B-FP8 W8A8 input_scales → cccc sidecar bin")
+    parser.add_argument("--hf_dir",  required=True, help="下载的 HF 权重目录（含 *.safetensors）")
+    parser.add_argument("--output",  default="hy_mt2_input_scales.bin", help="输出 sidecar .bin 文件路径")
+    args = parser.parse_args()
+    extract(args.hf_dir, args.output)
+
+
+if __name__ == "__main__":
+    main()

Hy-MT2/hy_mt2_7b_fp8_cccc.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a85c8e813e648c9d95f86f1eec021923f150fc1a35628ba042a84217fc2728b
+size 8029835724

Hy-MT2/hy_mt2_7b_w8a16.ini

@@ -0,0 +1,148 @@
+[train]
+# official FP8 E4M3 weights (from tencent/Hy-MT2-7B-FP8)
+load_file = hy_mt2_7b_fp8_cccc.bin
+load_net = 1
+WorkType = 0
+train_epochs = 0
+Batch = 1
+output_net = 1
+gpu = 1
+mp = 1
+mp_device = 0
+named_weights = 1
+data_type = bfloat16
+# fp8_e5m2: use_fp8_cache=true (loads E4M3 bytes from file) + act_dt=BFLOAT16 → W8A16
+weight_data_type = fp8_e5m2
+#disable_activation_memory_reuse=1
+
+[llm]
+# W8A16: data_type=fp8_e4m3 + load_file pointing to .fp8.bin triggers FP8 weight load
+# Weights occupy ~8 GB GPU VRAM instead of ~16 GB (BF16)
+# Activations remain BF16; GEMM dequantizes weight FP8→float just-in-time
+
+tokenizer = tokenizer.json
+
+# EOS token: <|eos|> = 127960
+eos_tokens = 127960
+
+# No thinking support
+think_open_id = -1
+think_close_id = -1
+no_think_str =
+
+# Chat format for Hy-MT2-7B
+# System:  <|startoftext|>{content}<|extra_4|>
+# User:    <|startoftext|>{content}<|extra_0|>
+# Asst:    {content}<|eos|>
+sys_prefix = <|startoftext|>
+sys_suffix = <|extra_4|>
+user_prefix = <|startoftext|>
+user_suffix = <|extra_0|>
+asst_prefix =
+asst_suffix = <|eos|>
+
+[net]
+net_num = 2
+
+# ── prefill net (T tokens at once) ──────────────────────────────────────────
+structure0='
+    D    = 4096;
+    Dq   = 4096;
+    Dkv  = 1024;
+    H    = 32;
+    Hkv  = 8;
+    hd   = 128;
+    I    = 14336;
+    V    = 128167;
+    T    = 1024;
+    HB   = 32;
+    HkvB = 8;
+
+    W_emb = MatrixWithName("W_emb", D, 1, 1, V);
+    token_ids = MatrixF(T, 1, 1, 1);
+    X = embed(token_ids, W_emb);
+
+    cos_tab = ropeCosTbl(T, hd, 10000.0);
+    sin_tab = ropeSinTbl(T, hd, 10000.0);
+
+    for (i = 0; i < 32; i++)
+    {
+        W_rms_attn = MatrixWithName("W_rms_attn_" + to_string(i), D, 1);
+        X_norm = rmsNorm(X, W_rms_attn);
+
+        W_q = MatrixWithName("W_q_" + to_string(i), D, Dq, 1, 1);
+        Q_dq = batchedMul(W_q, X_norm, 1, 0);
+        Q_hHT = reshape(Q_dq, {hd, H, T, 1});
+        W_qnorm = MatrixWithName("W_qnorm_" + to_string(i), hd, 1);
+        Q_qnorm = rmsNorm(Q_hHT, W_qnorm);
+        Q_hTH = permute(Q_qnorm, {0, 2, 1, 3});
+        Q_hb = reshapeBatch(Q_hTH, {hd, T, 1, HB});
+        Q_r = rope(Q_hb, cos_tab, sin_tab);
+
+        W_k = MatrixWithName("W_k_" + to_string(i), D, Dkv, 1, 1);
+        K_dkv = batchedMul(W_k, X_norm, 1, 0);
+        K_hHkvT = reshape(K_dkv, {hd, Hkv, T, 1});
+        W_knorm = MatrixWithName("W_knorm_" + to_string(i), hd, 1);
+        K_knorm = rmsNorm(K_hHkvT, W_knorm);
+        K_hTHkv = permute(K_knorm, {0, 2, 1, 3});
+        K_hb = reshapeBatch(K_hTHkv, {hd, T, 1, HkvB});
+        K_r = rope(K_hb, cos_tab, sin_tab);
+
+        W_v = MatrixWithName("W_v_" + to_string(i), D, Dkv, 1, 1);
+        V_dkv = batchedMul(W_v, X_norm, 1, 0);
+        V_hHkvT = reshape(V_dkv, {hd, Hkv, T, 1});
+        V_hTHkv = permute(V_hHkvT, {0, 2, 1, 3});
+        V_hb = reshapeBatch(V_hTHkv, {hd, T, 1, HkvB});
+
+        Kcache = MatrixWithName("Kcache_" + to_string(i), hd, T, 1, HkvB);
+        setIsWeight(Kcache, 0);
+        registerMatrix("Kcache_" + to_string(i), Kcache);
+        K_cached = kvcache(K_r, Kcache);
+
+        Vcache = MatrixWithName("Vcache_" + to_string(i), hd, T, 1, HkvB);
+        setIsWeight(Vcache, 0);
+        registerMatrix("Vcache_" + to_string(i), Vcache);
+        V_cached = kvcache(V_hb, Vcache);
+
+        K_r2   = reshapeBatch(K_cached,  {hd, T, HkvB, 1});
+        K_r3   = tile(K_r2, {1, 1, 1, 4});
+        K_r4   = permute(K_r3, {0, 1, 3, 2});
+        K_tiled = reshapeBatch(K_r4, {hd, T, 1, HB});
+
+        V_r2   = reshapeBatch(V_cached, {hd, T, HkvB, 1});
+        V_r3   = tile(V_r2, {1, 1, 1, 4});
+        V_r4   = permute(V_r3, {0, 1, 3, 2});
+        V_tiled = reshapeBatch(V_r4, {hd, T, 1, HB});
+
+        Attn = attention(Q_r, K_tiled, V_tiled, hd, 1);
+
+        Attn_hTH = reshapeBatch(Attn, {hd, T, H, 1});
+        Attn_hHT = permute(Attn_hTH, {0, 2, 1, 3});
+        Attn_flat = reshape(Attn_hHT, {Dq, T, 1, 1});
+        W_o = MatrixWithName("W_o_" + to_string(i), Dq, D, 1, 1);
+        O_out = batchedMul(W_o, Attn_flat, 1, 0);
+
+        R1 = X + O_out;
+
+        W_rms_ffn = MatrixWithName("W_rms_ffn_" + to_string(i), D, 1);
+        R1_norm = rmsNorm(R1, W_rms_ffn);
+
+        W_gate = MatrixWithName("W_gate_" + to_string(i), D, I, 1, 1);
+        W_up   = MatrixWithName("W_up_" + to_string(i), D, I, 1, 1);
+        gate_out = silu(batchedMul(W_gate, R1_norm, 1, 0));
+        up_out   = batchedMul(W_up, R1_norm, 1, 0);
+        gated    = elementMul(gate_out, up_out);
+        W_down = MatrixWithName("W_down_" + to_string(i), I, D, 1, 1);
+        ffn_out = batchedMul(W_down, gated, 1, 0);
+
+        X = R1 + ffn_out;
+    }
+
+    W_rms_final = MatrixWithName("W_rms_final", D, 1);
+    X_final = rmsNorm(X, W_rms_final);
+    // tie_word_embeddings=true: reshape W_emb to (D, V, 1, 1) view for lm_head
+    W_lm_head = reshapeBatch(W_emb, {D, V, 1, 1});
+    logits = batchedMul(W_lm_head, X_final, 1, 0);
+
+    setXY(token_ids, logits);
+'

Hy-MT2/hy_mt2_7b_w8a8.ini

@@ -0,0 +1,146 @@
+[train]
+# W8A8: FP8 E4M3 weights + FP8 E4M3 activations (static input_scale from HF calibration)
+load_file = hy_mt2_7b_fp8_cccc.bin
+# Sidecar file with per-layer activation input_scales (generate with extract_input_scales.py)
+input_scale_file = hy_mt2_input_scales.bin
+load_net = 1
+WorkType = 0
+train_epochs = 0
+Batch = 1
+output_net = 1
+gpu = 1
+mp = 1
+mp_device = 0
+named_weights = 1
+data_type = bfloat16
+# fp8_e4m3: triggers FP8 weight cache load AND sets act_dt=FP8_E4M3 → W8A8 GEMM path
+weight_data_type = fp8_e4m3
+#disable_activation_memory_reuse=1
+
+[llm]
+tokenizer = tokenizer.json
+
+# EOS token: <|eos|> = 127960
+eos_tokens = 127960
+
+# No thinking support
+think_open_id = -1
+think_close_id = -1
+no_think_str =
+
+# Chat format for Hy-MT2-7B
+# System:  <|startoftext|>{content}<|extra_4|>
+# User:    <|startoftext|>{content}<|extra_0|>
+# Asst:    {content}<|eos|>
+sys_prefix = <|startoftext|>
+sys_suffix = <|extra_4|>
+user_prefix = <|startoftext|>
+user_suffix = <|extra_0|>
+asst_prefix =
+asst_suffix = <|eos|>
+
+[net]
+net_num = 2
+
+# ── prefill net (T tokens at once) ──────────────────────────────────────────
+structure0='
+    D    = 4096;
+    Dq   = 4096;
+    Dkv  = 1024;
+    H    = 32;
+    Hkv  = 8;
+    hd   = 128;
+    I    = 14336;
+    V    = 128167;
+    T    = 1024;
+    HB   = 32;
+    HkvB = 8;
+
+    W_emb = MatrixWithName("W_emb", D, 1, 1, V);
+    token_ids = MatrixF(T, 1, 1, 1);
+    X = embed(token_ids, W_emb);
+
+    cos_tab = ropeCosTbl(T, hd, 10000.0);
+    sin_tab = ropeSinTbl(T, hd, 10000.0);
+
+    for (i = 0; i < 32; i++)
+    {
+        W_rms_attn = MatrixWithName("W_rms_attn_" + to_string(i), D, 1);
+        X_norm = rmsNorm(X, W_rms_attn);
+
+        W_q = MatrixWithName("W_q_" + to_string(i), D, Dq, 1, 1);
+        Q_dq = batchedMul(W_q, X_norm, 1, 0);
+        Q_hHT = reshape(Q_dq, {hd, H, T, 1});
+        W_qnorm = MatrixWithName("W_qnorm_" + to_string(i), hd, 1);
+        Q_qnorm = rmsNorm(Q_hHT, W_qnorm);
+        Q_hTH = permute(Q_qnorm, {0, 2, 1, 3});
+        Q_hb = reshapeBatch(Q_hTH, {hd, T, 1, HB});
+        Q_r = rope(Q_hb, cos_tab, sin_tab);
+
+        W_k = MatrixWithName("W_k_" + to_string(i), D, Dkv, 1, 1);
+        K_dkv = batchedMul(W_k, X_norm, 1, 0);
+        K_hHkvT = reshape(K_dkv, {hd, Hkv, T, 1});
+        W_knorm = MatrixWithName("W_knorm_" + to_string(i), hd, 1);
+        K_knorm = rmsNorm(K_hHkvT, W_knorm);
+        K_hTHkv = permute(K_knorm, {0, 2, 1, 3});
+        K_hb = reshapeBatch(K_hTHkv, {hd, T, 1, HkvB});
+        K_r = rope(K_hb, cos_tab, sin_tab);
+
+        W_v = MatrixWithName("W_v_" + to_string(i), D, Dkv, 1, 1);
+        V_dkv = batchedMul(W_v, X_norm, 1, 0);
+        V_hHkvT = reshape(V_dkv, {hd, Hkv, T, 1});
+        V_hTHkv = permute(V_hHkvT, {0, 2, 1, 3});
+        V_hb = reshapeBatch(V_hTHkv, {hd, T, 1, HkvB});
+
+        Kcache = MatrixWithName("Kcache_" + to_string(i), hd, T, 1, HkvB);
+        setIsWeight(Kcache, 0);
+        registerMatrix("Kcache_" + to_string(i), Kcache);
+        K_cached = kvcache(K_r, Kcache);
+
+        Vcache = MatrixWithName("Vcache_" + to_string(i), hd, T, 1, HkvB);
+        setIsWeight(Vcache, 0);
+        registerMatrix("Vcache_" + to_string(i), Vcache);
+        V_cached = kvcache(V_hb, Vcache);
+
+        K_r2   = reshapeBatch(K_cached,  {hd, T, HkvB, 1});
+        K_r3   = tile(K_r2, {1, 1, 1, 4});
+        K_r4   = permute(K_r3, {0, 1, 3, 2});
+        K_tiled = reshapeBatch(K_r4, {hd, T, 1, HB});
+
+        V_r2   = reshapeBatch(V_cached, {hd, T, HkvB, 1});
+        V_r3   = tile(V_r2, {1, 1, 1, 4});
+        V_r4   = permute(V_r3, {0, 1, 3, 2});
+        V_tiled = reshapeBatch(V_r4, {hd, T, 1, HB});
+
+        Attn = attention(Q_r, K_tiled, V_tiled, hd, 1);
+
+        Attn_hTH = reshapeBatch(Attn, {hd, T, H, 1});
+        Attn_hHT = permute(Attn_hTH, {0, 2, 1, 3});
+        Attn_flat = reshape(Attn_hHT, {Dq, T, 1, 1});
+        W_o = MatrixWithName("W_o_" + to_string(i), Dq, D, 1, 1);
+        O_out = batchedMul(W_o, Attn_flat, 1, 0);
+
+        R1 = X + O_out;
+
+        W_rms_ffn = MatrixWithName("W_rms_ffn_" + to_string(i), D, 1);
+        R1_norm = rmsNorm(R1, W_rms_ffn);
+
+        W_gate = MatrixWithName("W_gate_" + to_string(i), D, I, 1, 1);
+        W_up   = MatrixWithName("W_up_" + to_string(i), D, I, 1, 1);
+        gate_out = silu(batchedMul(W_gate, R1_norm, 1, 0));
+        up_out   = batchedMul(W_up, R1_norm, 1, 0);
+        gated    = elementMul(gate_out, up_out);
+        W_down = MatrixWithName("W_down_" + to_string(i), I, D, 1, 1);
+        ffn_out = batchedMul(W_down, gated, 1, 0);
+
+        X = R1 + ffn_out;
+    }
+
+    W_rms_final = MatrixWithName("W_rms_final", D, 1);
+    X_final = rmsNorm(X, W_rms_final);
+    // tie_word_embeddings=true: reshape W_emb to (D, V, 1, 1) view for lm_head
+    W_lm_head = reshapeBatch(W_emb, {D, V, 1, 1});
+    logits = batchedMul(W_lm_head, X_final, 1, 0);
+
+    setXY(token_ids, logits);
+'

Hy-MT2/hy_mt2_input_scales.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e302cdedfbf28e96575d7cb47928c16d7649376b1e62adf8832ca19a6e2e6f1
+size 7110

llama3.1-8b-fp4/chat_template.jinja

@@ -0,0 +1,5 @@
+{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>
+
+'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{{ '<|start_header_id|>assistant<|end_header_id|>
+
+' }}

llama3.1-8b-fp4/config.json

@@ -0,0 +1,74 @@
+{
+    "architectures": [
+        "LlamaForCausalLM"
+    ],
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 128000,
+    "eos_token_id": [
+        128001,
+        128008,
+        128009
+    ],
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 4096,
+    "initializer_range": 0.02,
+    "intermediate_size": 14336,
+    "max_position_embeddings": 131072,
+    "mlp_bias": false,
+    "model_type": "llama",
+    "num_attention_heads": 32,
+    "num_hidden_layers": 32,
+    "num_key_value_heads": 8,
+    "pretraining_tp": 1,
+    "rms_norm_eps": 1e-05,
+    "rope_scaling": {
+        "factor": 8.0,
+        "high_freq_factor": 4.0,
+        "low_freq_factor": 1.0,
+        "original_max_position_embeddings": 8192,
+        "rope_type": "llama3"
+    },
+    "rope_theta": 500000.0,
+    "tie_word_embeddings": false,
+    "torch_dtype": "bfloat16",
+    "transformers_version": "4.55.0",
+    "use_cache": true,
+    "vocab_size": 128256,
+    "quantization_config": {
+        "config_groups": {
+            "group_0": {
+                "input_activations": {
+                    "dynamic": false,
+                    "num_bits": 4,
+                    "type": "float",
+                    "group_size": 16
+                },
+                "weights": {
+                    "dynamic": false,
+                    "num_bits": 4,
+                    "type": "float",
+                    "group_size": 16
+                },
+                "targets": [
+                    "Linear"
+                ]
+            }
+        },
+        "ignore": [
+            "lm_head"
+        ],
+        "quant_algo": "NVFP4",
+        "kv_cache_scheme": {
+            "dynamic": false,
+            "num_bits": 8,
+            "type": "float"
+        },
+        "producer": {
+            "name": "modelopt",
+            "version": "0.37.0.dev5+g76fb12d47.d20250905"
+        },
+        "quant_method": "modelopt"
+    }
+}

llama3.1-8b-fp4/convert_weights_fp4bs.py

@@ -0,0 +1,333 @@
+"""
+convert_weights_fp4bs.py — 生成 cccc 纯 block-scale FP4 权重文件（绕过 BF16 中间格式）
+
+来源：nvidia/Llama-3.1-8B-Instruct-FP4 的 safetensors（MXFP4 格式）
+  <name>.weight           → FP4 E2M1 packed nibble（uint8，2元素/字节）
+  <name>.weight_scale_inv → FP8 E4M3，shape [out, in//16]，per-block scale
+  <name>.weight_scale_2   → float32 outer scalar（可选，两级缩放）
+
+输出格式（cccc INIReaderBin）：
+  data_type = fp4_e2m1
+  named_weights = 1
+  weight_fp4_e2m1_<cccc_name>  → 打包 FP4 nibbles，ceil(n/2) 字节
+  blockscale_<cccc_name>       → FP8 E4M3 block dequant scales，ceil(n/16) × 1 字节
+  weight_bf16_<cccc_name>      → 小权重（norm/emb/lm_head）保留 BF16
+
+工作流：
+  NVFP4 → dequant(float32) → 我们自己的 block-scale FP4 E2M1 (group=16)
+
+用法：
+  python convert_weights_fp4bs.py [--hf_dir DIR] [--output FILE] [--dry_run]
+  默认 hf_dir = 脚本所在目录
+  默认 output = llama31_8b_fp4bs_cccc.bin
+"""
+
+import argparse
+import os
+import struct
+import sys
+import numpy as np
+
+try:
+    import torch
+except ImportError:
+    print("请先安装: pip install torch")
+    sys.exit(1)
+
+try:
+    from safetensors import safe_open
+except ImportError:
+    print("请先安装: pip install safetensors")
+    sys.exit(1)
+
+
+# ── FP4 E2M1 查值表（16个合法值） ────────────────────────────────────────────
+#  nibble bit layout: [sign(3), exp(2:1), mantissa(0)]
+#  正数：0=0, 1=0.5, 2=1, 3=1.5, 4=2, 5=3, 6=4, 7=6
+FP4_TABLE = np.array([
+    0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0,
+    -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0,
+], dtype=np.float32)
+
+FP4_CANDIDATES = np.array([
+    0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0,
+    -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0,
+], dtype=np.float32)
+
+FP4_KMAX = 6.0  # 最大绝对值
+
+# FP4 E2M1 正值量化边界（用于 searchsorted，避免创建 [N,16] 临时数组）
+# 正码字: 0→0, 1→0.5, 2→1, 3→1.5, 4→2, 5→3, 6→4, 7→6
+# 边界取相邻值中点：0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0
+_FP4_BOUNDARIES = np.array([0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0], dtype=np.float32)
+
+
+def float32_to_fp4_packed(data: np.ndarray, group_size: int = 16):
+    """
+    float32 数组 → FP4 packed nibbles (uint8, ceil(n/2) 字节) + float32 block scales。
+    每 group_size 个元素共享一个 scale = absmax / FP4_KMAX。
+    低 nibble = 偶数索引元素，高 nibble = 奇数索引元素。
+    返回: (nibbles: bytes, scales: np.ndarray[float32])
+    """
+    n = len(data)
+    # 补零到 group_size 的整数倍
+    pad = (-n) % group_size
+    if pad:
+        data = np.concatenate([data, np.zeros(pad, dtype=np.float32)])
+    n_padded = len(data)
+    n_groups = n_padded // group_size
+
+    data_2d = data.reshape(n_groups, group_size)
+
+    # Per-group absmax
+    absmax = np.abs(data_2d).max(axis=1)  # [n_groups]
+    scales = np.where(absmax > 1e-10, absmax / FP4_KMAX, 1.0).astype(np.float32)
+    fwd_scales = np.where(absmax > 1e-10, FP4_KMAX / absmax, 1.0)
+
+    # 缩放到 [-6, 6] 范围
+    scaled = data_2d * fwd_scales[:, np.newaxis]   # [n_groups, group_size]
+    scaled_flat = scaled.ravel()   # [n_padded]
+
+    # 最近邻量化：用边界二分查找，避免 [N,16] 临时矩阵
+    sign_bits = (scaled_flat < 0).astype(np.uint8) * 8  # 符号位放 bit3
+    pos_codes = np.searchsorted(_FP4_BOUNDARIES, np.abs(scaled_flat)).astype(np.uint8)  # 0-7
+    nibbles_flat = (pos_codes | sign_bits).astype(np.uint8)  # [n_padded]
+
+    # 打包 nibbles：低 nibble = 偶数，高 nibble = 奇数
+    nibbles_even = nibbles_flat[0::2]  # [n_padded // 2]
+    nibbles_odd  = nibbles_flat[1::2]
+    packed = (nibbles_even | (nibbles_odd << 4)).astype(np.uint8)
+
+    # 取前 ceil(n/2) 字节（去掉填充部分的影响只影响尾部字节，已经是零填充）
+    n_bytes = (n + 1) // 2
+    packed = packed[:n_bytes]
+
+    # scales 只保留有效 group 数（基于原始 n 而非 padded）
+    n_valid_groups = (n + group_size - 1) // group_size
+    scales = scales[:n_valid_groups]
+
+    return packed.tobytes(), scales
+
+
+def fp4_packed_to_float32(packed: np.ndarray, n_elems: int) -> np.ndarray:
+    """packed uint8 (2 FP4/字节) → float32 数组。"""
+    lo = packed & 0x0F
+    hi = (packed >> 4) & 0x0F
+    out = np.empty(len(packed) * 2, dtype=np.float32)
+    out[0::2] = FP4_TABLE[lo]
+    out[1::2] = FP4_TABLE[hi]
+    return out[:n_elems]
+
+
+def fp8_e4m3_to_float32(tensor: "torch.Tensor") -> np.ndarray:
+    """FP8 E4M3 tensor → float32 numpy 数组。"""
+    return tensor.to(torch.float32).numpy()
+
+
+def dequant_nvfp4(tensor: "torch.Tensor", scale_t: "torch.Tensor",
+                  scale2: float, in_features: int, block_size: int = 16) -> np.ndarray:
+    """NVFP4 dequant → float32 flat 数组（全向量化，无 Python 循环）。
+    tensor:  packed FP4 uint8, shape [out, in//2]
+    scale_t: FP8 E4M3 block scales, shape [out, in//block_size]
+    """
+    try:
+        packed = tensor.view(torch.uint8).numpy()
+    except Exception:
+        packed = tensor.numpy().view(np.uint8)
+    out_features = packed.shape[0]
+    scale_np = fp8_e4m3_to_float32(scale_t).reshape(out_features, -1) * scale2  # [out, n_blocks]
+
+    # 向量化 FP4 dequant：低 nibble = 偶数元素，高 nibble = 奇数元素
+    lo = (packed & 0x0F)              # [out, in//2]
+    hi = (packed >> 4) & 0x0F         # [out, in//2]
+    result = np.empty((out_features, in_features), dtype=np.float32)
+    result[:, 0::2] = FP4_TABLE[lo]   # 偶数列
+    result[:, 1::2] = FP4_TABLE[hi]   # 奇数列
+
+    # 广播 block scales：scale_np [out, n_blocks] → [out, in]
+    scale_expanded = np.repeat(scale_np, block_size, axis=1)[:, :in_features]
+    result *= scale_expanded
+    return result.ravel()
+
+
+# ── 权重名称映射：HuggingFace → cccc ─────────────────────────────────────────
+def build_weight_map(num_layers: int = 32):
+    mapping = []
+    mapping.append(("model.embed_tokens.weight", "W_emb",       False, False))
+    mapping.append(("model.norm.weight",          "W_rms_final", False, False))
+    mapping.append(("lm_head.weight",             "W_lm_head",   False, False))
+    for i in range(num_layers):
+        pfx = f"model.layers.{i}"
+        mapping.extend([
+            (f"{pfx}.self_attn.q_proj.weight",  f"W_q_{i}",    False, True),
+            (f"{pfx}.self_attn.k_proj.weight",  f"W_k_{i}",    False, True),
+            (f"{pfx}.self_attn.v_proj.weight",  f"W_v_{i}",    False, True),
+            (f"{pfx}.self_attn.o_proj.weight",  f"W_o_{i}",    False, True),
+            (f"{pfx}.mlp.gate_proj.weight",     f"W_gate_{i}", False, True),
+            (f"{pfx}.mlp.up_proj.weight",       f"W_up_{i}",   False, True),
+            (f"{pfx}.mlp.down_proj.weight",     f"W_down_{i}", False, True),
+            (f"{pfx}.input_layernorm.weight",          f"W_rms_attn_{i}", False, False),
+            (f"{pfx}.post_attention_layernorm.weight", f"W_rms_ffn_{i}",  False, False),
+        ])
+    return mapping
+
+
+# ── INIReaderBin 序列化 ────────────────────────────────────────────────────────
+def build_inireaderbin(named_blobs: dict) -> bytes:
+    blob_parts = []
+    index_lines = []
+    offset = 0
+    for key, data in named_blobs.items():
+        blob_parts.append(data)
+        index_lines.append(f"{key} = {offset},{len(data)}\n")
+        offset += len(data)
+    ini_text = "".join(index_lines)
+    ini_bytes = ini_text.encode("utf-8")
+    binary_content = b"".join(blob_parts)
+    header = b"CFG_BIN INI" + b"\x00" * (32 - len("CFG_BIN INI"))
+    size_ini = struct.pack("<Q", len(ini_bytes))
+    return header + size_ini + ini_bytes + binary_content
+
+
+def convert(hf_dir: str, output_path: str, dry_run: bool = False, group_size: int = 16):
+    mapping = build_weight_map(num_layers=32)
+
+    shard_files = sorted(f for f in os.listdir(hf_dir) if f.endswith(".safetensors"))
+    if not shard_files:
+        print(f"在 {hf_dir} 中未找到 .safetensors 文件")
+        sys.exit(1)
+    print(f"找到 {len(shard_files)} 个分片：{shard_files}")
+
+    handles: dict = {}
+    for sf in shard_files:
+        h = safe_open(os.path.join(hf_dir, sf), framework="pt", device="cpu")
+        for key in h.keys():
+            handles[key] = h
+
+    if dry_run:
+        print("\n── 所有 key 列表 ──")
+        for k in sorted(handles.keys()):
+            t = handles[k].get_tensor(k)
+            print(f"  {k}: {t.dtype}  {list(t.shape)}")
+        return
+
+    def get_scale_key(wk):
+        for sfx in (".weight_scale_inv", ".weight_scale"):
+            k = wk.replace(".weight", sfx)
+            if k in handles:
+                return k
+        return None
+
+    def get_scale2(wk):
+        k = wk.replace(".weight", ".weight_scale_2")
+        if k in handles:
+            return float(handles[k].get_tensor(k).item())
+        return 1.0
+
+    named_blobs: dict = {}
+    named_blobs["data_type"] = b"fp4_e2m1"
+    named_blobs["named_weights"] = b"1"
+
+    missing = []
+    fp4_count = 0
+    bf16_count = 0
+    total_fp4_bytes = 0
+    total_scale_bytes = 0
+
+    for hf_name, cccc_name, needs_transpose, is_quantized in mapping:
+        if hf_name not in handles:
+            missing.append(hf_name)
+            print(f"  MISS  {hf_name}")
+            continue
+
+        tensor = handles[hf_name].get_tensor(hf_name)
+
+        if is_quantized:
+            # ── NVFP4 dequant → float32 → 我们的 block-scale FP4 ────────
+            scale_key = get_scale_key(hf_name)
+            if scale_key is None:
+                print(f"  WARNING: no scale for {hf_name}, treating as BF16")
+                # 回退到 BF16
+                t_bf16 = tensor.to(torch.bfloat16)
+                blob = t_bf16.view(torch.int16).numpy().tobytes()
+                named_blobs[f"weight_bf16_{cccc_name}"] = blob
+                bf16_count += 1
+                continue
+
+            scale_t = handles[scale_key].get_tensor(scale_key)
+            scale2  = get_scale2(hf_name)
+
+            orig_shape = list(tensor.shape)
+            # in_features = packed_dim * 2
+            try:
+                packed_tmp = tensor.view(torch.uint8).numpy()
+            except Exception:
+                packed_tmp = tensor.numpy().view(np.uint8)
+            out_features = packed_tmp.shape[0]
+            in_features  = packed_tmp.shape[1] * 2  # packed bytes → elements
+
+            # Dequant NVFP4 → float32
+            data_f32 = dequant_nvfp4(tensor, scale_t, scale2, in_features, block_size=16)
+            # shape: [out_features * in_features] (row-major = column-major transpose)
+            # cccc expects [out, in] in memory order (same as HF [out, in] C-order = cccc BF16 column-major)
+            # No transpose needed (matches cccc memory layout as noted in user memory).
+
+            # Re-quantize: float32 → block-scale FP4 (group_size=16)
+            nibbles_bytes, scales_f32 = float32_to_fp4_packed(data_f32, group_size=group_size)
+            scales_fp8 = torch.from_numpy(scales_f32).to(torch.float8_e4m3fn)
+            scales_bytes = scales_fp8.view(torch.uint8).numpy().tobytes()  # FP8 E4M3 block scales (1 byte each)
+
+            named_blobs[f"weight_fp4_e2m1_{cccc_name}"] = nibbles_bytes
+            named_blobs[f"blockscale_{cccc_name}"]       = scales_bytes
+
+            fp4_count += 1
+            total_fp4_bytes   += len(nibbles_bytes)
+            total_scale_bytes += len(scales_bytes)
+            n_elems = out_features * in_features
+            print(f"  FP4   {hf_name:60s} → {cccc_name:18s}  "
+                  f"shape={orig_shape}  n={n_elems}  "
+                  f"nibbles={len(nibbles_bytes)//1024}KB  scales={len(scales_bytes)//1024}KB")
+        else:
+            # ── 非量化 → BF16 ────────────────────────────────────────────
+            if needs_transpose and tensor.ndim == 2:
+                tensor = tensor.T.contiguous()
+            t_bf16 = tensor.to(torch.bfloat16)
+            blob = t_bf16.view(torch.int16).numpy().tobytes()
+            named_blobs[f"weight_bf16_{cccc_name}"] = blob
+            bf16_count += 1
+            print(f"  BF16  {hf_name:60s} → {cccc_name:18s}  "
+                  f"shape={list(tensor.shape)}  {len(blob)//1024}KB")
+
+    if missing:
+        print(f"\n警告：以下权重未找到：{missing}")
+
+    fp4_total_mb  = (total_fp4_bytes + total_scale_bytes) / 1024**2
+    print(f"\nFP4 权重: {fp4_count}  BF16 权重: {bf16_count}")
+    print(f"nibbles: {total_fp4_bytes//1024**2}MB  scales: {total_scale_bytes//1024**2}MB  "
+          f"FP4 合计: {fp4_total_mb:.1f}MB")
+    print(f"\n正在写入 {output_path} ...")
+    content = build_inireaderbin(named_blobs)
+    with open(output_path, "wb") as f:
+        f.write(content)
+    size_gb = os.path.getsize(output_path) / 1024**3
+    print(f"完成！文件大小：{size_gb:.2f} GB")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="NVFP4 HF model → cccc block-scale FP4 bin")
+    parser.add_argument("--hf_dir",    default=os.path.dirname(os.path.abspath(__file__)),
+                        help="HuggingFace 模型目录（含 .safetensors 分片）")
+    parser.add_argument("--output",    default="llama31_8b_fp4bs_cccc.bin",
+                        help="输出文件名（相对于 hf_dir）")
+    parser.add_argument("--group_size", type=int, default=16,
+                        help="FP4 block size（每组共享一个 scale，默认 16）")
+    parser.add_argument("--dry_run",   action="store_true",
+                        help="仅列出 key，不写文件")
+    args = parser.parse_args()
+
+    output_path = args.output if os.path.isabs(args.output) else os.path.join(args.hf_dir, args.output)
+    convert(args.hf_dir, output_path, dry_run=args.dry_run, group_size=args.group_size)
+
+
+if __name__ == "__main__":
+    main()

llama3.1-8b-fp4/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.55.0"
+}

llama3.1-8b-fp4/hf_quant_config.json

@@ -0,0 +1,14 @@
+{
+    "producer": {
+        "name": "modelopt",
+        "version": "0.37.0.dev5+g76fb12d47.d20250905"
+    },
+    "quantization": {
+        "quant_algo": "NVFP4",
+        "kv_cache_quant_algo": "FP8",
+        "group_size": 16,
+        "exclude_modules": [
+            "lm_head"
+        ]
+    }
+}

llama3.1-8b-fp4/llama31_8b_fp4bs.ini

@@ -0,0 +1,132 @@
+[train]
+load_file = llama31_8b_fp4bs_cccc.bin
+load_net = 1
+WorkType = 0
+train_epochs = 0
+Batch = 1
+output_net = 1
+gpu = 1
+mp = 1
+mp_device = 0
+named_weights = 1
+data_type = bfloat16
+# block-scale FP4 bin：直接加载，无需 weight_data_type（bin 文件本身就是 FP4）
+
+[llm]
+tokenizer = tokenizer.json
+
+# Llama 3.1 EOS tokens: <|end_of_text|>=128001, <|eot_id|>=128009
+eos_tokens = 128001,128009
+
+# Llama 3.1 没有 think tokens
+think_open_id  = -1
+think_close_id = -1
+
+# Chat format (Llama 3.1 Instruct)
+# BOS <|begin_of_text|> 通过 sys_prefix 带入（只出现一次，在 system 轮头部）
+sys_prefix  = <|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n
+sys_suffix  = <|eot_id|>
+user_prefix = <|start_header_id|>user<|end_header_id|>\n\n
+user_suffix = <|eot_id|>
+asst_prefix = <|start_header_id|>assistant<|end_header_id|>\n\n
+asst_suffix = <|eot_id|>
+
+[net]
+net_num = 2
+
+structure0='
+    D    = 4096;
+    Dq   = 4096;
+    Dkv  = 1024;
+    H    = 32;
+    Hkv  = 8;
+    hd   = 128;
+    I    = 14336;
+    V    = 128256;
+    T    = 1024;
+    HB   = 32;
+    HkvB = 8;
+
+    W_emb = MatrixWithName("W_emb", D, 1, 1, V);
+    token_ids = MatrixF(T, 1, 1, 1);
+    X = embed(token_ids, W_emb);
+
+    cos_tab = ropeCosTbl(T, hd, 500000.0);
+    sin_tab = ropeSinTbl(T, hd, 500000.0);
+
+    for (i = 0; i < 32; i++)
+    {
+        W_rms_attn = MatrixWithName("W_rms_attn_" + to_string(i), D, 1);
+        X_norm = rmsNorm(X, W_rms_attn);
+
+        W_q = MatrixWithName("W_q_" + to_string(i), D, Dq, 1, 1);
+        Q_dq = batchedMul(W_q, X_norm, 1, 0);
+        Q_hHT = reshape(Q_dq, {hd, H, T, 1});
+        Q_hTH = permute(Q_hHT, {0, 2, 1, 3});
+        Q_hb = reshapeBatch(Q_hTH, {hd, T, 1, HB});
+        Q_r = rope(Q_hb, cos_tab, sin_tab);
+
+        W_k = MatrixWithName("W_k_" + to_string(i), D, Dkv, 1, 1);
+        K_dkv = batchedMul(W_k, X_norm, 1, 0);
+        K_hHkvT = reshape(K_dkv, {hd, Hkv, T, 1});
+        K_hTHkv = permute(K_hHkvT, {0, 2, 1, 3});
+        K_hb = reshapeBatch(K_hTHkv, {hd, T, 1, HkvB});
+        K_r = rope(K_hb, cos_tab, sin_tab);
+
+        W_v = MatrixWithName("W_v_" + to_string(i), D, Dkv, 1, 1);
+        V_dkv = batchedMul(W_v, X_norm, 1, 0);
+        V_hHkvT = reshape(V_dkv, {hd, Hkv, T, 1});
+        V_hTHkv = permute(V_hHkvT, {0, 2, 1, 3});
+        V_hb = reshapeBatch(V_hTHkv, {hd, T, 1, HkvB});
+
+        Kcache = MatrixWithName("Kcache_" + to_string(i), hd, T, 1, HkvB);
+        setIsWeight(Kcache, 0);
+        registerMatrix("Kcache_" + to_string(i), Kcache);
+        K_cached = kvcache(K_r, Kcache);
+
+        Vcache = MatrixWithName("Vcache_" + to_string(i), hd, T, 1, HkvB);
+        setIsWeight(Vcache, 0);
+        registerMatrix("Vcache_" + to_string(i), Vcache);
+        V_cached = kvcache(V_hb, Vcache);
+
+        K_r2    = reshapeBatch(K_cached,  {hd, T, HkvB, 1});
+        K_r3    = tile(K_r2, {1, 1, 1, 4});
+        K_r4    = permute(K_r3, {0, 1, 3, 2});
+        K_tiled = reshapeBatch(K_r4, {hd, T, 1, HB});
+
+        V_r2    = reshapeBatch(V_cached, {hd, T, HkvB, 1});
+        V_r3    = tile(V_r2, {1, 1, 1, 4});
+        V_r4    = permute(V_r3, {0, 1, 3, 2});
+        V_tiled = reshapeBatch(V_r4, {hd, T, 1, HB});
+
+        Attn = attention(Q_r, K_tiled, V_tiled, hd, 1);
+
+        Attn_hTH = reshapeBatch(Attn, {hd, T, H, 1});
+        Attn_hHT = permute(Attn_hTH, {0, 2, 1, 3});
+        Attn_flat = reshape(Attn_hHT, {Dq, T, 1, 1});
+        W_o = MatrixWithName("W_o_" + to_string(i), Dq, D, 1, 1);
+        O_out = batchedMul(W_o, Attn_flat, 1, 0);
+
+        R1 = X + O_out;
+
+        W_rms_ffn = MatrixWithName("W_rms_ffn_" + to_string(i), D, 1);
+        R1_norm = rmsNorm(R1, W_rms_ffn);
+
+        W_gate = MatrixWithName("W_gate_" + to_string(i), D, I, 1, 1);
+        W_up   = MatrixWithName("W_up_" + to_string(i), D, I, 1, 1);
+        gate_out = silu(batchedMul(W_gate, R1_norm, 1, 0));
+        up_out   = batchedMul(W_up, R1_norm, 1, 0);
+        gated    = elementMul(gate_out, up_out);
+        W_down = MatrixWithName("W_down_" + to_string(i), I, D, 1, 1);
+        ffn_out = batchedMul(W_down, gated, 1, 0);
+
+        X = R1 + ffn_out;
+    }
+
+    W_rms_final = MatrixWithName("W_rms_final", D, 1);
+    X_final = rmsNorm(X, W_rms_final);
+    W_lm_head = MatrixWithName("W_lm_head", D, V, 1, 1);
+    logits = batchedMul(W_lm_head, X_final, 1, 0);
+
+    setXY(token_ids, logits);
+'

llama3.1-8b-fp4/llama31_8b_fp4bs_cccc.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:442a21e0bd8dfc4b3c19f5ac8cc4c7eaf204e94226c6505687074d2920d1b325
+size 6027769230

llama3.1-8b-fp4/special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|eot_id|>"
+}

llama3.1-8b-fp4/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

llama3.1-8b-fp4/tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

qwen3/qwen3_8b_w8a16.ini

@@ -0,0 +1,136 @@
+[train]
+# 直接加载 W8A16 FP8 E5M2 缓存（BF16 原始 bin 已删除）
+load_file = qwen3_8b_w8a16_cccc.bin
+load_net = 1
+WorkType = 0
+train_epochs = 0
+Batch = 1
+output_net = 1
+gpu = 1
+mp = 1
+mp_device = 0
+named_weights = 1
+data_type = bfloat16
+weight_data_type = fp8_e5m2
+[llm]
+tokenizer = tokenizer.json
+
+# EOS tokens: <|im_end|>=151645, <|endoftext|>=151643
+eos_tokens = 151645,151643
+
+# Thinking tokens (Qwen3)
+think_open_id = 151667
+think_close_id = 151668
+no_think_str = <think>\n</think>\n
+
+# Chat format (ChatML)
+sys_prefix = <|im_start|>system\n
+sys_suffix = <|im_end|>\n
+user_prefix = <|im_start|>user\n
+user_suffix = <|im_end|>\n
+asst_prefix = <|im_start|>assistant\n
+asst_suffix = <|im_end|>\n
+
+[net]
+net_num = 2
+
+structure0='
+    D    = 4096;
+    Dq   = 4096;
+    Dkv  = 1024;
+    H    = 32;
+    Hkv  = 8;
+    hd   = 128;
+    I    = 12288;
+    V    = 151936;
+    T    = 1024;
+    HB   = 32;
+    HkvB = 8;
+
+    W_emb = MatrixWithName("W_emb", D, 1, 1, V);
+    token_ids = MatrixF(T, 1, 1, 1);
+    X = embed(token_ids, W_emb);
+
+    cos_tab = ropeCosTbl(T, hd, 1000000.0);
+    sin_tab = ropeSinTbl(T, hd, 1000000.0);
+
+    for (i = 0; i < 36; i++)
+    {
+        W_rms_attn = MatrixWithName("W_rms_attn_" + to_string(i), D, 1);
+        X_norm = rmsNorm(X, W_rms_attn);
+
+        W_q = MatrixWithName("W_q_" + to_string(i), D, Dq, 1, 1);
+        Q_dq = batchedMul(W_q, X_norm, 1, 0);
+        Q_hHT = reshape(Q_dq, {hd, H, T, 1});
+        W_qnorm = MatrixWithName("W_qnorm_" + to_string(i), hd, 1);
+        Q_qnorm = rmsNorm(Q_hHT, W_qnorm);
+        Q_hTH = permute(Q_qnorm, {0, 2, 1, 3});
+        Q_hb = reshapeBatch(Q_hTH, {hd, T, 1, HB});
+        Q_r = rope(Q_hb, cos_tab, sin_tab);
+
+        W_k = MatrixWithName("W_k_" + to_string(i), D, Dkv, 1, 1);
+        K_dkv = batchedMul(W_k, X_norm, 1, 0);
+        K_hHkvT = reshape(K_dkv, {hd, Hkv, T, 1});
+        W_knorm = MatrixWithName("W_knorm_" + to_string(i), hd, 1);
+        K_knorm = rmsNorm(K_hHkvT, W_knorm);
+        K_hTHkv = permute(K_knorm, {0, 2, 1, 3});
+        K_hb = reshapeBatch(K_hTHkv, {hd, T, 1, HkvB});
+        K_r = rope(K_hb, cos_tab, sin_tab);
+
+        W_v = MatrixWithName("W_v_" + to_string(i), D, Dkv, 1, 1);
+        V_dkv = batchedMul(W_v, X_norm, 1, 0);
+        V_hHkvT = reshape(V_dkv, {hd, Hkv, T, 1});
+        V_hTHkv = permute(V_hHkvT, {0, 2, 1, 3});
+        V_hb = reshapeBatch(V_hTHkv, {hd, T, 1, HkvB});
+
+        Kcache = MatrixWithName("Kcache_" + to_string(i), hd, T, 1, HkvB);
+        setIsWeight(Kcache, 0);
+        registerMatrix("Kcache_" + to_string(i), Kcache);
+        K_cached = kvcache(K_r, Kcache);
+
+        Vcache = MatrixWithName("Vcache_" + to_string(i), hd, T, 1, HkvB);
+        setIsWeight(Vcache, 0);
+        registerMatrix("Vcache_" + to_string(i), Vcache);
+        V_cached = kvcache(V_hb, Vcache);
+
+        K_r2   = reshapeBatch(K_cached,  {hd, T, HkvB, 1});
+        K_r3   = tile(K_r2, {1, 1, 1, 4});
+        K_r4   = permute(K_r3, {0, 1, 3, 2});
+        K_tiled = reshapeBatch(K_r4, {hd, T, 1, HB});
+
+        V_r2   = reshapeBatch(V_cached, {hd, T, HkvB, 1});
+        V_r3   = tile(V_r2, {1, 1, 1, 4});
+        V_r4   = permute(V_r3, {0, 1, 3, 2});
+        V_tiled = reshapeBatch(V_r4, {hd, T, 1, HB});
+
+        Attn = attention(Q_r, K_tiled, V_tiled, hd, 1);
+
+        Attn_hTH = reshapeBatch(Attn, {hd, T, H, 1});
+        Attn_hHT = permute(Attn_hTH, {0, 2, 1, 3});
+        Attn_flat = reshape(Attn_hHT, {Dq, T, 1, 1});
+        W_o = MatrixWithName("W_o_" + to_string(i), Dq, D, 1, 1);
+        O_out = batchedMul(W_o, Attn_flat, 1, 0);
+
+        R1 = X + O_out;
+
+        W_rms_ffn = MatrixWithName("W_rms_ffn_" + to_string(i), D, 1);
+        R1_norm = rmsNorm(R1, W_rms_ffn);
+
+        W_gate = MatrixWithName("W_gate_" + to_string(i), D, I, 1, 1);
+        W_up   = MatrixWithName("W_up_" + to_string(i), D, I, 1, 1);
+        gate_out = silu(batchedMul(W_gate, R1_norm, 1, 0));
+        up_out   = batchedMul(W_up, R1_norm, 1, 0);
+        gated    = elementMul(gate_out, up_out);
+        W_down = MatrixWithName("W_down_" + to_string(i), I, D, 1, 1);
+        ffn_out = batchedMul(W_down, gated, 1, 0);
+
+        X = R1 + ffn_out;
+    }
+
+    W_rms_final = MatrixWithName("W_rms_final", D, 1);
+    X_final = rmsNorm(X, W_rms_final);
+    W_lm_head = MatrixWithName("W_lm_head", D, V, 1, 1);
+    logits = batchedMul(W_lm_head, X_final, 1, 0);
+
+    setXY(token_ids, logits);
+'

qwen3/qwen3_8b_w8a16_cccc.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ace5c29c99eb2ab7cb058283c4200ca9b6fe2cebdbaf755296c894d696384a01
+size 8964398070