Upload folder using huggingface_hub

qwen3/qwen3_0.5b_fp16.ini

@@ -0,0 +1,220 @@
+[train]
+load_file = qwen3_decoder_cccc.bin
+load_net = 1
+WorkType = 0
+train_epochs = 0
+Batch = 1
+output_net = 1
+gpu = 1
+mp = 1
+mp_device = 0
+named_weights = 1
+data_type = half
+
+[llm]
+tokenizer = tokenizer.json
+
+[net]
+net_num = 2
+
+structure0='
+    D    = 1024;
+    Dq   = 2048;
+    Dkv  = 1024;
+    H    = 16;
+    Hkv  = 8;
+    hd   = 128;
+    I    = 3072;
+    V    = 151936;
+    T    = 1024;
+    HB   = 16;
+    HkvB = 8;
+
+    W_emb = MatrixWithName("W_emb", D, 1, 1, V);
+    token_ids = MatrixF(T, 1, 1, 1);
+    X = embed(token_ids, W_emb);
+
+    cos_tab = ropeCosTbl(T, hd, 1000000.0);
+    sin_tab = ropeSinTbl(T, hd, 1000000.0);
+
+    for (i = 0; i < 28; i++)
+    {
+        W_rms_attn = MatrixWithName("W_rms_attn_" + to_string(i), D, 1);
+        X_norm = rmsNorm(X, W_rms_attn);
+
+        W_q = MatrixWithName("W_q_" + to_string(i), D, Dq, 1, 1);
+        Q_dq = batchedMul(W_q, X_norm, 1, 0);
+        Q_hHT = reshape(Q_dq, {hd, H, T, 1});
+        W_qnorm = MatrixWithName("W_qnorm_" + to_string(i), hd, 1);
+        Q_qnorm = rmsNorm(Q_hHT, W_qnorm);
+        Q_hTH = permute(Q_qnorm, {0, 2, 1, 3});
+        Q_hb = reshapeBatch(Q_hTH, {hd, T, 1, HB});
+        Q_r = rope(Q_hb, cos_tab, sin_tab);
+
+        W_k = MatrixWithName("W_k_" + to_string(i), D, Dkv, 1, 1);
+        K_dkv = batchedMul(W_k, X_norm, 1, 0);
+        K_hHkvT = reshape(K_dkv, {hd, Hkv, T, 1});
+        W_knorm = MatrixWithName("W_knorm_" + to_string(i), hd, 1);
+        K_knorm = rmsNorm(K_hHkvT, W_knorm);
+        K_hTHkv = permute(K_knorm, {0, 2, 1, 3});
+        K_hb = reshapeBatch(K_hTHkv, {hd, T, 1, HkvB});
+        K_r = rope(K_hb, cos_tab, sin_tab);
+
+        W_v = MatrixWithName("W_v_" + to_string(i), D, Dkv, 1, 1);
+        V_dkv = batchedMul(W_v, X_norm, 1, 0);
+        V_hHkvT = reshape(V_dkv, {hd, Hkv, T, 1});
+        V_hTHkv = permute(V_hHkvT, {0, 2, 1, 3});
+        V_hb = reshapeBatch(V_hTHkv, {hd, T, 1, HkvB});
+
+        Kcache = MatrixWithName("Kcache_" + to_string(i), hd, T, 1, HkvB);
+        setIsWeight(Kcache, 0);
+        registerMatrix("Kcache_" + to_string(i), Kcache);
+        K_cached = kvcache(K_r, Kcache);
+
+        Vcache = MatrixWithName("Vcache_" + to_string(i), hd, T, 1, HkvB);
+        setIsWeight(Vcache, 0);
+        registerMatrix("Vcache_" + to_string(i), Vcache);
+        V_cached = kvcache(V_hb, Vcache);
+
+        K_r2   = reshapeBatch(K_cached,  {hd, T, HkvB, 1});
+        K_r3   = tile(K_r2, {1, 1, 1, 2});
+        K_r4   = permute(K_r3, {0, 1, 3, 2});
+        K_tiled = reshapeBatch(K_r4, {hd, T, 1, HB});
+
+        V_r2   = reshapeBatch(V_cached, {hd, T, HkvB, 1});
+        V_r3   = tile(V_r2, {1, 1, 1, 2});
+        V_r4   = permute(V_r3, {0, 1, 3, 2});
+        V_tiled = reshapeBatch(V_r4, {hd, T, 1, HB});
+
+        Attn = attention(Q_r, K_tiled, V_tiled, hd, 1);
+
+        Attn_hTH = reshapeBatch(Attn, {hd, T, H, 1});
+        Attn_hHT = permute(Attn_hTH, {0, 2, 1, 3});
+        Attn_flat = reshape(Attn_hHT, {Dq, T, 1, 1});
+        W_o = MatrixWithName("W_o_" + to_string(i), Dq, D, 1, 1);
+        O_out = batchedMul(W_o, Attn_flat, 1, 0);
+
+        R1 = X + O_out;
+
+        W_rms_ffn = MatrixWithName("W_rms_ffn_" + to_string(i), D, 1);
+        R1_norm = rmsNorm(R1, W_rms_ffn);
+
+        W_gate = MatrixWithName("W_gate_" + to_string(i), D, I, 1, 1);
+        W_up   = MatrixWithName("W_up_" + to_string(i), D, I, 1, 1);
+        gate_out = silu(batchedMul(W_gate, R1_norm, 1, 0));
+        up_out   = batchedMul(W_up, R1_norm, 1, 0);
+        gated    = elementMul(gate_out, up_out);
+        W_down = MatrixWithName("W_down_" + to_string(i), I, D, 1, 1);
+        ffn_out = batchedMul(W_down, gated, 1, 0);
+
+        X = R1 + ffn_out;
+    }
+
+    W_rms_final = MatrixWithName("W_rms_final", D, 1);
+    X_final = rmsNorm(X, W_rms_final);
+    W_lm_head = MatrixWithName("W_lm_head", D, V, 1, 1);
+    logits = batchedMul(W_lm_head, X_final, 1, 0);
+
+    setXY(token_ids, logits);
+'
+
+structure1='
+    D    = 1024;
+    Dq   = 2048;
+    Dkv  = 1024;
+    H    = 16;
+    Hkv  = 8;
+    hd   = 128;
+    I    = 3072;
+    V    = 151936;
+    T    = 1;
+    T_kv = 1024;
+    HB   = 16;
+    HkvB = 8;
+
+    W_emb = MatrixWithName("W_emb", D, 1, 1, V);
+    token_ids = MatrixF(T, 1, 1, 1);
+    X = embed(token_ids, W_emb);
+
+    cos_tab = ropeCosTbl(T_kv, hd, 1000000.0);
+    sin_tab = ropeSinTbl(T_kv, hd, 1000000.0);
+
+    for (i = 0; i < 28; i++)
+    {
+        W_rms_attn = MatrixWithName("W_rms_attn_" + to_string(i), D, 1);
+        X_norm = rmsNorm(X, W_rms_attn);
+
+        W_q = MatrixWithName("W_q_" + to_string(i), D, Dq, 1, 1);
+        Q_dq = batchedMul(W_q, X_norm, 1, 0);
+        Q_hHT = reshape(Q_dq, {hd, H, T, 1});
+        W_qnorm = MatrixWithName("W_qnorm_" + to_string(i), hd, 1);
+        Q_qnorm = rmsNorm(Q_hHT, W_qnorm);
+        Q_hTH = permute(Q_qnorm, {0, 2, 1, 3});
+        Q_hb = reshapeBatch(Q_hTH, {hd, T, 1, HB});
+        Q_r = rope(Q_hb, cos_tab, sin_tab);
+
+        W_k = MatrixWithName("W_k_" + to_string(i), D, Dkv, 1, 1);
+        K_dkv = batchedMul(W_k, X_norm, 1, 0);
+        K_hHkvT = reshape(K_dkv, {hd, Hkv, T, 1});
+        W_knorm = MatrixWithName("W_knorm_" + to_string(i), hd, 1);
+        K_knorm = rmsNorm(K_hHkvT, W_knorm);
+        K_hTHkv = permute(K_knorm, {0, 2, 1, 3});
+        K_hb = reshapeBatch(K_hTHkv, {hd, T, 1, HkvB});
+        K_r = rope(K_hb, cos_tab, sin_tab);
+
+        W_v = MatrixWithName("W_v_" + to_string(i), D, Dkv, 1, 1);
+        V_dkv = batchedMul(W_v, X_norm, 1, 0);
+        V_hHkvT = reshape(V_dkv, {hd, Hkv, T, 1});
+        V_hTHkv = permute(V_hHkvT, {0, 2, 1, 3});
+        V_hb = reshapeBatch(V_hTHkv, {hd, T, 1, HkvB});
+
+        // 通过 MatrixWithName 找到 group 0 预载的共享 KV cache，无需重新分配显存
+        Kcache = MatrixWithName("Kcache_" + to_string(i), hd, T_kv, 1, HkvB);
+        setIsWeight(Kcache, 0);
+        K_cached = kvcache(K_r, Kcache);
+
+        Vcache = MatrixWithName("Vcache_" + to_string(i), hd, T_kv, 1, HkvB);
+        setIsWeight(Vcache, 0);
+        V_cached = kvcache(V_hb, Vcache);
+
+        K_r2   = reshapeBatch(K_cached, {hd, T_kv, HkvB, 1});
+        K_r3   = tile(K_r2, {1, 1, 1, 2});
+        K_r4   = permute(K_r3, {0, 1, 3, 2});
+        K_tiled = reshapeBatch(K_r4, {hd, T_kv, 1, HB});
+
+        V_r2   = reshapeBatch(V_cached, {hd, T_kv, HkvB, 1});
+        V_r3   = tile(V_r2, {1, 1, 1, 2});
+        V_r4   = permute(V_r3, {0, 1, 3, 2});
+        V_tiled = reshapeBatch(V_r4, {hd, T_kv, 1, HB});
+
+        Attn = attention(Q_r, K_tiled, V_tiled, hd, 1);
+
+        Attn_hTH = reshapeBatch(Attn, {hd, T, H, 1});
+        Attn_hHT = permute(Attn_hTH, {0, 2, 1, 3});
+        Attn_flat = reshape(Attn_hHT, {Dq, T, 1, 1});
+        W_o = MatrixWithName("W_o_" + to_string(i), Dq, D, 1, 1);
+        O_out = batchedMul(W_o, Attn_flat, 1, 0);
+
+        R1 = X + O_out;
+
+        W_rms_ffn = MatrixWithName("W_rms_ffn_" + to_string(i), D, 1);
+        R1_norm = rmsNorm(R1, W_rms_ffn);
+
+        W_gate = MatrixWithName("W_gate_" + to_string(i), D, I, 1, 1);
+        W_up   = MatrixWithName("W_up_" + to_string(i), D, I, 1, 1);
+        gate_out = silu(batchedMul(W_gate, R1_norm, 1, 0));
+        up_out   = batchedMul(W_up, R1_norm, 1, 0);
+        gated    = elementMul(gate_out, up_out);
+        W_down = MatrixWithName("W_down_" + to_string(i), I, D, 1, 1);
+        ffn_out = batchedMul(W_down, gated, 1, 0);
+
+        X = R1 + ffn_out;
+    }
+
+    W_rms_final = MatrixWithName("W_rms_final", D, 1);
+    X_final = rmsNorm(X, W_rms_final);
+    W_lm_head = MatrixWithName("W_lm_head", D, V, 1, 1);
+    logits = batchedMul(W_lm_head, X_final, 1, 0);
+
+    setXY(token_ids, logits);
+'

qwen3/qwen3_8b_cccc.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:528987e32b63901db0fc36cb4f75fd643f8de13e65cc816178095b37d1ded3e1
+size 16381484834

qwen3/qwen3_8b_fp16.ini

@@ -0,0 +1,220 @@
+[train]
+load_file = qwen3_8b_cccc.bin
+load_net = 1
+WorkType = 0
+train_epochs = 0
+Batch = 1
+output_net = 1
+gpu = 1
+mp = 1
+mp_device = 0
+named_weights = 1
+data_type = half
+
+[llm]
+tokenizer = tokenizer.json
+
+[net]
+net_num = 2
+
+structure0='
+    D    = 4096;
+    Dq   = 4096;
+    Dkv  = 1024;
+    H    = 32;
+    Hkv  = 8;
+    hd   = 128;
+    I    = 12288;
+    V    = 151936;
+    T    = 1024;
+    HB   = 32;
+    HkvB = 8;
+
+    W_emb = MatrixWithName("W_emb", D, 1, 1, V);
+    token_ids = MatrixF(T, 1, 1, 1);
+    X = embed(token_ids, W_emb);
+
+    cos_tab = ropeCosTbl(T, hd, 1000000.0);
+    sin_tab = ropeSinTbl(T, hd, 1000000.0);
+
+    for (i = 0; i < 36; i++)
+    {
+        W_rms_attn = MatrixWithName("W_rms_attn_" + to_string(i), D, 1);
+        X_norm = rmsNorm(X, W_rms_attn);
+
+        W_q = MatrixWithName("W_q_" + to_string(i), D, Dq, 1, 1);
+        Q_dq = batchedMul(W_q, X_norm, 1, 0);
+        Q_hHT = reshape(Q_dq, {hd, H, T, 1});
+        W_qnorm = MatrixWithName("W_qnorm_" + to_string(i), hd, 1);
+        Q_qnorm = rmsNorm(Q_hHT, W_qnorm);
+        Q_hTH = permute(Q_qnorm, {0, 2, 1, 3});
+        Q_hb = reshapeBatch(Q_hTH, {hd, T, 1, HB});
+        Q_r = rope(Q_hb, cos_tab, sin_tab);
+
+        W_k = MatrixWithName("W_k_" + to_string(i), D, Dkv, 1, 1);
+        K_dkv = batchedMul(W_k, X_norm, 1, 0);
+        K_hHkvT = reshape(K_dkv, {hd, Hkv, T, 1});
+        W_knorm = MatrixWithName("W_knorm_" + to_string(i), hd, 1);
+        K_knorm = rmsNorm(K_hHkvT, W_knorm);
+        K_hTHkv = permute(K_knorm, {0, 2, 1, 3});
+        K_hb = reshapeBatch(K_hTHkv, {hd, T, 1, HkvB});
+        K_r = rope(K_hb, cos_tab, sin_tab);
+
+        W_v = MatrixWithName("W_v_" + to_string(i), D, Dkv, 1, 1);
+        V_dkv = batchedMul(W_v, X_norm, 1, 0);
+        V_hHkvT = reshape(V_dkv, {hd, Hkv, T, 1});
+        V_hTHkv = permute(V_hHkvT, {0, 2, 1, 3});
+        V_hb = reshapeBatch(V_hTHkv, {hd, T, 1, HkvB});
+
+        Kcache = MatrixWithName("Kcache_" + to_string(i), hd, T, 1, HkvB);
+        setIsWeight(Kcache, 0);
+        registerMatrix("Kcache_" + to_string(i), Kcache);
+        K_cached = kvcache(K_r, Kcache);
+
+        Vcache = MatrixWithName("Vcache_" + to_string(i), hd, T, 1, HkvB);
+        setIsWeight(Vcache, 0);
+        registerMatrix("Vcache_" + to_string(i), Vcache);
+        V_cached = kvcache(V_hb, Vcache);
+
+        K_r2   = reshapeBatch(K_cached,  {hd, T, HkvB, 1});
+        K_r3   = tile(K_r2, {1, 1, 1, 4});
+        K_r4   = permute(K_r3, {0, 1, 3, 2});
+        K_tiled = reshapeBatch(K_r4, {hd, T, 1, HB});
+
+        V_r2   = reshapeBatch(V_cached, {hd, T, HkvB, 1});
+        V_r3   = tile(V_r2, {1, 1, 1, 4});
+        V_r4   = permute(V_r3, {0, 1, 3, 2});
+        V_tiled = reshapeBatch(V_r4, {hd, T, 1, HB});
+
+        Attn = attention(Q_r, K_tiled, V_tiled, hd, 1);
+
+        Attn_hTH = reshapeBatch(Attn, {hd, T, H, 1});
+        Attn_hHT = permute(Attn_hTH, {0, 2, 1, 3});
+        Attn_flat = reshape(Attn_hHT, {Dq, T, 1, 1});
+        W_o = MatrixWithName("W_o_" + to_string(i), Dq, D, 1, 1);
+        O_out = batchedMul(W_o, Attn_flat, 1, 0);
+
+        R1 = X + O_out;
+
+        W_rms_ffn = MatrixWithName("W_rms_ffn_" + to_string(i), D, 1);
+        R1_norm = rmsNorm(R1, W_rms_ffn);
+
+        W_gate = MatrixWithName("W_gate_" + to_string(i), D, I, 1, 1);
+        W_up   = MatrixWithName("W_up_" + to_string(i), D, I, 1, 1);
+        gate_out = silu(batchedMul(W_gate, R1_norm, 1, 0));
+        up_out   = batchedMul(W_up, R1_norm, 1, 0);
+        gated    = elementMul(gate_out, up_out);
+        W_down = MatrixWithName("W_down_" + to_string(i), I, D, 1, 1);
+        ffn_out = batchedMul(W_down, gated, 1, 0);
+
+        X = R1 + ffn_out;
+    }
+
+    W_rms_final = MatrixWithName("W_rms_final", D, 1);
+    X_final = rmsNorm(X, W_rms_final);
+    W_lm_head = MatrixWithName("W_lm_head", D, V, 1, 1);
+    logits = batchedMul(W_lm_head, X_final, 1, 0);
+
+    setXY(token_ids, logits);
+'
+
+structure1='
+    D    = 4096;
+    Dq   = 4096;
+    Dkv  = 1024;
+    H    = 32;
+    Hkv  = 8;
+    hd   = 128;
+    I    = 12288;
+    V    = 151936;
+    T    = 1;
+    T_kv = 1024;
+    HB   = 32;
+    HkvB = 8;
+
+    W_emb = MatrixWithName("W_emb", D, 1, 1, V);
+    token_ids = MatrixF(T, 1, 1, 1);
+    X = embed(token_ids, W_emb);
+
+    cos_tab = ropeCosTbl(T_kv, hd, 1000000.0);
+    sin_tab = ropeSinTbl(T_kv, hd, 1000000.0);
+
+    for (i = 0; i < 36; i++)
+    {
+        W_rms_attn = MatrixWithName("W_rms_attn_" + to_string(i), D, 1);
+        X_norm = rmsNorm(X, W_rms_attn);
+
+        W_q = MatrixWithName("W_q_" + to_string(i), D, Dq, 1, 1);
+        Q_dq = batchedMul(W_q, X_norm, 1, 0);
+        Q_hHT = reshape(Q_dq, {hd, H, T, 1});
+        W_qnorm = MatrixWithName("W_qnorm_" + to_string(i), hd, 1);
+        Q_qnorm = rmsNorm(Q_hHT, W_qnorm);
+        Q_hTH = permute(Q_qnorm, {0, 2, 1, 3});
+        Q_hb = reshapeBatch(Q_hTH, {hd, T, 1, HB});
+        Q_r = rope(Q_hb, cos_tab, sin_tab);
+
+        W_k = MatrixWithName("W_k_" + to_string(i), D, Dkv, 1, 1);
+        K_dkv = batchedMul(W_k, X_norm, 1, 0);
+        K_hHkvT = reshape(K_dkv, {hd, Hkv, T, 1});
+        W_knorm = MatrixWithName("W_knorm_" + to_string(i), hd, 1);
+        K_knorm = rmsNorm(K_hHkvT, W_knorm);
+        K_hTHkv = permute(K_knorm, {0, 2, 1, 3});
+        K_hb = reshapeBatch(K_hTHkv, {hd, T, 1, HkvB});
+        K_r = rope(K_hb, cos_tab, sin_tab);
+
+        W_v = MatrixWithName("W_v_" + to_string(i), D, Dkv, 1, 1);
+        V_dkv = batchedMul(W_v, X_norm, 1, 0);
+        V_hHkvT = reshape(V_dkv, {hd, Hkv, T, 1});
+        V_hTHkv = permute(V_hHkvT, {0, 2, 1, 3});
+        V_hb = reshapeBatch(V_hTHkv, {hd, T, 1, HkvB});
+
+        // 通过 MatrixWithName 找到 group 0 预载的共享 KV cache，无需重新分配显存
+        Kcache = MatrixWithName("Kcache_" + to_string(i), hd, T_kv, 1, HkvB);
+        setIsWeight(Kcache, 0);
+        K_cached = kvcache(K_r, Kcache);
+
+        Vcache = MatrixWithName("Vcache_" + to_string(i), hd, T_kv, 1, HkvB);
+        setIsWeight(Vcache, 0);
+        V_cached = kvcache(V_hb, Vcache);
+
+        K_r2   = reshapeBatch(K_cached, {hd, T_kv, HkvB, 1});
+        K_r3   = tile(K_r2, {1, 1, 1, 4});
+        K_r4   = permute(K_r3, {0, 1, 3, 2});
+        K_tiled = reshapeBatch(K_r4, {hd, T_kv, 1, HB});
+
+        V_r2   = reshapeBatch(V_cached, {hd, T_kv, HkvB, 1});
+        V_r3   = tile(V_r2, {1, 1, 1, 4});
+        V_r4   = permute(V_r3, {0, 1, 3, 2});
+        V_tiled = reshapeBatch(V_r4, {hd, T_kv, 1, HB});
+
+        Attn = attention(Q_r, K_tiled, V_tiled, hd, 1);
+
+        Attn_hTH = reshapeBatch(Attn, {hd, T, H, 1});
+        Attn_hHT = permute(Attn_hTH, {0, 2, 1, 3});
+        Attn_flat = reshape(Attn_hHT, {Dq, T, 1, 1});
+        W_o = MatrixWithName("W_o_" + to_string(i), Dq, D, 1, 1);
+        O_out = batchedMul(W_o, Attn_flat, 1, 0);
+
+        R1 = X + O_out;
+
+        W_rms_ffn = MatrixWithName("W_rms_ffn_" + to_string(i), D, 1);
+        R1_norm = rmsNorm(R1, W_rms_ffn);
+
+        W_gate = MatrixWithName("W_gate_" + to_string(i), D, I, 1, 1);
+        W_up   = MatrixWithName("W_up_" + to_string(i), D, I, 1, 1);
+        gate_out = silu(batchedMul(W_gate, R1_norm, 1, 0));
+        up_out   = batchedMul(W_up, R1_norm, 1, 0);
+        gated    = elementMul(gate_out, up_out);
+        W_down = MatrixWithName("W_down_" + to_string(i), I, D, 1, 1);
+        ffn_out = batchedMul(W_down, gated, 1, 0);
+
+        X = R1 + ffn_out;
+    }
+
+    W_rms_final = MatrixWithName("W_rms_final", D, 1);
+    X_final = rmsNorm(X, W_rms_final);
+    W_lm_head = MatrixWithName("W_lm_head", D, V, 1, 1);
+    logits = batchedMul(W_lm_head, X_final, 1, 0);
+
+    setXY(token_ids, logits);
+'

qwen3/qwen3_decoder_cccc.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:43efb535cfff9bfab19705aba4650df58a9d3b0b6975e6270d68ad0cd504945a
-size 3006540214
+oid sha256:0244cd483b026a7e461c0cf40cdcea24e808a0dada44778f2310ba143b373ccf
+size 1503275183