Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+qwen3/tokenizer.json filter=lfs diff=lfs merge=lfs -text

qwen3/qwen3_0.5b.ini

@@ -0,0 +1,124 @@
+[train]
+load_file = qwen3\qwen3_decoder_cccc.bin
+load_net = 1
+WorkType = 0
+train_epochs = 0
+Batch = 1
+output_net = 1
+gpu = 1
+mp = 1
+mp_device = 0
+cifa = 1
+named_weights = 1
+
+[net]
+cifa = 1
+
+[cifa]
+structure='
+    D    = 1024;
+    Dq   = 2048;
+    Dkv  = 1024;
+    H    = 16;
+    Hkv  = 8;
+    hd   = 128;
+    I    = 3072;
+    V    = 151936;
+    T    = 512;
+    HB   = 16;
+    HkvB = 8;
+
+    W_emb = Matrix(D, 1, 1, V);
+    setWeightName(W_emb, "W_emb");
+    token_ids = Matrix(T, 1, 1, 1);
+    X = embed(token_ids, W_emb);
+
+    cos_tab = ropeCosTbl(T, hd, 1000000.0);
+    sin_tab = ropeSinTbl(T, hd, 1000000.0);
+
+    for (i = 0; i < 28; i++)
+    {
+        W_rms_attn = Matrix(D, 1);
+        setWeightName(W_rms_attn, "W_rms_attn_" + to_string(i));
+        X_norm = rmsNorm(X, W_rms_attn);
+
+        W_q = Matrix(D, Dq, 1, 1);
+        setWeightName(W_q, "W_q_" + to_string(i));
+        Q_dq = batchedMul(W_q, X_norm, 1, 0);
+        Q_hHT = reshape(Q_dq, {hd, H, T, 1});
+        W_qnorm = Matrix(hd, 1);
+        setWeightName(W_qnorm, "W_qnorm_" + to_string(i));
+        Q_qnorm = rmsNorm(Q_hHT, W_qnorm);
+        Q_hTH = permute(Q_qnorm, {0, 2, 1, 3});
+        Q_hb = reshapeBatch(Q_hTH, {hd, T, 1, HB});
+        Q_r = rope(Q_hb, cos_tab, sin_tab);
+
+        W_k = Matrix(D, Dkv, 1, 1);
+        setWeightName(W_k, "W_k_" + to_string(i));
+        K_dkv = batchedMul(W_k, X_norm, 1, 0);
+        K_hHkvT = reshape(K_dkv, {hd, Hkv, T, 1});
+        W_knorm = Matrix(hd, 1);
+        setWeightName(W_knorm, "W_knorm_" + to_string(i));
+        K_knorm = rmsNorm(K_hHkvT, W_knorm);
+        K_hTHkv = permute(K_knorm, {0, 2, 1, 3});
+        K_hb = reshapeBatch(K_hTHkv, {hd, T, 1, HkvB});
+        K_r = rope(K_hb, cos_tab, sin_tab);
+
+        W_v = Matrix(D, Dkv, 1, 1);
+        setWeightName(W_v, "W_v_" + to_string(i));
+        V_dkv = batchedMul(W_v, X_norm, 1, 0);
+        V_hHkvT = reshape(V_dkv, {hd, Hkv, T, 1});
+        V_hTHkv = permute(V_hHkvT, {0, 2, 1, 3});
+        V_hb = reshapeBatch(V_hTHkv, {hd, T, 1, HkvB});
+
+        // GQA: standard repeat_kv grouping (Q head h uses K/V head h//2)
+        // Equivalent to: for each K/V head k, tile it groups=2 times consecutively
+        // Steps: reshapeBatch(Hkv→channel) + tile(number 1→groups) + permute(swap chan/num) + reshapeBatch
+        K_r2   = reshapeBatch(K_r,  {hd, T, HkvB, 1});
+        K_r3   = tile(K_r2, {1, 1, 1, 2});
+        K_r4   = permute(K_r3, {0, 1, 3, 2});
+        K_tiled = reshapeBatch(K_r4, {hd, T, 1, HB});
+
+        V_r2   = reshapeBatch(V_hb, {hd, T, HkvB, 1});
+        V_r3   = tile(V_r2, {1, 1, 1, 2});
+        V_r4   = permute(V_r3, {0, 1, 3, 2});
+        V_tiled = reshapeBatch(V_r4, {hd, T, 1, HB});
+
+        Attn = attention(Q_r, K_tiled, V_tiled, hd, 1);
+
+        Attn_hTH = reshapeBatch(Attn, {hd, T, H, 1});
+        Attn_hHT = permute(Attn_hTH, {0, 2, 1, 3});
+        Attn_flat = reshape(Attn_hHT, {Dq, T, 1, 1});
+        W_o = Matrix(Dq, D, 1, 1);
+        setWeightName(W_o, "W_o_" + to_string(i));
+        O_out = batchedMul(W_o, Attn_flat, 1, 0);
+
+        R1 = X + O_out;
+
+        W_rms_ffn = Matrix(D, 1);
+        setWeightName(W_rms_ffn, "W_rms_ffn_" + to_string(i));
+        R1_norm = rmsNorm(R1, W_rms_ffn);
+
+        W_gate = Matrix(D, I, 1, 1);
+        setWeightName(W_gate, "W_gate_" + to_string(i));
+        W_up   = Matrix(D, I, 1, 1);
+        setWeightName(W_up, "W_up_" + to_string(i));
+        gate_out = silu(batchedMul(W_gate, R1_norm, 1, 0));
+        up_out   = batchedMul(W_up, R1_norm, 1, 0);
+        gated    = elementMul(gate_out, up_out);
+        W_down = Matrix(I, D, 1, 1);
+        setWeightName(W_down, "W_down_" + to_string(i));
+        ffn_out = batchedMul(W_down, gated, 1, 0);
+
+        X = R1 + ffn_out;
+    }
+
+    W_rms_final = Matrix(D, 1);
+    setWeightName(W_rms_final, "W_rms_final");
+    X_final = rmsNorm(X, W_rms_final);
+    W_lm_head = Matrix(D, V, 1, 1);
+    setWeightName(W_lm_head, "W_lm_head");
+    logits = batchedMul(W_lm_head, X_final, 1, 0);
+
+    setXY(token_ids, logits);
+'

qwen3/qwen3_decoder_cccc.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43efb535cfff9bfab19705aba4650df58a9d3b0b6975e6270d68ad0cd504945a
+size 3006540214

qwen3/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
+size 11422654