Upload folder using huggingface_hub

README.md

@@ -0,0 +1,15 @@
+# flint CPU optimization
+
+CPU-optimized build of [unsloth/flint-GGUF](https://huggingface.co/unsloth/flint-GGUF) (flint-UD-Q4_K_XL.gguf) using a llama.cpp CPU patch in the style of [HelloSun/Qwen3.6-35B-A3B-GGUF_CPU_OPT](https://huggingface.co/HelloSun/Qwen3.6-35B-A3B-GGUF_CPU_OPT).
+
+## Apply
+```bash
+git clone https://github.com/ggerganov/llama.cpp
+cd llama.cpp
+git checkout $(cat patches/BASE_COMMIT)
+git apply patches/flint-cpu-opt.patch
+./build.sh
+```
+
+## Results
+See benchmarks/summary.md. Mean KLD vs stock: 0.2058.

benchmarks/kld_final.log

@@ -0,0 +1,80 @@
+0.01.766.660 I cmn          init: llama threadpool init, n_threads = 16
+0.01.845.751 I 
+0.01.845.855 I system_info: n_threads = 16 (n_threads_batch = 16) / 192 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | AMX_INT8 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | 
+0.01.886.689 I kl_divergence: computing over 32 chunks, n_ctx=512, batch_size=512, n_seq=1
+0.04.588.983 I kl_divergence: 2.70 seconds per pass - ETA 1.43 minutes
+
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   1      31.8950 ±   10.7913      -0.01919 ±    0.12414       0.47995 ±    0.07714    16.215 ±  2.303 %    78.824 ±  2.564 %
+   2      18.7693 ±    3.8844       0.00294 ±    0.06270       0.25980 ±    0.04020    12.069 ±  1.576 %    85.294 ±  1.570 %
+   3      15.4713 ±    2.4439       0.01637 ±    0.04302       0.21289 ±    0.03283    11.207 ±  1.271 %    87.712 ±  1.188 %
+   4      14.1940 ±    1.9658       0.03541 ±    0.03515       0.22116 ±    0.02998    10.684 ±  1.035 %    88.431 ±  1.002 %
+   5      10.8930 ±    1.3155       0.01565 ±    0.03140       0.23288 ±    0.02782    11.670 ±  0.962 %    89.020 ±  0.876 %
+   6       7.3329 ±    0.7564       0.01319 ±    0.02617       0.19435 ±    0.02328    10.668 ±  0.879 %    90.850 ±  0.737 %
+   7       5.5187 ±    0.4963       0.01079 ±    0.02243       0.16887 ±    0.02007     9.970 ±  0.809 %    92.157 ±  0.637 %
+   8       4.4589 ±    0.3553       0.00925 ±    0.01963       0.14834 ±    0.01760     9.372 ±  0.755 %    93.088 ±  0.562 %
+   9       4.0899 ±    0.2965       0.00299 ±    0.01793       0.14117 ±    0.01709     9.272 ±  0.730 %    93.595 ±  0.511 %
+  10       4.9979 ±    0.3706       0.01408 ±    0.02749       0.25037 ±    0.02457    10.943 ±  0.721 %    91.647 ±  0.548 %
+  11       4.5721 ±    0.3136       0.01320 ±    0.02500       0.22823 ±    0.02238    10.458 ±  0.686 %    92.228 ±  0.506 %
+  12       4.1937 ±    0.2670       0.01381 ±    0.02296       0.21160 ±    0.02057    10.275 ±  0.655 %    92.647 ±  0.472 %
+  13       3.9080 ±    0.2322       0.01163 ±    0.02146       0.20177 ±    0.01926    10.305 ±  0.634 %    93.032 ±  0.442 %
+  14       3.7166 ±    0.2082       0.01202 ±    0.01996       0.18824 ±    0.01791    10.031 ±  0.610 %    93.389 ±  0.416 %
+  15       3.5969 ±    0.1918       0.01192 ±    0.01865       0.17664 ±    0.01673     9.818 ±  0.585 %    93.725 ±  0.392 %
+  16       3.6195 ±    0.1845       0.00955 ±    0.01752       0.16778 ±    0.01571     9.655 ±  0.562 %    93.775 ±  0.378 %
+  17       3.7784 ±    0.1872       0.00322 ±    0.01773       0.19185 ±    0.01589    10.467 ±  0.555 %    93.218 ±  0.382 %
+  18       3.9020 ±    0.1906       0.01415 ±    0.01814       0.22527 ±    0.01680    11.350 ±  0.545 %    92.702 ±  0.384 %
+  19       3.9563 ±    0.1872       0.01320 ±    0.01723       0.21543 ±    0.01595    11.213 ±  0.531 %    92.859 ±  0.370 %
+  20       4.0206 ±    0.1852       0.00258 ±    0.01733       0.22077 ±    0.01576    11.437 ±  0.520 %    92.647 ±  0.366 %
+  21       4.2279 ±    0.1948       0.01021 ±    0.01782       0.25106 ±    0.01716    12.200 ±  0.521 %    92.176 ±  0.367 %
+  22       4.2137 ±    0.1895       0.01000 ±    0.01712       0.24988 ±    0.01667    12.241 ±  0.508 %    92.103 ±  0.360 %
+  23       4.1466 ±    0.1817       0.01604 ±    0.01705       0.24787 ±    0.01625    12.207 ±  0.495 %    92.242 ±  0.349 %
+  24       4.1800 ±    0.1779       0.01373 ±    0.01640       0.23897 ±    0.01559    11.969 ±  0.484 %    92.435 ±  0.338 %
+  25       4.1542 ±    0.1722       0.01618 ±    0.01584       0.23559 ±    0.01505    12.003 ±  0.474 %    92.502 ±  0.330 %
+  26       4.0994 ±    0.1649       0.01558 ±    0.01527       0.22948 ±    0.01454    11.892 ±  0.466 %    92.609 ±  0.321 %
+  27       4.1120 ±    0.1618       0.01565 ±    0.01479       0.22470 ±    0.01405    11.881 ±  0.458 %    92.694 ±  0.314 %
+  28       4.2240 ±    0.1640       0.01531 ±    0.01436       0.22426 ±    0.01375    11.893 ±  0.449 %    92.591 ±  0.310 %
+  29       4.2403 ±    0.1614       0.01391 ±    0.01395       0.22072 ±    0.01336    11.775 ±  0.440 %    92.630 ±  0.304 %
+  30       4.2686 ±    0.1591       0.01269 ±    0.01350       0.21384 ±    0.01292    11.600 ±  0.432 %    92.745 ±  0.297 %
+  31       4.2673 ±    0.1558       0.01106 ±    0.01308       0.20801 ±    0.01252    11.505 ±  0.425 %    92.865 ±  0.290 %
+  32       4.1299 ±    0.1470       0.01347 ±    0.01282       0.20576 ±    0.01230    11.498 ±  0.420 %    92.990 ±  0.283 %
+
+====== Perplexity statistics ======
+Mean PPL(Q)                   :   4.129912 ±   0.147024
+Mean PPL(base)                :   4.074668 ±   0.139399
+Cor(ln(PPL(Q)), ln(PPL(base))):  93.34%
+Mean ln(PPL(Q)/PPL(base))     :   0.013467 ±   0.012816
+Mean PPL(Q)/PPL(base)         :   1.013558 ±   0.012990
+Mean PPL(Q)-PPL(base)         :   0.055243 ±   0.052818
+
+====== KL divergence statistics ======
+Mean    KLD:   0.205759 ±   0.012297
+Maximum KLD:  20.963177
+99.9%   KLD:  14.032636
+99.0%   KLD:   5.719375
+95.0%   KLD:   0.771354
+90.0%   KLD:   0.092588
+Median  KLD:   0.000773
+10.0%   KLD:   0.000002
+ 5.0%   KLD:   0.000000
+ 1.0%   KLD:  -0.000000
+ 0.1%   KLD:  -0.000001
+Minimum KLD:  -0.000013
+
+====== Token probability statistics ======
+Mean    Δp: -0.023 ± 0.127 %
+Maximum Δp: 99.114%
+99.9%   Δp: 96.666%
+99.0%   Δp: 42.103%
+95.0%   Δp:  4.663%
+90.0%   Δp:  1.578%
+75.0%   Δp:  0.070%
+Median  Δp: -0.000%
+25.0%   Δp: -0.066%
+10.0%   Δp: -1.639%
+ 5.0%   Δp: -5.594%
+ 1.0%   Δp: -38.947%
+ 0.1%   Δp: -98.266%
+Minimum Δp: -99.912%
+RMS Δp    : 11.498 ± 0.420 %
+Same top p: 92.990 ± 0.283 %
+

benchmarks/summary.md

@@ -0,0 +1,14 @@
+# flint CPU optimization benchmarks
+
+Model: unsloth/flint-GGUF flint-UD-Q4_K_XL.gguf (25.05 GiB)
+Hardware: Intel Xeon Platinum 8559C, 16 threads, taskset -c 0-15, -fa 1
+
+| metric | stock (t/s) | optimized (t/s) | speedup |
+|---|---|---|---|
+| pp256 | 221.5 | 260.9 | 1.15x |
+| tg128 | 32.6  | 35.4  | 1.07x |
+| pp2048 | 215.4 | 263.4 | 1.22x |
+| tg128 (pp2048 run) | 32.9 | 37.3 | 1.13x |
+
+Mean KLD vs stock: 0.2058 ± 0.0123 (prefill+decode, 32 chunks x 512)
+Mean PPL(Q)/PPL(base): 1.0136

build.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -e
+cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=ON -DGGML_CUDA=OFF -DLLAMA_CURL=OFF
+cmake --build build -j$(nproc) --target llama-bench llama-cli llama-perplexity

machine.txt

@@ -0,0 +1,6 @@
+CPU: Intel Xeon Platinum 8559C (Emerald Rapids)
+Threads used: 16 (cgroup cpu.max 1600000/100000)
+Memory cap: ~104 GiB (cgroup memory.max)
+Build: GCC 14.2.0, cmake 3.31.6, -march=native (AVX512, VNNI, AMX-INT8)
+llama.cpp base commit: 338fce651 (b11942)
+AMX buffer: 1716 MiB (Q8_0 2D tensors); CPU_REPACK: 864 MiB (Q4_K 8x8)

patches/BASE_COMMIT

@@ -0,0 +1 @@
+338fce6511c744893cc0d3d958b275f40552c7a4

patches/flint-cpu-opt.patch

@@ -0,0 +1,513 @@
+diff --git a/ggml/src/ggml-cpu/amx/amx.cpp b/ggml/src/ggml-cpu/amx/amx.cpp
+index e90c2797f..e8b61a2eb 100644
+--- a/ggml/src/ggml-cpu/amx/amx.cpp
++++ b/ggml/src/ggml-cpu/amx/amx.cpp
+@@ -161,6 +161,17 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {
+         if (op->ne[0] % (TILE_N * 2)) {
+             return false;
+         }
++        // machine-specific knobs: keep selected types out of the AMX buffer (decode then uses the VNNI vec_dot kernels)
++        {
++            // Q6_K (lm_head): the AVX512-VNNI vec_dot is ~9% faster for decode than the AMX M=1 kernel; opt back in with GGML_AMX_Q6K=1
++            static const char * keep_q6 = getenv("GGML_AMX_Q6K");
++            if (src0->type == GGML_TYPE_Q6_K && !(keep_q6 && keep_q6[0] == '1')) { return false; }
++            static const char * skip = getenv("GGML_AMX_SKIP");
++            if (skip) {
++                if (src0->type == GGML_TYPE_Q6_K && strstr(skip, "q6_K")) { return false; }
++                if (src0->type == GGML_TYPE_Q8_0 && strstr(skip, "q8_0")) { return false; }
++            }
++        }
+         int alignment;
+         switch (src0->type) {
+             case GGML_TYPE_Q4_0:
+diff --git a/ggml/src/ggml-cpu/amx/mmq.cpp b/ggml/src/ggml-cpu/amx/mmq.cpp
+index 9f3a744b5..d23e4d971 100644
+--- a/ggml/src/ggml-cpu/amx/mmq.cpp
++++ b/ggml/src/ggml-cpu/amx/mmq.cpp
+@@ -1989,6 +1989,91 @@ struct tinygemm_kernel_vnni<block_q8_K, block_iq4_xs, float, BLOCK_M, BLOCK_N, B
+         (const char *)src0->data + src0_offset + PACKED_INDEX(nb * kTilesN, 0, KB, TILE_SIZE), \
+         (float *) dst->data + dst_offset + nb_start, ldc)
+ 
++// Q8_0 x Q8_0 AMX gemm with register accumulation: one 16x16 output tile at a time, its 16 float row
++// accumulators live in zmm registers across all K blocks (the generic kernel read-modify-writes C through memory
++// every 32-wide K block and converts the A scale per row). The AMX product of block i+1 is issued before the AVX
++// epilogue of block i so the two overlap. Per-block math (cvt(int32) * (dW*dA) accumulated with an fma, blocks in
++// order) is identical to acc_C, so results are bit-identical. Returns false when KB is too large for the scale cache.
++// tile registers are pasted into the asm string by the intrinsics, so they must be literals: one function per slot
++#define Q8_REGACC_ISSUE_FN(NAME, TB_, TA_, TC_, SLOT)                                                                 \
++static ALWAYS_INLINE void NAME(const block_q8_0 * RESTRICT A, const char * RESTRICT B, int KB, int u, int t,          \
++                               int mrows, int i, int8_t (*TileA)[TILE_M * TILE_K], int32_t (*TileC)[TILE_M * TILE_N]) { \
++    const int TILE_SIZE = get_tile_size<block_q8_0>(); /* 16 x 34 B + 16 x int32 compensation */                                                            \
++    const char * Bb = B + PACKED_INDEX(t, i, KB, TILE_SIZE);                                                          \
++    _tile_loadd(TB_, Bb, TILE_N * VNNI_BLK);                                                                          \
++    const block_q8_0 * Ab = A + (size_t) (u * TILE_M) * KB + i;                                                       \
++    if (mrows == TILE_M) {                                                                                            \
++        _tile_loadd(TA_, Ab->qs, KB * sizeof(block_q8_0));                                                            \
++    } else {                                                                                                          \
++        unpack_A(TileA[SLOT], Ab, KB, mrows);                                                                         \
++        _tile_loadd(TA_, TileA[SLOT], TILE_K);                                                                        \
++    }                                                                                                                 \
++    _tile_zero(TC_);                                                                                                  \
++    _tile_dpbssd(TC_, TA_, TB_);                                                                                      \
++    _tile_stored(TC_, TileC[SLOT], TILE_N * sizeof(int32_t));                                                         \
++}
++Q8_REGACC_ISSUE_FN(q8_regacc_issue0, 0, 2, 4, 0)
++Q8_REGACC_ISSUE_FN(q8_regacc_issue1, 1, 3, 5, 1)
++#undef Q8_REGACC_ISSUE_FN
++
++static bool tinygemm_kernel_amx_q8_0_regacc(int M, int KB, const block_q8_0 * RESTRICT A, const char * RESTRICT B,
++                                            float * RESTRICT C, int ldc) {
++    constexpr int MAX_KB = 256;
++    if (KB > MAX_KB || KB < 1) {
++        return false;
++    }
++    const int TILE_SIZE = get_tile_size<block_q8_0>(); /* 16 x 34 B + 16 x int32 compensation */
++    constexpr int SCALE_OFF = TILE_N * TILE_K;
++
++    alignas(64) static thread_local int8_t  TileA[2][TILE_M * TILE_K];
++    alignas(64) static thread_local int32_t TileC[2][TILE_M * TILE_N];
++    alignas(64) static thread_local float   sA[2 * TILE_M][MAX_KB];
++
++    const int rows = std::min(M, 2 * TILE_M);
++    for (int r = 0; r < 2 * TILE_M; ++r) {
++        for (int i = 0; i < KB; ++i) {
++            sA[r][i] = r < rows ? GGML_CPU_FP16_TO_FP32(A[(size_t) r * KB + i].d) : 0.0f;
++        }
++    }
++
++    const int nu = rows > TILE_M ? 2 : 1;
++    for (int u = 0; u < nu; ++u) {
++        const int mrows = std::min(TILE_M, rows - u * TILE_M);
++        for (int t = 0; t < 2; ++t) {
++            __m512 acc[TILE_M];
++            #pragma GCC unroll 16
++            for (int m = 0; m < TILE_M; ++m) {
++                acc[m] = _mm512_setzero_ps();
++            }
++
++            q8_regacc_issue0(A, B, KB, u, t, mrows, 0, TileA, TileC);
++            for (int i = 0; i < KB; ++i) {
++                const int slot = i & 1;
++                if (i + 1 < KB) {
++                    if (slot == 0) {
++                        q8_regacc_issue1(A, B, KB, u, t, mrows, i + 1, TileA, TileC);
++                    } else {
++                        q8_regacc_issue0(A, B, KB, u, t, mrows, i + 1, TileA, TileC);
++                    }
++                }
++                const __m512 vd0 = _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *) (B + PACKED_INDEX(t, i, KB, TILE_SIZE) + SCALE_OFF)));
++                const int32_t * tile = TileC[slot];
++                const float * sa = &sA[u * TILE_M][0];
++                #pragma GCC unroll 16
++                for (int m = 0; m < TILE_M; ++m) {
++                    const __m512 vtile = _mm512_cvtepi32_ps(_mm512_loadu_si512(tile + m * TILE_N));
++                    const __m512 vd1   = _mm512_set1_ps(sa[(size_t) m * MAX_KB + i]);
++                    acc[m] = _mm512_fmadd_ps(vtile, _mm512_mul_ps(vd0, vd1), acc[m]);
++                }
++            }
++            for (int m = 0; m < mrows; ++m) {
++                _mm512_storeu_ps(C + (size_t) (u * TILE_M + m) * ldc + t * TILE_N, acc[m]);
++            }
++        }
++    }
++    return true;
++}
++
+ template <typename TA, typename TB, typename TC, int BLOCK_K,
+           typename std::enable_if<!is_type_qkk<TB>::value, int>::type = 0>
+ void tinygemm_kernel_amx(int M, int N, int KB, const void * RESTRICT _A, const void * RESTRICT _B, TC * RESTRICT C, int ldc) {
+@@ -1997,6 +2082,12 @@ void tinygemm_kernel_amx(int M, int N, int KB, const void * RESTRICT _A, const v
+     const bool need_unpack = do_unpack<TB>::value;
+ 
+     GGML_ASSERT(M <= 2 * TILE_M && N == 2 * TILE_N);
++    if constexpr (std::is_same<TB, block_q8_0>::value && std::is_same<TA, block_q8_0>::value) {
++        static const bool regacc = [] { const char * e = getenv("GGML_AMX_Q8_REGACC"); return !(e && e[0] == '0'); }();
++        if (regacc && tinygemm_kernel_amx_q8_0_regacc(M, KB, static_cast<const block_q8_0 *>(_A), static_cast<const char *>(_B), C, ldc)) {
++            return;
++        }
++    }
+     const TA * RESTRICT A = static_cast<const TA *>(_A);
+     const char * RESTRICT B = static_cast<const char *>(_B);
+ 
+@@ -2434,8 +2525,9 @@ void ggml_backend_amx_mul_mat(const ggml_compute_params * params, struct ggml_te
+ 
+     if (M == 1) {
+         // MB = 1 and handle 8 tiles in each block
+-        constexpr int kTilesN = 4;
+-        constexpr int BLOCK_N = TILE_N * kTilesN;
++        // small N (e.g. 512) would give fewer tasks than threads with 64-wide blocks; use 32-wide blocks
++        const int BLOCK_N = (N <= 1024) ? 32 : 64;
++        const int kTilesN = BLOCK_N / TILE_N;
+         const int NB = div_up(N, BLOCK_N);
+ 
+         parallel_for_ggml(params, n_batch * NB, [&](int begin, int end) {
+diff --git a/ggml/src/ggml-cpu/arch/x86/quants.c b/ggml/src/ggml-cpu/arch/x86/quants.c
+index 21cd6312a..0e8aafa76 100644
+--- a/ggml/src/ggml-cpu/arch/x86/quants.c
++++ b/ggml/src/ggml-cpu/arch/x86/quants.c
+@@ -2118,7 +2118,89 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
+ 
+     uint32_t utmp[4];
+ 
+-#if defined __AVX2__
++#if defined(__AVX512F__) && defined(__AVX512BW__) && defined(__AVX512VL__) && defined(__AVX512VNNI__)
++    // AVX512-VNNI path (Q4_K x Q8_K): 64 weights per vector, two 32-wide sub-blocks per vector.
++    const __m512i m4 = _mm512_set1_epi8(0xF);
++    #define K4_SCALE_IDX(a, b) _mm512_inserti64x4( \
++        _mm512_castsi256_si512(_mm256_set_m128i(_mm_set1_epi16((short)((2*(a)) | ((2*(a)+1) << 8))), _mm_set1_epi16((short)((2*(a)) | ((2*(a)+1) << 8))))), \
++        _mm256_set_m128i(_mm_set1_epi16((short)((2*(b)) | ((2*(b)+1) << 8))), _mm_set1_epi16((short)((2*(b)) | ((2*(b)+1) << 8)))), 1)
++    const __m512i sidx0 = K4_SCALE_IDX(0, 2);
++    const __m512i sidx1 = K4_SCALE_IDX(1, 3);
++    const __m512i sidx2 = K4_SCALE_IDX(4, 6);
++    const __m512i sidx3 = K4_SCALE_IDX(5, 7);
++    #undef K4_SCALE_IDX
++
++    __m512 acc = _mm512_setzero_ps();
++    __m128 acc_m = _mm_setzero_ps();
++
++    for (int i = 0; i < nb; ++i) {
++#ifndef K4_PF_DIST
++#define K4_PF_DIST 40   // ~9 KB ahead, tuned for Xeon 8559C (prefetch into L2)
++#endif
++        static int k4pf = -1;
++        if (k4pf < 0) { const char * e = getenv("GGML_K4_PF"); k4pf = e ? atoi(e) : K4_PF_DIST; }
++        {
++            const char * pf = (const char *)(x + i + k4pf);
++            _mm_prefetch(pf,       _MM_HINT_T1);
++            _mm_prefetch(pf +  64, _MM_HINT_T1);
++            _mm_prefetch(pf + 128, _MM_HINT_T1);
++        }
++
++        const float d    =  y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);
++        const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);
++
++        memcpy(utmp, x[i].scales, 12);
++        utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
++        const uint32_t uaux = utmp[1] & kmask1;
++        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
++        utmp[2] = uaux;
++        utmp[0] &= kmask1;
++
++        const uint8_t * GGML_RESTRICT q4 = x[i].qs;
++        const int8_t  * GGML_RESTRICT q8 = y[i].qs;
++
++        const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]));
++
++        const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);
++        const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));
++        const __m128i prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s);
++        acc_m = _mm_fmadd_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod), acc_m);
++
++        const __m512i scales = _mm512_broadcast_i32x4(_mm256_extracti128_si256(mins_and_scales, 0));
++
++        const __m512i v0 = _mm512_loadu_si512((const void *)(q4));
++        const __m512i v1 = _mm512_loadu_si512((const void *)(q4 + 64));
++
++        const __m512i Q0 = _mm512_loadu_si512((const void *)(q8));
++        const __m512i Q1 = _mm512_loadu_si512((const void *)(q8 + 64));
++        const __m512i Q2 = _mm512_loadu_si512((const void *)(q8 + 128));
++        const __m512i Q3 = _mm512_loadu_si512((const void *)(q8 + 192));
++
++        const __m512i a0 = _mm512_and_si512(v0, m4);
++        const __m512i a1 = _mm512_and_si512(_mm512_srli_epi16(v0, 4), m4);
++        const __m512i a2 = _mm512_and_si512(v1, m4);
++        const __m512i a3 = _mm512_and_si512(_mm512_srli_epi16(v1, 4), m4);
++
++        const __m512i b0 = _mm512_shuffle_i64x2(Q0, Q1, 0x44);
++        const __m512i b1 = _mm512_shuffle_i64x2(Q0, Q1, 0xEE);
++        const __m512i b2 = _mm512_shuffle_i64x2(Q2, Q3, 0x44);
++        const __m512i b3 = _mm512_shuffle_i64x2(Q2, Q3, 0xEE);
++
++        __m512i sumi = _mm512_setzero_si512();
++        sumi = _mm512_dpwssd_epi32(sumi, _mm512_maddubs_epi16(a0, b0), _mm512_shuffle_epi8(scales, sidx0));
++        sumi = _mm512_dpwssd_epi32(sumi, _mm512_maddubs_epi16(a1, b1), _mm512_shuffle_epi8(scales, sidx1));
++        sumi = _mm512_dpwssd_epi32(sumi, _mm512_maddubs_epi16(a2, b2), _mm512_shuffle_epi8(scales, sidx2));
++        sumi = _mm512_dpwssd_epi32(sumi, _mm512_maddubs_epi16(a3, b3), _mm512_shuffle_epi8(scales, sidx3));
++
++        acc = _mm512_fmadd_ps(_mm512_set1_ps(d), _mm512_cvtepi32_ps(sumi), acc);
++    }
++
++    acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));
++    acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));
++
++    *s = _mm512_reduce_add_ps(acc) + _mm_cvtss_f32(acc_m);
++
++#elif defined __AVX2__
+ 
+     const __m256i m4 = _mm256_set1_epi8(0xF);
+ 
+@@ -2296,7 +2378,97 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
+ 
+     uint32_t utmp[4];
+ 
+-#if defined __AVX2__
++#if defined(__AVX512F__) && defined(__AVX512BW__) && defined(__AVX512VL__) && defined(__AVX512VNNI__)
++    // AVX512-VNNI path (Q5_K x Q8_K), same structure as the Q4_K kernel: 64 weights per vector, two 32-wide
++    // sub-blocks per vector. The 5th bit of sub-block s is bit s of qh[l]; it is merged in as +16 before maddubs.
++    const __m512i m4  = _mm512_set1_epi8(0xF);
++    const __m512i m10 = _mm512_set1_epi8(0x10);
++    #define K5_SCALE_IDX(a, b) _mm512_inserti64x4( \
++        _mm512_castsi256_si512(_mm256_set_m128i(_mm_set1_epi16((short)((2*(a)) | ((2*(a)+1) << 8))), _mm_set1_epi16((short)((2*(a)) | ((2*(a)+1) << 8))))), \
++        _mm256_set_m128i(_mm_set1_epi16((short)((2*(b)) | ((2*(b)+1) << 8))), _mm_set1_epi16((short)((2*(b)) | ((2*(b)+1) << 8)))), 1)
++    const __m512i sidx0 = K5_SCALE_IDX(0, 2);
++    const __m512i sidx1 = K5_SCALE_IDX(1, 3);
++    const __m512i sidx2 = K5_SCALE_IDX(4, 6);
++    const __m512i sidx3 = K5_SCALE_IDX(5, 7);
++    #undef K5_SCALE_IDX
++    #define K5_CNT(a, b) _mm512_inserti64x4(_mm512_castsi256_si512(_mm256_set1_epi16(a)), _mm256_set1_epi16(b), 1)
++    const __m512i cl0 = K5_CNT(4, 2);   // sub-blocks 0,2 : bit s -> bit 4 via left shift 4-s
++    const __m512i cl1 = K5_CNT(3, 1);   // sub-blocks 1,3
++    const __m512i cr2 = K5_CNT(0, 2);   // sub-blocks 4,6 : right shift s-4
++    const __m512i cr3 = K5_CNT(1, 3);   // sub-blocks 5,7
++    #undef K5_CNT
++
++    static int k5pf = -1;
++    if (k5pf < 0) { const char * e = getenv("GGML_K5_PF"); k5pf = e ? atoi(e) : 48; }
++
++    __m512 acc = _mm512_setzero_ps();
++    __m128 acc_m = _mm_setzero_ps();
++
++    for (int i = 0; i < nb; ++i) {
++        {
++            const char * pf = (const char *)(x + i + k5pf);
++            _mm_prefetch(pf,       _MM_HINT_T1);
++            _mm_prefetch(pf +  64, _MM_HINT_T1);
++            _mm_prefetch(pf + 128, _MM_HINT_T1);
++        }
++
++        const float d    =  y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);
++        const float dmin = -y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);
++
++        memcpy(utmp, x[i].scales, 12);
++        utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
++        const uint32_t uaux = utmp[1] & kmask1;
++        utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
++        utmp[2] = uaux;
++        utmp[0] &= kmask1;
++
++        const uint8_t * GGML_RESTRICT q5 = x[i].qs;
++        const int8_t  * GGML_RESTRICT q8 = y[i].qs;
++
++        const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0]));
++
++        const __m256i q8sums = _mm256_loadu_si256((const __m256i*)y[i].bsums);
++        const __m128i q8s = _mm_hadd_epi16(_mm256_extracti128_si256(q8sums, 0), _mm256_extracti128_si256(q8sums, 1));
++        const __m128i prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s);
++        acc_m = _mm_fmadd_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod), acc_m);
++
++        const __m512i scales = _mm512_broadcast_i32x4(_mm256_extracti128_si256(mins_and_scales, 0));
++
++        const __m512i qhv = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i *)x[i].qh));
++
++        const __m512i v0 = _mm512_loadu_si512((const void *)(q5));
++        const __m512i v1 = _mm512_loadu_si512((const void *)(q5 + 64));
++
++        const __m512i Q0 = _mm512_loadu_si512((const void *)(q8));
++        const __m512i Q1 = _mm512_loadu_si512((const void *)(q8 + 64));
++        const __m512i Q2 = _mm512_loadu_si512((const void *)(q8 + 128));
++        const __m512i Q3 = _mm512_loadu_si512((const void *)(q8 + 192));
++
++        const __m512i a0 = _mm512_or_si512(_mm512_and_si512(v0, m4),                       _mm512_and_si512(_mm512_sllv_epi16(qhv, cl0), m10));
++        const __m512i a1 = _mm512_or_si512(_mm512_and_si512(_mm512_srli_epi16(v0, 4), m4), _mm512_and_si512(_mm512_sllv_epi16(qhv, cl1), m10));
++        const __m512i a2 = _mm512_or_si512(_mm512_and_si512(v1, m4),                       _mm512_and_si512(_mm512_srlv_epi16(qhv, cr2), m10));
++        const __m512i a3 = _mm512_or_si512(_mm512_and_si512(_mm512_srli_epi16(v1, 4), m4), _mm512_and_si512(_mm512_srlv_epi16(qhv, cr3), m10));
++
++        const __m512i b0 = _mm512_shuffle_i64x2(Q0, Q1, 0x44);
++        const __m512i b1 = _mm512_shuffle_i64x2(Q0, Q1, 0xEE);
++        const __m512i b2 = _mm512_shuffle_i64x2(Q2, Q3, 0x44);
++        const __m512i b3 = _mm512_shuffle_i64x2(Q2, Q3, 0xEE);
++
++        __m512i sumi = _mm512_setzero_si512();
++        sumi = _mm512_dpwssd_epi32(sumi, _mm512_maddubs_epi16(a0, b0), _mm512_shuffle_epi8(scales, sidx0));
++        sumi = _mm512_dpwssd_epi32(sumi, _mm512_maddubs_epi16(a1, b1), _mm512_shuffle_epi8(scales, sidx1));
++        sumi = _mm512_dpwssd_epi32(sumi, _mm512_maddubs_epi16(a2, b2), _mm512_shuffle_epi8(scales, sidx2));
++        sumi = _mm512_dpwssd_epi32(sumi, _mm512_maddubs_epi16(a3, b3), _mm512_shuffle_epi8(scales, sidx3));
++
++        acc = _mm512_fmadd_ps(_mm512_set1_ps(d), _mm512_cvtepi32_ps(sumi), acc);
++    }
++
++    acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m));
++    acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m));
++
++    *s = _mm512_reduce_add_ps(acc) + _mm_cvtss_f32(acc_m);
++
++#elif defined __AVX2__
+ 
+     const __m256i m4 = _mm256_set1_epi8(0xF);
+     const __m128i mzero = _mm_setzero_si128();
+@@ -2500,7 +2672,81 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
+ 
+     const int nb = n / QK_K;
+ 
+-#if defined __AVX2__
++#if defined(__AVX512F__) && defined(__AVX512BW__) && defined(__AVX512VL__) && defined(__AVX512VNNI__)
++    // AVX512-VNNI path (Q6_K x Q8_K): each 64-byte vector holds two 32-wide chunks.
++    const __m512i m15 = _mm512_set1_epi8(0xF);
++    const __m512i m30 = _mm512_set1_epi8(0x30);
++    const __m512i shl_cnt = _mm512_inserti64x4(_mm512_castsi256_si512(_mm256_set1_epi16(4)), _mm256_set1_epi16(2), 1);
++    const __m512i shr_cnt = _mm512_inserti64x4(_mm512_castsi256_si512(_mm256_set1_epi16(0)), _mm256_set1_epi16(2), 1);
++    #define Q6_SIDX(c1, c2) _mm512_inserti64x4( \
++        _mm512_castsi256_si512(_mm256_set_m128i(_mm_set1_epi16((short)(2*(c1)+1)), _mm_set1_epi16((short)(2*(c1))))), \
++        _mm256_set_m128i(_mm_set1_epi16((short)(2*(c2)+1)), _mm_set1_epi16((short)(2*(c2)))), 1)
++    const __m512i sidx0 = Q6_SIDX(0, 1);
++    const __m512i sidx1 = Q6_SIDX(2, 3);
++    const __m512i sidx2 = Q6_SIDX(4, 5);
++    const __m512i sidx3 = Q6_SIDX(6, 7);
++    #undef Q6_SIDX
++
++    __m512 acc = _mm512_setzero_ps();
++
++    for (int i = 0; i < nb; ++i) {
++#ifndef Q6_PF_DIST
++#define Q6_PF_DIST 44   // ~9 KB ahead, tuned for Xeon 8559C (prefetch into L2)
++#endif
++        static int q6pf = -1;
++        if (q6pf < 0) { const char * e = getenv("GGML_Q6_PF"); q6pf = e ? atoi(e) : Q6_PF_DIST; }
++        {
++            const char * pf = (const char *)(x + i + q6pf);
++            _mm_prefetch(pf,       _MM_HINT_T1);
++            _mm_prefetch(pf +  64, _MM_HINT_T1);
++            _mm_prefetch(pf + 128, _MM_HINT_T1);
++            _mm_prefetch(pf + 192, _MM_HINT_T1);
++        }
++
++        const float d = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);
++
++        const uint8_t * GGML_RESTRICT ql = x[i].ql;
++        const uint8_t * GGML_RESTRICT qh = x[i].qh;
++        const int8_t  * GGML_RESTRICT q8 = y[i].qs;
++
++        // -32 offset handled with bsums
++        const __m256i q8sums    = _mm256_loadu_si256((const __m256i*)y[i].bsums);
++        const __m256i scales_16 = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)x[i].scales));
++        const __m256i q8sclsub  = _mm256_slli_epi32(_mm256_madd_epi16(q8sums, scales_16), 5);
++        const __m512i scales512 = _mm512_zextsi256_si512(scales_16);
++
++        const __m512i QL0 = _mm512_loadu_si512((const void *)(ql));
++        const __m512i QL1 = _mm512_loadu_si512((const void *)(ql + 64));
++        const __m512i H0  = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)(qh)));
++        const __m512i H1  = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i*)(qh + 32)));
++
++        const __m512i X0 = _mm512_loadu_si512((const void *)(q8));
++        const __m512i X1 = _mm512_loadu_si512((const void *)(q8 + 64));
++        const __m512i X2 = _mm512_loadu_si512((const void *)(q8 + 128));
++        const __m512i X3 = _mm512_loadu_si512((const void *)(q8 + 192));
++
++        // 6-bit values (unsigned 0..63): low nibble / high nibble of ql combined with 2 bits of qh
++        const __m512i q6_0 = _mm512_or_si512(_mm512_and_si512(QL0, m15),
++                               _mm512_and_si512(_mm512_sllv_epi16(H0, shl_cnt), m30));
++        const __m512i q6_1 = _mm512_or_si512(_mm512_and_si512(_mm512_srli_epi16(QL0, 4), m15),
++                               _mm512_and_si512(_mm512_srlv_epi16(H0, shr_cnt), m30));
++        const __m512i q6_2 = _mm512_or_si512(_mm512_and_si512(QL1, m15),
++                               _mm512_and_si512(_mm512_sllv_epi16(H1, shl_cnt), m30));
++        const __m512i q6_3 = _mm512_or_si512(_mm512_and_si512(_mm512_srli_epi16(QL1, 4), m15),
++                               _mm512_and_si512(_mm512_srlv_epi16(H1, shr_cnt), m30));
++
++        __m512i sumi = _mm512_zextsi256_si512(_mm256_sub_epi32(_mm256_setzero_si256(), q8sclsub));
++        sumi = _mm512_dpwssd_epi32(sumi, _mm512_maddubs_epi16(q6_0, X0), _mm512_permutexvar_epi16(sidx0, scales512));
++        sumi = _mm512_dpwssd_epi32(sumi, _mm512_maddubs_epi16(q6_1, X1), _mm512_permutexvar_epi16(sidx1, scales512));
++        sumi = _mm512_dpwssd_epi32(sumi, _mm512_maddubs_epi16(q6_2, X2), _mm512_permutexvar_epi16(sidx2, scales512));
++        sumi = _mm512_dpwssd_epi32(sumi, _mm512_maddubs_epi16(q6_3, X3), _mm512_permutexvar_epi16(sidx3, scales512));
++
++        acc = _mm512_fmadd_ps(_mm512_set1_ps(d), _mm512_cvtepi32_ps(sumi), acc);
++    }
++
++    *s = _mm512_reduce_add_ps(acc);
++
++#elif defined __AVX2__
+ 
+     const __m256i m3 = _mm256_set1_epi8(3);
+     const __m256i m15 = _mm256_set1_epi8(15);
+diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
+index 53d39d249..02889fd24 100644
+--- a/ggml/src/ggml-cpu/ggml-cpu.c
++++ b/ggml/src/ggml-cpu/ggml-cpu.c
+@@ -35,6 +35,49 @@
+ #include <limits.h>
+ #include <stdarg.h>
+ #include <signal.h>
++
++// ---- lightweight per-op profiling (GGML_PROF=1), diagnostic only ----
++static int g_prof = -1;
++struct ggml_prof_entry { const char * op; const char * type; const char * ty0; int64_t ne0, ne1, ne2; int64_t us; int64_t calls; };
++static struct ggml_prof_entry g_prof_tab[1024];
++static int g_prof_n = 0;
++static int64_t g_prof_total = 0;
++
++static void ggml_prof_dump(void) {
++    if (g_prof_n == 0) return;
++    fprintf(stderr, "\n[GGML_PROF] total_us=%" PRId64 "\n", g_prof_total);
++    for (int i = 0; i < g_prof_n; ++i) {
++        fprintf(stderr, "[GGML_PROF] %-20s %-6s<%-6s [%6" PRId64 ",%5" PRId64 ",%4" PRId64 "] %9.2f ms %6.2f%% calls=%" PRId64 " us/call=%.1f\n",
++            g_prof_tab[i].op, g_prof_tab[i].type, g_prof_tab[i].ty0, g_prof_tab[i].ne0, g_prof_tab[i].ne1, g_prof_tab[i].ne2,
++            g_prof_tab[i].us/1000.0,
++            100.0*g_prof_tab[i].us/(g_prof_total?g_prof_total:1),
++            g_prof_tab[i].calls,
++            (double)g_prof_tab[i].us/(g_prof_tab[i].calls?g_prof_tab[i].calls:1));
++    }
++    fflush(stderr);
++}
++
++static void ggml_prof_add(const struct ggml_tensor * node, int64_t us) {
++    g_prof_total += us;
++    const char * op = ggml_op_name(node->op);
++    const char * ty = ggml_type_name(node->type);
++    const char * ty0 = node->src[0] ? ggml_type_name(node->src[0]->type) : "-";
++    const int64_t ne0 = node->ne[0], ne1 = node->ne[1], ne2 = node->ne[2];
++    for (int i = 0; i < g_prof_n; ++i) {
++        if (strcmp(g_prof_tab[i].op, op) == 0 && strcmp(g_prof_tab[i].type, ty) == 0 && strcmp(g_prof_tab[i].ty0, ty0) == 0 &&
++                g_prof_tab[i].ne0 == ne0 && g_prof_tab[i].ne1 == ne1 && g_prof_tab[i].ne2 == ne2) {
++            g_prof_tab[i].us += us; g_prof_tab[i].calls++;
++            return;
++        }
++    }
++    if (g_prof_n < 1024) {
++        g_prof_tab[g_prof_n].op = op; g_prof_tab[g_prof_n].type = ty; g_prof_tab[g_prof_n].ty0 = ty0;
++        g_prof_tab[g_prof_n].ne0 = ne0; g_prof_tab[g_prof_n].ne1 = ne1; g_prof_tab[g_prof_n].ne2 = ne2;
++        g_prof_tab[g_prof_n].us = us; g_prof_tab[g_prof_n].calls = 1;
++        g_prof_n++;
++    }
++}
++// ----------------------------------------------------------------
+ #if defined(__gnu_linux__)
+ #include <syscall.h>
+ #endif
+@@ -3148,6 +3191,13 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
+             continue;
+         }
+ 
++        if (g_prof < 0) {
++            const char * e = getenv("GGML_PROF");
++            g_prof = e && e[0] == '1' ? 1 : 0;
++            if (g_prof) atexit(ggml_prof_dump);
++        }
++        const int64_t prof_t0 = (g_prof && state->ith == 0) ? ggml_time_us() : 0;
++
+         // TODO: move fused-op detection into ggml_graph_plan so fusion decisions are made once at planning time
+         // Try fused ops, fall back to normal compute
+         const int n_fused = ggml_cpu_try_fuse_ops(cgraph, node_n, &params, cplan);
+@@ -3166,6 +3216,9 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
+         if (node_n + 1 < cgraph->n_nodes) {
+             ggml_barrier(state->threadpool);
+         }
++        if (g_prof && state->ith == 0) {
++            ggml_prof_add(node, ggml_time_us() - prof_t0);
++        }
+     }
+ 
+ #ifdef GGML_USE_OPENMP
+diff --git a/ggml/src/ggml-cpu/tiled/tiled.cpp b/ggml/src/ggml-cpu/tiled/tiled.cpp
+index 77e1a9945..06fd9a2d5 100644
+--- a/ggml/src/ggml-cpu/tiled/tiled.cpp
++++ b/ggml/src/ggml-cpu/tiled/tiled.cpp
+@@ -716,7 +716,9 @@ static void tiled_store_window(const float * buf, int n_src0, int n_src1, int bu
+ // unpack and microtile are reused unchanged from mul_mat.
+ 
+ // src0 rows per g group; finer than the TILED_TILE_K kernel tile so all threads stay busy at small ne01
+-#define TILED_MMID_GROUP 64
++#ifndef TILED_MMID_GROUP
++#define TILED_MMID_GROUP 32
++#endif
+ 
+ // like tiled_store_window, but the dst columns are not contiguous: col_ptrs[j] points at the
+ // start of dst column j, whose rows are contiguous (dim 0, nb0 == 4 bytes).

patches/optional-profiler.patch

@@ -0,0 +1,78 @@
+diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
+index 53d39d249..02889fd24 100644
+--- a/ggml/src/ggml-cpu/ggml-cpu.c
++++ b/ggml/src/ggml-cpu/ggml-cpu.c
+@@ -35,6 +35,49 @@
+ #include <limits.h>
+ #include <stdarg.h>
+ #include <signal.h>
++
++// ---- lightweight per-op profiling (GGML_PROF=1), diagnostic only ----
++static int g_prof = -1;
++struct ggml_prof_entry { const char * op; const char * type; const char * ty0; int64_t ne0, ne1, ne2; int64_t us; int64_t calls; };
++static struct ggml_prof_entry g_prof_tab[1024];
++static int g_prof_n = 0;
++static int64_t g_prof_total = 0;
++
++static void ggml_prof_dump(void) {
++    if (g_prof_n == 0) return;
++    fprintf(stderr, "\n[GGML_PROF] total_us=%" PRId64 "\n", g_prof_total);
++    for (int i = 0; i < g_prof_n; ++i) {
++        fprintf(stderr, "[GGML_PROF] %-20s %-6s<%-6s [%6" PRId64 ",%5" PRId64 ",%4" PRId64 "] %9.2f ms %6.2f%% calls=%" PRId64 " us/call=%.1f\n",
++            g_prof_tab[i].op, g_prof_tab[i].type, g_prof_tab[i].ty0, g_prof_tab[i].ne0, g_prof_tab[i].ne1, g_prof_tab[i].ne2,
++            g_prof_tab[i].us/1000.0,
++            100.0*g_prof_tab[i].us/(g_prof_total?g_prof_total:1),
++            g_prof_tab[i].calls,
++            (double)g_prof_tab[i].us/(g_prof_tab[i].calls?g_prof_tab[i].calls:1));
++    }
++    fflush(stderr);
++}
++
++static void ggml_prof_add(const struct ggml_tensor * node, int64_t us) {
++    g_prof_total += us;
++    const char * op = ggml_op_name(node->op);
++    const char * ty = ggml_type_name(node->type);
++    const char * ty0 = node->src[0] ? ggml_type_name(node->src[0]->type) : "-";
++    const int64_t ne0 = node->ne[0], ne1 = node->ne[1], ne2 = node->ne[2];
++    for (int i = 0; i < g_prof_n; ++i) {
++        if (strcmp(g_prof_tab[i].op, op) == 0 && strcmp(g_prof_tab[i].type, ty) == 0 && strcmp(g_prof_tab[i].ty0, ty0) == 0 &&
++                g_prof_tab[i].ne0 == ne0 && g_prof_tab[i].ne1 == ne1 && g_prof_tab[i].ne2 == ne2) {
++            g_prof_tab[i].us += us; g_prof_tab[i].calls++;
++            return;
++        }
++    }
++    if (g_prof_n < 1024) {
++        g_prof_tab[g_prof_n].op = op; g_prof_tab[g_prof_n].type = ty; g_prof_tab[g_prof_n].ty0 = ty0;
++        g_prof_tab[g_prof_n].ne0 = ne0; g_prof_tab[g_prof_n].ne1 = ne1; g_prof_tab[g_prof_n].ne2 = ne2;
++        g_prof_tab[g_prof_n].us = us; g_prof_tab[g_prof_n].calls = 1;
++        g_prof_n++;
++    }
++}
++// ----------------------------------------------------------------
+ #if defined(__gnu_linux__)
+ #include <syscall.h>
+ #endif
+@@ -3148,6 +3191,13 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
+             continue;
+         }
+ 
++        if (g_prof < 0) {
++            const char * e = getenv("GGML_PROF");
++            g_prof = e && e[0] == '1' ? 1 : 0;
++            if (g_prof) atexit(ggml_prof_dump);
++        }
++        const int64_t prof_t0 = (g_prof && state->ith == 0) ? ggml_time_us() : 0;
++
+         // TODO: move fused-op detection into ggml_graph_plan so fusion decisions are made once at planning time
+         // Try fused ops, fall back to normal compute
+         const int n_fused = ggml_cpu_try_fuse_ops(cgraph, node_n, &params, cplan);
+@@ -3166,6 +3216,9 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
+         if (node_n + 1 < cgraph->n_nodes) {
+             ggml_barrier(state->threadpool);
+         }
++        if (g_prof && state->ith == 0) {
++            ggml_prof_add(node, ggml_time_us() - prof_t0);
++        }
+     }
+ 
+ #ifdef GGML_USE_OPENMP