diff --git a/README.md b/README.md index 760fc3bc652de5fbe63c4e461673b4d1014e0234..e944b94465e159385eaf2b5671f5fe0f2ef2ae6b 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,13 @@ tags: CPU expert paging for the Unsloth `flint-UD-Q4_K_XL.gguf` checkpoint. Routed gate/up/down expert weights stay on disk and are copied into a bounded RAM arena as needed. Flint's router, selected-expert weights, shared expert, attention and dense layers remain on the upstream llama.cpp graph. +Two versions are measured throughout: + +- **v1**: expert paging with the per-expert custom op ("mode C"). `SDQ_PRELOAD=0 SDQ_ROW_PARALLEL=0`. +- **v2**: v1 plus preload of all experts into the arena and row-parallel decode for gated (gate/up/down) experts. `SDQ_PRELOAD=1 SDQ_ROW_PARALLEL=1`, arena ≥ 28.6 GiB. + +Native llama.cpp (`SDQ_PAGER=0`) is the reference. + ## Source model - GGUF: [`unsloth/flint-GGUF`](https://huggingface.co/unsloth/flint-GGUF), `flint-UD-Q4_K_XL.gguf` (25,054,688,704 bytes) @@ -28,13 +35,13 @@ System information recorded on 2026-10-23: |---|---| | CPU | Intel Xeon Platinum 8559C, 2 sockets × 48 cores, SMT enabled (192 host logical CPUs) | | CPU features | AVX2, AVX-512, VNNI, BF16; AMX available on host but disabled for this build | -| Container CPU quota | 16 CPUs (`cpu.max` = 1,600,000 / 100,000); benchmarks used 4 threads | +| Container CPU quota | 16 CPUs (`cpu.max` = 1,600,000 / 100,000); v1 memory tiers used 16 threads, v2 and the throughput runs used 16 threads | | Host memory | 2.0 TiB reported by `free` | | Container memory limit | 104,000,000,000 bytes (about 96.9 GiB), cgroup v2 | | Container swap limit | 2,936,848,384 bytes (about 2.74 GiB); measured test processes used 0 swap | | GPU | No GPU used; CPU-only build and inference | | OS / kernel | Amazon Linux 2023 x86_64, Linux 6.18.48 | -| llama.cpp | `970b51ce7c9ba5fff0726c5e9994f1c2f0e4eedc` (`970b51c`) + `flint-epq.patch` | +| llama.cpp | `970b51ce7c9ba5fff0726c5e9994f1c2f0e4eedc` (`970b51c`) + `flint-epq.patch` (sha256 `15c6554271d0c77357d96e628546f8ba624e6834c4c90c69615c90c2ac2310ea`) | ## Build @@ -53,77 +60,100 @@ cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DLLAMA_CURL=OFF \ cmake --build build -j 8 --target llama-cli llama-server llama-completion llama-perplexity ``` -Download the GGUF and run the smoke test exactly as it was verified (peak process-tree RSS 2.39 GiB, swap 0, answer produced): +v1 smoke test (verified; peak process-tree RSS 2.39 GiB, swap 0, answer produced): ```bash hf download unsloth/flint-GGUF flint-UD-Q4_K_XL.gguf --local-dir models -SDQ_PAGER=1 SDQ_ARENA_MB=512 SDQ_PREFETCH=0 SDQ_PRELOAD=0 \ +SDQ_PAGER=1 SDQ_ARENA_MB=512 SDQ_PREFETCH=0 SDQ_PRELOAD=0 SDQ_ROW_PARALLEL=0 \ build/bin/llama-cli -m models/flint-UD-Q4_K_XL.gguf -c 2048 \ -p 'What is 1 + 1? Answer briefly.' -n 32 -t 16 --temp 0 --single-turn --simple-io ``` -Set `SDQ_PAGER=0` to use the native llama.cpp MoE path. `SDQ_ARENA_MB` sets the expert cache size; `SDQ_PREFETCH=0` is the tested default. The pager emits JSON statistics when `SDQ_STATS_FILE=/path/to/stats.jsonl` is set. +v2 (preload + row-parallel). The arena must hold all 12,032 experts (≥ 28.6 GiB). Row-parallel applies to decode steps where the number of needed experts is below the thread count, so use at least 7 threads: + +```bash +SDQ_PAGER=1 SDQ_PREFETCH=0 SDQ_ARENA_MB=29400 SDQ_PRELOAD=1 SDQ_ROW_PARALLEL=1 \ + build/bin/llama-cli -m models/flint-UD-Q4_K_XL.gguf -c 512 -b 128 -ub 32 \ + -p 'Explain in one sentence why the sky is blue.' -n 32 -t 16 --temp 0 --single-turn --simple-io +``` + +`SDQ_PAGER=0` selects the native llama.cpp MoE path. `SDQ_ARENA_MB` sets the expert cache size. `SDQ_STATS_FILE=/path/to/stats.jsonl` writes JSON statistics. ## Numerical validation -EPQ is compared against native llama.cpp (`SDQ_PAGER=0`) on the same binary. Reference text: the first 6,000 bytes of llama.cpp's `README.md` (`results/logs/ppl_corpus.txt`), 4 chunks of 256 tokens. +Reference: native llama.cpp (`SDQ_PAGER=0`) on the same binary. Text: the first 6,000 bytes of llama.cpp's `README.md` (`results/logs/common/ppl_corpus.txt`). All logs are in `results/logs/{native,v1,v2}/`. -| Test | Result | -|---|---| -| Greedy generation, 24 tokens, `The capital of France is` | Identical to native (`results/logs/greedy_*.txt`) | -| Perplexity, native | 10.7790 ± 1.6026 | -| Perplexity, EPQ | 10.8014 ± 1.6076 (+0.21 %) | -| KL divergence vs native, EPQ | mean 0.01912 ± 0.00198; median 0.00429; max 0.455 | -| Top-1 token agreement vs native, EPQ | 92.3 % ± 1.2 %; RMS Δp 2.90 % | -| **Control:** native with ubatch 16 vs native with ubatch 64 (only float summation order changes) | mean KLD 0.02401 ± 0.00339; max 1.136; top-1 agreement 93.1 % | -| Layer-wise MoE output, layers 2–6, EPQ vs native (same input) | max relative difference ≤ 1.6e-5 | +| Test | Native | v1 | v2 | +|---|---|---|---| +| Greedy, 24 tokens, 8 threads (`The capital of France is`) | reference | identical to native | identical to native | +| Greedy, 24 tokens, 16 threads | reference | differs from native at char 13 | identical to v1 | +| Prefill perplexity (`-ub 64`, 8 threads, 4 chunks × 256 tokens) | 10.7790 ± 1.6026 | 10.8014 ± 1.6076 (+0.21 %) | 10.8014 ± 1.6076 | +| Prefill KL vs native (mean / median / max) | — | 0.019120 / 0.004286 / 0.455 | 0.019120 / 0.004286 / 0.455 | +| Prefill top-1 agreement vs native | — | 92.3 % | 92.3 % | +| Decode KL vs native (`-ub 1`, 16 threads, 2 chunks) | — | 0.044057; top-1 92.5 % | 0.044057; top-1 92.5 % | +| **Control:** native `-ub 64` vs native `-ub 1` (only float order changes) | — | — | mean KLD 0.034464; top-1 90.2 % | +| Decode v2 vs v1 (`-ub 1`, 16 threads) | — | — | KLD 1e-6 (max 3.8e-5); top-1 100 % | +| Layer-wise MoE output, layers 2–6, EPQ vs native (same input) | — | max relative diff ≤ 1.6e-5 | — | -The EPQ result is **not bit-identical** to native. The EPQ MoE sums the selected experts in expert-id order, while upstream sums in top-k order, so the outputs differ at the float-rounding level (~1e-7). Activation quantization to 8 bits then amplifies this small difference through the layers, so the difference grows with depth (layers 7 and later). The ubatch control shows that native llama.cpp itself gives a KL divergence and top-1 disagreement of the same size when only float summation order changes. The EPQ deviation is therefore in the same range as ordinary reordering sensitivity for this checkpoint. This is a measured conclusion from these runs, not a proof of exact equivalence. +Findings: -## 2/4/8/16/32 GiB memory benchmark +- **v2 row-parallel is numerically equivalent to v1** in decode (KL 1e-6, top-1 100 %) and identical to v1 in prefill (prefill does not use row-parallel). It is not bit-identical. +- **EPQ is not bit-identical to native.** Float summation order differs, and 8-bit activation quantization amplifies the difference through the layers. The native `-ub 1` vs `-ub 64` control gives a KL divergence of the same magnitude (0.034 vs 0.044). The deviation is in the range of ordinary reordering sensitivity for this checkpoint; it is a measured conclusion from these runs, not a proof of exact equivalence. +- **Greedy text identity is fragile.** At 8 threads the 24-token output matches native exactly; at 16 threads it diverges after 12 characters, for v1 and v2 alike. The divergence is a near-tie argmax flip caused by the reordering above. + +## Memory benchmark (RSS watchdog, v1) `bench.sh ` runs a fixed prompt and samples process-tree RSS every 20 ms. If sampled RSS exceeds the requested limit, the watchdog kills the run. This is a sampled RSS watchdog, not a kernel/cgroup hard limit: RSS may briefly exceed the threshold between samples, and it counts file-backed pages as reported by `/proc`. `tools/runmem_cgroup.py` (cgroup-v2 `MemoryMax`) is included but **untested in this environment**; it requires a host with writable cgroup v2 and systemd. ```bash -for cap in 2 4 8 16 32; do ./bench.sh "$cap" 1; done +for cap in 2 4 8 16 32; do ./bench.sh "$cap" 1; done # THREADS defaults to 16 ``` -Override model path, arena size and thread count with `MODEL`, `ARENA_MB` and `THREADS`. The workload is short (context 128, batch 32, ubatch 16, 8 generated tokens) and the benchmark sets `SDQ_PRELOAD=0`, so the preload/row-parallel path is not exercised by these numbers. Keep enough memory for non-expert weights, KV/state and compute buffers; a limit below that floor is expected to fail to load. +Override with `MODEL`, `ARENA_MB`, `THREADS`. Workload: context 128, batch 32, ubatch 16, 8 generated tokens, prefetch off. ### Results -Each tier was run 3 times with the same checkpoint, prompt (`Explain in one sentence why the sky is blue.`), 4 CPU threads, pager enabled, prefetch disabled. Table values are medians; all 15 runs exited 0 with 0 swap and never exceeded their watchdog threshold. Raw logs: `results/logs/tier_*g_run*.log`, summary `results/logs/tiers_3runs.txt`. +Each row is the median of 3 runs. All runs exited 0, swap 0, and stayed under their threshold. Logs: `results/logs/v1/mem_v1_*.log`, `results/logs/v2/v2_32g_run*.log`, summaries in `results/logs/common/mem_summary.txt`. + +| Tier | Version | Expert arena (slots) | Peak process-tree RSS | Prompt | Generation | +|---:|---|---:|---:|---:|---:| +| 2 GiB | v1 | 256 MiB (105) | 2.014 GiB | 6.9 t/s | 3.9 t/s | +| 4 GiB | v1 | 1 GiB (420) | 2.765 GiB | 7.1 t/s | 4.4 t/s | +| 8 GiB | v1 | 3 GiB (1,260) | 4.766 GiB | 9.4 t/s | 4.4 t/s | +| 16 GiB | v1 | 10 GiB (4,201) | 11.767 GiB | 12.5 t/s | 5.2 t/s | +| 32 GiB | v1 | 18 GiB (7,561 of 12,032) | 19.766 GiB | 12.0 t/s | 5.2 t/s | +| 32 GiB | v2 | 28.7 GiB (12,061, all experts) | 30.473 GiB | 33.7 t/s | 13.8 t/s | -| RSS watchdog tier | Expert arena (slots) | Peak process-tree RSS (3 runs) | Swap | Prompt (median) | Generation (median) | -|---:|---:|---:|---:|---:|---:| -| 2 GiB | 256 MiB (105) | 1.970 GiB | 0 | 5.5 t/s | 3.2 t/s | -| 4 GiB | 1 GiB (420) | 2.721 GiB | 0 | 5.8 t/s | 3.1 t/s | -| 8 GiB | 3 GiB (1,260) | 4.721 GiB | 0 | 5.9 t/s | 3.3 t/s | -| 16 GiB | 10 GiB (4,201) | 11.721 GiB | 0 | 6.5 t/s | 3.8 t/s | -| 32 GiB | 18 GiB (7,561 of 12,032 experts) | 19.720 GiB | 0 | 6.7 t/s | 3.9 t/s | +v2 is only possible in the 32 GiB tier, since preloading all experts needs at least 28.6 GiB of arena. Throughput varies by about ±15 % between runs; treat t/s as approximate. -Throughput varied by about ±15 % between runs of the same tier; treat the t/s columns as approximate. These are measurements on a shared virtualized host, not cgroup-enforced hard limits. Repeat on the target machine for representative throughput. +## Throughput (v1, v2, native) -## Validation status +Separate workload from the memory tiers: context 512, batch 128, ubatch 32, 32 generated tokens, 16 threads, 3 runs each (median). -See [`results/test-status.md`](results/test-status.md) for the full system and validation record, including what was not tested. +| Configuration | Prompt | Generation | Peak RSS | Logs | +|---|---:|---:|---:|---| +| Native (`SDQ_PAGER=0`) | 54.4 t/s | 21.5 t/s | 24.31 GiB | `results/logs/native/perf_c512_*` | +| v1, arena 18 GiB | 13.0 t/s | 5.6 t/s | 19.79 GiB | `results/logs/v1/perf_c512_v1_arena18g_*` | +| v1, arena 28.7 GiB, no preload | 12.3 t/s | 5.5 t/s | 30.51 GiB | `results/logs/v1/perf_c512_v1_arena28g_*` | +| **v2**, arena 28.7 GiB, preload + row-parallel | **32.4 t/s** | **16.2 t/s** | 30.50 GiB | `results/logs/v2/perf_c512_v2_arena28g_*` | -## Which EPQ version is ported +v2 generates about 2.9× faster than v1 and about 75 % of native generation speed. The prompt phase is also faster in v2 (likely because the preload removes SSD reads during prefill, which still uses mode C; not separately measured). The v1 and v2 rows at 28.7 GiB have nearly the same RSS because the prompt already touches most experts in both runs. -The source is [HelloSun's Nemotron EPQ](https://huggingface.co/HelloSun/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF_EPQ) with two patches applied in order: `0001-epq-nemotron-h-moe.patch` (EPQ v1) and `epqv2.patch` (EPQ v2 increment). The Flint port applies both, but only these parts are active for Flint: +## Version coverage -| Feature | Source | Status for Flint | +| Feature | Version | Status for Flint | |---|---|---| -| Expert paging, SSD→arena, per-expert custom op ("mode C") | v1 | Active; this is what all measurements use | -| Preload all experts into the arena | v2 | Code present; runs only when the arena holds all 12,032 experts (about 28.6 GiB at 2.44 MiB/slot). No measured tier reaches this, so it is not exercised here | -| Row-parallel MoE (16-thread row split) | v2 | Inactive: gated only for no-gate (Nemotron) experts; Flint has gate/up/down | +| Expert paging, SSD→arena, per-expert custom op ("mode C") | v1 | Active in all v1 and v2 runs (prefill always uses it) | +| Preload all experts into the arena | v2 | Active when the arena holds all 12,032 experts (≥ 28.6 GiB) | +| Row-parallel MoE, decode (16-row blocks shared across threads) | v2 | Active for gated Flint experts when the thread count exceeds the number of needed experts (decode, ≥ 7 threads). Ported in this version. | +| Nemotron no-gate / ReLU² path | v1, v2 | Not used by Flint | -The Flint-specific hook routes gated SiLU experts through the pager. Nemotron's no-gate/ReLU-squared path is not used by Flint. +The Flint hook routes gated SiLU experts through the pager. The v2 row-parallel gated path computes gate and up for the same row block and applies the same SwiGLU function as the mode C path. -Flint's upstream `build_moe_ffn` also applies a power-of-two amax rescale of the down-projection input and undoes it on the output. This rescale is exact in F32 and Q8_K quantization is scale-invariant, so the paged path omits it. The layer-2–6 MoE comparison against native (which includes the rescale) confirms that the omission changes no results beyond float-summation noise. +Flint's upstream `build_moe_ffn` also applies a power-of-two amax rescale of the down-projection input and undoes it on the output. This rescale is exact in F32 and Q8_K quantization is scale-invariant, so the paged path omits it. The layer-2–6 MoE comparison against native (which includes the rescale) confirms the omission changes nothing beyond float-summation noise. ## Attribution -The pager is adapted from HelloSun's Nemotron EPQ v1 and v2 (linked above). +The pager is adapted from HelloSun's Nemotron EPQ [v1 and v2](https://huggingface.co/HelloSun/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-GGUF_EPQ). The GGUF weights remain hosted by Unsloth. This repository contains source, patch, scripts, logs and test notes only. diff --git a/flint-epq.patch b/flint-epq.patch index 67f948f11f41da6a9949e6a3e386a01d184b74ab..c452a9d082bd80f50ad4d9a3259a4fa31a04d7ac 100644 --- a/flint-epq.patch +++ b/flint-epq.patch @@ -463,10 +463,10 @@ index 0000000..11c6bc5 +} // namespace sdq diff --git a/src/sdq_moe.cpp b/src/sdq_moe.cpp new file mode 100644 -index 0000000..38ad235 +index 0000000..14cc23e --- /dev/null +++ b/src/sdq_moe.cpp -@@ -0,0 +1,1150 @@ +@@ -0,0 +1,1182 @@ +// SDQ 被分頁的 MoE FFN(自訂 ggml op) +// +// 為什麼要自己算:上游 build_moe_ffn() 用 ggml_mul_mat_id 直接對 @@ -798,6 +798,19 @@ index 0000000..38ad235 + +// ---------------------------------------------------------------- 單一 expert 的數學 +// 從 arena 的槽位記憶體算出一個 expert 對多個 token 的輸出(加權後直接累加到 out) ++// SwiGLU 啟動,與上游 build_moe_ffn 一致(見 llama-graph.cpp 的 LLM_FFN_SILU 分支): ++// limit > eps 時 up 兩側限制、silu(gate) 只限制上界;否則 silu(g)*u。 ++// expert_forward 與 row-parallel 共用,確保兩條路徑數學完全相同。 ++static inline float gated_act(float g, float u, float swiglu_limit) { ++ if (swiglu_limit > 1e-6f) { ++ const float uc = std::max(-swiglu_limit, std::min(swiglu_limit, u)); ++ float gc = g / (1.0f + std::exp(-g)); ++ gc = std::min(swiglu_limit, gc); ++ return gc * uc; ++ } ++ return (g / (1.0f + std::exp(-g))) * u; ++} ++ +static void expert_forward(const expert_desc & ed, const uint8_t * base, + const float * cur, int64_t n_embd, + const std::vector> & toks, @@ -874,19 +887,7 @@ index 0000000..38ad235 + + PROF_T0_x(_td); // SiLU + 加權求和 + for (int64_t r = 0; r < ed.down_ne0; r++) { -+ const float g = gate[(size_t) r]; -+ const float u = upv[(size_t) r]; -+ float act; -+ if (swiglu_limit > 1e-6f) { -+ const float uc = std::max(-swiglu_limit, std::min(swiglu_limit, u)); -+ float gc = g / (1.0f + std::exp(-g)); -+ // 與上游 build_moe_ffn 一致:silu(gate) 只限制上界(ggml_clamp(-INF, limit)) -+ gc = std::min(swiglu_limit, gc); -+ act = gc * uc; -+ } else { -+ act = (g / (1.0f + std::exp(-g))) * u; -+ } -+ h[(size_t) r] = act; ++ h[(size_t) r] = gated_act(gate[(size_t) r], upv[(size_t) r], swiglu_limit); + } + quantize_act(act_down, h.data(), hq, ed.down_ne0); + @@ -1083,10 +1084,13 @@ index 0000000..38ad235 + struct ed_info { + const uint8_t * wu = nullptr; + const uint8_t * wd = nullptr; ++ const uint8_t * wg = nullptr; // gated 模型才有:gate 權重 + size_t up_row = 0; + size_t dn_row = 0; ++ size_t g_row = 0; + vec_dot_fn f_up = nullptr; + vec_dot_fn f_down = nullptr; ++ vec_dot_fn f_gate = nullptr; + const expert_desc * ed = nullptr; + int64_t n_te = 0; + }; @@ -1102,9 +1106,16 @@ index 0000000..38ad235 + q.dn_row = ggml_row_size(ed.type[part::DOWN], ed.down_ne0); + q.f_up = dot_for(ed.type[part::UP]); + q.f_down = dot_for(ed.type[part::DOWN]); ++ if (!R->no_gate) { ++ q.g_row = ggml_row_size(ed.type[part::GATE], ed.ne0); ++ q.f_gate = dot_for(ed.type[part::GATE]); ++ } + if (sl >= 0) { + q.wu = P.slot_ptr(sl) + ed.slot_off[part::UP]; + q.wd = P.slot_ptr(sl) + ed.slot_off[part::DOWN]; ++ if (!R->no_gate) { ++ q.wg = P.slot_ptr(sl) + ed.slot_off[part::GATE]; ++ } + } + } + @@ -1124,6 +1135,9 @@ index 0000000..38ad235 + if (!q.wu || !q.f_up) { + continue; + } ++ if (!R->no_gate && (!q.wg || !q.f_gate)) { ++ continue; ++ } + const auto & toks = R->tok_of_expert[(size_t) R->need[(size_t) j]]; + for (size_t ti = 0; ti < toks.size(); ti++) { + const int32_t t = toks[ti].first; @@ -1138,8 +1152,17 @@ index 0000000..38ad235 + float acc = 0.0f; + dot_row(q.f_up, q.wu + (size_t) r * q.up_row, xq, + q.up_row, (int) q.ed->ne0, acc); -+ acc = acc > 0.0f ? acc : 0.0f; -+ h[(size_t) r] = acc * acc; ++ if (R->no_gate) { ++ // Nemotron LatentMoE:relu(x)^2,沒有 gate ++ acc = acc > 0.0f ? acc : 0.0f; ++ h[(size_t) r] = acc * acc; ++ } else { ++ // gated(Flint):silu(gate)*up,與 expert_forward 相同的 SwiGLU ++ float g = 0.0f; ++ dot_row(q.f_gate, q.wg + (size_t) r * q.g_row, xq, ++ q.g_row, (int) q.ed->ne0, g); ++ h[(size_t) r] = gated_act(g, acc, R->swiglu_limit); ++ } + } + } + } @@ -1309,7 +1332,14 @@ index 0000000..38ad235 + if (ith == 0) { + R->use_rowpar = false; + R->rowpar_miss = 0; -+ if (rowpar_env && R->no_gate && n_need > 0 && n_need < nth) { ++ // gated(Flint)需要 gate 與 up 共用同一個 activation 量化型別,才能共用 xq。 ++ bool types_ok = true; ++ if (!R->no_gate && n_need > 0) { ++ const expert_desc & e0 = L->experts[(size_t) R->need[0]]; ++ types_ok = !e0.fused_gate_up && dot_for(e0.type[part::GATE]) && dot_for(e0.type[part::UP]) && ++ act_type_for(e0.type[part::GATE]) == act_type_for(e0.type[part::UP]); ++ } ++ if (rowpar_env && n_need > 0 && n_need < nth && types_ok) { + int miss = 0; + for (int i = 0; i < n_need; i++) { + if (!P.resident(R->il, R->need[(size_t) i])) { @@ -1557,8 +1587,10 @@ index 0000000..38ad235 +ggml_tensor * sdq_build_moe(ggml_cgraph * gf, ggml_context * ctx, ggml_tensor * cur, + ggml_tensor * ids, ggml_tensor * weights, int il, int64_t n_expert, + int64_t n_expert_used, float swiglu_limit) { -+ (void) swiglu_limit; -+ return sdq_build_moe_ex(gf, ctx, cur, ids, weights, il, n_expert, n_expert_used, false, false); ++ ggml_tensor * out = sdq_build_moe_ex(gf, ctx, cur, ids, weights, il, n_expert, n_expert_used, false, false); ++ // 把 swiglu limit 傳給 op(先前這裡 (void) 丟掉,造成 clamp 永遠不生效) ++ rec_for(il)->swiglu_limit = swiglu_limit; ++ return out; +} + +} // namespace sdq diff --git a/results/logs/common/mem_summary.txt b/results/logs/common/mem_summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..caa22a34f76dcc517cc48f01c64767033150cce7 --- /dev/null +++ b/results/logs/common/mem_summary.txt @@ -0,0 +1,18 @@ +v1 2 GiB run1 Prompt: 7.4 t/s | Generation: 3.9 "peak_gib": 2.014 "swap_kb": 0 "limit_exceeded": false "exit": 0 +v1 2 GiB run2 Prompt: 6.9 t/s | Generation: 3.8 "peak_gib": 2.013 "swap_kb": 0 "limit_exceeded": false "exit": 0 +v1 2 GiB run3 Prompt: 6.8 t/s | Generation: 3.9 "peak_gib": 2.014 "swap_kb": 0 "limit_exceeded": false "exit": 0 +v1 4 GiB run1 Prompt: 7.1 t/s | Generation: 4.4 "peak_gib": 2.765 "swap_kb": 0 "limit_exceeded": false "exit": 0 +v1 4 GiB run2 Prompt: 7.1 t/s | Generation: 4.7 "peak_gib": 2.765 "swap_kb": 0 "limit_exceeded": false "exit": 0 +v1 4 GiB run3 Prompt: 7.1 t/s | Generation: 4.4 "peak_gib": 2.765 "swap_kb": 0 "limit_exceeded": false "exit": 0 +v1 8 GiB run1 Prompt: 8.4 t/s | Generation: 4.5 "peak_gib": 4.765 "swap_kb": 0 "limit_exceeded": false "exit": 0 +v1 8 GiB run2 Prompt: 9.4 t/s | Generation: 4.4 "peak_gib": 4.766 "swap_kb": 0 "limit_exceeded": false "exit": 0 +v1 8 GiB run3 Prompt: 9.6 t/s | Generation: 4.4 "peak_gib": 4.766 "swap_kb": 0 "limit_exceeded": false "exit": 0 +v1 16 GiB run1 Prompt: 12.0 t/s | Generation: 5.2 "peak_gib": 11.767 "swap_kb": 0 "limit_exceeded": false "exit": 0 +v1 16 GiB run2 Prompt: 13.3 t/s | Generation: 5.2 "peak_gib": 11.766 "swap_kb": 0 "limit_exceeded": false "exit": 0 +v1 16 GiB run3 Prompt: 12.5 t/s | Generation: 5.4 "peak_gib": 11.767 "swap_kb": 0 "limit_exceeded": false "exit": 0 +v1 32 GiB run1 Prompt: 12.7 t/s | Generation: 5.2 "peak_gib": 19.766 "swap_kb": 0 "limit_exceeded": false "exit": 0 +v1 32 GiB run2 Prompt: 11.6 t/s | Generation: 5.1 "peak_gib": 19.766 "swap_kb": 0 "limit_exceeded": false "exit": 0 +v1 32 GiB run3 Prompt: 12.0 t/s | Generation: 5.3 "peak_gib": 19.765 "swap_kb": 0 "limit_exceeded": false "exit": 0 +v2 32 GiB run1 Prompt: 33.7 t/s | Generation: 17.3 "peak_gib": 30.473 "swap_kb": 0 "limit_exceeded": false "exit": 0 +v2 32 GiB run2 Prompt: 30.8 t/s | Generation: 13.7 "peak_gib": 30.473 "swap_kb": 0 "limit_exceeded": false "exit": 0 +v2 32 GiB run3 Prompt: 34.5 t/s | Generation: 13.8 "peak_gib": 30.473 "swap_kb": 0 "limit_exceeded": false "exit": 0 diff --git a/results/logs/common/perf_c512_summary.txt b/results/logs/common/perf_c512_summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..ff3edc642e364fa4e2d20598266f8aeb8da03fe4 --- /dev/null +++ b/results/logs/common/perf_c512_summary.txt @@ -0,0 +1,12 @@ +v1_arena18g run1 Prompt: 12.0 t/s | Generation: 5.5 "peak_gib": 19.794 "swap_kb": 0 "exit": 0 +v1_arena18g run2 Prompt: 13.0 t/s | Generation: 5.6 "peak_gib": 19.794 "swap_kb": 0 "exit": 0 +v1_arena18g run3 Prompt: 13.0 t/s | Generation: 5.7 "peak_gib": 19.794 "swap_kb": 0 "exit": 0 +v1_arena28g run1 Prompt: 11.5 t/s | Generation: 5.5 "peak_gib": 30.507 "swap_kb": 0 "exit": 0 +v1_arena28g run2 Prompt: 12.6 t/s | Generation: 5.5 "peak_gib": 30.507 "swap_kb": 0 "exit": 0 +v1_arena28g run3 Prompt: 12.3 t/s | Generation: 6.2 "peak_gib": 30.507 "swap_kb": 0 "exit": 0 +v2_arena28g run1 Prompt: 31.1 t/s | Generation: 14.5 "peak_gib": 30.499 "swap_kb": 0 "exit": 0 +v2_arena28g run2 Prompt: 32.4 t/s | Generation: 16.2 "peak_gib": 30.499 "swap_kb": 0 "exit": 0 +v2_arena28g run3 Prompt: 40.0 t/s | Generation: 17.3 "peak_gib": 30.498 "swap_kb": 0 "exit": 0 +native run1 Prompt: 61.4 t/s | Generation: 23.3 "peak_gib": 24.315 "swap_kb": 0 "exit": 0 +native run2 Prompt: 54.4 t/s | Generation: 21.5 "peak_gib": 24.311 "swap_kb": 0 "exit": 0 +native run3 Prompt: 44.7 t/s | Generation: 19.8 "peak_gib": 24.31 "swap_kb": 0 "exit": 0 diff --git a/results/logs/ppl_corpus.txt b/results/logs/common/ppl_corpus.txt similarity index 100% rename from results/logs/ppl_corpus.txt rename to results/logs/common/ppl_corpus.txt diff --git a/results/logs/greedy_epq_fresh.txt b/results/logs/native/greedy_t16.txt similarity index 100% rename from results/logs/greedy_epq_fresh.txt rename to results/logs/native/greedy_t16.txt diff --git a/results/logs/greedy_native.txt b/results/logs/native/greedy_t8.txt similarity index 100% rename from results/logs/greedy_native.txt rename to results/logs/native/greedy_t8.txt diff --git a/results/logs/kl_native_ub16_vs_native.log b/results/logs/native/kl_ub16_vs_native_ub64.log similarity index 100% rename from results/logs/kl_native_ub16_vs_native.log rename to results/logs/native/kl_ub16_vs_native_ub64.log diff --git a/results/logs/native/kl_ub64_vs_native_ub1_t16.log b/results/logs/native/kl_ub64_vs_native_ub1_t16.log new file mode 100644 index 0000000000000000000000000000000000000000..6c84048913b7347f504705d5f710ede3095f56d9 --- /dev/null +++ b/results/logs/native/kl_ub64_vs_native_ub1_t16.log @@ -0,0 +1,50 @@ +0.01.488.946 I cmn init: llama threadpool init, n_threads = 16 +0.01.544.647 I +0.01.544.771 I system_info: n_threads = 16 (n_threads_batch = 16) / 192 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.01.562.447 I kl_divergence: computing over 2 chunks, n_ctx=256, batch_size=256, n_seq=1 +0.04.557.676 I kl_divergence: 3.00 seconds per pass - ETA 0.08 minutes + +chunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p + 1 13.8921 ± 4.7298 0.03108 ± 0.03658 0.03632 ± 0.01491 3.273 ± 0.529 % 90.551 ± 2.606 % + 2 14.8507 ± 3.2268 0.01606 ± 0.03056 0.03446 ± 0.00844 2.798 ± 0.338 % 90.157 ± 1.873 % + +====== Perplexity statistics ====== +Mean PPL(Q) : 14.850718 ± 3.226807 +Mean PPL(base) : 14.614086 ± 3.091134 +Cor(ln(PPL(Q)), ln(PPL(base))): 99.02% +Mean ln(PPL(Q)/PPL(base)) : 0.016062 ± 0.030563 +Mean PPL(Q)/PPL(base) : 1.016192 ± 0.031058 +Mean PPL(Q)-PPL(base) : 0.236632 ± 0.462519 + +====== KL divergence statistics ====== +Mean KLD: 0.034464 ± 0.008439 +Maximum KLD: 1.809971 +99.9% KLD: 1.497865 +99.0% KLD: 0.475971 +95.0% KLD: 0.113679 +90.0% KLD: 0.052229 +Median KLD: 0.005795 +10.0% KLD: 0.000036 + 5.0% KLD: 0.000006 + 1.0% KLD: -0.000000 + 0.1% KLD: -0.000001 +Minimum KLD: -0.000001 + +====== Token probability statistics ====== +Mean Δp: 0.014 ± 0.176 % +Maximum Δp: 17.039% +99.9% Δp: 15.461% +99.0% Δp: 10.002% +95.0% Δp: 2.763% +90.0% Δp: 1.551% +75.0% Δp: 0.226% +Median Δp: 0.000% +25.0% Δp: -0.155% +10.0% Δp: -1.820% + 5.0% Δp: -3.285% + 1.0% Δp: -10.016% + 0.1% Δp: -14.516% +Minimum Δp: -15.501% +RMS Δp : 2.798 ± 0.338 % +Same top p: 90.157 ± 1.873 % + diff --git a/results/logs/native/perf_c512_native_run1.log b/results/logs/native/perf_c512_native_run1.log new file mode 100644 index 0000000000000000000000000000000000000000..7138ddbac9e59fa86eaac03d4bda2581caf8c6bd --- /dev/null +++ b/results/logs/native/perf_c512_native_run1.log @@ -0,0 +1,41 @@ + + +Loading model... + +▄▄ ▄▄ +██ ██ +██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄ +██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██ +██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀ + ██ ██ + ▀▀ ▀▀ + +build : b1-970b51c +model : models/flint-UD-Q4_K_XL.gguf +ftype : Q4_K - Medium +modalities : text + +available commands: + /exit or Ctrl+C stop or exit + /regen regenerate the last response + /clear clear the chat history + /read add a text file + /glob add text files using globbing pattern + + + +> Explain in one sentence why the sky is blue. + +[Start thinking] + +The user is asking for a one-sentence explanation of why the sky is blue. + +This is a classic physics question about Rayleigh scattering. + +The key points: + +[ Prompt: 61.4 t/s | Generation: 23.3 t/s ] + + +Exiting... +RUNMEM {"peak_kb": 25495692, "peak_gib": 24.315, "limit_gib": null, "limit_exceeded": false, "swap_kb": 0, "wall_s": 63.97, "exit": 0, "cmd": ["env", "SDQ_PREFETCH=0", "SDQ_PAGER=0", "build/bin/llama-cli", "-m", "models/flint-UD-Q4_K_XL.gguf", "-c", "512", "-b", "128", "-ub", "32", "-p", "Explain in one sentence why the sky is blue.", "-n", "32", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/native/perf_c512_native_run2.log b/results/logs/native/perf_c512_native_run2.log new file mode 100644 index 0000000000000000000000000000000000000000..f413f9807cf0b9688f26678e71dded2cc42235db --- /dev/null +++ b/results/logs/native/perf_c512_native_run2.log @@ -0,0 +1,41 @@ + + +Loading model... + +▄▄ ▄▄ +██ ██ +██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄ +██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██ +██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀ + ██ ██ + ▀▀ ▀▀ + +build : b1-970b51c +model : models/flint-UD-Q4_K_XL.gguf +ftype : Q4_K - Medium +modalities : text + +available commands: + /exit or Ctrl+C stop or exit + /regen regenerate the last response + /clear clear the chat history + /read add a text file + /glob add text files using globbing pattern + + + +> Explain in one sentence why the sky is blue. + +[Start thinking] + +The user is asking for a one-sentence explanation of why the sky is blue. + +This is a classic physics question about Rayleigh scattering. + +The key points: + +[ Prompt: 54.4 t/s | Generation: 21.5 t/s ] + + +Exiting... +RUNMEM {"peak_kb": 25491508, "peak_gib": 24.311, "limit_gib": null, "limit_exceeded": false, "swap_kb": 0, "wall_s": 3.74, "exit": 0, "cmd": ["env", "SDQ_PREFETCH=0", "SDQ_PAGER=0", "build/bin/llama-cli", "-m", "models/flint-UD-Q4_K_XL.gguf", "-c", "512", "-b", "128", "-ub", "32", "-p", "Explain in one sentence why the sky is blue.", "-n", "32", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/native/perf_c512_native_run3.log b/results/logs/native/perf_c512_native_run3.log new file mode 100644 index 0000000000000000000000000000000000000000..13b54bc556f5bdb68b29dbc398bea460760140cb --- /dev/null +++ b/results/logs/native/perf_c512_native_run3.log @@ -0,0 +1,41 @@ + + +Loading model... + +▄▄ ▄▄ +██ ██ +██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄ +██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██ +██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀ + ██ ██ + ▀▀ ▀▀ + +build : b1-970b51c +model : models/flint-UD-Q4_K_XL.gguf +ftype : Q4_K - Medium +modalities : text + +available commands: + /exit or Ctrl+C stop or exit + /regen regenerate the last response + /clear clear the chat history + /read add a text file + /glob add text files using globbing pattern + + + +> Explain in one sentence why the sky is blue. + +[Start thinking] + +The user is asking for a one-sentence explanation of why the sky is blue. + +This is a classic physics question about Rayleigh scattering. + +The key points: + +[ Prompt: 44.7 t/s | Generation: 19.8 t/s ] + + +Exiting... +RUNMEM {"peak_kb": 25490932, "peak_gib": 24.31, "limit_gib": null, "limit_exceeded": false, "swap_kb": 0, "wall_s": 3.96, "exit": 0, "cmd": ["env", "SDQ_PREFETCH=0", "SDQ_PAGER=0", "build/bin/llama-cli", "-m", "models/flint-UD-Q4_K_XL.gguf", "-c", "512", "-b", "128", "-ub", "32", "-p", "Explain in one sentence why the sky is blue.", "-n", "32", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/native/ppl_kl_base_decode_ub1_t16.log b/results/logs/native/ppl_kl_base_decode_ub1_t16.log new file mode 100644 index 0000000000000000000000000000000000000000..4647340da2baabf0dd5084c2aac502991ad86981 --- /dev/null +++ b/results/logs/native/ppl_kl_base_decode_ub1_t16.log @@ -0,0 +1,11 @@ +0.01.530.395 I cmn init: llama threadpool init, n_threads = 16 +0.01.656.534 I +0.01.656.668 I system_info: n_threads = 16 (n_threads_batch = 16) / 192 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.01.656.779 I perplexity: saving all logits to /tmp/opencode/flint-work/kt1/base_native_ub1.bin +0.01.656.787 I perplexity: tokenizing the input .. +0.01.666.455 I perplexity: tokenization took 9.663 ms +0.01.666.471 I perplexity: calculating perplexity over 2 chunks, n_ctx=256, batch_size=256, n_seq=1 +0.11.685.672 I perplexity: 9.99 seconds per pass - ETA 0.32 minutes +[1]13.6918,[2]14.7355, +0.21.643.935 I Final estimate: PPL = 14.7355 +/- 3.14946 + diff --git a/results/logs/ppl_kl_native_base_ub64.log b/results/logs/native/ppl_kl_base_prefill_ub64_t8.log similarity index 100% rename from results/logs/ppl_kl_native_base_ub64.log rename to results/logs/native/ppl_kl_base_prefill_ub64_t8.log diff --git a/results/logs/tiers_3runs.txt b/results/logs/tiers_3runs.txt deleted file mode 100644 index 07cd7514ee89078721bd5dc7d22b50aa7487663a..0000000000000000000000000000000000000000 --- a/results/logs/tiers_3runs.txt +++ /dev/null @@ -1,15 +0,0 @@ -2 1 Prompt: 6.3 t/s | Generation: 3.2 "peak_gib": 1.97 "swap_kb": 0 "limit_exceeded": false "exit": 0 -2 2 Prompt: 5.5 t/s | Generation: 3.2 "peak_gib": 1.97 "swap_kb": 0 "limit_exceeded": false "exit": 0 -2 3 Prompt: 5.5 t/s | Generation: 3.0 "peak_gib": 1.97 "swap_kb": 0 "limit_exceeded": false "exit": 0 -4 1 Prompt: 6.0 t/s | Generation: 3.1 "peak_gib": 2.72 "swap_kb": 0 "limit_exceeded": false "exit": 0 -4 2 Prompt: 5.8 t/s | Generation: 2.8 "peak_gib": 2.72 "swap_kb": 0 "limit_exceeded": false "exit": 0 -4 3 Prompt: 5.8 t/s | Generation: 3.3 "peak_gib": 2.72 "swap_kb": 0 "limit_exceeded": false "exit": 0 -8 1 Prompt: 5.7 t/s | Generation: 3.3 "peak_gib": 4.72 "swap_kb": 0 "limit_exceeded": false "exit": 0 -8 2 Prompt: 5.9 t/s | Generation: 3.5 "peak_gib": 4.721 "swap_kb": 0 "limit_exceeded": false "exit": 0 -8 3 Prompt: 5.9 t/s | Generation: 3.3 "peak_gib": 4.72 "swap_kb": 0 "limit_exceeded": false "exit": 0 -16 1 Prompt: 6.3 t/s | Generation: 3.7 "peak_gib": 11.721 "swap_kb": 0 "limit_exceeded": false "exit": 0 -16 2 Prompt: 6.6 t/s | Generation: 3.8 "peak_gib": 11.721 "swap_kb": 0 "limit_exceeded": false "exit": 0 -16 3 Prompt: 6.5 t/s | Generation: 3.9 "peak_gib": 11.721 "swap_kb": 0 "limit_exceeded": false "exit": 0 -32 1 Prompt: 6.7 t/s | Generation: 3.9 "peak_gib": 19.72 "swap_kb": 0 "limit_exceeded": false "exit": 0 -32 2 Prompt: 6.7 t/s | Generation: 3.7 "peak_gib": 19.72 "swap_kb": 0 "limit_exceeded": false "exit": 0 -32 3 Prompt: 6.0 t/s | Generation: 3.9 "peak_gib": 19.72 "swap_kb": 0 "limit_exceeded": false "exit": 0 diff --git a/results/logs/v1/greedy_t16.txt b/results/logs/v1/greedy_t16.txt new file mode 100644 index 0000000000000000000000000000000000000000..84c4fd2f82b06d02f7c64b81cd118201ac404e24 --- /dev/null +++ b/results/logs/v1/greedy_t16.txt @@ -0,0 +1,2 @@ + Paris is a true statement. The capital of France is London is a false. The capital of France is Berlin is a + diff --git a/results/logs/v1/greedy_t8.txt b/results/logs/v1/greedy_t8.txt new file mode 100644 index 0000000000000000000000000000000000000000..f5e3f96f9a1ce6be6680de660e7cff069d078c32 --- /dev/null +++ b/results/logs/v1/greedy_t8.txt @@ -0,0 +1,2 @@ + Paris is a simple true/false question. The answer is true. The capital of France is indeed Paris. The user is + diff --git a/results/logs/v1/kl_decode_ub1_t16_vs_native.log b/results/logs/v1/kl_decode_ub1_t16_vs_native.log new file mode 100644 index 0000000000000000000000000000000000000000..a4dfd26e0523ffa0786e72f4e0dc99200130e67e --- /dev/null +++ b/results/logs/v1/kl_decode_ub1_t16_vs_native.log @@ -0,0 +1,70 @@ +[sdq] MoE 層 47 / 共 49 層(dense lead 2):expert 權重合計 21.47 GiB、12032 個 expert +[sdq] 統一槽位 2555904 B(最大 expert 大小;平均 1915708 B) +[sdq] 使用 O_DIRECT 讀取模型檔(1 個 shard,不佔用 kernel page cache) +[sdq] I/O 段平行:24 條讀取執行緒 × 1096 KB 對齊緩衝(gate/up/down 同時讀,每段再切 1 塊) +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.76 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 +0.27.048.482 I cmn init: llama threadpool init, n_threads = 16 +0.27.205.762 I +0.27.205.855 I system_info: n_threads = 16 (n_threads_batch = 16) / 192 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.27.221.544 I kl_divergence: computing over 2 chunks, n_ctx=256, batch_size=256, n_seq=1 +0.48.384.347 I kl_divergence: 21.16 seconds per pass - ETA 0.70 minutes + +chunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p + 1 13.7740 ± 4.6245 0.02254 ± 0.03440 0.03177 ± 0.01493 4.997 ± 2.174 % 95.276 ± 1.890 % + 2 14.8027 ± 3.1804 0.01283 ± 0.02276 0.04406 ± 0.01523 4.051 ± 1.350 % 92.520 ± 1.654 % + +====== Perplexity statistics ====== +Mean PPL(Q) : 14.802734 ± 3.180425 +Mean PPL(base) : 14.614086 ± 3.091134 +Cor(ln(PPL(Q)), ln(PPL(base))): 99.44% +Mean ln(PPL(Q)/PPL(base)) : 0.012826 ± 0.022762 +Mean PPL(Q)/PPL(base) : 1.012909 ± 0.023056 +Mean PPL(Q)-PPL(base) : 0.188648 ± 0.342994 + +====== KL divergence statistics ====== +Mean KLD: 0.044057 ± 0.015234 +Maximum KLD: 3.161583 +99.9% KLD: 2.810790 +99.0% KLD: 0.854633 +95.0% KLD: 0.086102 +90.0% KLD: 0.048945 +Median KLD: 0.005596 +10.0% KLD: 0.000030 + 5.0% KLD: 0.000006 + 1.0% KLD: 0.000000 + 0.1% KLD: -0.000003 +Minimum KLD: -0.000004 + +====== Token probability statistics ====== +Mean Δp: -0.309 ± 0.254 % +Maximum Δp: 12.097% +99.9% Δp: 11.854% +99.0% Δp: 7.841% +95.0% Δp: 2.800% +90.0% Δp: 1.211% +75.0% Δp: 0.182% +Median Δp: 0.000% +25.0% Δp: -0.223% +10.0% Δp: -2.155% + 5.0% Δp: -3.443% + 1.0% Δp: -9.554% + 0.1% Δp: -43.359% +Minimum Δp: -52.532% +RMS Δp : 4.051 ± 1.350 % +Same top p: 92.520 ± 1.654 % + +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.83 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 diff --git a/results/logs/kl_epq_vs_native.log b/results/logs/v1/kl_prefill_ub64_t8_vs_native.log similarity index 100% rename from results/logs/kl_epq_vs_native.log rename to results/logs/v1/kl_prefill_ub64_t8_vs_native.log diff --git a/results/logs/tier_16g_run3.log b/results/logs/v1/mem_v1_16g_run1.log similarity index 86% rename from results/logs/tier_16g_run3.log rename to results/logs/v1/mem_v1_16g_run1.log index 52eb2226807431c88df9c1f4e381ae199b2b3401..2f68f0e492655d15eb2b43ef18fe8dd944e905e9 100644 --- a/results/logs/tier_16g_run3.log +++ b/results/logs/v1/mem_v1_16g_run1.log @@ -43,13 +43,13 @@ The user is asking for a one [sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 [sdq] expert pager ready: /tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf [sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) -[sdq] slot=2.44 MiB (2555904 B) slots=4201 arena=10.00 GiB (RAM 實測 RSS=10.13 GiB) +[sdq] slot=2.44 MiB (2555904 B) slots=4201 arena=10.00 GiB (RAM 實測 RSS=10.17 GiB) [sdq] evict_policy=lru stale_window=640 [sdq] 槽位:單一尺寸 2.44 MiB × 4201 [sdq] prefetch=off fadvise_dontneed=1 -[ Prompt: 6.5 t/s | Generation: 3.9 t/s ] +[ Prompt: 12.0 t/s | Generation: 5.2 t/s ] Exiting... -RUNMEM {"peak_kb": 12290252, "peak_gib": 11.721, "limit_gib": 16.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 12.0, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=10240", "SDQ_PRELOAD=0", "/tmp/opencode/flint-fresh/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "4", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} +RUNMEM {"peak_kb": 12338316, "peak_gib": 11.767, "limit_gib": 16.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 10.13, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=10240", "SDQ_PRELOAD=0", "/tmp/opencode/flint-work/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/tier_16g_run1.log b/results/logs/v1/mem_v1_16g_run2.log similarity index 83% rename from results/logs/tier_16g_run1.log rename to results/logs/v1/mem_v1_16g_run2.log index 5a4a8ce381ed6ad3ee462a9036b9a1e5633395d2..efe099d24ba4a7ce570eb53c64bd1f81304a4886 100644 --- a/results/logs/tier_16g_run1.log +++ b/results/logs/v1/mem_v1_16g_run2.log @@ -43,13 +43,13 @@ The user is asking for a one [sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 [sdq] expert pager ready: /tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf [sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) -[sdq] slot=2.44 MiB (2555904 B) slots=4201 arena=10.00 GiB (RAM 實測 RSS=10.13 GiB) +[sdq] slot=2.44 MiB (2555904 B) slots=4201 arena=10.00 GiB (RAM 實測 RSS=10.17 GiB) [sdq] evict_policy=lru stale_window=640 [sdq] 槽位:單一尺寸 2.44 MiB × 4201 [sdq] prefetch=off fadvise_dontneed=1 -[ Prompt: 6.3 t/s | Generation: 3.7 t/s ] +[ Prompt: 13.3 t/s | Generation: 5.2 t/s ] Exiting... -RUNMEM {"peak_kb": 12290444, "peak_gib": 11.721, "limit_gib": 16.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 12.29, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=10240", "SDQ_PRELOAD=0", "/tmp/opencode/flint-fresh/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "4", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} +RUNMEM {"peak_kb": 12338064, "peak_gib": 11.766, "limit_gib": 16.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 10.35, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=10240", "SDQ_PRELOAD=0", "/tmp/opencode/flint-work/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/tier_16g_run2.log b/results/logs/v1/mem_v1_16g_run3.log similarity index 83% rename from results/logs/tier_16g_run2.log rename to results/logs/v1/mem_v1_16g_run3.log index d893e538f6a9e2c9c79e86583b74505f6485b264..32faf5e40342ba71563f14e33c1a8fd7b53b7e2e 100644 --- a/results/logs/tier_16g_run2.log +++ b/results/logs/v1/mem_v1_16g_run3.log @@ -43,13 +43,13 @@ The user is asking for a one [sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 [sdq] expert pager ready: /tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf [sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) -[sdq] slot=2.44 MiB (2555904 B) slots=4201 arena=10.00 GiB (RAM 實測 RSS=10.13 GiB) +[sdq] slot=2.44 MiB (2555904 B) slots=4201 arena=10.00 GiB (RAM 實測 RSS=10.17 GiB) [sdq] evict_policy=lru stale_window=640 [sdq] 槽位:單一尺寸 2.44 MiB × 4201 [sdq] prefetch=off fadvise_dontneed=1 -[ Prompt: 6.6 t/s | Generation: 3.8 t/s ] +[ Prompt: 12.5 t/s | Generation: 5.4 t/s ] Exiting... -RUNMEM {"peak_kb": 12290756, "peak_gib": 11.721, "limit_gib": 16.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 12.55, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=10240", "SDQ_PRELOAD=0", "/tmp/opencode/flint-fresh/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "4", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} +RUNMEM {"peak_kb": 12338768, "peak_gib": 11.767, "limit_gib": 16.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 10.08, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=10240", "SDQ_PRELOAD=0", "/tmp/opencode/flint-work/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/tier_2g_run2.log b/results/logs/v1/mem_v1_2g_run1.log similarity index 86% rename from results/logs/tier_2g_run2.log rename to results/logs/v1/mem_v1_2g_run1.log index a15151023afd117053713e06ad65ba40245899ef..6b65b302eed869f07a00b6d57a6d5cf0e7151533 100644 --- a/results/logs/tier_2g_run2.log +++ b/results/logs/v1/mem_v1_2g_run1.log @@ -43,13 +43,13 @@ The user is asking for a one [sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 [sdq] expert pager ready: /tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf [sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) -[sdq] slot=2.44 MiB (2555904 B) slots=105 arena=0.25 GiB (RAM 實測 RSS=0.38 GiB) +[sdq] slot=2.44 MiB (2555904 B) slots=105 arena=0.25 GiB (RAM 實測 RSS=0.42 GiB) [sdq] evict_policy=lru stale_window=640 [sdq] 槽位:單一尺寸 2.44 MiB × 105 [sdq] prefetch=off fadvise_dontneed=1 -[ Prompt: 5.5 t/s | Generation: 3.2 t/s ] +[ Prompt: 7.4 t/s | Generation: 3.9 t/s ] Exiting... -RUNMEM {"peak_kb": 2066160, "peak_gib": 1.97, "limit_gib": 2.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 6.11, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=256", "SDQ_PRELOAD=0", "/tmp/opencode/flint-fresh/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "4", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} +RUNMEM {"peak_kb": 2112124, "peak_gib": 2.014, "limit_gib": 2.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 4.98, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=256", "SDQ_PRELOAD=0", "/tmp/opencode/flint-work/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/tier_2g_run3.log b/results/logs/v1/mem_v1_2g_run2.log similarity index 86% rename from results/logs/tier_2g_run3.log rename to results/logs/v1/mem_v1_2g_run2.log index 6e69ec60b5654c6befe069b0e6b08fa8634f27c3..5a8a7482a2a0675cbe2e02f513b9e80b5d103005 100644 --- a/results/logs/tier_2g_run3.log +++ b/results/logs/v1/mem_v1_2g_run2.log @@ -43,13 +43,13 @@ The user is asking for a one [sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 [sdq] expert pager ready: /tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf [sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) -[sdq] slot=2.44 MiB (2555904 B) slots=105 arena=0.25 GiB (RAM 實測 RSS=0.38 GiB) +[sdq] slot=2.44 MiB (2555904 B) slots=105 arena=0.25 GiB (RAM 實測 RSS=0.42 GiB) [sdq] evict_policy=lru stale_window=640 [sdq] 槽位:單一尺寸 2.44 MiB × 105 [sdq] prefetch=off fadvise_dontneed=1 -[ Prompt: 5.5 t/s | Generation: 3.0 t/s ] +[ Prompt: 6.9 t/s | Generation: 3.8 t/s ] Exiting... -RUNMEM {"peak_kb": 2065892, "peak_gib": 1.97, "limit_gib": 2.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 6.21, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=256", "SDQ_PRELOAD=0", "/tmp/opencode/flint-fresh/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "4", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} +RUNMEM {"peak_kb": 2110928, "peak_gib": 2.013, "limit_gib": 2.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 5.14, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=256", "SDQ_PRELOAD=0", "/tmp/opencode/flint-work/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/tier_2g_run1.log b/results/logs/v1/mem_v1_2g_run3.log similarity index 86% rename from results/logs/tier_2g_run1.log rename to results/logs/v1/mem_v1_2g_run3.log index bb53056065ca9f5d0d82bcd433c75133aba23468..87650194b51d619c857831568558cf7a2a32503f 100644 --- a/results/logs/tier_2g_run1.log +++ b/results/logs/v1/mem_v1_2g_run3.log @@ -43,13 +43,13 @@ The user is asking for a one [sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 [sdq] expert pager ready: /tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf [sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) -[sdq] slot=2.44 MiB (2555904 B) slots=105 arena=0.25 GiB (RAM 實測 RSS=0.38 GiB) +[sdq] slot=2.44 MiB (2555904 B) slots=105 arena=0.25 GiB (RAM 實測 RSS=0.42 GiB) [sdq] evict_policy=lru stale_window=640 [sdq] 槽位:單一尺寸 2.44 MiB × 105 [sdq] prefetch=off fadvise_dontneed=1 -[ Prompt: 6.3 t/s | Generation: 3.2 t/s ] +[ Prompt: 6.8 t/s | Generation: 3.9 t/s ] Exiting... -RUNMEM {"peak_kb": 2065880, "peak_gib": 1.97, "limit_gib": 2.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 5.84, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=256", "SDQ_PRELOAD=0", "/tmp/opencode/flint-fresh/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "4", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} +RUNMEM {"peak_kb": 2111384, "peak_gib": 2.014, "limit_gib": 2.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 5.11, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=256", "SDQ_PRELOAD=0", "/tmp/opencode/flint-work/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/tier_32g_run2.log b/results/logs/v1/mem_v1_32g_run1.log similarity index 86% rename from results/logs/tier_32g_run2.log rename to results/logs/v1/mem_v1_32g_run1.log index c8827a66614cdf087d5ce5e800af961cc43fd568..9d27dd8eb68af19ee5a2ff5ceef05cd9a312289f 100644 --- a/results/logs/tier_32g_run2.log +++ b/results/logs/v1/mem_v1_32g_run1.log @@ -43,13 +43,13 @@ The user is asking for a one [sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 [sdq] expert pager ready: /tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf [sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) -[sdq] slot=2.44 MiB (2555904 B) slots=7561 arena=18.00 GiB (RAM 實測 RSS=18.13 GiB) +[sdq] slot=2.44 MiB (2555904 B) slots=7561 arena=18.00 GiB (RAM 實測 RSS=18.17 GiB) [sdq] evict_policy=lru stale_window=640 [sdq] 槽位:單一尺寸 2.44 MiB × 7561 [sdq] prefetch=off fadvise_dontneed=1 -[ Prompt: 6.7 t/s | Generation: 3.7 t/s ] +[ Prompt: 12.7 t/s | Generation: 5.2 t/s ] Exiting... -RUNMEM {"peak_kb": 20677612, "peak_gib": 19.72, "limit_gib": 32.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 17.28, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=18432", "SDQ_PRELOAD=0", "/tmp/opencode/flint-fresh/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "4", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} +RUNMEM {"peak_kb": 20725676, "peak_gib": 19.766, "limit_gib": 32.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 15.36, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=18432", "SDQ_PRELOAD=0", "/tmp/opencode/flint-work/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/tier_32g_run3.log b/results/logs/v1/mem_v1_32g_run2.log similarity index 86% rename from results/logs/tier_32g_run3.log rename to results/logs/v1/mem_v1_32g_run2.log index 93bfd88e611a70e90c4bfe857b2217f76071d26b..8662bf6ee2823aa2d9826d9aeaeed6a2593955ec 100644 --- a/results/logs/tier_32g_run3.log +++ b/results/logs/v1/mem_v1_32g_run2.log @@ -43,13 +43,13 @@ The user is asking for a one [sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 [sdq] expert pager ready: /tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf [sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) -[sdq] slot=2.44 MiB (2555904 B) slots=7561 arena=18.00 GiB (RAM 實測 RSS=18.13 GiB) +[sdq] slot=2.44 MiB (2555904 B) slots=7561 arena=18.00 GiB (RAM 實測 RSS=18.17 GiB) [sdq] evict_policy=lru stale_window=640 [sdq] 槽位:單一尺寸 2.44 MiB × 7561 [sdq] prefetch=off fadvise_dontneed=1 -[ Prompt: 6.0 t/s | Generation: 3.9 t/s ] +[ Prompt: 11.6 t/s | Generation: 5.1 t/s ] Exiting... -RUNMEM {"peak_kb": 20678068, "peak_gib": 19.72, "limit_gib": 32.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 17.72, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=18432", "SDQ_PRELOAD=0", "/tmp/opencode/flint-fresh/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "4", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} +RUNMEM {"peak_kb": 20725876, "peak_gib": 19.766, "limit_gib": 32.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 15.45, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=18432", "SDQ_PRELOAD=0", "/tmp/opencode/flint-work/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/tier_32g_run1.log b/results/logs/v1/mem_v1_32g_run3.log similarity index 86% rename from results/logs/tier_32g_run1.log rename to results/logs/v1/mem_v1_32g_run3.log index 170c5a681aa5e9e8b33647b4a140e25c54ed65a1..9050cadeefba89193fb7f5c783e936753d75c9ae 100644 --- a/results/logs/tier_32g_run1.log +++ b/results/logs/v1/mem_v1_32g_run3.log @@ -43,13 +43,13 @@ The user is asking for a one [sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 [sdq] expert pager ready: /tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf [sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) -[sdq] slot=2.44 MiB (2555904 B) slots=7561 arena=18.00 GiB (RAM 實測 RSS=18.13 GiB) +[sdq] slot=2.44 MiB (2555904 B) slots=7561 arena=18.00 GiB (RAM 實測 RSS=18.17 GiB) [sdq] evict_policy=lru stale_window=640 [sdq] 槽位:單一尺寸 2.44 MiB × 7561 [sdq] prefetch=off fadvise_dontneed=1 -[ Prompt: 6.7 t/s | Generation: 3.9 t/s ] +[ Prompt: 12.0 t/s | Generation: 5.3 t/s ] Exiting... -RUNMEM {"peak_kb": 20677644, "peak_gib": 19.72, "limit_gib": 32.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 17.06, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=18432", "SDQ_PRELOAD=0", "/tmp/opencode/flint-fresh/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "4", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} +RUNMEM {"peak_kb": 20725484, "peak_gib": 19.765, "limit_gib": 32.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 15.68, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=18432", "SDQ_PRELOAD=0", "/tmp/opencode/flint-work/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/tier_4g_run2.log b/results/logs/v1/mem_v1_4g_run1.log similarity index 86% rename from results/logs/tier_4g_run2.log rename to results/logs/v1/mem_v1_4g_run1.log index 7458920c1ecde3a194e0ac4269d1e6b6f7dfbde6..17a8ee1e136d021d592286e31f69f4cdc9b4e2ed 100644 --- a/results/logs/tier_4g_run2.log +++ b/results/logs/v1/mem_v1_4g_run1.log @@ -43,13 +43,13 @@ The user is asking for a one [sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 [sdq] expert pager ready: /tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf [sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) -[sdq] slot=2.44 MiB (2555904 B) slots=420 arena=1.00 GiB (RAM 實測 RSS=1.13 GiB) +[sdq] slot=2.44 MiB (2555904 B) slots=420 arena=1.00 GiB (RAM 實測 RSS=1.17 GiB) [sdq] evict_policy=lru stale_window=640 [sdq] 槽位:單一尺寸 2.44 MiB × 420 [sdq] prefetch=off fadvise_dontneed=1 -[ Prompt: 5.8 t/s | Generation: 2.8 t/s ] +[ Prompt: 7.1 t/s | Generation: 4.4 t/s ] Exiting... -RUNMEM {"peak_kb": 2852268, "peak_gib": 2.72, "limit_gib": 4.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 6.93, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=1024", "SDQ_PRELOAD=0", "/tmp/opencode/flint-fresh/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "4", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} +RUNMEM {"peak_kb": 2899620, "peak_gib": 2.765, "limit_gib": 4.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 5.31, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=1024", "SDQ_PRELOAD=0", "/tmp/opencode/flint-work/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/tier_4g_run3.log b/results/logs/v1/mem_v1_4g_run2.log similarity index 80% rename from results/logs/tier_4g_run3.log rename to results/logs/v1/mem_v1_4g_run2.log index 4bc2c0f790268cc934ae9d76a75bdff4000e9b5a..de57496454f1be1894ae86551ea54890ec705b90 100644 --- a/results/logs/tier_4g_run3.log +++ b/results/logs/v1/mem_v1_4g_run2.log @@ -43,13 +43,13 @@ The user is asking for a one [sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 [sdq] expert pager ready: /tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf [sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) -[sdq] slot=2.44 MiB (2555904 B) slots=420 arena=1.00 GiB (RAM 實測 RSS=1.13 GiB) +[sdq] slot=2.44 MiB (2555904 B) slots=420 arena=1.00 GiB (RAM 實測 RSS=1.17 GiB) [sdq] evict_policy=lru stale_window=640 [sdq] 槽位:單一尺寸 2.44 MiB × 420 [sdq] prefetch=off fadvise_dontneed=1 -[ Prompt: 5.8 t/s | Generation: 3.3 t/s ] +[ Prompt: 7.1 t/s | Generation: 4.7 t/s ] Exiting... -RUNMEM {"peak_kb": 2852068, "peak_gib": 2.72, "limit_gib": 4.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 6.6, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=1024", "SDQ_PRELOAD=0", "/tmp/opencode/flint-fresh/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "4", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} +RUNMEM {"peak_kb": 2899720, "peak_gib": 2.765, "limit_gib": 4.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 5.18, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=1024", "SDQ_PRELOAD=0", "/tmp/opencode/flint-work/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/tier_4g_run1.log b/results/logs/v1/mem_v1_4g_run3.log similarity index 80% rename from results/logs/tier_4g_run1.log rename to results/logs/v1/mem_v1_4g_run3.log index a3ea65fd612c7a06dd51ca90278022a6d798610b..60289df520972d8ef44f194b2bcdffa2de830a94 100644 --- a/results/logs/tier_4g_run1.log +++ b/results/logs/v1/mem_v1_4g_run3.log @@ -43,13 +43,13 @@ The user is asking for a one [sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 [sdq] expert pager ready: /tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf [sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) -[sdq] slot=2.44 MiB (2555904 B) slots=420 arena=1.00 GiB (RAM 實測 RSS=1.13 GiB) +[sdq] slot=2.44 MiB (2555904 B) slots=420 arena=1.00 GiB (RAM 實測 RSS=1.17 GiB) [sdq] evict_policy=lru stale_window=640 [sdq] 槽位:單一尺寸 2.44 MiB × 420 [sdq] prefetch=off fadvise_dontneed=1 -[ Prompt: 6.0 t/s | Generation: 3.1 t/s ] +[ Prompt: 7.1 t/s | Generation: 4.4 t/s ] Exiting... -RUNMEM {"peak_kb": 2852152, "peak_gib": 2.72, "limit_gib": 4.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 6.6, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=1024", "SDQ_PRELOAD=0", "/tmp/opencode/flint-fresh/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "4", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} +RUNMEM {"peak_kb": 2899752, "peak_gib": 2.765, "limit_gib": 4.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 5.33, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=1024", "SDQ_PRELOAD=0", "/tmp/opencode/flint-work/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/tier_8g_run1.log b/results/logs/v1/mem_v1_8g_run1.log similarity index 86% rename from results/logs/tier_8g_run1.log rename to results/logs/v1/mem_v1_8g_run1.log index 886840dc4705789721c1458af61e7d5b71cbdd2f..af0fca3b102ca0c4529953f81fc381076de832b8 100644 --- a/results/logs/tier_8g_run1.log +++ b/results/logs/v1/mem_v1_8g_run1.log @@ -43,13 +43,13 @@ The user is asking for a one [sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 [sdq] expert pager ready: /tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf [sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) -[sdq] slot=2.44 MiB (2555904 B) slots=1260 arena=3.00 GiB (RAM 實測 RSS=3.13 GiB) +[sdq] slot=2.44 MiB (2555904 B) slots=1260 arena=3.00 GiB (RAM 實測 RSS=3.17 GiB) [sdq] evict_policy=lru stale_window=640 [sdq] 槽位:單一尺寸 2.44 MiB × 1260 [sdq] prefetch=off fadvise_dontneed=1 -[ Prompt: 5.7 t/s | Generation: 3.3 t/s ] +[ Prompt: 8.4 t/s | Generation: 4.5 t/s ] Exiting... -RUNMEM {"peak_kb": 4949192, "peak_gib": 4.72, "limit_gib": 8.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 7.79, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=3072", "SDQ_PRELOAD=0", "/tmp/opencode/flint-fresh/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "4", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} +RUNMEM {"peak_kb": 4996628, "peak_gib": 4.765, "limit_gib": 8.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 6.15, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=3072", "SDQ_PRELOAD=0", "/tmp/opencode/flint-work/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/tier_8g_run3.log b/results/logs/v1/mem_v1_8g_run2.log similarity index 86% rename from results/logs/tier_8g_run3.log rename to results/logs/v1/mem_v1_8g_run2.log index abd4416e2c04e15843c5f75459ce7b18c9641f44..af6bdb3c5dc6944e60984b60d6b44df09e394148 100644 --- a/results/logs/tier_8g_run3.log +++ b/results/logs/v1/mem_v1_8g_run2.log @@ -43,13 +43,13 @@ The user is asking for a one [sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 [sdq] expert pager ready: /tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf [sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) -[sdq] slot=2.44 MiB (2555904 B) slots=1260 arena=3.00 GiB (RAM 實測 RSS=3.13 GiB) +[sdq] slot=2.44 MiB (2555904 B) slots=1260 arena=3.00 GiB (RAM 實測 RSS=3.17 GiB) [sdq] evict_policy=lru stale_window=640 [sdq] 槽位:單一尺寸 2.44 MiB × 1260 [sdq] prefetch=off fadvise_dontneed=1 -[ Prompt: 5.9 t/s | Generation: 3.3 t/s ] +[ Prompt: 9.4 t/s | Generation: 4.4 t/s ] Exiting... -RUNMEM {"peak_kb": 4949056, "peak_gib": 4.72, "limit_gib": 8.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 7.86, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=3072", "SDQ_PRELOAD=0", "/tmp/opencode/flint-fresh/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "4", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} +RUNMEM {"peak_kb": 4997400, "peak_gib": 4.766, "limit_gib": 8.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 6.26, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=3072", "SDQ_PRELOAD=0", "/tmp/opencode/flint-work/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/tier_8g_run2.log b/results/logs/v1/mem_v1_8g_run3.log similarity index 83% rename from results/logs/tier_8g_run2.log rename to results/logs/v1/mem_v1_8g_run3.log index e72dd5f0a997abee135bd5133af7e55042e1cb98..fadb347141a07e7f3a6cb1e520c24265bf745dc3 100644 --- a/results/logs/tier_8g_run2.log +++ b/results/logs/v1/mem_v1_8g_run3.log @@ -43,13 +43,13 @@ The user is asking for a one [sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 [sdq] expert pager ready: /tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf [sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) -[sdq] slot=2.44 MiB (2555904 B) slots=1260 arena=3.00 GiB (RAM 實測 RSS=3.13 GiB) +[sdq] slot=2.44 MiB (2555904 B) slots=1260 arena=3.00 GiB (RAM 實測 RSS=3.17 GiB) [sdq] evict_policy=lru stale_window=640 [sdq] 槽位:單一尺寸 2.44 MiB × 1260 [sdq] prefetch=off fadvise_dontneed=1 -[ Prompt: 5.9 t/s | Generation: 3.5 t/s ] +[ Prompt: 9.6 t/s | Generation: 4.4 t/s ] Exiting... -RUNMEM {"peak_kb": 4950432, "peak_gib": 4.721, "limit_gib": 8.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 7.78, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=3072", "SDQ_PRELOAD=0", "/tmp/opencode/flint-fresh/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "4", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} +RUNMEM {"peak_kb": 4996996, "peak_gib": 4.766, "limit_gib": 8.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 6.25, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=3072", "SDQ_PRELOAD=0", "/tmp/opencode/flint-work/llama.cpp/build/bin/llama-cli", "-m", "/tmp/opencode/flint-work/llama.cpp/models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/v1/perf_c512_v1_arena18g_run1.log b/results/logs/v1/perf_c512_v1_arena18g_run1.log new file mode 100644 index 0000000000000000000000000000000000000000..484504ccf9a175715e7053a0a23a3e0964550774 --- /dev/null +++ b/results/logs/v1/perf_c512_v1_arena18g_run1.log @@ -0,0 +1,59 @@ +[sdq] MoE 層 47 / 共 49 層(dense lead 2):expert 權重合計 21.47 GiB、12032 個 expert +[sdq] 統一槽位 2555904 B(最大 expert 大小;平均 1915708 B) +[sdq] 使用 O_DIRECT 讀取模型檔(1 個 shard,不佔用 kernel page cache) +[sdq] I/O 段平行:24 條讀取執行緒 × 1096 KB 對齊緩衝(gate/up/down 同時讀,每段再切 1 塊) +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=7561 arena=18.00 GiB (RAM 實測 RSS=18.06 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 7561 +[sdq] prefetch=off fadvise_dontneed=1 + + +Loading model... + +▄▄ ▄▄ +██ ██ +██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄ +██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██ +██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀ + ██ ██ + ▀▀ ▀▀ + +build : b1-970b51c +model : models/flint-UD-Q4_K_XL.gguf +ftype : Q4_K - Medium +modalities : text + +available commands: + /exit or Ctrl+C stop or exit + /regen regenerate the last response + /clear clear the chat history + /read add a text file + /glob add text files using globbing pattern + + + +> Explain in one sentence why the sky is blue. + +[Start thinking] + +The user is asking for a one-sentence explanation of why the sky is blue. + +This is a classic physics question about Rayleigh scattering. + +The key points: +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=7561 arena=18.00 GiB (RAM 實測 RSS=18.17 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 7561 +[sdq] prefetch=off fadvise_dontneed=1 + +[ Prompt: 12.0 t/s | Generation: 5.5 t/s ] + + +Exiting... +RUNMEM {"peak_kb": 20755568, "peak_gib": 19.794, "limit_gib": null, "limit_exceeded": false, "swap_kb": 0, "wall_s": 19.15, "exit": 0, "cmd": ["env", "SDQ_PREFETCH=0", "SDQ_PAGER=1", "SDQ_ARENA_MB=18432", "SDQ_PRELOAD=0", "SDQ_ROW_PARALLEL=0", "build/bin/llama-cli", "-m", "models/flint-UD-Q4_K_XL.gguf", "-c", "512", "-b", "128", "-ub", "32", "-p", "Explain in one sentence why the sky is blue.", "-n", "32", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/v1/perf_c512_v1_arena18g_run2.log b/results/logs/v1/perf_c512_v1_arena18g_run2.log new file mode 100644 index 0000000000000000000000000000000000000000..c5143f55376222426954f5448c916e205461aa30 --- /dev/null +++ b/results/logs/v1/perf_c512_v1_arena18g_run2.log @@ -0,0 +1,59 @@ +[sdq] MoE 層 47 / 共 49 層(dense lead 2):expert 權重合計 21.47 GiB、12032 個 expert +[sdq] 統一槽位 2555904 B(最大 expert 大小;平均 1915708 B) +[sdq] 使用 O_DIRECT 讀取模型檔(1 個 shard,不佔用 kernel page cache) +[sdq] I/O 段平行:24 條讀取執行緒 × 1096 KB 對齊緩衝(gate/up/down 同時讀,每段再切 1 塊) +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=7561 arena=18.00 GiB (RAM 實測 RSS=18.06 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 7561 +[sdq] prefetch=off fadvise_dontneed=1 + + +Loading model... + +▄▄ ▄▄ +██ ██ +██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄ +██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██ +██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀ + ██ ██ + ▀▀ ▀▀ + +build : b1-970b51c +model : models/flint-UD-Q4_K_XL.gguf +ftype : Q4_K - Medium +modalities : text + +available commands: + /exit or Ctrl+C stop or exit + /regen regenerate the last response + /clear clear the chat history + /read add a text file + /glob add text files using globbing pattern + + + +> Explain in one sentence why the sky is blue. + +[Start thinking] + +The user is asking for a one-sentence explanation of why the sky is blue. + +This is a classic physics question about Rayleigh scattering. + +The key points: +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=7561 arena=18.00 GiB (RAM 實測 RSS=18.17 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 7561 +[sdq] prefetch=off fadvise_dontneed=1 + +[ Prompt: 13.0 t/s | Generation: 5.6 t/s ] + + +Exiting... +RUNMEM {"peak_kb": 20755352, "peak_gib": 19.794, "limit_gib": null, "limit_exceeded": false, "swap_kb": 0, "wall_s": 19.54, "exit": 0, "cmd": ["env", "SDQ_PREFETCH=0", "SDQ_PAGER=1", "SDQ_ARENA_MB=18432", "SDQ_PRELOAD=0", "SDQ_ROW_PARALLEL=0", "build/bin/llama-cli", "-m", "models/flint-UD-Q4_K_XL.gguf", "-c", "512", "-b", "128", "-ub", "32", "-p", "Explain in one sentence why the sky is blue.", "-n", "32", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/v1/perf_c512_v1_arena18g_run3.log b/results/logs/v1/perf_c512_v1_arena18g_run3.log new file mode 100644 index 0000000000000000000000000000000000000000..0ebd2f1e351b7d4ce07090aa2088bdb9345bedac --- /dev/null +++ b/results/logs/v1/perf_c512_v1_arena18g_run3.log @@ -0,0 +1,59 @@ +[sdq] MoE 層 47 / 共 49 層(dense lead 2):expert 權重合計 21.47 GiB、12032 個 expert +[sdq] 統一槽位 2555904 B(最大 expert 大小;平均 1915708 B) +[sdq] 使用 O_DIRECT 讀取模型檔(1 個 shard,不佔用 kernel page cache) +[sdq] I/O 段平行:24 條讀取執行緒 × 1096 KB 對齊緩衝(gate/up/down 同時讀,每段再切 1 塊) +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=7561 arena=18.00 GiB (RAM 實測 RSS=18.06 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 7561 +[sdq] prefetch=off fadvise_dontneed=1 + + +Loading model... + +▄▄ ▄▄ +██ ██ +██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄ +██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██ +██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀ + ██ ██ + ▀▀ ▀▀ + +build : b1-970b51c +model : models/flint-UD-Q4_K_XL.gguf +ftype : Q4_K - Medium +modalities : text + +available commands: + /exit or Ctrl+C stop or exit + /regen regenerate the last response + /clear clear the chat history + /read add a text file + /glob add text files using globbing pattern + + + +> Explain in one sentence why the sky is blue. + +[Start thinking] + +The user is asking for a one-sentence explanation of why the sky is blue. + +This is a classic physics question about Rayleigh scattering. + +The key points: +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=7561 arena=18.00 GiB (RAM 實測 RSS=18.17 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 7561 +[sdq] prefetch=off fadvise_dontneed=1 + +[ Prompt: 13.0 t/s | Generation: 5.7 t/s ] + + +Exiting... +RUNMEM {"peak_kb": 20755712, "peak_gib": 19.794, "limit_gib": null, "limit_exceeded": false, "swap_kb": 0, "wall_s": 21.96, "exit": 0, "cmd": ["env", "SDQ_PREFETCH=0", "SDQ_PAGER=1", "SDQ_ARENA_MB=18432", "SDQ_PRELOAD=0", "SDQ_ROW_PARALLEL=0", "build/bin/llama-cli", "-m", "models/flint-UD-Q4_K_XL.gguf", "-c", "512", "-b", "128", "-ub", "32", "-p", "Explain in one sentence why the sky is blue.", "-n", "32", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/v1/perf_c512_v1_arena28g_run1.log b/results/logs/v1/perf_c512_v1_arena28g_run1.log new file mode 100644 index 0000000000000000000000000000000000000000..f5923bf40a0e2a5ad5f3b3b6d2fddc18d0980339 --- /dev/null +++ b/results/logs/v1/perf_c512_v1_arena28g_run1.log @@ -0,0 +1,59 @@ +[sdq] MoE 層 47 / 共 49 層(dense lead 2):expert 權重合計 21.47 GiB、12032 個 expert +[sdq] 統一槽位 2555904 B(最大 expert 大小;平均 1915708 B) +[sdq] 使用 O_DIRECT 讀取模型檔(1 個 shard,不佔用 kernel page cache) +[sdq] I/O 段平行:24 條讀取執行緒 × 1096 KB 對齊緩衝(gate/up/down 同時讀,每段再切 1 塊) +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.77 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 + + +Loading model... + +▄▄ ▄▄ +██ ██ +██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄ +██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██ +██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀ + ██ ██ + ▀▀ ▀▀ + +build : b1-970b51c +model : models/flint-UD-Q4_K_XL.gguf +ftype : Q4_K - Medium +modalities : text + +available commands: + /exit or Ctrl+C stop or exit + /regen regenerate the last response + /clear clear the chat history + /read add a text file + /glob add text files using globbing pattern + + + +> Explain in one sentence why the sky is blue. + +[Start thinking] + +The user is asking for a one-sentence explanation of why the sky is blue. + +This is a classic physics question about Rayleigh scattering. + +The key points: +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.88 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 + +[ Prompt: 11.5 t/s | Generation: 5.5 t/s ] + + +Exiting... +RUNMEM {"peak_kb": 31988764, "peak_gib": 30.507, "limit_gib": null, "limit_exceeded": false, "swap_kb": 0, "wall_s": 29.95, "exit": 0, "cmd": ["env", "SDQ_PREFETCH=0", "SDQ_PAGER=1", "SDQ_ARENA_MB=29400", "SDQ_PRELOAD=0", "SDQ_ROW_PARALLEL=0", "build/bin/llama-cli", "-m", "models/flint-UD-Q4_K_XL.gguf", "-c", "512", "-b", "128", "-ub", "32", "-p", "Explain in one sentence why the sky is blue.", "-n", "32", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/v1/perf_c512_v1_arena28g_run2.log b/results/logs/v1/perf_c512_v1_arena28g_run2.log new file mode 100644 index 0000000000000000000000000000000000000000..139846735b22430fe1da0a052ad42dc932c4fec7 --- /dev/null +++ b/results/logs/v1/perf_c512_v1_arena28g_run2.log @@ -0,0 +1,59 @@ +[sdq] MoE 層 47 / 共 49 層(dense lead 2):expert 權重合計 21.47 GiB、12032 個 expert +[sdq] 統一槽位 2555904 B(最大 expert 大小;平均 1915708 B) +[sdq] 使用 O_DIRECT 讀取模型檔(1 個 shard,不佔用 kernel page cache) +[sdq] I/O 段平行:24 條讀取執行緒 × 1096 KB 對齊緩衝(gate/up/down 同時讀,每段再切 1 塊) +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.77 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 + + +Loading model... + +▄▄ ▄▄ +██ ██ +██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄ +██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██ +██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀ + ██ ██ + ▀▀ ▀▀ + +build : b1-970b51c +model : models/flint-UD-Q4_K_XL.gguf +ftype : Q4_K - Medium +modalities : text + +available commands: + /exit or Ctrl+C stop or exit + /regen regenerate the last response + /clear clear the chat history + /read add a text file + /glob add text files using globbing pattern + + + +> Explain in one sentence why the sky is blue. + +[Start thinking] + +The user is asking for a one-sentence explanation of why the sky is blue. + +This is a classic physics question about Rayleigh scattering. + +The key points: +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.88 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 + +[ Prompt: 12.6 t/s | Generation: 5.5 t/s ] + + +Exiting... +RUNMEM {"peak_kb": 31988676, "peak_gib": 30.507, "limit_gib": null, "limit_exceeded": false, "swap_kb": 0, "wall_s": 29.03, "exit": 0, "cmd": ["env", "SDQ_PREFETCH=0", "SDQ_PAGER=1", "SDQ_ARENA_MB=29400", "SDQ_PRELOAD=0", "SDQ_ROW_PARALLEL=0", "build/bin/llama-cli", "-m", "models/flint-UD-Q4_K_XL.gguf", "-c", "512", "-b", "128", "-ub", "32", "-p", "Explain in one sentence why the sky is blue.", "-n", "32", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/v1/perf_c512_v1_arena28g_run3.log b/results/logs/v1/perf_c512_v1_arena28g_run3.log new file mode 100644 index 0000000000000000000000000000000000000000..6c40f28c580c2db70386ab8dfd7163c3e7de23db --- /dev/null +++ b/results/logs/v1/perf_c512_v1_arena28g_run3.log @@ -0,0 +1,59 @@ +[sdq] MoE 層 47 / 共 49 層(dense lead 2):expert 權重合計 21.47 GiB、12032 個 expert +[sdq] 統一槽位 2555904 B(最大 expert 大小;平均 1915708 B) +[sdq] 使用 O_DIRECT 讀取模型檔(1 個 shard,不佔用 kernel page cache) +[sdq] I/O 段平行:24 條讀取執行緒 × 1096 KB 對齊緩衝(gate/up/down 同時讀,每段再切 1 塊) +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.77 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 + + +Loading model... + +▄▄ ▄▄ +██ ██ +██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄ +██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██ +██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀ + ██ ██ + ▀▀ ▀▀ + +build : b1-970b51c +model : models/flint-UD-Q4_K_XL.gguf +ftype : Q4_K - Medium +modalities : text + +available commands: + /exit or Ctrl+C stop or exit + /regen regenerate the last response + /clear clear the chat history + /read add a text file + /glob add text files using globbing pattern + + + +> Explain in one sentence why the sky is blue. + +[Start thinking] + +The user is asking for a one-sentence explanation of why the sky is blue. + +This is a classic physics question about Rayleigh scattering. + +The key points: +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.88 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 + +[ Prompt: 12.3 t/s | Generation: 6.2 t/s ] + + +Exiting... +RUNMEM {"peak_kb": 31988828, "peak_gib": 30.507, "limit_gib": null, "limit_exceeded": false, "swap_kb": 0, "wall_s": 27.87, "exit": 0, "cmd": ["env", "SDQ_PREFETCH=0", "SDQ_PAGER=1", "SDQ_ARENA_MB=29400", "SDQ_PRELOAD=0", "SDQ_ROW_PARALLEL=0", "build/bin/llama-cli", "-m", "models/flint-UD-Q4_K_XL.gguf", "-c", "512", "-b", "128", "-ub", "32", "-p", "Explain in one sentence why the sky is blue.", "-n", "32", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/ppl_epq.log b/results/logs/v1/ppl_prefill_ub64_t8.log similarity index 100% rename from results/logs/ppl_epq.log rename to results/logs/v1/ppl_prefill_ub64_t8.log diff --git a/results/logs/v2/greedy_t16.txt b/results/logs/v2/greedy_t16.txt new file mode 100644 index 0000000000000000000000000000000000000000..84c4fd2f82b06d02f7c64b81cd118201ac404e24 --- /dev/null +++ b/results/logs/v2/greedy_t16.txt @@ -0,0 +1,2 @@ + Paris is a true statement. The capital of France is London is a false. The capital of France is Berlin is a + diff --git a/results/logs/v2/greedy_t8.txt b/results/logs/v2/greedy_t8.txt new file mode 100644 index 0000000000000000000000000000000000000000..f5e3f96f9a1ce6be6680de660e7cff069d078c32 --- /dev/null +++ b/results/logs/v2/greedy_t8.txt @@ -0,0 +1,2 @@ + Paris is a simple true/false question. The answer is true. The capital of France is indeed Paris. The user is + diff --git a/results/logs/v2/kl_decode_ub1_t16_vs_native.log b/results/logs/v2/kl_decode_ub1_t16_vs_native.log new file mode 100644 index 0000000000000000000000000000000000000000..842eafbfc65320e4247a6fd5b5dca04a3b247ae9 --- /dev/null +++ b/results/logs/v2/kl_decode_ub1_t16_vs_native.log @@ -0,0 +1,70 @@ +[sdq] MoE 層 47 / 共 49 層(dense lead 2):expert 權重合計 21.47 GiB、12032 個 expert +[sdq] 統一槽位 2555904 B(最大 expert 大小;平均 1915708 B) +[sdq] 使用 O_DIRECT 讀取模型檔(1 個 shard,不佔用 kernel page cache) +[sdq] I/O 段平行:24 條讀取執行緒 × 1096 KB 對齊緩衝(gate/up/down 同時讀,每段再切 1 塊) +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.76 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 +0.30.239.333 I cmn init: llama threadpool init, n_threads = 16 +0.30.433.929 I +0.30.434.050 I system_info: n_threads = 16 (n_threads_batch = 16) / 192 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.30.450.827 I kl_divergence: computing over 2 chunks, n_ctx=256, batch_size=256, n_seq=1 +0.43.870.456 I kl_divergence: 13.42 seconds per pass - ETA 0.43 minutes + +chunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p + 1 13.7740 ± 4.6245 0.02254 ± 0.03440 0.03177 ± 0.01493 4.997 ± 2.174 % 95.276 ± 1.890 % + 2 14.8027 ± 3.1804 0.01283 ± 0.02276 0.04406 ± 0.01523 4.051 ± 1.350 % 92.520 ± 1.654 % + +====== Perplexity statistics ====== +Mean PPL(Q) : 14.802734 ± 3.180425 +Mean PPL(base) : 14.614086 ± 3.091134 +Cor(ln(PPL(Q)), ln(PPL(base))): 99.44% +Mean ln(PPL(Q)/PPL(base)) : 0.012826 ± 0.022762 +Mean PPL(Q)/PPL(base) : 1.012909 ± 0.023056 +Mean PPL(Q)-PPL(base) : 0.188648 ± 0.342994 + +====== KL divergence statistics ====== +Mean KLD: 0.044057 ± 0.015234 +Maximum KLD: 3.161583 +99.9% KLD: 2.810790 +99.0% KLD: 0.854633 +95.0% KLD: 0.086102 +90.0% KLD: 0.048945 +Median KLD: 0.005596 +10.0% KLD: 0.000030 + 5.0% KLD: 0.000006 + 1.0% KLD: 0.000000 + 0.1% KLD: -0.000003 +Minimum KLD: -0.000004 + +====== Token probability statistics ====== +Mean Δp: -0.309 ± 0.254 % +Maximum Δp: 12.097% +99.9% Δp: 11.854% +99.0% Δp: 7.841% +95.0% Δp: 2.800% +90.0% Δp: 1.211% +75.0% Δp: 0.182% +Median Δp: 0.000% +25.0% Δp: -0.223% +10.0% Δp: -2.155% + 5.0% Δp: -3.443% + 1.0% Δp: -9.554% + 0.1% Δp: -43.359% +Minimum Δp: -52.532% +RMS Δp : 4.051 ± 1.350 % +Same top p: 92.520 ± 1.654 % + +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.82 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 diff --git a/results/logs/v2/kl_decode_ub1_t16_vs_v1.log b/results/logs/v2/kl_decode_ub1_t16_vs_v1.log new file mode 100644 index 0000000000000000000000000000000000000000..9061bfddcd508af6ebfd6e2d476ff9ec99a5fb26 --- /dev/null +++ b/results/logs/v2/kl_decode_ub1_t16_vs_v1.log @@ -0,0 +1,70 @@ +[sdq] MoE 層 47 / 共 49 層(dense lead 2):expert 權重合計 21.47 GiB、12032 個 expert +[sdq] 統一槽位 2555904 B(最大 expert 大小;平均 1915708 B) +[sdq] 使用 O_DIRECT 讀取模型檔(1 個 shard,不佔用 kernel page cache) +[sdq] I/O 段平行:24 條讀取執行緒 × 1096 KB 對齊緩衝(gate/up/down 同時讀,每段再切 1 塊) +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.76 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 +0.30.646.682 I cmn init: llama threadpool init, n_threads = 16 +0.30.827.678 I +0.30.827.803 I system_info: n_threads = 16 (n_threads_batch = 16) / 192 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.30.844.516 I kl_divergence: computing over 2 chunks, n_ctx=256, batch_size=256, n_seq=1 +0.46.567.395 I kl_divergence: 15.72 seconds per pass - ETA 0.52 minutes + +chunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p + 1 13.7740 ± 4.6245 0.02395 ± 0.02233 0.00000 ± 0.00000 0.001 ± 0.000 % 100.000 ± 0.000 % + 2 14.8027 ± 3.1804 0.01198 ± 0.01117 0.00000 ± 0.00000 0.000 ± 0.000 % 100.000 ± 0.000 % + +====== Perplexity statistics ====== +Mean PPL(Q) : 14.802734 ± 3.180425 +Mean PPL(base) : 14.626524 ± 3.094576 +Cor(ln(PPL(Q)), ln(PPL(base))): 99.87% +Mean ln(PPL(Q)/PPL(base)) : 0.011975 ± 0.011170 +Mean PPL(Q)/PPL(base) : 1.012047 ± 0.011305 +Mean PPL(Q)-PPL(base) : 0.176210 ± 0.179040 + +====== KL divergence statistics ====== +Mean KLD: 0.000001 ± 0.000001 +Maximum KLD: 0.000038 +99.9% KLD: 0.000037 +99.0% KLD: 0.000034 +95.0% KLD: 0.000022 +90.0% KLD: 0.000015 +Median KLD: 0.000000 +10.0% KLD: -0.000011 + 5.0% KLD: -0.000020 + 1.0% KLD: -0.000034 + 0.1% KLD: -0.000039 +Minimum KLD: -0.000040 + +====== Token probability statistics ====== +Mean Δp: -0.000 ± 0.000 % +Maximum Δp: 0.003% +99.9% Δp: 0.002% +99.0% Δp: 0.001% +95.0% Δp: 0.001% +90.0% Δp: 0.000% +75.0% Δp: 0.000% +Median Δp: -0.000% +25.0% Δp: -0.000% +10.0% Δp: -0.000% + 5.0% Δp: -0.001% + 1.0% Δp: -0.002% + 0.1% Δp: -0.003% +Minimum Δp: -0.004% +RMS Δp : 0.000 ± 0.000 % +Same top p: 100.000 ± 0.000 % + +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.82 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 diff --git a/results/logs/v2/kl_prefill_ub64_t8_vs_native.log b/results/logs/v2/kl_prefill_ub64_t8_vs_native.log new file mode 100644 index 0000000000000000000000000000000000000000..b942f00f227a988d1519edb826e969724b4552c3 --- /dev/null +++ b/results/logs/v2/kl_prefill_ub64_t8_vs_native.log @@ -0,0 +1,72 @@ +[sdq] MoE 層 47 / 共 49 層(dense lead 2):expert 權重合計 21.47 GiB、12032 個 expert +[sdq] 統一槽位 2555904 B(最大 expert 大小;平均 1915708 B) +[sdq] 使用 O_DIRECT 讀取模型檔(1 個 shard,不佔用 kernel page cache) +[sdq] I/O 段平行:24 條讀取執行緒 × 1096 KB 對齊緩衝(gate/up/down 同時讀,每段再切 1 塊) +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.76 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 +0.27.947.775 I cmn init: llama threadpool init, n_threads = 8 +0.28.110.989 I +0.28.111.120 I system_info: n_threads = 8 (n_threads_batch = 8) / 192 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.28.130.995 I kl_divergence: computing over 4 chunks, n_ctx=256, batch_size=256, n_seq=1 +0.34.415.654 I kl_divergence: 6.28 seconds per pass - ETA 0.42 minutes + +chunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p + 1 13.4089 ± 4.4805 -0.00934 ± 0.04792 0.02043 ± 0.00445 2.355 ± 0.333 % 92.913 ± 2.286 % + 2 14.5720 ± 3.1356 -0.00591 ± 0.03286 0.02463 ± 0.00364 3.396 ± 0.658 % 90.157 ± 1.873 % + 3 9.0785 ± 1.5308 0.00134 ± 0.02267 0.01938 ± 0.00251 3.090 ± 0.488 % 91.864 ± 1.402 % + 4 10.8014 ± 1.6076 0.00923 ± 0.01822 0.01912 ± 0.00198 2.903 ± 0.398 % 92.323 ± 1.182 % + +====== Perplexity statistics ====== +Mean PPL(Q) : 10.801407 ± 1.607644 +Mean PPL(base) : 10.702135 ± 1.575857 +Cor(ln(PPL(Q)), ln(PPL(base))): 99.25% +Mean ln(PPL(Q)/PPL(base)) : 0.009233 ± 0.018216 +Mean PPL(Q)/PPL(base) : 1.009276 ± 0.018385 +Mean PPL(Q)-PPL(base) : 0.099271 ± 0.197680 + +====== KL divergence statistics ====== +Mean KLD: 0.019120 ± 0.001984 +Maximum KLD: 0.455408 +99.9% KLD: 0.412640 +99.0% KLD: 0.236318 +95.0% KLD: 0.072383 +90.0% KLD: 0.045335 +Median KLD: 0.004286 +10.0% KLD: 0.000022 + 5.0% KLD: 0.000003 + 1.0% KLD: -0.000000 + 0.1% KLD: -0.000001 +Minimum KLD: -0.000001 + +====== Token probability statistics ====== +Mean Δp: 0.088 ± 0.129 % +Maximum Δp: 28.966% +99.9% Δp: 21.585% +99.0% Δp: 7.792% +95.0% Δp: 3.637% +90.0% Δp: 1.888% +75.0% Δp: 0.336% +Median Δp: 0.000% +25.0% Δp: -0.142% +10.0% Δp: -1.699% + 5.0% Δp: -3.238% + 1.0% Δp: -7.473% + 0.1% Δp: -20.033% +Minimum Δp: -26.901% +RMS Δp : 2.903 ± 0.398 % +Same top p: 92.323 ± 1.182 % + +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=29.01 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 diff --git a/results/logs/v2/perf_c512_v2_arena28g_run1.log b/results/logs/v2/perf_c512_v2_arena28g_run1.log new file mode 100644 index 0000000000000000000000000000000000000000..822b497b6651f0686351750ecd53f46003ab7927 --- /dev/null +++ b/results/logs/v2/perf_c512_v2_arena28g_run1.log @@ -0,0 +1,61 @@ +[sdq] MoE 層 47 / 共 49 層(dense lead 2):expert 權重合計 21.47 GiB、12032 個 expert +[sdq] 統一槽位 2555904 B(最大 expert 大小;平均 1915708 B) +[sdq] 使用 O_DIRECT 讀取模型檔(1 個 shard,不佔用 kernel page cache) +[sdq] I/O 段平行:24 條讀取執行緒 × 1096 KB 對齊緩衝(gate/up/down 同時讀,每段再切 1 塊) +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.77 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 + + +Loading model... + +▄▄ ▄▄ +██ ██ +██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄ +██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██ +██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀ + ██ ██ + ▀▀ ▀▀ + +build : b1-970b51c +model : models/flint-UD-Q4_K_XL.gguf +ftype : Q4_K - Medium +modalities : text + +available commands: + /exit or Ctrl+C stop or exit + /regen regenerate the last response + /clear clear the chat history + /read add a text file + /glob add text files using globbing pattern + + + +> Explain in one sentence why the sky is blue. + +[Start thinking] + +The user is asking for a one-sentence explanation of why the sky is blue. + +This is a classic physics question about Rayleigh scattering. + +The key points: +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.87 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 + +[ Prompt: 31.1 t/s | Generation: 14.5 t/s ] + + +Exiting... +RUNMEM {"peak_kb": 31980088, "peak_gib": 30.499, "limit_gib": null, "limit_exceeded": false, "swap_kb": 0, "wall_s": 45.04, "exit": 0, "cmd": ["env", "SDQ_PREFETCH=0", "SDQ_PAGER=1", "SDQ_ARENA_MB=29400", "SDQ_PRELOAD=1", "SDQ_ROW_PARALLEL=1", "build/bin/llama-cli", "-m", "models/flint-UD-Q4_K_XL.gguf", "-c", "512", "-b", "128", "-ub", "32", "-p", "Explain in one sentence why the sky is blue.", "-n", "32", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/v2/perf_c512_v2_arena28g_run2.log b/results/logs/v2/perf_c512_v2_arena28g_run2.log new file mode 100644 index 0000000000000000000000000000000000000000..920a169cdf1f1db77d0bcbb900e4c575a679a8e3 --- /dev/null +++ b/results/logs/v2/perf_c512_v2_arena28g_run2.log @@ -0,0 +1,61 @@ +[sdq] MoE 層 47 / 共 49 層(dense lead 2):expert 權重合計 21.47 GiB、12032 個 expert +[sdq] 統一槽位 2555904 B(最大 expert 大小;平均 1915708 B) +[sdq] 使用 O_DIRECT 讀取模型檔(1 個 shard,不佔用 kernel page cache) +[sdq] I/O 段平行:24 條讀取執行緒 × 1096 KB 對齊緩衝(gate/up/down 同時讀,每段再切 1 塊) +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.77 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 + + +Loading model... + +▄▄ ▄▄ +██ ██ +██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄ +██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██ +██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀ + ██ ██ + ▀▀ ▀▀ + +build : b1-970b51c +model : models/flint-UD-Q4_K_XL.gguf +ftype : Q4_K - Medium +modalities : text + +available commands: + /exit or Ctrl+C stop or exit + /regen regenerate the last response + /clear clear the chat history + /read add a text file + /glob add text files using globbing pattern + + + +> Explain in one sentence why the sky is blue. + +[Start thinking] + +The user is asking for a one-sentence explanation of why the sky is blue. + +This is a classic physics question about Rayleigh scattering. + +The key points: +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.87 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 + +[ Prompt: 32.4 t/s | Generation: 16.2 t/s ] + + +Exiting... +RUNMEM {"peak_kb": 31980008, "peak_gib": 30.499, "limit_gib": null, "limit_exceeded": false, "swap_kb": 0, "wall_s": 32.44, "exit": 0, "cmd": ["env", "SDQ_PREFETCH=0", "SDQ_PAGER=1", "SDQ_ARENA_MB=29400", "SDQ_PRELOAD=1", "SDQ_ROW_PARALLEL=1", "build/bin/llama-cli", "-m", "models/flint-UD-Q4_K_XL.gguf", "-c", "512", "-b", "128", "-ub", "32", "-p", "Explain in one sentence why the sky is blue.", "-n", "32", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/v2/perf_c512_v2_arena28g_run3.log b/results/logs/v2/perf_c512_v2_arena28g_run3.log new file mode 100644 index 0000000000000000000000000000000000000000..aacc8f0dbd78553fe355c13d4858499faba88ad3 --- /dev/null +++ b/results/logs/v2/perf_c512_v2_arena28g_run3.log @@ -0,0 +1,61 @@ +[sdq] MoE 層 47 / 共 49 層(dense lead 2):expert 權重合計 21.47 GiB、12032 個 expert +[sdq] 統一槽位 2555904 B(最大 expert 大小;平均 1915708 B) +[sdq] 使用 O_DIRECT 讀取模型檔(1 個 shard,不佔用 kernel page cache) +[sdq] I/O 段平行:24 條讀取執行緒 × 1096 KB 對齊緩衝(gate/up/down 同時讀,每段再切 1 塊) +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.77 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 + + +Loading model... + +▄▄ ▄▄ +██ ██ +██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄ +██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██ +██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀ + ██ ██ + ▀▀ ▀▀ + +build : b1-970b51c +model : models/flint-UD-Q4_K_XL.gguf +ftype : Q4_K - Medium +modalities : text + +available commands: + /exit or Ctrl+C stop or exit + /regen regenerate the last response + /clear clear the chat history + /read add a text file + /glob add text files using globbing pattern + + + +> Explain in one sentence why the sky is blue. + +[Start thinking] + +The user is asking for a one-sentence explanation of why the sky is blue. + +This is a classic physics question about Rayleigh scattering. + +The key points: +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.87 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 + +[ Prompt: 40.0 t/s | Generation: 17.3 t/s ] + + +Exiting... +RUNMEM {"peak_kb": 31979436, "peak_gib": 30.498, "limit_gib": null, "limit_exceeded": false, "swap_kb": 0, "wall_s": 30.47, "exit": 0, "cmd": ["env", "SDQ_PREFETCH=0", "SDQ_PAGER=1", "SDQ_ARENA_MB=29400", "SDQ_PRELOAD=1", "SDQ_ROW_PARALLEL=1", "build/bin/llama-cli", "-m", "models/flint-UD-Q4_K_XL.gguf", "-c", "512", "-b", "128", "-ub", "32", "-p", "Explain in one sentence why the sky is blue.", "-n", "32", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/v2/ppl_prefill_ub64_t8.log b/results/logs/v2/ppl_prefill_ub64_t8.log new file mode 100644 index 0000000000000000000000000000000000000000..78d5b272b664ac7c173baf77374ce62ec9f8c455 --- /dev/null +++ b/results/logs/v2/ppl_prefill_ub64_t8.log @@ -0,0 +1,30 @@ +[sdq] MoE 層 47 / 共 49 層(dense lead 2):expert 權重合計 21.47 GiB、12032 個 expert +[sdq] 統一槽位 2555904 B(最大 expert 大小;平均 1915708 B) +[sdq] 使用 O_DIRECT 讀取模型檔(1 個 shard,不佔用 kernel page cache) +[sdq] I/O 段平行:24 條讀取執行緒 × 1096 KB 對齊緩衝(gate/up/down 同時讀,每段再切 1 塊) +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.76 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 +0.27.382.921 I cmn init: llama threadpool init, n_threads = 8 +0.27.578.587 I +0.27.578.686 I system_info: n_threads = 8 (n_threads_batch = 8) / 192 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.27.578.703 I perplexity: tokenizing the input .. +0.27.585.073 I perplexity: tokenization took 6.365 ms +0.27.585.087 I perplexity: calculating perplexity over 4 chunks, n_ctx=256, batch_size=256, n_seq=1 +0.33.385.452 I perplexity: 5.80 seconds per pass - ETA 0.38 minutes +[1]13.4089,[2]14.5720,[3]9.0785,[4]10.8014, +0.50.490.263 I Final estimate: PPL = 10.8014 +/- 1.60764 + +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=29.01 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 diff --git a/results/logs/v2/v2_32g_run1.log b/results/logs/v2/v2_32g_run1.log new file mode 100644 index 0000000000000000000000000000000000000000..672f41e6a4b8ef051aba2d80e505430a6b7695db --- /dev/null +++ b/results/logs/v2/v2_32g_run1.log @@ -0,0 +1,57 @@ +[sdq] MoE 層 47 / 共 49 層(dense lead 2):expert 權重合計 21.47 GiB、12032 個 expert +[sdq] 統一槽位 2555904 B(最大 expert 大小;平均 1915708 B) +[sdq] 使用 O_DIRECT 讀取模型檔(1 個 shard,不佔用 kernel page cache) +[sdq] I/O 段平行:24 條讀取執行緒 × 1096 KB 對齊緩衝(gate/up/down 同時讀,每段再切 1 塊) +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.77 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 + + +Loading model... + +▄▄ ▄▄ +██ ██ +██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄ +██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██ +██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀ + ██ ██ + ▀▀ ▀▀ + +build : b1-970b51c +model : models/flint-UD-Q4_K_XL.gguf +ftype : Q4_K - Medium +modalities : text + +available commands: + /exit or Ctrl+C stop or exit + /regen regenerate the last response + /clear clear the chat history + /read add a text file + /glob add text files using globbing pattern + + + +> Explain in one sentence why the sky is blue. + +[Start thinking] + +The user is asking for a one +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.88 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 + +[ Prompt: 33.7 t/s | Generation: 17.3 t/s ] + + +Exiting... +RUNMEM {"peak_kb": 31953556, "peak_gib": 30.473, "limit_gib": 32.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 30.09, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=29400", "SDQ_PRELOAD=1", "SDQ_ROW_PARALLEL=1", "build/bin/llama-cli", "-m", "models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/v2/v2_32g_run2.log b/results/logs/v2/v2_32g_run2.log new file mode 100644 index 0000000000000000000000000000000000000000..9852be4662ff6363da8620a0f623ad8f2bb29690 --- /dev/null +++ b/results/logs/v2/v2_32g_run2.log @@ -0,0 +1,57 @@ +[sdq] MoE 層 47 / 共 49 層(dense lead 2):expert 權重合計 21.47 GiB、12032 個 expert +[sdq] 統一槽位 2555904 B(最大 expert 大小;平均 1915708 B) +[sdq] 使用 O_DIRECT 讀取模型檔(1 個 shard,不佔用 kernel page cache) +[sdq] I/O 段平行:24 條讀取執行緒 × 1096 KB 對齊緩衝(gate/up/down 同時讀,每段再切 1 塊) +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.77 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 + + +Loading model... + +▄▄ ▄▄ +██ ██ +██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄ +██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██ +██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀ + ██ ██ + ▀▀ ▀▀ + +build : b1-970b51c +model : models/flint-UD-Q4_K_XL.gguf +ftype : Q4_K - Medium +modalities : text + +available commands: + /exit or Ctrl+C stop or exit + /regen regenerate the last response + /clear clear the chat history + /read add a text file + /glob add text files using globbing pattern + + + +> Explain in one sentence why the sky is blue. + +[Start thinking] + +The user is asking for a one +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.88 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 + +[ Prompt: 30.8 t/s | Generation: 13.7 t/s ] + + +Exiting... +RUNMEM {"peak_kb": 31953620, "peak_gib": 30.473, "limit_gib": 32.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 30.2, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=29400", "SDQ_PRELOAD=1", "SDQ_ROW_PARALLEL=1", "build/bin/llama-cli", "-m", "models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/logs/v2/v2_32g_run3.log b/results/logs/v2/v2_32g_run3.log new file mode 100644 index 0000000000000000000000000000000000000000..80be63f0dac2719ba9306436097fe545d7133c29 --- /dev/null +++ b/results/logs/v2/v2_32g_run3.log @@ -0,0 +1,57 @@ +[sdq] MoE 層 47 / 共 49 層(dense lead 2):expert 權重合計 21.47 GiB、12032 個 expert +[sdq] 統一槽位 2555904 B(最大 expert 大小;平均 1915708 B) +[sdq] 使用 O_DIRECT 讀取模型檔(1 個 shard,不佔用 kernel page cache) +[sdq] I/O 段平行:24 條讀取執行緒 × 1096 KB 對齊緩衝(gate/up/down 同時讀,每段再切 1 塊) +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.77 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 +[sdq] 已預載全部 expert(12061 slots)→ 之後 row-parallel 可用 + + +Loading model... + +▄▄ ▄▄ +██ ██ +██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄ +██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██ +██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀ + ██ ██ + ▀▀ ▀▀ + +build : b1-970b51c +model : models/flint-UD-Q4_K_XL.gguf +ftype : Q4_K - Medium +modalities : text + +available commands: + /exit or Ctrl+C stop or exit + /regen regenerate the last response + /clear clear the chat history + /read add a text file + /glob add text files using globbing pattern + + + +> Explain in one sentence why the sky is blue. + +[Start thinking] + +The user is asking for a one +[sdq] page-map: n_layer=49 n_expert=256 n_expert_used=6 n_embd=2048 n_ff_exp=512 | L2E0 gate=986105536/450560 up=1104733888/450560 down=799327936/720896 (off/size) ne0=2048 ne1=512 d_ne0=512 d_ne1=2048 +[sdq] expert pager ready: models/flint-UD-Q4_K_XL.gguf +[sdq] 熱門 expert 釘選:0 個(無剖面,用動態 LRU/LFU) +[sdq] slot=2.44 MiB (2555904 B) slots=12061 arena=28.71 GiB (RAM 實測 RSS=28.88 GiB) +[sdq] evict_policy=lru stale_window=640 +[sdq] 槽位:單一尺寸 2.44 MiB × 12061 +[sdq] prefetch=off fadvise_dontneed=1 + +[ Prompt: 34.5 t/s | Generation: 13.8 t/s ] + + +Exiting... +RUNMEM {"peak_kb": 31953600, "peak_gib": 30.473, "limit_gib": 32.0, "limit_exceeded": false, "swap_kb": 0, "wall_s": 30.34, "exit": 0, "cmd": ["env", "SDQ_PAGER=1", "SDQ_PREFETCH=0", "SDQ_ARENA_MB=29400", "SDQ_PRELOAD=1", "SDQ_ROW_PARALLEL=1", "build/bin/llama-cli", "-m", "models/flint-UD-Q4_K_XL.gguf", "-c", "128", "-b", "32", "-ub", "16", "-p", "Explain in one sentence why the sky is blue.", "-n", "8", "-t", "16", "--temp", "0", "-st", "--simple-io"], "method": "procfs-rss-watchdog"} diff --git a/results/test-status.md b/results/test-status.md index 70aa9b3c94deb30783570d041d938546aa7f7e42..c0c5c94c631189f86184928f51611e9e6c4bc589 100644 --- a/results/test-status.md +++ b/results/test-status.md @@ -1,47 +1,63 @@ # Validation status -Re-verified on 2026-10-23 after a correctness audit. Environment: Intel Xeon Platinum 8559C, container quota 16 CPUs (benchmarks used 4 threads), cgroup v2 `memory.max` 104,000,000,000 bytes (~96.9 GiB), `memory.swap.max` 2,936,848,384 bytes (~2.74 GiB), Amazon Linux 2023, kernel 6.18.48, CPU only. The container's cgroup v2 mount is read-only and there is no systemd manager, so no cgroup-enforced memory limit could be applied. +Environment: Intel Xeon Platinum 8559C, container quota 16 CPUs, cgroup v2 `memory.max` 104,000,000,000 bytes (~96.9 GiB), `memory.swap.max` 2,936,848,384 bytes (~2.74 GiB), Amazon Linux 2023, kernel 6.18.48, CPU only. The container's cgroup v2 mount is read-only and there is no systemd manager, so no cgroup-enforced memory limit could be applied. -llama.cpp base: `970b51ce7c9ba5fff0726c5e9994f1c2f0e4eedc` plus `flint-epq.patch`. +llama.cpp base: `970b51ce7c9ba5fff0726c5e9994f1c2f0e4eedc` plus `flint-epq.patch` (sha256 `15c6554271d0c77357d96e628546f8ba624e6834c4c90c69615c90c2ac2310ea`). + +Versions: **v1** = expert paging, mode C (`SDQ_PRELOAD=0 SDQ_ROW_PARALLEL=0`). **v2** = v1 + preload + row-parallel for gated experts (`SDQ_PRELOAD=1 SDQ_ROW_PARALLEL=1`, arena ≥ 28.6 GiB). **native** = `SDQ_PAGER=0`. ## Checks -| Check | Result | -|---|---| -| Clean checkout of `970b51c` + `flint-epq.patch` applies; CPU build of `llama-cli`, `llama-server`, `llama-completion`, `llama-perplexity` | PASS (fresh worktree, configure command from README) | -| Flint GGUF header / expert map | PASS: 49 layers (2 dense + 47 MoE), 256 experts, top-6, 12,032 routed experts, 21.47 GiB routed expert weights | -| Expert type coverage (`IQ3_S`, `IQ4_XS`, `Q4_K`, `Q5_K`, `Q6_K`, `Q8_0`) | PASS: all types have dot-product kernels; gate and up share one activation type in all 47 layers | -| Smoke test (README command, `-c 2048`, arena 512 MiB, `--single-turn`) | PASS: exit 0, answer produced, peak process-tree RSS 2.392 GiB, swap 0 | -| Greedy output, 24 tokens, EPQ vs native | PASS: byte-identical (fresh build) | -| Perplexity, 4 × 256-token chunks: native / EPQ | 10.7790 / 10.8014 (+0.21 %) | -| KL divergence vs native, EPQ | mean 0.019120 ± 0.001984; median 0.004286; max 0.455; top-1 agreement 92.32 % | -| Control: native ubatch 16 vs 64 | mean KLD 0.024005 ± 0.003389; max 1.136; top-1 agreement 93.11 % | -| Layer-wise MoE output vs native, layers 2–6 (same input) | PASS: max relative difference ≤ 1.6e-5 | -| Memory tiers 2 / 4 / 8 / 16 / 32 GiB (RSS watchdog, 3 runs each) | PASS: all 15 runs exit 0, swap 0, never over threshold; peaks 1.970 / 2.721 / 4.721 / 11.721 / 19.720 GiB | -| cgroup-v2 `MemoryMax` runs (`tools/runmem_cgroup.py`) | NOT RUN: no writable cgroup v2 / systemd here. The script is untested | - -## Defects found and fixed in this pass - -1. **Bad debug hook (debug path only):** `sdq_maybe_dump_tensor` (enabled by `SDQ_MOE_OUT_DUMP`) wrote the tensor to a file but never copied it into its output. Downstream nodes read uninitialized memory whenever the dump was enabled, which made early layer comparisons look wrong. Fixed: the dump node now copies its input. The production path (no dump env) never used this node and was unaffected. -2. **SwiGLU clamp mismatch:** the pager clamped `silu(gate)` to `[-limit, limit]`; upstream `build_moe_ffn` clamps only the upper bound (`ggml_clamp(-INF, limit)`). Fixed to match upstream. This checkpoint has no `swiglu_clamp` metadata, so the clamp is inactive here and the numbers above are unaffected. -3. **Published patch was stale:** the patch on Hugging Face predated fixes 1 and 2. Re-generated from the verified tree and re-uploaded. -4. **Smoke command in the model card was not the one that was run** (missing `--simple-io`, different context). Model card now shows the command that was executed. -5. **`tools/runmem_cgroup.py` and the cgroup claim were presented as usable.** Now labelled untested. -6. **Earlier tier throughput was from single runs** and varied by ~±15 % between runs. Replaced with medians of 3 runs. - -## EPQ version coverage - -- v1 (expert paging, mode C): active in all runs. -- v2 preload: code present, requires an arena of at least 28.6 GiB; never triggered in these tests (the 24 GiB run had 10,082 slots, below the 12,032 needed). -- v2 row-parallel: Nemotron no-gate only; inactive for Flint. +| Check | Version | Result | +|---|---|---| +| Clean checkout of `970b51c` + patch applies; CPU build | v1, v2 | PASS (fresh worktree, configure command from README) | +| Flint GGUF header / expert map | — | PASS: 49 layers (2 dense + 47 MoE), 256 experts, top-6, 12,032 routed experts, 21.47 GiB routed expert weights | +| Expert type coverage (`IQ3_S`, `IQ4_XS`, `Q4_K`, `Q5_K`, `Q6_K`, `Q8_0`) | — | PASS: all types have dot-product kernels; gate and up share one activation type in all 47 layers | +| Smoke test (README command, `-c 2048`, arena 512 MiB, `--single-turn`) | v1 | PASS: exit 0, answer produced, peak process-tree RSS 2.392 GiB, swap 0 | +| Greedy, 24 tokens, 8 threads | v1, v2 | PASS: identical to native | +| Greedy, 24 tokens, 16 threads | v1, v2 | v1 = v2; both differ from native at char 13 (near-tie argmax flip) | +| Prefill perplexity, `-ub 64`, 8 threads, 4 chunks | native / v1 / v2 | 10.7790 / 10.8014 / 10.8014 | +| Prefill KL vs native | v1, v2 | mean 0.019120 ± 0.001984; median 0.004286; max 0.455; top-1 92.32 % (identical for v1 and v2) | +| Decode KL vs native, `-ub 1`, 16 threads, 2 chunks | v1 | mean 0.044057; max 3.16; top-1 92.52 % | +| Decode KL vs native, `-ub 1`, 16 threads, 2 chunks | v2 | mean 0.044057; max 3.16; top-1 92.52 % | +| Decode v2 vs v1, `-ub 1`, 16 threads | v2 vs v1 | mean KLD 1e-6 (max 3.8e-5); top-1 100 %. Row-parallel is numerically equivalent to mode C but not bit-identical | +| Control: native `-ub 64` vs native `-ub 1`, 16 threads | native | mean KLD 0.034464 ± 0.008439; top-1 90.16 % | +| Control: native `-ub 16` vs `-ub 64`, prefill | native | mean KLD 0.024005 ± 0.003389; top-1 93.11 % | +| Layer-wise MoE output vs native, layers 2–6 (same input) | v1 | max relative difference ≤ 1.6e-5 | +| Row-parallel active check (`SDQ_DBG_OP`) | v2 | `rowpar=1` on decode ops with 6 needed experts at 16 threads; `rowpar=0` with `SDQ_ROW_PARALLEL=0` | +| Preload check | v2 | "已預載全部 expert(12061 slots)" with a 28.7 GiB arena | +| Throughput, context 512, batch 128, ubatch 32, 32 tokens, 16 threads (3 runs, median) | native / v1 / v2 | Generation 21.5 / 5.6 / 16.2 t/s; prompt 54.4 / 13.0 / 32.4 t/s. Peak RSS 24.3 / 19.8 / 30.5 GiB | +| Memory tiers 2 / 4 / 8 / 16 / 32 GiB, 16 threads (3 runs each) | v1 | PASS: all 15 runs exit 0, swap 0, never over threshold; peaks 2.014 / 2.765 / 4.766 / 11.767 / 19.766 GiB | +| 32 GiB tier, 28.7 GiB arena, 16 threads (3 runs) | v2 | PASS: peak 30.473 GiB, swap 0, never over threshold | +| cgroup-v2 `MemoryMax` runs (`tools/runmem_cgroup.py`) | — | NOT RUN: no writable cgroup v2 / systemd here. The script is untested | + +## Defects found and fixed + +1. **Bad debug hook (debug path only):** `sdq_maybe_dump_tensor` (enabled by `SDQ_MOE_OUT_DUMP`) wrote the tensor to a file but never copied it into its output, so downstream nodes read uninitialized memory. Fixed; the production path never used this node. +2. **SwiGLU clamp mismatch:** the pager clamped `silu(gate)` to `[-limit, limit]`; upstream clamps only the upper bound. Fixed to match upstream. +3. **`swiglu_limit` was dropped (found in this pass):** `sdq_build_moe` ignored its `swiglu_limit` argument and the op always used 0. Fixed; the value now reaches the op. This checkpoint has limit 0, so the numbers are unaffected. +4. **Published patch was stale:** the patch on Hugging Face predated these fixes. Re-generated from the verified tree and re-uploaded. +5. **Smoke command in the model card was not the one that was run.** The card now shows the executed command. +6. **`tools/runmem_cgroup.py` and the cgroup claim were presented as usable.** Now labeled untested. +7. **Earlier memory-tier throughput came from single runs** and varied by ~±15 %. Replaced with medians of 3 runs. +8. **Earlier tier tables used 4 threads; `bench.sh` defaults to 16.** The tier tables are now labeled with 16 threads; the earlier 4-thread numbers are superseded. +9. **Row-parallel was inactive for Flint.** Added the gated (gate/up/down) path so v2 applies to Flint; verified active and numerically equivalent to v1 (see checks). ## Known limits and open items -- EPQ output is not bit-identical to native llama.cpp. Differences come from float summation order and are amplified by 8-bit activation quantization; native llama.cpp itself shows the same magnitude of KLD under a ubatch change (control above). No functional mismatch was found. -- The memory tiers use an RSS watchdog that samples every 20 ms and counts file-backed pages as reported by `/proc`. They are not kernel-enforced limits. A true cgroup-enforced test must be run on a host with a writable cgroup v2 hierarchy and systemd (`tools/runmem_cgroup.py`, untested). -- Benchmarks set `SDQ_PRELOAD=0`, so the preload / row-parallel path is not exercised by the tier numbers. The 18 GiB arena at the 32 GiB tier holds 7,561 of 12,032 experts, so eviction occurs. -- The accuracy tests use about 1,000 scored tokens from one text sample. They do not cover long-context behavior or other prompts beyond the greedy check. +- EPQ output is not bit-identical to native llama.cpp. Float summation order differs, and 8-bit activation quantization amplifies the difference through the layers. Native itself shows a KL divergence of the same magnitude under a ubatch change. No functional mismatch was found. +- Greedy identity is fragile: at 16 threads both v1 and v2 diverge from native after 12 characters. +- The memory tiers use an RSS watchdog that samples every 20 ms and counts file-backed pages as reported by `/proc`. They are not kernel-enforced limits. A true cgroup-enforced test requires a host with a writable cgroup v2 hierarchy and systemd (`tools/runmem_cgroup.py`, untested). +- v2 only runs in the 32 GiB tier, where the arena can hold all experts. The 2–16 GiB tiers are v1 only. +- Accuracy tests use about 1,000 scored tokens from one text sample. They do not cover long-context behavior or other prompts beyond the greedy checks. +- The v2 prompt-phase speedup is likely due to preload removing SSD reads during prefill; this was not isolated by a separate measurement. ## Raw logs -`results/logs/`: KL and perplexity logs (`kl_epq_vs_native.log`, `kl_native_ub16_vs_native.log`, `ppl_kl_native_base_ub64.log`, `ppl_epq.log`), greedy outputs (`greedy_native.txt`, `greedy_epq_fresh.txt`), the text corpus (`ppl_corpus.txt`), and per-run memory-tier logs with the 3-run summary (`tiers_3runs.txt`). +`results/logs/{native,v1,v2}/` hold the per-version logs; `results/logs/common/` holds the corpus and summaries: + +- Prefill accuracy: `v1/ppl_prefill_ub64_t8.log`, `v1/kl_prefill_ub64_t8_vs_native.log`, `v2/ppl_prefill_ub64_t8.log`, `v2/kl_prefill_ub64_t8_vs_native.log`, `native/ppl_kl_base_prefill_ub64_t8.log`, `native/kl_ub16_vs_native_ub64.log` +- Decode accuracy: `v1/kl_decode_ub1_t16_vs_native.log`, `v2/kl_decode_ub1_t16_vs_native.log`, `v2/kl_decode_ub1_t16_vs_v1.log`, `native/ppl_kl_base_decode_ub1_t16.log`, `native/kl_ub64_vs_native_ub1_t16.log` +- Greedy: `native/greedy_t8.txt`, `native/greedy_t16.txt`, `v1/greedy_t8.txt`, `v1/greedy_t16.txt`, `v2/greedy_t8.txt`, `v2/greedy_t16.txt` +- Throughput: `*/perf_c512_*.log`, `common/perf_c512_summary.txt` +- Memory: `v1/mem_v1_*.log`, `v2/v2_32g_run*.log`, `common/mem_summary.txt` diff --git a/src/sdq_moe.cpp b/src/sdq_moe.cpp index 38ad23572e527bed39de1bd98472de6eace3c490..14cc23e500e9f23c352d2ce479c10312c5d4207e 100644 --- a/src/sdq_moe.cpp +++ b/src/sdq_moe.cpp @@ -329,6 +329,19 @@ static size_t env_partial_cap_mb() { // ---------------------------------------------------------------- 單一 expert 的數學 // 從 arena 的槽位記憶體算出一個 expert 對多個 token 的輸出(加權後直接累加到 out) +// SwiGLU 啟動,與上游 build_moe_ffn 一致(見 llama-graph.cpp 的 LLM_FFN_SILU 分支): +// limit > eps 時 up 兩側限制、silu(gate) 只限制上界;否則 silu(g)*u。 +// expert_forward 與 row-parallel 共用,確保兩條路徑數學完全相同。 +static inline float gated_act(float g, float u, float swiglu_limit) { + if (swiglu_limit > 1e-6f) { + const float uc = std::max(-swiglu_limit, std::min(swiglu_limit, u)); + float gc = g / (1.0f + std::exp(-g)); + gc = std::min(swiglu_limit, gc); + return gc * uc; + } + return (g / (1.0f + std::exp(-g))) * u; +} + static void expert_forward(const expert_desc & ed, const uint8_t * base, const float * cur, int64_t n_embd, const std::vector> & toks, @@ -405,19 +418,7 @@ static void expert_forward(const expert_desc & ed, const uint8_t * base, PROF_T0_x(_td); // SiLU + 加權求和 for (int64_t r = 0; r < ed.down_ne0; r++) { - const float g = gate[(size_t) r]; - const float u = upv[(size_t) r]; - float act; - if (swiglu_limit > 1e-6f) { - const float uc = std::max(-swiglu_limit, std::min(swiglu_limit, u)); - float gc = g / (1.0f + std::exp(-g)); - // 與上游 build_moe_ffn 一致:silu(gate) 只限制上界(ggml_clamp(-INF, limit)) - gc = std::min(swiglu_limit, gc); - act = gc * uc; - } else { - act = (g / (1.0f + std::exp(-g))) * u; - } - h[(size_t) r] = act; + h[(size_t) r] = gated_act(gate[(size_t) r], upv[(size_t) r], swiglu_limit); } quantize_act(act_down, h.data(), hq, ed.down_ne0); @@ -614,10 +615,13 @@ static void moe_row_parallel(ggml_tensor * dst, int ith, int nth, layer_rec * R, struct ed_info { const uint8_t * wu = nullptr; const uint8_t * wd = nullptr; + const uint8_t * wg = nullptr; // gated 模型才有:gate 權重 size_t up_row = 0; size_t dn_row = 0; + size_t g_row = 0; vec_dot_fn f_up = nullptr; vec_dot_fn f_down = nullptr; + vec_dot_fn f_gate = nullptr; const expert_desc * ed = nullptr; int64_t n_te = 0; }; @@ -633,9 +637,16 @@ static void moe_row_parallel(ggml_tensor * dst, int ith, int nth, layer_rec * R, q.dn_row = ggml_row_size(ed.type[part::DOWN], ed.down_ne0); q.f_up = dot_for(ed.type[part::UP]); q.f_down = dot_for(ed.type[part::DOWN]); + if (!R->no_gate) { + q.g_row = ggml_row_size(ed.type[part::GATE], ed.ne0); + q.f_gate = dot_for(ed.type[part::GATE]); + } if (sl >= 0) { q.wu = P.slot_ptr(sl) + ed.slot_off[part::UP]; q.wd = P.slot_ptr(sl) + ed.slot_off[part::DOWN]; + if (!R->no_gate) { + q.wg = P.slot_ptr(sl) + ed.slot_off[part::GATE]; + } } } @@ -655,6 +666,9 @@ static void moe_row_parallel(ggml_tensor * dst, int ith, int nth, layer_rec * R, if (!q.wu || !q.f_up) { continue; } + if (!R->no_gate && (!q.wg || !q.f_gate)) { + continue; + } const auto & toks = R->tok_of_expert[(size_t) R->need[(size_t) j]]; for (size_t ti = 0; ti < toks.size(); ti++) { const int32_t t = toks[ti].first; @@ -669,8 +683,17 @@ static void moe_row_parallel(ggml_tensor * dst, int ith, int nth, layer_rec * R, float acc = 0.0f; dot_row(q.f_up, q.wu + (size_t) r * q.up_row, xq, q.up_row, (int) q.ed->ne0, acc); - acc = acc > 0.0f ? acc : 0.0f; - h[(size_t) r] = acc * acc; + if (R->no_gate) { + // Nemotron LatentMoE:relu(x)^2,沒有 gate + acc = acc > 0.0f ? acc : 0.0f; + h[(size_t) r] = acc * acc; + } else { + // gated(Flint):silu(gate)*up,與 expert_forward 相同的 SwiGLU + float g = 0.0f; + dot_row(q.f_gate, q.wg + (size_t) r * q.g_row, xq, + q.g_row, (int) q.ed->ne0, g); + h[(size_t) r] = gated_act(g, acc, R->swiglu_limit); + } } } } @@ -840,7 +863,14 @@ static void moe_custom_op(ggml_tensor * dst, int ith, int nth, void * userdata) if (ith == 0) { R->use_rowpar = false; R->rowpar_miss = 0; - if (rowpar_env && R->no_gate && n_need > 0 && n_need < nth) { + // gated(Flint)需要 gate 與 up 共用同一個 activation 量化型別,才能共用 xq。 + bool types_ok = true; + if (!R->no_gate && n_need > 0) { + const expert_desc & e0 = L->experts[(size_t) R->need[0]]; + types_ok = !e0.fused_gate_up && dot_for(e0.type[part::GATE]) && dot_for(e0.type[part::UP]) && + act_type_for(e0.type[part::GATE]) == act_type_for(e0.type[part::UP]); + } + if (rowpar_env && n_need > 0 && n_need < nth && types_ok) { int miss = 0; for (int i = 0; i < n_need; i++) { if (!P.resident(R->il, R->need[(size_t) i])) { @@ -1088,8 +1118,10 @@ ggml_tensor * sdq_build_moe_ex(ggml_cgraph * gf, ggml_context * ctx, ggml_tensor ggml_tensor * sdq_build_moe(ggml_cgraph * gf, ggml_context * ctx, ggml_tensor * cur, ggml_tensor * ids, ggml_tensor * weights, int il, int64_t n_expert, int64_t n_expert_used, float swiglu_limit) { - (void) swiglu_limit; - return sdq_build_moe_ex(gf, ctx, cur, ids, weights, il, n_expert, n_expert_used, false, false); + ggml_tensor * out = sdq_build_moe_ex(gf, ctx, cur, ids, weights, il, n_expert, n_expert_used, false, false); + // 把 swiglu limit 傳給 op(先前這裡 (void) 丟掉,造成 clamp 永遠不生效) + rec_for(il)->swiglu_limit = swiglu_limit; + return out; } } // namespace sdq