HelloSun
/

ling30tiny_EPQ

diff --git a/ggml/src/ggml-cpu/amx/amx.cpp b/ggml/src/ggml-cpu/amx/amx.cpp
index e90c279..49be9e1 100644
--- a/ggml/src/ggml-cpu/amx/amx.cpp
+++ b/ggml/src/ggml-cpu/amx/amx.cpp
@@ -152,6 +152,13 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {
         if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) {
             return false;
         }
+        // SDQ: the AMX weight repack (ggml_backend_amx_convert_weight) only converts the
+        // ne0 x ne1 slice of a tensor, so a 3D/4D weight would be left with unconverted
+        // data in the remaining slices. MLA attention feeds a 3D wv_b to MUL_MAT, which
+        // lands here and then reads garbage (NaN) - keep 3D weights in the CPU buffer.
+        if (src0->ne[2] > 1 || src0->ne[3] > 1) {
+            return false;
+        }
         if (!src0->buffer || src0->buffer->buft != ggml_backend_amx_buffer_type()) {
             return false;
         }
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 97250c2..1fcfb6e 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -18,6 +18,9 @@ set(LLAMA_CORE_SOURCES
     llama-cparams.cpp
     llama-grammar.cpp
     llama-graph.cpp
+    sdq_pager.cpp
+    sdq_buft.cpp
+    sdq_moe.cpp
     llama-hparams.cpp
     llama-impl.cpp
     llama-io.cpp
@@ -51,6 +54,11 @@ add_library(llama
             ${LLAMA_MODELS_SOURCES}
             )
 
+# SDQ: chat / benchmark front end (links only llama + ggml)
+add_executable(sdq-chat sdq_cli.cpp)
+target_link_libraries(sdq-chat PRIVATE llama)
+target_include_directories(sdq-chat PRIVATE .)
+
 set_target_properties(llama PROPERTIES
     VERSION ${LLAMA_VERSION_BASE}
     SOVERSION ${LLAMA_VERSION_MAJOR}
@@ -61,11 +69,14 @@ set_target_properties(llama PROPERTIES
 
 # exclude non-model sources from unity build
 set_source_files_properties(${LLAMA_CORE_SOURCES} ../include/llama.h unicode.h
+    sdq_pager.cpp sdq_buft.cpp sdq_moe.cpp
     PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
 
 configure_file(llama-version.h.in ${CMAKE_CURRENT_BINARY_DIR}/llama-version.h @ONLY)
 
 target_include_directories(llama PRIVATE . ${CMAKE_CURRENT_BINARY_DIR})
+# SDQ needs ggml internal headers (custom buffer type + quantized dot kernels)
+target_include_directories(llama PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../ggml/src)
 target_include_directories(llama PUBLIC ../include)
 target_compile_features   (llama PRIVATE cxx_std_17) # don't bump
 
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index 9dcaf05..4fd893d 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -4,6 +4,9 @@
 #include "llama-arch.h"
 #include "llama-graph.h"
 #include "llama-impl.h"
+
+// SDQ: expert pager
+#include "sdq_pager.h"
 #include "llama-batch.h"
 #include "llama-io.h"
 #include "llama-memory.h"
@@ -2122,6 +2125,12 @@ int llama_context::decode(const llama_batch_ext & batch_inp) {
     // wait for the computation to finish (automatically done when obtaining the model output)
     //synchronize();
 
+    // ---- SDQ: 收掉這一步 decode 的 accounting。
+    // 這是所有前端都會經過的單一出口；sdq-chat 自己另外驅動，所以不會重複計數。
+    if (sdq::sdq_enabled()) {
+        sdq::sdq_on_decode((int) n_tokens_all);
+    }
+
     return 0;
 }
 
diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index 23e4c90..4769929 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -1,5 +1,7 @@
 #include "llama-graph.h"
 
+#include "sdq_pager.h"
+
 #include "llama-impl.h"
 #include "llama-model.h"
 #include "llama-moe-cache.h"
@@ -2211,6 +2213,40 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
     //call early so that topk-moe can be used
     ggml_build_forward_expand(gf, weights);
 
+    // SDQ expert pager: route the selected experts through the SSD->RAM pager
+    // instead of ggml_mul_mat_id, so the expert weights never have to be resident.
+    // The router (logits / gating / group top-k / weight norm) is still computed
+    // by ggml above; only "evaluate the selected experts and accumulate" moves.
+    if (const char * dp = getenv("SDQ_GRAPH_DUMP")) {
+        const int dl = getenv("SDQ_GRAPH_DUMP_LAYER") ? atoi(getenv("SDQ_GRAPH_DUMP_LAYER")) : -1;
+        if (dl < 0 || il == dl) {
+            ggml_tensor * c3 = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);
+            ggml_build_forward_expand(gf, c3);
+            ggml_build_forward_expand(gf, selected_experts);
+            ggml_build_forward_expand(gf, weights);
+            std::string path = std::string(dp) + "." + std::to_string(il);
+            FILE * f = fopen(path.c_str(), "wb");
+            if (f) {
+                const int32_t meta[8] = { (int32_t) n_embd, (int32_t) n_tokens, (int32_t) n_expert_used,
+                                          (int32_t) n_expert, (int32_t) (sdq::sdq_pager_active_for_layer(il) ? 1 : 0), 0, 0, 0 };
+                fwrite(meta, sizeof(int32_t), 8, f);
+                fwrite(c3->data, sizeof(float), (size_t) (n_embd * n_tokens), f);
+                fwrite(selected_experts->data, sizeof(int32_t), (size_t) (n_expert_used * n_tokens), f);
+                fwrite(weights->data, sizeof(float), (size_t) (n_expert_used * n_tokens), f);
+                fclose(f);
+            }
+        }
+    }
+
+    if (sdq::sdq_pager_active_for_layer(il) && cur->ne[0] == n_embd) {
+        const float swiglu_limit = il >= 0 ? hparams.swiglu_clamp_exp[il] : 0.0f;
+        ggml_tensor * moe_out = sdq::sdq_build_moe(gf, ctx0, cur, selected_experts, weights, il,
+                n_expert, n_expert_used, swiglu_limit);
+        cb(moe_out, "ffn_moe_out", il);
+        moe_out = sdq::sdq_maybe_dump_tensor(ctx0, moe_out, il);
+        return moe_out;
+    }
+
     // the experts of host-resident layers may be read from the MoE cache
     ggml_tensor * slots = build_moe_cache_slots(selected_experts, up_exps, gate_exps, down_exps, gate_up_exps, il);
 
@@ -2415,6 +2451,9 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
 
     cb(moe_out, "ffn_moe_out", il);
 
+    // dump the native path too, so it can be diffed against the paged path
+    moe_out = sdq::sdq_maybe_dump_tensor(ctx0, moe_out, il);
+
     return moe_out;
 }
 
@@ -2798,7 +2837,16 @@ ggml_tensor * llm_graph_context::build_attn_mha(
 #else
             // It's preferable to do the calculation as a matrix-matrix multiplication with n_tokens in dimension 1.
             // The permutations are noops and only change how the tensor data is interpreted.
+            // FLASH_ATTN_EXT produces F16, but the CPU backend cannot run MUL_MAT with a
+            // quantized v_mla and F16 activations (widening requires vec_dot_type == F32),
+            // and a quantized weight in the AMX buffer requires contiguous activations.
+            if (cur->type != GGML_TYPE_F32 && v_mla->type != GGML_TYPE_F16 && v_mla->type != GGML_TYPE_F32) {
+                cur = ggml_cast(ctx0, cur, GGML_TYPE_F32);
+            }
             cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
+            if (!ggml_is_contiguous(cur)) {
+                cur = ggml_cont(ctx0, cur);
+            }
             cur = ggml_mul_mat(ctx0, v_mla, cur);
             cb(cur, "fattn_mla", il);
             cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 2e86914..fbf06cb 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -1,5 +1,7 @@
 #include "llama-model.h"
 
+#include "sdq_pager.h"
+
 #include "llama-arch.h"
 #include "llama-ext.h"
 #include "llama-hparams.h"
@@ -1834,7 +1836,9 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
     // per-tensor activation precision policy
     prec_policy.load(ml, *this);
 
-    ml.init_mappings(true, use_mlock ? &pimpl->mlock_mmaps : nullptr);
+    // SDQ: the expert weights are read by the pager with its own pread calls, so the
+    // loader must not MAP_POPULATE / WILLNEED the whole model file into page cache.
+    ml.init_mappings(!sdq::sdq_enabled(), use_mlock ? &pimpl->mlock_mmaps : nullptr);
     pimpl->mappings.reserve(ml.mappings.size());
 
     // create the backend buffers
diff --git a/src/llama.cpp b/src/llama.cpp
index 9beb148..2633d96 100644
--- a/src/llama.cpp
+++ b/src/llama.cpp
@@ -11,6 +11,9 @@
 #include "llama-model-saver.h"
 #include "llama-model.h"
 
+// SDQ: expert pager（SSD -> RAM 分頁），見 sdq-ling/0001-sdq-ling.patch
+#include "sdq_pager.h"
+
 #include "ggml.h"
 #include "ggml-cpp.h"
 #include "ggml-backend.h"
@@ -404,6 +407,27 @@ static struct llama_model * llama_model_load_from_file_impl(
     // TODO: remove
     ggml_time_init();
 
+    // ---- SDQ: 把 MoE expert 權重交給 pager（SSD -> RAM arena 分頁）----
+    // 放在模型載入的唯一入口，這樣 sdq-chat / llama-server / llama-cli 行為一致。
+    //   SDQ_PAGER=0        關閉，回到上游行為（A/B 對照用）
+    //   SDQ_RAM_BUDGET_MB  RAM 總預算，arena 會配合縮減（沒有給就用實體 RAM 的一部分）
+    //   SDQ_ARENA_MB       直接指定 arena 大小
+    //   SDQ_RESERVE_MB     KV/SSM state + compute buffer 的預留
+    //   SDQ_IO_THREADS     I/O 執行緒數、SDQ_PREFETCH=0 關背景預取、SDQ_STATS_FILE 指標輸出
+    {
+        const char * sdq_off = getenv("SDQ_PAGER");
+        if ((!sdq_off || strcmp(sdq_off, "0") != 0) && !params.vocab_only && !path_model.empty()) {
+            if (sdq::sdq_init(path_model)) {
+                // 沒有這一行，load_tensors() 會把整個模型檔要求預取進 page cache，
+                // 正是分頁要避免的事（I/O 稽核見 results/io_audit.md）。
+                sdq::sdq_set_enabled(true);
+                sdq::sdq_install_buft_overrides(&params);
+                // 權重是 pread 進 arena 的，不該再讓後端自己 offload 一次。
+                params.n_gpu_layers = 0;
+            }
+        }
+    }
+
     if (!params.vocab_only && ggml_backend_reg_count() == 0) {
         LLAMA_LOG_ERROR("%s: no backends are loaded. hint: use ggml_backend_load() or ggml_backend_load_all() to load a backend before calling this function\n", __func__);
         return nullptr;

	diff --git a/ggml/src/ggml-cpu/amx/amx.cpp b/ggml/src/ggml-cpu/amx/amx.cpp
	index e90c279..49be9e1 100644
	--- a/ggml/src/ggml-cpu/amx/amx.cpp
	+++ b/ggml/src/ggml-cpu/amx/amx.cpp
	@@ -152,6 +152,13 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {
	if (!ggml_is_contiguous(src0) \|\| !ggml_is_contiguous(src1)) {
	return false;
	}
	+ // SDQ: the AMX weight repack (ggml_backend_amx_convert_weight) only converts the
	+ // ne0 x ne1 slice of a tensor, so a 3D/4D weight would be left with unconverted
	+ // data in the remaining slices. MLA attention feeds a 3D wv_b to MUL_MAT, which
	+ // lands here and then reads garbage (NaN) - keep 3D weights in the CPU buffer.
	+ if (src0->ne[2] > 1 \|\| src0->ne[3] > 1) {
	+ return false;
	+ }
	if (!src0->buffer \|\| src0->buffer->buft != ggml_backend_amx_buffer_type()) {
	return false;
	}
	diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
	index 97250c2..1fcfb6e 100644
	--- a/src/CMakeLists.txt
	+++ b/src/CMakeLists.txt
	@@ -18,6 +18,9 @@ set(LLAMA_CORE_SOURCES
	llama-cparams.cpp
	llama-grammar.cpp
	llama-graph.cpp
	+ sdq_pager.cpp
	+ sdq_buft.cpp
	+ sdq_moe.cpp
	llama-hparams.cpp
	llama-impl.cpp
	llama-io.cpp
	@@ -51,6 +54,11 @@ add_library(llama
	${LLAMA_MODELS_SOURCES}
	)

	+# SDQ: chat / benchmark front end (links only llama + ggml)
	+add_executable(sdq-chat sdq_cli.cpp)
	+target_link_libraries(sdq-chat PRIVATE llama)
	+target_include_directories(sdq-chat PRIVATE .)
	+
	set_target_properties(llama PROPERTIES
	VERSION ${LLAMA_VERSION_BASE}
	SOVERSION ${LLAMA_VERSION_MAJOR}
	@@ -61,11 +69,14 @@ set_target_properties(llama PROPERTIES

	# exclude non-model sources from unity build
	set_source_files_properties(${LLAMA_CORE_SOURCES} ../include/llama.h unicode.h
	+ sdq_pager.cpp sdq_buft.cpp sdq_moe.cpp
	PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)

	configure_file(llama-version.h.in ${CMAKE_CURRENT_BINARY_DIR}/llama-version.h @ONLY)

	target_include_directories(llama PRIVATE . ${CMAKE_CURRENT_BINARY_DIR})
	+# SDQ needs ggml internal headers (custom buffer type + quantized dot kernels)
	+target_include_directories(llama PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../ggml/src)
	target_include_directories(llama PUBLIC ../include)
	target_compile_features (llama PRIVATE cxx_std_17) # don't bump

	diff --git a/src/llama-context.cpp b/src/llama-context.cpp
	index 9dcaf05..4fd893d 100644
	--- a/src/llama-context.cpp
	+++ b/src/llama-context.cpp
	@@ -4,6 +4,9 @@
	#include "llama-arch.h"
	#include "llama-graph.h"
	#include "llama-impl.h"
	+
	+// SDQ: expert pager
	+#include "sdq_pager.h"
	#include "llama-batch.h"
	#include "llama-io.h"
	#include "llama-memory.h"
	@@ -2122,6 +2125,12 @@ int llama_context::decode(const llama_batch_ext & batch_inp) {
	// wait for the computation to finish (automatically done when obtaining the model output)
	//synchronize();

	+ // ---- SDQ: 收掉這一步 decode 的 accounting。
	+ // 這是所有前端都會經過的單一出口；sdq-chat 自己另外驅動，所以不會重複計數。
	+ if (sdq::sdq_enabled()) {
	+ sdq::sdq_on_decode((int) n_tokens_all);
	+ }
	+
	return 0;
	}

	diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
	index 23e4c90..4769929 100644
	--- a/src/llama-graph.cpp
	+++ b/src/llama-graph.cpp
	@@ -1,5 +1,7 @@
	#include "llama-graph.h"

	+#include "sdq_pager.h"
	+
	#include "llama-impl.h"
	#include "llama-model.h"
	#include "llama-moe-cache.h"
	@@ -2211,6 +2213,40 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
	//call early so that topk-moe can be used
	ggml_build_forward_expand(gf, weights);

	+ // SDQ expert pager: route the selected experts through the SSD->RAM pager
	+ // instead of ggml_mul_mat_id, so the expert weights never have to be resident.
	+ // The router (logits / gating / group top-k / weight norm) is still computed
	+ // by ggml above; only "evaluate the selected experts and accumulate" moves.
	+ if (const char * dp = getenv("SDQ_GRAPH_DUMP")) {
	+ const int dl = getenv("SDQ_GRAPH_DUMP_LAYER") ? atoi(getenv("SDQ_GRAPH_DUMP_LAYER")) : -1;
	+ if (dl < 0 \|\| il == dl) {
	+ ggml_tensor * c3 = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);
	+ ggml_build_forward_expand(gf, c3);
	+ ggml_build_forward_expand(gf, selected_experts);
	+ ggml_build_forward_expand(gf, weights);
	+ std::string path = std::string(dp) + "." + std::to_string(il);
	+ FILE * f = fopen(path.c_str(), "wb");
	+ if (f) {
	+ const int32_t meta[8] = { (int32_t) n_embd, (int32_t) n_tokens, (int32_t) n_expert_used,
	+ (int32_t) n_expert, (int32_t) (sdq::sdq_pager_active_for_layer(il) ? 1 : 0), 0, 0, 0 };
	+ fwrite(meta, sizeof(int32_t), 8, f);
	+ fwrite(c3->data, sizeof(float), (size_t) (n_embd * n_tokens), f);
	+ fwrite(selected_experts->data, sizeof(int32_t), (size_t) (n_expert_used * n_tokens), f);
	+ fwrite(weights->data, sizeof(float), (size_t) (n_expert_used * n_tokens), f);
	+ fclose(f);
	+ }
	+ }
	+ }
	+
	+ if (sdq::sdq_pager_active_for_layer(il) && cur->ne[0] == n_embd) {
	+ const float swiglu_limit = il >= 0 ? hparams.swiglu_clamp_exp[il] : 0.0f;
	+ ggml_tensor * moe_out = sdq::sdq_build_moe(gf, ctx0, cur, selected_experts, weights, il,
	+ n_expert, n_expert_used, swiglu_limit);
	+ cb(moe_out, "ffn_moe_out", il);
	+ moe_out = sdq::sdq_maybe_dump_tensor(ctx0, moe_out, il);
	+ return moe_out;
	+ }
	+
	// the experts of host-resident layers may be read from the MoE cache
	ggml_tensor * slots = build_moe_cache_slots(selected_experts, up_exps, gate_exps, down_exps, gate_up_exps, il);

	@@ -2415,6 +2451,9 @@ ggml_tensor * llm_graph_context::build_moe_ffn(

	cb(moe_out, "ffn_moe_out", il);

	+ // dump the native path too, so it can be diffed against the paged path
	+ moe_out = sdq::sdq_maybe_dump_tensor(ctx0, moe_out, il);
	+
	return moe_out;
	}

	@@ -2798,7 +2837,16 @@ ggml_tensor * llm_graph_context::build_attn_mha(
	#else
	// It's preferable to do the calculation as a matrix-matrix multiplication with n_tokens in dimension 1.
	// The permutations are noops and only change how the tensor data is interpreted.
	+ // FLASH_ATTN_EXT produces F16, but the CPU backend cannot run MUL_MAT with a
	+ // quantized v_mla and F16 activations (widening requires vec_dot_type == F32),
	+ // and a quantized weight in the AMX buffer requires contiguous activations.
	+ if (cur->type != GGML_TYPE_F32 && v_mla->type != GGML_TYPE_F16 && v_mla->type != GGML_TYPE_F32) {
	+ cur = ggml_cast(ctx0, cur, GGML_TYPE_F32);
	+ }
	cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
	+ if (!ggml_is_contiguous(cur)) {
	+ cur = ggml_cont(ctx0, cur);
	+ }
	cur = ggml_mul_mat(ctx0, v_mla, cur);
	cb(cur, "fattn_mla", il);
	cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
	diff --git a/src/llama-model.cpp b/src/llama-model.cpp
	index 2e86914..fbf06cb 100644
	--- a/src/llama-model.cpp
	+++ b/src/llama-model.cpp
	@@ -1,5 +1,7 @@
	#include "llama-model.h"

	+#include "sdq_pager.h"
	+
	#include "llama-arch.h"
	#include "llama-ext.h"
	#include "llama-hparams.h"
	@@ -1834,7 +1836,9 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
	// per-tensor activation precision policy
	prec_policy.load(ml, *this);

	- ml.init_mappings(true, use_mlock ? &pimpl->mlock_mmaps : nullptr);
	+ // SDQ: the expert weights are read by the pager with its own pread calls, so the
	+ // loader must not MAP_POPULATE / WILLNEED the whole model file into page cache.
	+ ml.init_mappings(!sdq::sdq_enabled(), use_mlock ? &pimpl->mlock_mmaps : nullptr);
	pimpl->mappings.reserve(ml.mappings.size());

	// create the backend buffers
	diff --git a/src/llama.cpp b/src/llama.cpp
	index 9beb148..2633d96 100644
	--- a/src/llama.cpp
	+++ b/src/llama.cpp
	@@ -11,6 +11,9 @@
	#include "llama-model-saver.h"
	#include "llama-model.h"

	+// SDQ: expert pager（SSD -> RAM 分頁），見 sdq-ling/0001-sdq-ling.patch
	+#include "sdq_pager.h"
	+
	#include "ggml.h"
	#include "ggml-cpp.h"
	#include "ggml-backend.h"
	@@ -404,6 +407,27 @@ static struct llama_model * llama_model_load_from_file_impl(
	// TODO: remove
	ggml_time_init();

	+ // ---- SDQ: 把 MoE expert 權重交給 pager（SSD -> RAM arena 分頁）----
	+ // 放在模型載入的唯一入口，這樣 sdq-chat / llama-server / llama-cli 行為一致。
	+ // SDQ_PAGER=0 關閉，回到上游行為（A/B 對照用）
	+ // SDQ_RAM_BUDGET_MB RAM 總預算，arena 會配合縮減（沒有給就用實體 RAM 的一部分）
	+ // SDQ_ARENA_MB 直接指定 arena 大小
	+ // SDQ_RESERVE_MB KV/SSM state + compute buffer 的預留
	+ // SDQ_IO_THREADS I/O 執行緒數、SDQ_PREFETCH=0 關背景預取、SDQ_STATS_FILE 指標輸出
	+ {
	+ const char * sdq_off = getenv("SDQ_PAGER");
	+ if ((!sdq_off \|\| strcmp(sdq_off, "0") != 0) && !params.vocab_only && !path_model.empty()) {
	+ if (sdq::sdq_init(path_model)) {
	+ // 沒有這一行，load_tensors() 會把整個模型檔要求預取進 page cache，
	+ // 正是分頁要避免的事（I/O 稽核見 results/io_audit.md）。
	+ sdq::sdq_set_enabled(true);
	+ sdq::sdq_install_buft_overrides(&params);
	+ // 權重是 pread 進 arena 的，不該再讓後端自己 offload 一次。
	+ params.n_gpu_layers = 0;
	+ }
	+ }
	+ }
	+
	if (!params.vocab_only && ggml_backend_reg_count() == 0) {
	LLAMA_LOG_ERROR("%s: no backends are loaded. hint: use ggml_backend_load() or ggml_backend_load_all() to load a backend before calling this function\n", __func__);
	return nullptr;