diff --git a/ggml/src/ggml-cpu/amx/amx.cpp b/ggml/src/ggml-cpu/amx/amx.cpp index e90c279..49be9e1 100644 --- a/ggml/src/ggml-cpu/amx/amx.cpp +++ b/ggml/src/ggml-cpu/amx/amx.cpp @@ -152,6 +152,13 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type { if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { return false; } + // SDQ: the AMX weight repack (ggml_backend_amx_convert_weight) only converts the + // ne0 x ne1 slice of a tensor, so a 3D/4D weight would be left with unconverted + // data in the remaining slices. MLA attention feeds a 3D wv_b to MUL_MAT, which + // lands here and then reads garbage (NaN) - keep 3D weights in the CPU buffer. + if (src0->ne[2] > 1 || src0->ne[3] > 1) { + return false; + } if (!src0->buffer || src0->buffer->buft != ggml_backend_amx_buffer_type()) { return false; } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 97250c2..1fcfb6e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -18,6 +18,9 @@ set(LLAMA_CORE_SOURCES llama-cparams.cpp llama-grammar.cpp llama-graph.cpp + sdq_pager.cpp + sdq_buft.cpp + sdq_moe.cpp llama-hparams.cpp llama-impl.cpp llama-io.cpp @@ -51,6 +54,11 @@ add_library(llama ${LLAMA_MODELS_SOURCES} ) +# SDQ: chat / benchmark front end (links only llama + ggml) +add_executable(sdq-chat sdq_cli.cpp) +target_link_libraries(sdq-chat PRIVATE llama) +target_include_directories(sdq-chat PRIVATE .) + set_target_properties(llama PROPERTIES VERSION ${LLAMA_VERSION_BASE} SOVERSION ${LLAMA_VERSION_MAJOR} @@ -61,11 +69,14 @@ set_target_properties(llama PROPERTIES # exclude non-model sources from unity build set_source_files_properties(${LLAMA_CORE_SOURCES} ../include/llama.h unicode.h + sdq_pager.cpp sdq_buft.cpp sdq_moe.cpp PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) configure_file(llama-version.h.in ${CMAKE_CURRENT_BINARY_DIR}/llama-version.h @ONLY) target_include_directories(llama PRIVATE . ${CMAKE_CURRENT_BINARY_DIR}) +# SDQ needs ggml internal headers (custom buffer type + quantized dot kernels) +target_include_directories(llama PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../ggml/src) target_include_directories(llama PUBLIC ../include) target_compile_features (llama PRIVATE cxx_std_17) # don't bump diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 9dcaf05..4fd893d 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -4,6 +4,9 @@ #include "llama-arch.h" #include "llama-graph.h" #include "llama-impl.h" + +// SDQ: expert pager +#include "sdq_pager.h" #include "llama-batch.h" #include "llama-io.h" #include "llama-memory.h" @@ -2122,6 +2125,12 @@ int llama_context::decode(const llama_batch_ext & batch_inp) { // wait for the computation to finish (automatically done when obtaining the model output) //synchronize(); + // ---- SDQ: 收掉這一步 decode 的 accounting。 + // 這是所有前端都會經過的單一出口；sdq-chat 自己另外驅動，所以不會重複計數。 + if (sdq::sdq_enabled()) { + sdq::sdq_on_decode((int) n_tokens_all); + } + return 0; } diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 23e4c90..4769929 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1,5 +1,7 @@ #include "llama-graph.h" +#include "sdq_pager.h" + #include "llama-impl.h" #include "llama-model.h" #include "llama-moe-cache.h" @@ -2211,6 +2213,40 @@ ggml_tensor * llm_graph_context::build_moe_ffn( //call early so that topk-moe can be used ggml_build_forward_expand(gf, weights); + // SDQ expert pager: route the selected experts through the SSD->RAM pager + // instead of ggml_mul_mat_id, so the expert weights never have to be resident. + // The router (logits / gating / group top-k / weight norm) is still computed + // by ggml above; only "evaluate the selected experts and accumulate" moves. + if (const char * dp = getenv("SDQ_GRAPH_DUMP")) { + const int dl = getenv("SDQ_GRAPH_DUMP_LAYER") ? atoi(getenv("SDQ_GRAPH_DUMP_LAYER")) : -1; + if (dl < 0 || il == dl) { + ggml_tensor * c3 = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens); + ggml_build_forward_expand(gf, c3); + ggml_build_forward_expand(gf, selected_experts); + ggml_build_forward_expand(gf, weights); + std::string path = std::string(dp) + "." + std::to_string(il); + FILE * f = fopen(path.c_str(), "wb"); + if (f) { + const int32_t meta[8] = { (int32_t) n_embd, (int32_t) n_tokens, (int32_t) n_expert_used, + (int32_t) n_expert, (int32_t) (sdq::sdq_pager_active_for_layer(il) ? 1 : 0), 0, 0, 0 }; + fwrite(meta, sizeof(int32_t), 8, f); + fwrite(c3->data, sizeof(float), (size_t) (n_embd * n_tokens), f); + fwrite(selected_experts->data, sizeof(int32_t), (size_t) (n_expert_used * n_tokens), f); + fwrite(weights->data, sizeof(float), (size_t) (n_expert_used * n_tokens), f); + fclose(f); + } + } + } + + if (sdq::sdq_pager_active_for_layer(il) && cur->ne[0] == n_embd) { + const float swiglu_limit = il >= 0 ? hparams.swiglu_clamp_exp[il] : 0.0f; + ggml_tensor * moe_out = sdq::sdq_build_moe(gf, ctx0, cur, selected_experts, weights, il, + n_expert, n_expert_used, swiglu_limit); + cb(moe_out, "ffn_moe_out", il); + moe_out = sdq::sdq_maybe_dump_tensor(ctx0, moe_out, il); + return moe_out; + } + // the experts of host-resident layers may be read from the MoE cache ggml_tensor * slots = build_moe_cache_slots(selected_experts, up_exps, gate_exps, down_exps, gate_up_exps, il); @@ -2415,6 +2451,9 @@ ggml_tensor * llm_graph_context::build_moe_ffn( cb(moe_out, "ffn_moe_out", il); + // dump the native path too, so it can be diffed against the paged path + moe_out = sdq::sdq_maybe_dump_tensor(ctx0, moe_out, il); + return moe_out; } @@ -2798,7 +2837,16 @@ ggml_tensor * llm_graph_context::build_attn_mha( #else // It's preferable to do the calculation as a matrix-matrix multiplication with n_tokens in dimension 1. // The permutations are noops and only change how the tensor data is interpreted. + // FLASH_ATTN_EXT produces F16, but the CPU backend cannot run MUL_MAT with a + // quantized v_mla and F16 activations (widening requires vec_dot_type == F32), + // and a quantized weight in the AMX buffer requires contiguous activations. + if (cur->type != GGML_TYPE_F32 && v_mla->type != GGML_TYPE_F16 && v_mla->type != GGML_TYPE_F32) { + cur = ggml_cast(ctx0, cur, GGML_TYPE_F32); + } cur = ggml_permute(ctx0, cur, 0, 2, 1, 3); + if (!ggml_is_contiguous(cur)) { + cur = ggml_cont(ctx0, cur); + } cur = ggml_mul_mat(ctx0, v_mla, cur); cb(cur, "fattn_mla", il); cur = ggml_permute(ctx0, cur, 0, 2, 1, 3); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 2e86914..fbf06cb 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1,5 +1,7 @@ #include "llama-model.h" +#include "sdq_pager.h" + #include "llama-arch.h" #include "llama-ext.h" #include "llama-hparams.h" @@ -1834,7 +1836,9 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { // per-tensor activation precision policy prec_policy.load(ml, *this); - ml.init_mappings(true, use_mlock ? &pimpl->mlock_mmaps : nullptr); + // SDQ: the expert weights are read by the pager with its own pread calls, so the + // loader must not MAP_POPULATE / WILLNEED the whole model file into page cache. + ml.init_mappings(!sdq::sdq_enabled(), use_mlock ? &pimpl->mlock_mmaps : nullptr); pimpl->mappings.reserve(ml.mappings.size()); // create the backend buffers diff --git a/src/llama.cpp b/src/llama.cpp index 9beb148..2633d96 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -11,6 +11,9 @@ #include "llama-model-saver.h" #include "llama-model.h" +// SDQ: expert pager（SSD -> RAM 分頁），見 sdq-ling/0001-sdq-ling.patch +#include "sdq_pager.h" + #include "ggml.h" #include "ggml-cpp.h" #include "ggml-backend.h" @@ -404,6 +407,27 @@ static struct llama_model * llama_model_load_from_file_impl( // TODO: remove ggml_time_init(); + // ---- SDQ: 把 MoE expert 權重交給 pager（SSD -> RAM arena 分頁）---- + // 放在模型載入的唯一入口，這樣 sdq-chat / llama-server / llama-cli 行為一致。 + // SDQ_PAGER=0 關閉，回到上游行為（A/B 對照用） + // SDQ_RAM_BUDGET_MB RAM 總預算，arena 會配合縮減（沒有給就用實體 RAM 的一部分） + // SDQ_ARENA_MB 直接指定 arena 大小 + // SDQ_RESERVE_MB KV/SSM state + compute buffer 的預留 + // SDQ_IO_THREADS I/O 執行緒數、SDQ_PREFETCH=0 關背景預取、SDQ_STATS_FILE 指標輸出 + { + const char * sdq_off = getenv("SDQ_PAGER"); + if ((!sdq_off || strcmp(sdq_off, "0") != 0) && !params.vocab_only && !path_model.empty()) { + if (sdq::sdq_init(path_model)) { + // 沒有這一行，load_tensors() 會把整個模型檔要求預取進 page cache， + // 正是分頁要避免的事（I/O 稽核見 results/io_audit.md）。 + sdq::sdq_set_enabled(true); + sdq::sdq_install_buft_overrides(¶ms); + // 權重是 pread 進 arena 的，不該再讓後端自己 offload 一次。 + params.n_gpu_layers = 0; + } + } + } + if (!params.vocab_only && ggml_backend_reg_count() == 0) { LLAMA_LOG_ERROR("%s: no backends are loaded. hint: use ggml_backend_load() or ggml_backend_load_all() to load a backend before calling this function\n", __func__); return nullptr;