// SPDX-License-Identifier: Apache-2.0 // // RDT2-FM fused PREFIX op "qkvpost", reader (NCRISC, NoC 0). The core runs heads h0, h0 + P, ... of row tile r // (qp_share; NH = NQ + 2 NKV heads of the fused qkv projection): per head the 4 tiles of head h in row tile r of // qkv (1, 1, L, NH * 128) bf16; the RoPE tiles of row r (cos_0, cos_1, sin_0, sin_1 fp32: the halves of the mRoPE // tables are equal) once; a k head also needs the fp32 bias tables T and D of its kv head. v heads bypass the // compute. The CBs hold every page of the core's heads: all reads are issued at once (a transaction id per head) // and each head is handed over as soon as its own reads have landed. #include #include "api/dataflow/dataflow_api.h" #include "pxf_common.hpp" using namespace pxf; namespace { constexpr uint32_t MT = get_compile_time_arg_val(QP_CT_MT); constexpr uint32_t NQ = get_compile_time_arg_val(QP_CT_NQ); constexpr uint32_t NKV = get_compile_time_arg_val(QP_CT_NKV); constexpr uint32_t NH = NQ + 2 * NKV; constexpr uint32_t P = get_compile_time_arg_val(QP_CT_P); constexpr auto acc_qkv = TensorAccessorArgs(); constexpr auto acc_cs = TensorAccessorArgs(); constexpr auto acc_t = TensorAccessorArgs(); constexpr auto acc_d = TensorAccessorArgs(); } // namespace void kernel_main() { uint32_t r, h0; qp_share(get_absolute_logical_x(), get_absolute_logical_y(), get_compile_time_arg_val(QP_CT_GX), P, r, h0); const auto qkv = TensorAccessor(acc_qkv, get_common_arg_val(QP_C_QKV), T16); const auto cs = TensorAccessor(acc_cs, get_common_arg_val(QP_C_CS), T32); const auto ts = TensorAccessor(acc_t, get_common_arg_val(QP_C_T), T32); const auto ds = TensorAccessor(acc_d, get_common_arg_val(QP_C_D), T32); uint32_t nx = 0, nk = 0, nv = 0; for (uint32_t h = h0; h < NH; h += P) { nx += h < NQ + NKV ? 1 : 0; nk += (h >= NQ && h < NQ + NKV) ? 1 : 0; nv += h >= NQ + NKV ? 1 : 0; } if (nx) { cb_reserve_back(QP_CB_CS, HD_T); cb_reserve_back(QP_CB_X, nx * HD_T); } if (nk) { cb_reserve_back(QP_CB_T, nk * HD_T); cb_reserve_back(QP_CB_D, nk * HD_T); } if (nv) { cb_reserve_back(QP_CB_V, nv * HD_T); } uint32_t xa = get_write_ptr(QP_CB_X), ta = get_write_ptr(QP_CB_T), da = get_write_ptr(QP_CB_D), va = get_write_ptr(QP_CB_V); if (nx) { // cos_0, cos_1, sin_0, sin_1 of row tile r (trid 1) noc_async_read_set_trid(1); const uint32_t ca = get_write_ptr(QP_CB_CS); for (uint32_t t = 0; t < HD_T; ++t) { noc_async_read(cs.get_noc_addr(r * HD_T + t), ca + t * T32, T32); } } for (uint32_t h = h0; h < NH; h += P) { // trid 2 + i per head (<= 13 heads per core: qkv_plan) noc_async_read_set_trid(2 + (h - h0) / P); const uint32_t page = (r * NH + h) * HD_T; if (h >= NQ + NKV) { // v: straight to the writer for (uint32_t t = 0; t < HD_T; ++t, va += T16) { noc_async_read(qkv.get_noc_addr(page + t), va, T16); } continue; } for (uint32_t t = 0; t < HD_T; ++t, xa += T16) { noc_async_read(qkv.get_noc_addr(page + t), xa, T16); } if (h >= NQ) { // k: the bias tables of kv head h - NQ const uint32_t tp = ((h - NQ) * MT + r) * HD_T; for (uint32_t t = 0; t < HD_T; ++t, ta += T32, da += T32) { noc_async_read(ts.get_noc_addr(tp + t), ta, T32); noc_async_read(ds.get_noc_addr(tp + t), da, T32); } } } if (nx) { noc_async_read_barrier_with_trid(1); cb_push_back(QP_CB_CS, HD_T); } for (uint32_t h = h0; h < NH; h += P) { // hand each head over as soon as its reads have landed noc_async_read_barrier_with_trid(2 + (h - h0) / P); if (h >= NQ + NKV) { cb_push_back(QP_CB_V, HD_T); continue; } cb_push_back(QP_CB_X, HD_T); if (h >= NQ) { cb_push_back(QP_CB_T, HD_T); cb_push_back(QP_CB_D, HD_T); } } noc_async_read_set_trid(0); noc_async_read_barrier(); }