Download code/tt_rdt2/tt/kernels/pxf_qkv_reader.cpp from changh95/rdt2-base-p150: direct link, hf CLI and curl.
- Browser
- Download file 4.36 kB
-
https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/tt/kernels/pxf_qkv_reader.cpp
- Command line
-
hf download hf://changh95/rdt2-base-p150/code/tt_rdt2/tt/kernels/pxf_qkv_reader.cpp
-
curl -L -o pxf_qkv_reader.cpp https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/tt/kernels/pxf_qkv_reader.cpp
4.36 kB
| // SPDX-License-Identifier: Apache-2.0 | |
| // | |
| // RDT2-FM fused PREFIX op "qkvpost", reader (NCRISC, NoC 0). The core runs heads h0, h0 + P, ... of row tile r | |
| // (qp_share; NH = NQ + 2 NKV heads of the fused qkv projection): per head the 4 tiles of head h in row tile r of | |
| // qkv (1, 1, L, NH * 128) bf16; the RoPE tiles of row r (cos_0, cos_1, sin_0, sin_1 fp32: the halves of the mRoPE | |
| // tables are equal) once; a k head also needs the fp32 bias tables T and D of its kv head. v heads bypass the | |
| // compute. The CBs hold every page of the core's heads: all reads are issued at once (a transaction id per head) | |
| // and each head is handed over as soon as its own reads have landed. | |
| using namespace pxf; | |
| namespace { | |
| constexpr uint32_t MT = get_compile_time_arg_val(QP_CT_MT); | |
| constexpr uint32_t NQ = get_compile_time_arg_val(QP_CT_NQ); | |
| constexpr uint32_t NKV = get_compile_time_arg_val(QP_CT_NKV); | |
| constexpr uint32_t NH = NQ + 2 * NKV; | |
| constexpr uint32_t P = get_compile_time_arg_val(QP_CT_P); | |
| constexpr auto acc_qkv = TensorAccessorArgs<QP_CT_ACC0>(); | |
| constexpr auto acc_cs = TensorAccessorArgs<acc_qkv.next_compile_time_args_offset()>(); | |
| constexpr auto acc_t = TensorAccessorArgs<acc_cs.next_compile_time_args_offset()>(); | |
| constexpr auto acc_d = TensorAccessorArgs<acc_t.next_compile_time_args_offset()>(); | |
| } // namespace | |
| void kernel_main() { | |
| uint32_t r, h0; | |
| qp_share(get_absolute_logical_x(), get_absolute_logical_y(), get_compile_time_arg_val(QP_CT_GX), P, r, h0); | |
| const auto qkv = TensorAccessor(acc_qkv, get_common_arg_val<uint32_t>(QP_C_QKV), T16); | |
| const auto cs = TensorAccessor(acc_cs, get_common_arg_val<uint32_t>(QP_C_CS), T32); | |
| const auto ts = TensorAccessor(acc_t, get_common_arg_val<uint32_t>(QP_C_T), T32); | |
| const auto ds = TensorAccessor(acc_d, get_common_arg_val<uint32_t>(QP_C_D), T32); | |
| uint32_t nx = 0, nk = 0, nv = 0; | |
| for (uint32_t h = h0; h < NH; h += P) { | |
| nx += h < NQ + NKV ? 1 : 0; | |
| nk += (h >= NQ && h < NQ + NKV) ? 1 : 0; | |
| nv += h >= NQ + NKV ? 1 : 0; | |
| } | |
| if (nx) { | |
| cb_reserve_back(QP_CB_CS, HD_T); | |
| cb_reserve_back(QP_CB_X, nx * HD_T); | |
| } | |
| if (nk) { | |
| cb_reserve_back(QP_CB_T, nk * HD_T); | |
| cb_reserve_back(QP_CB_D, nk * HD_T); | |
| } | |
| if (nv) { | |
| cb_reserve_back(QP_CB_V, nv * HD_T); | |
| } | |
| uint32_t xa = get_write_ptr(QP_CB_X), ta = get_write_ptr(QP_CB_T), da = get_write_ptr(QP_CB_D), | |
| va = get_write_ptr(QP_CB_V); | |
| if (nx) { // cos_0, cos_1, sin_0, sin_1 of row tile r (trid 1) | |
| noc_async_read_set_trid(1); | |
| const uint32_t ca = get_write_ptr(QP_CB_CS); | |
| for (uint32_t t = 0; t < HD_T; ++t) { | |
| noc_async_read(cs.get_noc_addr(r * HD_T + t), ca + t * T32, T32); | |
| } | |
| } | |
| for (uint32_t h = h0; h < NH; h += P) { // trid 2 + i per head (<= 13 heads per core: qkv_plan) | |
| noc_async_read_set_trid(2 + (h - h0) / P); | |
| const uint32_t page = (r * NH + h) * HD_T; | |
| if (h >= NQ + NKV) { // v: straight to the writer | |
| for (uint32_t t = 0; t < HD_T; ++t, va += T16) { | |
| noc_async_read(qkv.get_noc_addr(page + t), va, T16); | |
| } | |
| continue; | |
| } | |
| for (uint32_t t = 0; t < HD_T; ++t, xa += T16) { | |
| noc_async_read(qkv.get_noc_addr(page + t), xa, T16); | |
| } | |
| if (h >= NQ) { // k: the bias tables of kv head h - NQ | |
| const uint32_t tp = ((h - NQ) * MT + r) * HD_T; | |
| for (uint32_t t = 0; t < HD_T; ++t, ta += T32, da += T32) { | |
| noc_async_read(ts.get_noc_addr(tp + t), ta, T32); | |
| noc_async_read(ds.get_noc_addr(tp + t), da, T32); | |
| } | |
| } | |
| } | |
| if (nx) { | |
| noc_async_read_barrier_with_trid(1); | |
| cb_push_back(QP_CB_CS, HD_T); | |
| } | |
| for (uint32_t h = h0; h < NH; h += P) { // hand each head over as soon as its reads have landed | |
| noc_async_read_barrier_with_trid(2 + (h - h0) / P); | |
| if (h >= NQ + NKV) { | |
| cb_push_back(QP_CB_V, HD_T); | |
| continue; | |
| } | |
| cb_push_back(QP_CB_X, HD_T); | |
| if (h >= NQ) { | |
| cb_push_back(QP_CB_T, HD_T); | |
| cb_push_back(QP_CB_D, HD_T); | |
| } | |
| } | |
| noc_async_read_set_trid(0); | |
| noc_async_read_barrier(); | |
| } | |