File size: 4,357 Bytes
9d8d25c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 | // SPDX-License-Identifier: Apache-2.0
//
// RDT2-FM fused PREFIX op "qkvpost", reader (NCRISC, NoC 0). The core runs heads h0, h0 + P, ... of row tile r
// (qp_share; NH = NQ + 2 NKV heads of the fused qkv projection): per head the 4 tiles of head h in row tile r of
// qkv (1, 1, L, NH * 128) bf16; the RoPE tiles of row r (cos_0, cos_1, sin_0, sin_1 fp32: the halves of the mRoPE
// tables are equal) once; a k head also needs the fp32 bias tables T and D of its kv head. v heads bypass the
// compute. The CBs hold every page of the core's heads: all reads are issued at once (a transaction id per head)
// and each head is handed over as soon as its own reads have landed.
#include <cstdint>
#include "api/dataflow/dataflow_api.h"
#include "pxf_common.hpp"
using namespace pxf;
namespace {
constexpr uint32_t MT = get_compile_time_arg_val(QP_CT_MT);
constexpr uint32_t NQ = get_compile_time_arg_val(QP_CT_NQ);
constexpr uint32_t NKV = get_compile_time_arg_val(QP_CT_NKV);
constexpr uint32_t NH = NQ + 2 * NKV;
constexpr uint32_t P = get_compile_time_arg_val(QP_CT_P);
constexpr auto acc_qkv = TensorAccessorArgs<QP_CT_ACC0>();
constexpr auto acc_cs = TensorAccessorArgs<acc_qkv.next_compile_time_args_offset()>();
constexpr auto acc_t = TensorAccessorArgs<acc_cs.next_compile_time_args_offset()>();
constexpr auto acc_d = TensorAccessorArgs<acc_t.next_compile_time_args_offset()>();
} // namespace
void kernel_main() {
uint32_t r, h0;
qp_share(get_absolute_logical_x(), get_absolute_logical_y(), get_compile_time_arg_val(QP_CT_GX), P, r, h0);
const auto qkv = TensorAccessor(acc_qkv, get_common_arg_val<uint32_t>(QP_C_QKV), T16);
const auto cs = TensorAccessor(acc_cs, get_common_arg_val<uint32_t>(QP_C_CS), T32);
const auto ts = TensorAccessor(acc_t, get_common_arg_val<uint32_t>(QP_C_T), T32);
const auto ds = TensorAccessor(acc_d, get_common_arg_val<uint32_t>(QP_C_D), T32);
uint32_t nx = 0, nk = 0, nv = 0;
for (uint32_t h = h0; h < NH; h += P) {
nx += h < NQ + NKV ? 1 : 0;
nk += (h >= NQ && h < NQ + NKV) ? 1 : 0;
nv += h >= NQ + NKV ? 1 : 0;
}
if (nx) {
cb_reserve_back(QP_CB_CS, HD_T);
cb_reserve_back(QP_CB_X, nx * HD_T);
}
if (nk) {
cb_reserve_back(QP_CB_T, nk * HD_T);
cb_reserve_back(QP_CB_D, nk * HD_T);
}
if (nv) {
cb_reserve_back(QP_CB_V, nv * HD_T);
}
uint32_t xa = get_write_ptr(QP_CB_X), ta = get_write_ptr(QP_CB_T), da = get_write_ptr(QP_CB_D),
va = get_write_ptr(QP_CB_V);
if (nx) { // cos_0, cos_1, sin_0, sin_1 of row tile r (trid 1)
noc_async_read_set_trid(1);
const uint32_t ca = get_write_ptr(QP_CB_CS);
for (uint32_t t = 0; t < HD_T; ++t) {
noc_async_read(cs.get_noc_addr(r * HD_T + t), ca + t * T32, T32);
}
}
for (uint32_t h = h0; h < NH; h += P) { // trid 2 + i per head (<= 13 heads per core: qkv_plan)
noc_async_read_set_trid(2 + (h - h0) / P);
const uint32_t page = (r * NH + h) * HD_T;
if (h >= NQ + NKV) { // v: straight to the writer
for (uint32_t t = 0; t < HD_T; ++t, va += T16) {
noc_async_read(qkv.get_noc_addr(page + t), va, T16);
}
continue;
}
for (uint32_t t = 0; t < HD_T; ++t, xa += T16) {
noc_async_read(qkv.get_noc_addr(page + t), xa, T16);
}
if (h >= NQ) { // k: the bias tables of kv head h - NQ
const uint32_t tp = ((h - NQ) * MT + r) * HD_T;
for (uint32_t t = 0; t < HD_T; ++t, ta += T32, da += T32) {
noc_async_read(ts.get_noc_addr(tp + t), ta, T32);
noc_async_read(ds.get_noc_addr(tp + t), da, T32);
}
}
}
if (nx) {
noc_async_read_barrier_with_trid(1);
cb_push_back(QP_CB_CS, HD_T);
}
for (uint32_t h = h0; h < NH; h += P) { // hand each head over as soon as its reads have landed
noc_async_read_barrier_with_trid(2 + (h - h0) / P);
if (h >= NQ + NKV) {
cb_push_back(QP_CB_V, HD_T);
continue;
}
cb_push_back(QP_CB_X, HD_T);
if (h >= NQ) {
cb_push_back(QP_CB_T, HD_T);
cb_push_back(QP_CB_D, HD_T);
}
}
noc_async_read_set_trid(0);
noc_async_read_barrier();
}
|