File size: 3,063 Bytes
9d8d25c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | // SPDX-License-Identifier: Apache-2.0
//
// RDT2-FM fused PREFIX op "addnorm", reader (NCRISC, NoC 0): the core's column tiles [c0, c0 + nc) of row tile r of
// x (fp32 or bf16), delta (bf16) and gamma (bf16), all reads in flight at once (a transaction id per pass).
#include <cstdint>
#include "api/dataflow/dataflow_api.h"
#include "pxf_common.hpp"
using namespace pxf;
namespace {
constexpr uint32_t NT = get_compile_time_arg_val(AN_CT_NT);
constexpr uint32_t XF32 = get_compile_time_arg_val(AN_CT_XF32);
constexpr auto acc_x = TensorAccessorArgs<AN_CT_ACC0>();
constexpr auto acc_d = TensorAccessorArgs<acc_x.next_compile_time_args_offset()>();
constexpr auto acc_g = TensorAccessorArgs<acc_d.next_compile_time_args_offset()>();
constexpr uint32_t XB = XF32 ? T32 : T16;
} // namespace
void kernel_main() {
const AnShare sh = an_share(get_absolute_logical_x(), get_absolute_logical_y(), get_compile_time_arg_val(AN_CT_GX),
get_compile_time_arg_val(AN_CT_P), get_compile_time_arg_val(AN_CT_NT));
const uint32_t r = sh.r;
const uint32_t c0 = sh.c0;
const uint32_t nc = sh.nc;
const auto xs = TensorAccessor(acc_x, get_common_arg_val<uint32_t>(AN_C_X), XB);
const auto ds = TensorAccessor(acc_d, get_common_arg_val<uint32_t>(AN_C_D), T16);
const auto gs = TensorAccessor(acc_g, get_common_arg_val<uint32_t>(AN_C_G), T16);
const uint32_t row0 = r * NT + c0;
// every read in flight at once (the CBs hold all nc tiles); one transaction id per pass of AN_CHUNK tiles, so the
// compute starts on pass 0 while the later passes are still in flight
cb_reserve_back(AN_CB_X, nc);
cb_reserve_back(AN_CB_D, nc);
cb_reserve_back(AN_CB_G, nc);
const uint32_t xa = get_write_ptr(AN_CB_X), da = get_write_ptr(AN_CB_D), ga = get_write_ptr(AN_CB_G);
uint32_t trid = 1;
for (uint32_t j0 = 0; j0 < nc; j0 += AN_CHUNK, ++trid) {
noc_async_read_set_trid(trid);
const uint32_t n = nc - j0 < AN_CHUNK ? nc - j0 : AN_CHUNK;
for (uint32_t i = j0; i < j0 + n; ++i) {
noc_async_read(xs.get_noc_addr(row0 + i), xa + i * XB, XB);
noc_async_read(ds.get_noc_addr(row0 + i), da + i * T16, T16);
}
}
noc_async_read_set_trid(trid);
for (uint32_t i = 0; i < nc; ++i) { // gamma: only row 0 of each tile is used (row broadcast): 64 B (DRAM read
// alignment) at the start of face 0 and of face 1 (the rest is never read)
const uint64_t g = gs.get_noc_addr(c0 + i);
noc_async_read(g, ga + i * T16, 64);
noc_async_read(g + 512, ga + i * T16 + 512, 64);
}
trid = 1;
for (uint32_t j0 = 0; j0 < nc; j0 += AN_CHUNK, ++trid) {
const uint32_t n = nc - j0 < AN_CHUNK ? nc - j0 : AN_CHUNK;
noc_async_read_barrier_with_trid(trid);
cb_push_back(AN_CB_X, n);
cb_push_back(AN_CB_D, n);
}
noc_async_read_barrier_with_trid(trid);
cb_push_back(AN_CB_G, nc);
noc_async_read_set_trid(0);
noc_async_read_barrier();
}
|