// SPDX-License-Identifier: Apache-2.0 // // RDT2-FM fused PREFIX op "addnorm", compute: x' = x + delta (fp32), h = bf16(x' * rsqrt(mean_row(x'^2) + eps) * g). // Everything in fp32 on the SFPU (fp32 DST, full sync: 8 tiles); one rounding of h. Row statistics: each core sums // x'^2 over its column tiles (sfpu_reduce, column 0 of one fp32 tile); the writer exchanges the P partials of the row // group; every core of the group adds the same P partials in the same order (slot 0 .. P-1). #include #include "api/compute/compute_kernel_api.h" #include "api/compute/eltwise_binary_sfpu.h" #include "api/compute/cb_api.h" #include "api/compute/reg_api.h" #include "api/compute/pack.h" #include "api/compute/common.h" #include "api/compute/bcast.h" #include "api/compute/tile_move_copy.h" #include "api/compute/sfpu_binary_bcast.h" #include "api/compute/eltwise_unary/rsqrt.h" #include "api/compute/eltwise_unary/binop_with_scalar.h" #include "api/compute/reconfig_data_format.h" #include "api/compute/compute_kernel_hw_startup.h" #include "pxf_common.hpp" #define NOINL __attribute__((noinline, noclone)) using namespace pxf; namespace { constexpr uint32_t P = get_compile_time_arg_val(AN_CT_P); constexpr uint32_t WRX = get_compile_time_arg_val(AN_CT_WRX); NOINL void ld(uint32_t cb, uint32_t t, uint32_t d) { reconfig_data_format_srca(cb); copy_init(cb); copy_tile(cb, t, d); } NOINL void fadd(uint32_t a, uint32_t b, uint32_t o) { add_binary_tile_init(); add_binary_tile(a, b, o); } NOINL void fmul(uint32_t a, uint32_t b, uint32_t o) { mul_binary_tile_init(); mul_binary_tile(a, b, o); } NOINL void rsum(uint32_t d, uint32_t ct) { sfpu_reduce_init(); sfpu_reduce(d, ct, 1); } NOINL void pk(uint32_t d, uint32_t cb, uint32_t idx) { pack_reconfig_data_format(cb); pack_tile(d, cb, idx); } FORCE_INLINE void acq() { tile_regs_acquire(); } FORCE_INLINE void done() { tile_regs_commit(); tile_regs_wait(); } FORCE_INLINE void rel() { tile_regs_release(); } } // namespace void kernel_main() { const AnShare sh = an_share(get_absolute_logical_x(), get_absolute_logical_y(), get_compile_time_arg_val(AN_CT_GX), get_compile_time_arg_val(AN_CT_P), get_compile_time_arg_val(AN_CT_NT)); const uint32_t nc = sh.nc; const uint32_t eps = get_common_arg_val(AN_C_EPS); const uint32_t invh = get_common_arg_val(AN_C_INVH); compute_kernel_hw_startup(AN_CB_X, AN_CB_D, AN_CB_XN); // phase 1: x' = x + delta -> XN (kept; the writer also sends it to DRAM); partial row sums of x'^2 per pass -> PC uint32_t npc = 0; for (uint32_t j0 = 0; j0 < nc; j0 += AN_CHUNK, ++npc) { const uint32_t n = nc - j0 < AN_CHUNK ? nc - j0 : AN_CHUNK; cb_wait_front(AN_CB_X, n); cb_wait_front(AN_CB_D, n); PXF_ZONE("AN_P1"); acq(); reconfig_data_format_srca(AN_CB_X); copy_init(AN_CB_X); for (uint32_t i = 0; i < n; ++i) { copy_tile(AN_CB_X, i, i); } reconfig_data_format_srca(AN_CB_D); copy_init(AN_CB_D); for (uint32_t i = 0; i < n; ++i) { copy_tile(AN_CB_D, i, AN_CHUNK + i); } add_binary_tile_init(); for (uint32_t i = 0; i < n; ++i) { add_binary_tile(i, AN_CHUNK + i, i); } mul_binary_tile_init(); for (uint32_t i = 0; i < n; ++i) { mul_binary_tile(i, i, AN_CHUNK + i); } add_binary_tile_init(); // x'^2 of the pass summed tile-wise (one row reduction per core, in PART) for (uint32_t i = 1; i < n; ++i) { add_binary_tile(AN_CHUNK, AN_CHUNK + i, AN_CHUNK); } done(); cb_reserve_back(AN_CB_XN, n); for (uint32_t i = 0; i < n; ++i) { pk(i, AN_CB_XN, i); } cb_push_back(AN_CB_XN, n); cb_reserve_back(AN_CB_PC, 1); pk(AN_CHUNK, AN_CB_PC, 0); cb_push_back(AN_CB_PC, 1); rel(); cb_pop_front(AN_CB_X, n); cb_pop_front(AN_CB_D, n); } // the core's partial (sum of the passes) -> PART (the writer sends it to the group) cb_wait_front(AN_CB_PC, npc); { PXF_ZONE("AN_PART"); acq(); reconfig_data_format_srca(AN_CB_PC); copy_init(AN_CB_PC); for (uint32_t k = 0; k < npc; ++k) { copy_tile(AN_CB_PC, k, k); } add_binary_tile_init(); for (uint32_t k = 1; k < npc; ++k) { add_binary_tile(0, k, 0); } rsum(0, 1); done(); cb_reserve_back(AN_CB_PART, 1); pk(0, AN_CB_PART, 0); cb_push_back(AN_CB_PART, 1); rel(); cb_pop_front(AN_CB_PC, npc); } // phase 2: r = rsqrt(sum of the P partials / H + eps) (column 0) cb_wait_front(AN_CB_STAT, P); { PXF_ZONE("AN_STAT"); acq(); reconfig_data_format_srca(AN_CB_STAT); copy_init(AN_CB_STAT); // slots 0..P-1 summed in the same order on every core of the group: up to 8 in DST at once for (uint32_t q = 0; q < P && q < 8; ++q) { copy_tile(AN_CB_STAT, q, q); } add_binary_tile_init(); for (uint32_t q = 1; q < P && q < 8; ++q) { add_binary_tile(0, q, 0); } for (uint32_t q = 8; q < P; ++q) { copy_init(AN_CB_STAT); copy_tile(AN_CB_STAT, q, 1); add_binary_tile_init(); add_binary_tile(0, 1, 0); } binop_with_scalar_tile_init(); mul_unary_tile(0, invh); add_unary_tile(0, eps); rsqrt_tile_init(); rsqrt_tile(0); done(); cb_reserve_back(AN_CB_R, 1); pk(0, AN_CB_R, 0); cb_push_back(AN_CB_R, 1); rel(); cb_pop_front(AN_CB_STAT, P); } // h = bf16(x' r g) cb_wait_front(AN_CB_R, 1); cb_wait_front(AN_CB_XN, nc); cb_wait_front(AN_CB_G, nc); for (uint32_t j0 = 0; j0 < nc; j0 += AN_CHUNK) { const uint32_t n = nc - j0 < AN_CHUNK ? nc - j0 : AN_CHUNK; PXF_ZONE("AN_P2"); acq(); reconfig_data_format_srca(AN_CB_XN); copy_init(AN_CB_XN); for (uint32_t i = 0; i < n; ++i) { copy_tile(AN_CB_XN, j0 + i, i); } reconfig_data_format_srca(AN_CB_R); copy_init(AN_CB_R); copy_tile(AN_CB_R, 0, AN_CHUNK); sfpu_mul_bcast_col_init(); for (uint32_t i = 0; i < n; ++i) { sfpu_mul_bcast_col(i, AN_CHUNK); } reconfig_data_format_srca(AN_CB_G); reconfig_data_format_srcb(AN_CB_G); unary_bcast_init(AN_CB_G); for (uint32_t i = 0; i < n; ++i) { unary_bcast(AN_CB_G, j0 + i, AN_CHUNK + i); } mul_binary_tile_init(); for (uint32_t i = 0; i < n; ++i) { mul_binary_tile(i, AN_CHUNK + i, i); } done(); cb_reserve_back(AN_CB_H, n); for (uint32_t i = 0; i < n; ++i) { pk(i, AN_CB_H, i); } cb_push_back(AN_CB_H, n); rel(); } cb_pop_front(AN_CB_R, 1); cb_pop_front(AN_CB_XN, nc); cb_pop_front(AN_CB_G, nc); }