rdt2-base-p150 / code /tt_rdt2 /tt /kernels /pxf_addnorm_compute.cpp
changh95's picture
tt-model push rdt2-base-p150 (container)
9d8d25c verified
Raw History Blame Contribute Delete
7.14 kB
// SPDX-License-Identifier: Apache-2.0
//
// RDT2-FM fused PREFIX op "addnorm", compute: x' = x + delta (fp32), h = bf16(x' * rsqrt(mean_row(x'^2) + eps) * g).
// Everything in fp32 on the SFPU (fp32 DST, full sync: 8 tiles); one rounding of h. Row statistics: each core sums
// x'^2 over its column tiles (sfpu_reduce, column 0 of one fp32 tile); the writer exchanges the P partials of the row
// group; every core of the group adds the same P partials in the same order (slot 0 .. P-1).
#include <cstdint>
#include "api/compute/compute_kernel_api.h"
#include "api/compute/eltwise_binary_sfpu.h"
#include "api/compute/cb_api.h"
#include "api/compute/reg_api.h"
#include "api/compute/pack.h"
#include "api/compute/common.h"
#include "api/compute/bcast.h"
#include "api/compute/tile_move_copy.h"
#include "api/compute/sfpu_binary_bcast.h"
#include "api/compute/eltwise_unary/rsqrt.h"
#include "api/compute/eltwise_unary/binop_with_scalar.h"
#include "api/compute/reconfig_data_format.h"
#include "api/compute/compute_kernel_hw_startup.h"
#include "pxf_common.hpp"
#define NOINL __attribute__((noinline, noclone))
using namespace pxf;
namespace {
constexpr uint32_t P = get_compile_time_arg_val(AN_CT_P);
constexpr uint32_t WRX = get_compile_time_arg_val(AN_CT_WRX);
NOINL void ld(uint32_t cb, uint32_t t, uint32_t d) {
reconfig_data_format_srca(cb);
copy_init(cb);
copy_tile(cb, t, d);
}
NOINL void fadd(uint32_t a, uint32_t b, uint32_t o) { add_binary_tile_init(); add_binary_tile(a, b, o); }
NOINL void fmul(uint32_t a, uint32_t b, uint32_t o) { mul_binary_tile_init(); mul_binary_tile(a, b, o); }
NOINL void rsum(uint32_t d, uint32_t ct) {
sfpu_reduce_init<PoolType::SUM, DataFormat::Float32>();
sfpu_reduce<PoolType::SUM, DataFormat::Float32, ReduceDim::REDUCE_ROW>(d, ct, 1);
}
NOINL void pk(uint32_t d, uint32_t cb, uint32_t idx) {
pack_reconfig_data_format(cb);
pack_tile<true>(d, cb, idx);
}
FORCE_INLINE void acq() { tile_regs_acquire(); }
FORCE_INLINE void done() {
tile_regs_commit();
tile_regs_wait();
}
FORCE_INLINE void rel() { tile_regs_release(); }
} // namespace
void kernel_main() {
const AnShare sh = an_share(get_absolute_logical_x(), get_absolute_logical_y(), get_compile_time_arg_val(AN_CT_GX),
get_compile_time_arg_val(AN_CT_P), get_compile_time_arg_val(AN_CT_NT));
const uint32_t nc = sh.nc;
const uint32_t eps = get_common_arg_val<uint32_t>(AN_C_EPS);
const uint32_t invh = get_common_arg_val<uint32_t>(AN_C_INVH);
compute_kernel_hw_startup(AN_CB_X, AN_CB_D, AN_CB_XN);
// phase 1: x' = x + delta -> XN (kept; the writer also sends it to DRAM); partial row sums of x'^2 per pass -> PC
uint32_t npc = 0;
for (uint32_t j0 = 0; j0 < nc; j0 += AN_CHUNK, ++npc) {
const uint32_t n = nc - j0 < AN_CHUNK ? nc - j0 : AN_CHUNK;
cb_wait_front(AN_CB_X, n);
cb_wait_front(AN_CB_D, n);
PXF_ZONE("AN_P1");
acq();
reconfig_data_format_srca(AN_CB_X);
copy_init(AN_CB_X);
for (uint32_t i = 0; i < n; ++i) {
copy_tile(AN_CB_X, i, i);
}
reconfig_data_format_srca(AN_CB_D);
copy_init(AN_CB_D);
for (uint32_t i = 0; i < n; ++i) {
copy_tile(AN_CB_D, i, AN_CHUNK + i);
}
add_binary_tile_init();
for (uint32_t i = 0; i < n; ++i) {
add_binary_tile(i, AN_CHUNK + i, i);
}
mul_binary_tile_init();
for (uint32_t i = 0; i < n; ++i) {
mul_binary_tile(i, i, AN_CHUNK + i);
}
add_binary_tile_init(); // x'^2 of the pass summed tile-wise (one row reduction per core, in PART)
for (uint32_t i = 1; i < n; ++i) {
add_binary_tile(AN_CHUNK, AN_CHUNK + i, AN_CHUNK);
}
done();
cb_reserve_back(AN_CB_XN, n);
for (uint32_t i = 0; i < n; ++i) {
pk(i, AN_CB_XN, i);
}
cb_push_back(AN_CB_XN, n);
cb_reserve_back(AN_CB_PC, 1);
pk(AN_CHUNK, AN_CB_PC, 0);
cb_push_back(AN_CB_PC, 1);
rel();
cb_pop_front(AN_CB_X, n);
cb_pop_front(AN_CB_D, n);
}
// the core's partial (sum of the passes) -> PART (the writer sends it to the group)
cb_wait_front(AN_CB_PC, npc);
{
PXF_ZONE("AN_PART");
acq();
reconfig_data_format_srca(AN_CB_PC);
copy_init(AN_CB_PC);
for (uint32_t k = 0; k < npc; ++k) {
copy_tile(AN_CB_PC, k, k);
}
add_binary_tile_init();
for (uint32_t k = 1; k < npc; ++k) {
add_binary_tile(0, k, 0);
}
rsum(0, 1);
done();
cb_reserve_back(AN_CB_PART, 1);
pk(0, AN_CB_PART, 0);
cb_push_back(AN_CB_PART, 1);
rel();
cb_pop_front(AN_CB_PC, npc);
}
// phase 2: r = rsqrt(sum of the P partials / H + eps) (column 0)
cb_wait_front(AN_CB_STAT, P);
{
PXF_ZONE("AN_STAT");
acq();
reconfig_data_format_srca(AN_CB_STAT);
copy_init(AN_CB_STAT);
// slots 0..P-1 summed in the same order on every core of the group: up to 8 in DST at once
for (uint32_t q = 0; q < P && q < 8; ++q) {
copy_tile(AN_CB_STAT, q, q);
}
add_binary_tile_init();
for (uint32_t q = 1; q < P && q < 8; ++q) {
add_binary_tile(0, q, 0);
}
for (uint32_t q = 8; q < P; ++q) {
copy_init(AN_CB_STAT);
copy_tile(AN_CB_STAT, q, 1);
add_binary_tile_init();
add_binary_tile(0, 1, 0);
}
binop_with_scalar_tile_init();
mul_unary_tile(0, invh);
add_unary_tile(0, eps);
rsqrt_tile_init();
rsqrt_tile(0);
done();
cb_reserve_back(AN_CB_R, 1);
pk(0, AN_CB_R, 0);
cb_push_back(AN_CB_R, 1);
rel();
cb_pop_front(AN_CB_STAT, P);
}
// h = bf16(x' r g)
cb_wait_front(AN_CB_R, 1);
cb_wait_front(AN_CB_XN, nc);
cb_wait_front(AN_CB_G, nc);
for (uint32_t j0 = 0; j0 < nc; j0 += AN_CHUNK) {
const uint32_t n = nc - j0 < AN_CHUNK ? nc - j0 : AN_CHUNK;
PXF_ZONE("AN_P2");
acq();
reconfig_data_format_srca(AN_CB_XN);
copy_init(AN_CB_XN);
for (uint32_t i = 0; i < n; ++i) {
copy_tile(AN_CB_XN, j0 + i, i);
}
reconfig_data_format_srca(AN_CB_R);
copy_init(AN_CB_R);
copy_tile(AN_CB_R, 0, AN_CHUNK);
sfpu_mul_bcast_col_init();
for (uint32_t i = 0; i < n; ++i) {
sfpu_mul_bcast_col(i, AN_CHUNK);
}
reconfig_data_format_srca(AN_CB_G);
reconfig_data_format_srcb(AN_CB_G);
unary_bcast_init<BroadcastType::ROW>(AN_CB_G);
for (uint32_t i = 0; i < n; ++i) {
unary_bcast<BroadcastType::ROW>(AN_CB_G, j0 + i, AN_CHUNK + i);
}
mul_binary_tile_init();
for (uint32_t i = 0; i < n; ++i) {
mul_binary_tile(i, AN_CHUNK + i, i);
}
done();
cb_reserve_back(AN_CB_H, n);
for (uint32_t i = 0; i < n; ++i) {
pk(i, AN_CB_H, i);
}
cb_push_back(AN_CB_H, n);
rel();
}
cb_pop_front(AN_CB_R, 1);
cb_pop_front(AN_CB_XN, nc);
cb_pop_front(AN_CB_G, nc);
}