Download code/tt_rdt2/tt/kernels/pxf_addnorm_compute.cpp from changh95/rdt2-base-p150: direct link, hf CLI and curl.
- Browser
- Download file 7.14 kB
-
https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/tt/kernels/pxf_addnorm_compute.cpp
- Command line
-
hf download hf://changh95/rdt2-base-p150/code/tt_rdt2/tt/kernels/pxf_addnorm_compute.cpp
-
curl -L -o pxf_addnorm_compute.cpp https://huggingface.co/changh95/rdt2-base-p150/resolve/main/code/tt_rdt2/tt/kernels/pxf_addnorm_compute.cpp
7.14 kB
| // SPDX-License-Identifier: Apache-2.0 | |
| // | |
| // RDT2-FM fused PREFIX op "addnorm", compute: x' = x + delta (fp32), h = bf16(x' * rsqrt(mean_row(x'^2) + eps) * g). | |
| // Everything in fp32 on the SFPU (fp32 DST, full sync: 8 tiles); one rounding of h. Row statistics: each core sums | |
| // x'^2 over its column tiles (sfpu_reduce, column 0 of one fp32 tile); the writer exchanges the P partials of the row | |
| // group; every core of the group adds the same P partials in the same order (slot 0 .. P-1). | |
| using namespace pxf; | |
| namespace { | |
| constexpr uint32_t P = get_compile_time_arg_val(AN_CT_P); | |
| constexpr uint32_t WRX = get_compile_time_arg_val(AN_CT_WRX); | |
| NOINL void ld(uint32_t cb, uint32_t t, uint32_t d) { | |
| reconfig_data_format_srca(cb); | |
| copy_init(cb); | |
| copy_tile(cb, t, d); | |
| } | |
| NOINL void fadd(uint32_t a, uint32_t b, uint32_t o) { add_binary_tile_init(); add_binary_tile(a, b, o); } | |
| NOINL void fmul(uint32_t a, uint32_t b, uint32_t o) { mul_binary_tile_init(); mul_binary_tile(a, b, o); } | |
| NOINL void rsum(uint32_t d, uint32_t ct) { | |
| sfpu_reduce_init<PoolType::SUM, DataFormat::Float32>(); | |
| sfpu_reduce<PoolType::SUM, DataFormat::Float32, ReduceDim::REDUCE_ROW>(d, ct, 1); | |
| } | |
| NOINL void pk(uint32_t d, uint32_t cb, uint32_t idx) { | |
| pack_reconfig_data_format(cb); | |
| pack_tile<true>(d, cb, idx); | |
| } | |
| FORCE_INLINE void acq() { tile_regs_acquire(); } | |
| FORCE_INLINE void done() { | |
| tile_regs_commit(); | |
| tile_regs_wait(); | |
| } | |
| FORCE_INLINE void rel() { tile_regs_release(); } | |
| } // namespace | |
| void kernel_main() { | |
| const AnShare sh = an_share(get_absolute_logical_x(), get_absolute_logical_y(), get_compile_time_arg_val(AN_CT_GX), | |
| get_compile_time_arg_val(AN_CT_P), get_compile_time_arg_val(AN_CT_NT)); | |
| const uint32_t nc = sh.nc; | |
| const uint32_t eps = get_common_arg_val<uint32_t>(AN_C_EPS); | |
| const uint32_t invh = get_common_arg_val<uint32_t>(AN_C_INVH); | |
| compute_kernel_hw_startup(AN_CB_X, AN_CB_D, AN_CB_XN); | |
| // phase 1: x' = x + delta -> XN (kept; the writer also sends it to DRAM); partial row sums of x'^2 per pass -> PC | |
| uint32_t npc = 0; | |
| for (uint32_t j0 = 0; j0 < nc; j0 += AN_CHUNK, ++npc) { | |
| const uint32_t n = nc - j0 < AN_CHUNK ? nc - j0 : AN_CHUNK; | |
| cb_wait_front(AN_CB_X, n); | |
| cb_wait_front(AN_CB_D, n); | |
| PXF_ZONE("AN_P1"); | |
| acq(); | |
| reconfig_data_format_srca(AN_CB_X); | |
| copy_init(AN_CB_X); | |
| for (uint32_t i = 0; i < n; ++i) { | |
| copy_tile(AN_CB_X, i, i); | |
| } | |
| reconfig_data_format_srca(AN_CB_D); | |
| copy_init(AN_CB_D); | |
| for (uint32_t i = 0; i < n; ++i) { | |
| copy_tile(AN_CB_D, i, AN_CHUNK + i); | |
| } | |
| add_binary_tile_init(); | |
| for (uint32_t i = 0; i < n; ++i) { | |
| add_binary_tile(i, AN_CHUNK + i, i); | |
| } | |
| mul_binary_tile_init(); | |
| for (uint32_t i = 0; i < n; ++i) { | |
| mul_binary_tile(i, i, AN_CHUNK + i); | |
| } | |
| add_binary_tile_init(); // x'^2 of the pass summed tile-wise (one row reduction per core, in PART) | |
| for (uint32_t i = 1; i < n; ++i) { | |
| add_binary_tile(AN_CHUNK, AN_CHUNK + i, AN_CHUNK); | |
| } | |
| done(); | |
| cb_reserve_back(AN_CB_XN, n); | |
| for (uint32_t i = 0; i < n; ++i) { | |
| pk(i, AN_CB_XN, i); | |
| } | |
| cb_push_back(AN_CB_XN, n); | |
| cb_reserve_back(AN_CB_PC, 1); | |
| pk(AN_CHUNK, AN_CB_PC, 0); | |
| cb_push_back(AN_CB_PC, 1); | |
| rel(); | |
| cb_pop_front(AN_CB_X, n); | |
| cb_pop_front(AN_CB_D, n); | |
| } | |
| // the core's partial (sum of the passes) -> PART (the writer sends it to the group) | |
| cb_wait_front(AN_CB_PC, npc); | |
| { | |
| PXF_ZONE("AN_PART"); | |
| acq(); | |
| reconfig_data_format_srca(AN_CB_PC); | |
| copy_init(AN_CB_PC); | |
| for (uint32_t k = 0; k < npc; ++k) { | |
| copy_tile(AN_CB_PC, k, k); | |
| } | |
| add_binary_tile_init(); | |
| for (uint32_t k = 1; k < npc; ++k) { | |
| add_binary_tile(0, k, 0); | |
| } | |
| rsum(0, 1); | |
| done(); | |
| cb_reserve_back(AN_CB_PART, 1); | |
| pk(0, AN_CB_PART, 0); | |
| cb_push_back(AN_CB_PART, 1); | |
| rel(); | |
| cb_pop_front(AN_CB_PC, npc); | |
| } | |
| // phase 2: r = rsqrt(sum of the P partials / H + eps) (column 0) | |
| cb_wait_front(AN_CB_STAT, P); | |
| { | |
| PXF_ZONE("AN_STAT"); | |
| acq(); | |
| reconfig_data_format_srca(AN_CB_STAT); | |
| copy_init(AN_CB_STAT); | |
| // slots 0..P-1 summed in the same order on every core of the group: up to 8 in DST at once | |
| for (uint32_t q = 0; q < P && q < 8; ++q) { | |
| copy_tile(AN_CB_STAT, q, q); | |
| } | |
| add_binary_tile_init(); | |
| for (uint32_t q = 1; q < P && q < 8; ++q) { | |
| add_binary_tile(0, q, 0); | |
| } | |
| for (uint32_t q = 8; q < P; ++q) { | |
| copy_init(AN_CB_STAT); | |
| copy_tile(AN_CB_STAT, q, 1); | |
| add_binary_tile_init(); | |
| add_binary_tile(0, 1, 0); | |
| } | |
| binop_with_scalar_tile_init(); | |
| mul_unary_tile(0, invh); | |
| add_unary_tile(0, eps); | |
| rsqrt_tile_init(); | |
| rsqrt_tile(0); | |
| done(); | |
| cb_reserve_back(AN_CB_R, 1); | |
| pk(0, AN_CB_R, 0); | |
| cb_push_back(AN_CB_R, 1); | |
| rel(); | |
| cb_pop_front(AN_CB_STAT, P); | |
| } | |
| // h = bf16(x' r g) | |
| cb_wait_front(AN_CB_R, 1); | |
| cb_wait_front(AN_CB_XN, nc); | |
| cb_wait_front(AN_CB_G, nc); | |
| for (uint32_t j0 = 0; j0 < nc; j0 += AN_CHUNK) { | |
| const uint32_t n = nc - j0 < AN_CHUNK ? nc - j0 : AN_CHUNK; | |
| PXF_ZONE("AN_P2"); | |
| acq(); | |
| reconfig_data_format_srca(AN_CB_XN); | |
| copy_init(AN_CB_XN); | |
| for (uint32_t i = 0; i < n; ++i) { | |
| copy_tile(AN_CB_XN, j0 + i, i); | |
| } | |
| reconfig_data_format_srca(AN_CB_R); | |
| copy_init(AN_CB_R); | |
| copy_tile(AN_CB_R, 0, AN_CHUNK); | |
| sfpu_mul_bcast_col_init(); | |
| for (uint32_t i = 0; i < n; ++i) { | |
| sfpu_mul_bcast_col(i, AN_CHUNK); | |
| } | |
| reconfig_data_format_srca(AN_CB_G); | |
| reconfig_data_format_srcb(AN_CB_G); | |
| unary_bcast_init<BroadcastType::ROW>(AN_CB_G); | |
| for (uint32_t i = 0; i < n; ++i) { | |
| unary_bcast<BroadcastType::ROW>(AN_CB_G, j0 + i, AN_CHUNK + i); | |
| } | |
| mul_binary_tile_init(); | |
| for (uint32_t i = 0; i < n; ++i) { | |
| mul_binary_tile(i, AN_CHUNK + i, i); | |
| } | |
| done(); | |
| cb_reserve_back(AN_CB_H, n); | |
| for (uint32_t i = 0; i < n; ++i) { | |
| pk(i, AN_CB_H, i); | |
| } | |
| cb_push_back(AN_CB_H, n); | |
| rel(); | |
| } | |
| cb_pop_front(AN_CB_R, 1); | |
| cb_pop_front(AN_CB_XN, nc); | |
| cb_pop_front(AN_CB_G, nc); | |
| } | |