File size: 3,063 Bytes
9d8d25c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
// SPDX-License-Identifier: Apache-2.0
//
// RDT2-FM fused PREFIX op "addnorm", reader (NCRISC, NoC 0): the core's column tiles [c0, c0 + nc) of row tile r of
// x (fp32 or bf16), delta (bf16) and gamma (bf16), all reads in flight at once (a transaction id per pass).
#include <cstdint>
#include "api/dataflow/dataflow_api.h"
#include "pxf_common.hpp"

using namespace pxf;

namespace {
constexpr uint32_t NT = get_compile_time_arg_val(AN_CT_NT);
constexpr uint32_t XF32 = get_compile_time_arg_val(AN_CT_XF32);
constexpr auto acc_x = TensorAccessorArgs<AN_CT_ACC0>();
constexpr auto acc_d = TensorAccessorArgs<acc_x.next_compile_time_args_offset()>();
constexpr auto acc_g = TensorAccessorArgs<acc_d.next_compile_time_args_offset()>();
constexpr uint32_t XB = XF32 ? T32 : T16;
}  // namespace

void kernel_main() {
    const AnShare sh = an_share(get_absolute_logical_x(), get_absolute_logical_y(), get_compile_time_arg_val(AN_CT_GX),
                                get_compile_time_arg_val(AN_CT_P), get_compile_time_arg_val(AN_CT_NT));
    const uint32_t r = sh.r;
    const uint32_t c0 = sh.c0;
    const uint32_t nc = sh.nc;
    const auto xs = TensorAccessor(acc_x, get_common_arg_val<uint32_t>(AN_C_X), XB);
    const auto ds = TensorAccessor(acc_d, get_common_arg_val<uint32_t>(AN_C_D), T16);
    const auto gs = TensorAccessor(acc_g, get_common_arg_val<uint32_t>(AN_C_G), T16);
    const uint32_t row0 = r * NT + c0;
    // every read in flight at once (the CBs hold all nc tiles); one transaction id per pass of AN_CHUNK tiles, so the
    // compute starts on pass 0 while the later passes are still in flight
    cb_reserve_back(AN_CB_X, nc);
    cb_reserve_back(AN_CB_D, nc);
    cb_reserve_back(AN_CB_G, nc);
    const uint32_t xa = get_write_ptr(AN_CB_X), da = get_write_ptr(AN_CB_D), ga = get_write_ptr(AN_CB_G);
    uint32_t trid = 1;
    for (uint32_t j0 = 0; j0 < nc; j0 += AN_CHUNK, ++trid) {
        noc_async_read_set_trid(trid);
        const uint32_t n = nc - j0 < AN_CHUNK ? nc - j0 : AN_CHUNK;
        for (uint32_t i = j0; i < j0 + n; ++i) {
            noc_async_read(xs.get_noc_addr(row0 + i), xa + i * XB, XB);
            noc_async_read(ds.get_noc_addr(row0 + i), da + i * T16, T16);
        }
    }
    noc_async_read_set_trid(trid);
    for (uint32_t i = 0; i < nc; ++i) {  // gamma: only row 0 of each tile is used (row broadcast): 64 B (DRAM read
                                         // alignment) at the start of face 0 and of face 1 (the rest is never read)
        const uint64_t g = gs.get_noc_addr(c0 + i);
        noc_async_read(g, ga + i * T16, 64);
        noc_async_read(g + 512, ga + i * T16 + 512, 64);
    }
    trid = 1;
    for (uint32_t j0 = 0; j0 < nc; j0 += AN_CHUNK, ++trid) {
        const uint32_t n = nc - j0 < AN_CHUNK ? nc - j0 : AN_CHUNK;
        noc_async_read_barrier_with_trid(trid);
        cb_push_back(AN_CB_X, n);
        cb_push_back(AN_CB_D, n);
    }
    noc_async_read_barrier_with_trid(trid);
    cb_push_back(AN_CB_G, nc);
    noc_async_read_set_trid(0);
    noc_async_read_barrier();
}