File size: 4,357 Bytes
9d8d25c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
// SPDX-License-Identifier: Apache-2.0
//
// RDT2-FM fused PREFIX op "qkvpost", reader (NCRISC, NoC 0). The core runs heads h0, h0 + P, ... of row tile r
// (qp_share; NH = NQ + 2 NKV heads of the fused qkv projection): per head the 4 tiles of head h in row tile r of
// qkv (1, 1, L, NH * 128) bf16; the RoPE tiles of row r (cos_0, cos_1, sin_0, sin_1 fp32: the halves of the mRoPE
// tables are equal) once; a k head also needs the fp32 bias tables T and D of its kv head. v heads bypass the
// compute. The CBs hold every page of the core's heads: all reads are issued at once (a transaction id per head)
// and each head is handed over as soon as its own reads have landed.
#include <cstdint>
#include "api/dataflow/dataflow_api.h"
#include "pxf_common.hpp"

using namespace pxf;

namespace {
constexpr uint32_t MT = get_compile_time_arg_val(QP_CT_MT);
constexpr uint32_t NQ = get_compile_time_arg_val(QP_CT_NQ);
constexpr uint32_t NKV = get_compile_time_arg_val(QP_CT_NKV);
constexpr uint32_t NH = NQ + 2 * NKV;
constexpr uint32_t P = get_compile_time_arg_val(QP_CT_P);
constexpr auto acc_qkv = TensorAccessorArgs<QP_CT_ACC0>();
constexpr auto acc_cs = TensorAccessorArgs<acc_qkv.next_compile_time_args_offset()>();
constexpr auto acc_t = TensorAccessorArgs<acc_cs.next_compile_time_args_offset()>();
constexpr auto acc_d = TensorAccessorArgs<acc_t.next_compile_time_args_offset()>();

}  // namespace

void kernel_main() {
    uint32_t r, h0;
    qp_share(get_absolute_logical_x(), get_absolute_logical_y(), get_compile_time_arg_val(QP_CT_GX), P, r, h0);
    const auto qkv = TensorAccessor(acc_qkv, get_common_arg_val<uint32_t>(QP_C_QKV), T16);
    const auto cs = TensorAccessor(acc_cs, get_common_arg_val<uint32_t>(QP_C_CS), T32);
    const auto ts = TensorAccessor(acc_t, get_common_arg_val<uint32_t>(QP_C_T), T32);
    const auto ds = TensorAccessor(acc_d, get_common_arg_val<uint32_t>(QP_C_D), T32);
    uint32_t nx = 0, nk = 0, nv = 0;
    for (uint32_t h = h0; h < NH; h += P) {
        nx += h < NQ + NKV ? 1 : 0;
        nk += (h >= NQ && h < NQ + NKV) ? 1 : 0;
        nv += h >= NQ + NKV ? 1 : 0;
    }
    if (nx) {
        cb_reserve_back(QP_CB_CS, HD_T);
        cb_reserve_back(QP_CB_X, nx * HD_T);
    }
    if (nk) {
        cb_reserve_back(QP_CB_T, nk * HD_T);
        cb_reserve_back(QP_CB_D, nk * HD_T);
    }
    if (nv) {
        cb_reserve_back(QP_CB_V, nv * HD_T);
    }
    uint32_t xa = get_write_ptr(QP_CB_X), ta = get_write_ptr(QP_CB_T), da = get_write_ptr(QP_CB_D),
             va = get_write_ptr(QP_CB_V);
    if (nx) {  // cos_0, cos_1, sin_0, sin_1 of row tile r (trid 1)
        noc_async_read_set_trid(1);
        const uint32_t ca = get_write_ptr(QP_CB_CS);
        for (uint32_t t = 0; t < HD_T; ++t) {
            noc_async_read(cs.get_noc_addr(r * HD_T + t), ca + t * T32, T32);
        }
    }
    for (uint32_t h = h0; h < NH; h += P) {  // trid 2 + i per head (<= 13 heads per core: qkv_plan)
        noc_async_read_set_trid(2 + (h - h0) / P);
        const uint32_t page = (r * NH + h) * HD_T;
        if (h >= NQ + NKV) {  // v: straight to the writer
            for (uint32_t t = 0; t < HD_T; ++t, va += T16) {
                noc_async_read(qkv.get_noc_addr(page + t), va, T16);
            }
            continue;
        }
        for (uint32_t t = 0; t < HD_T; ++t, xa += T16) {
            noc_async_read(qkv.get_noc_addr(page + t), xa, T16);
        }
        if (h >= NQ) {  // k: the bias tables of kv head h - NQ
            const uint32_t tp = ((h - NQ) * MT + r) * HD_T;
            for (uint32_t t = 0; t < HD_T; ++t, ta += T32, da += T32) {
                noc_async_read(ts.get_noc_addr(tp + t), ta, T32);
                noc_async_read(ds.get_noc_addr(tp + t), da, T32);
            }
        }
    }
    if (nx) {
        noc_async_read_barrier_with_trid(1);
        cb_push_back(QP_CB_CS, HD_T);
    }
    for (uint32_t h = h0; h < NH; h += P) {  // hand each head over as soon as its reads have landed
        noc_async_read_barrier_with_trid(2 + (h - h0) / P);
        if (h >= NQ + NKV) {
            cb_push_back(QP_CB_V, HD_T);
            continue;
        }
        cb_push_back(QP_CB_X, HD_T);
        if (h >= NQ) {
            cb_push_back(QP_CB_T, HD_T);
            cb_push_back(QP_CB_D, HD_T);
        }
    }
    noc_async_read_set_trid(0);
    noc_async_read_barrier();
}