rtl/src/fa_engine.sv

// FA_ENGINE: tiled FlashAttention with online softmax, driven by an in-memory descriptor.
//
//   FA_BEGIN descriptor (6 consecutive words at desc_addr):
//     [0] Q_ptr  [1] K_ptr  [2] V_ptr  [3] O_ptr  [4] sequence_length N  [5] head_dimension d
//   Q, K, V, O are N x d row-major word arrays. Q/K/V elements are Q4.4 int8 (low 8 bits of the
//   word); O elements are signed integers with 8 fractional bits.
//
//   FA_ENGINE
//   |-- Q_TILE_SRAM / K_TILE_SRAM / V_TILE_SRAM
//   |-- QK_DOT_PRODUCT (fa_qk_mac)  -> ROW_MAX (fa_row_max)
//   |-- ONLINE_SOFTMAX (m, l) with EXP_APPROX
//   |-- PV_ACCUMULATOR
//   |-- OUTPUT_NORMALIZER
//   `-- DMA / MEMORY_INTERFACE
//
// start/desc_addr are sampled while idle. done pulses when finished; err is set (until the next
// start) if the descriptor is invalid, in which case memory is not written.

module fa_engine
    import fa_pkg::*;
(
    input  logic        clk,
    input  logic        rst_n,

    input  logic        start,
    input  logic [31:0] desc_addr,
    output logic        busy,
    output logic        done,
    output logic        err,

    output logic        mem_req,
    output logic        mem_we,
    output logic [31:0] mem_addr,
    output logic [31:0] mem_wdata,
    input  logic [31:0] mem_rdata,
    input  logic        mem_ready
);

    localparam int RW = $clog2(BR);
    localparam int CW = $clog2(D_MAX);
    localparam logic [31:0] MAX_N = 32'd1_048_576;

    localparam logic [1:0] OP_LOAD = 2'd0, OP_RD = 2'd1, OP_WR = 2'd2;
    localparam logic [1:0] TGT_Q = 2'd0, TGT_K = 2'd1, TGT_V = 2'd2;

    typedef enum logic [5:0] {
        S_IDLE, S_DESC_REQ, S_DESC_WAIT, S_CHECK,
        S_Q_REQ, S_Q_WAIT, S_INIT,
        S_K_REQ, S_K_WAIT, S_V_REQ, S_V_WAIT,
        S_SC_CLR, S_SC_RD, S_SC_FEED, S_SC_WAIT,
        S_TILE_ST, S_ALPHA_WAIT, S_RESC, S_RESC_WAIT,
        S_PR_RD, S_PR_WAIT, S_PR_ACC, S_PR_ACCW,
        S_NRM_START, S_NRM_WAIT, S_NRM_STORE, S_NRM_STWAIT,
        S_FIN, S_ERR
    } state_t;
    state_t state;

    // descriptor
    logic [31:0] desc_base;
    logic [2:0]  dcnt;
    logic [31:0] q_ptr, k_ptr, v_ptr, o_ptr, seq_n, hd;

    // loop state
    logic [31:0] i0, j0, rows_q, keys_k, r, jj, nr, nc;
    logic [1:0]  tgt;
    logic signed [31:0] score_buf [BC];
    logic signed [31:0] nrm_out_q;
    logic err_q;

    assign busy = (state != S_IDLE);
    assign err  = err_q;

    function automatic logic [31:0] min32(input logic [31:0] a, input logic [31:0] b);
        return (a < b) ? a : b;
    endfunction

    // ------------------------------------------------------------------ DMA
    logic        dma_cmd_valid;
    logic [1:0]  dma_op;
    logic [31:0] dma_addr, dma_wdata;
    logic [3:0]  dma_rows;
    logic [4:0]  dma_cols;
    logic        dma_done;
    logic [31:0] dma_rd_data;
    logic        sram_we;
    logic [1:0]  sram_row;
    logic [3:0]  sram_col;
    logic [7:0]  sram_data;

    fa_dma u_dma (
        .clk, .rst_n,
        .cmd_valid(dma_cmd_valid), .cmd_op(dma_op), .cmd_addr(dma_addr),
        .cmd_rows(dma_rows), .cmd_cols(dma_cols), .cmd_wdata(dma_wdata),
        .cmd_done(dma_done), .rd_data(dma_rd_data),
        .sram_we, .sram_row, .sram_col, .sram_data,
        .mem_req, .mem_we, .mem_addr, .mem_wdata, .mem_rdata, .mem_ready
    );

    // ------------------------------------------------------------------ tile SRAMs
    logic [D_MAX-1:0][7:0] q_rd, k_rd, v_rd;

    fa_tile_sram #(.ROWS(BR), .COLS(D_MAX), .W(8)) u_q_sram (
        .clk, .wr_en(sram_we && tgt == TGT_Q), .wr_row(sram_row[RW-1:0]), .wr_col(sram_col[CW-1:0]),
        .wr_data(sram_data), .rd_row(r[RW-1:0]), .rd_data(q_rd));
    fa_tile_sram #(.ROWS(BC), .COLS(D_MAX), .W(8)) u_k_sram (
        .clk, .wr_en(sram_we && tgt == TGT_K), .wr_row(sram_row[$clog2(BC)-1:0]), .wr_col(sram_col[CW-1:0]),
        .wr_data(sram_data), .rd_row(jj[$clog2(BC)-1:0]), .rd_data(k_rd));
    fa_tile_sram #(.ROWS(BC), .COLS(D_MAX), .W(8)) u_v_sram (
        .clk, .wr_en(sram_we && tgt == TGT_V), .wr_row(sram_row[$clog2(BC)-1:0]), .wr_col(sram_col[CW-1:0]),
        .wr_data(sram_data), .rd_row(jj[$clog2(BC)-1:0]), .rd_data(v_rd));

    // ------------------------------------------------------------------ QK MAC + ROW_MAX
    logic               mac_in_valid, mac_out_valid;
    logic signed [23:0] mac_dot;
    fa_qk_mac u_mac (
        .clk, .rst_n, .in_valid(mac_in_valid), .q_vec(q_rd), .k_vec(k_rd), .d(hd[4:0]),
        .out_valid(mac_out_valid), .dot(mac_dot));

    logic signed [47:0] logit_wide;
    logic signed [31:0] logit_val;
    always_comb begin
        logit_wide = 48'(mac_dot) * $signed({39'b0, scale_q(hd[4:0])});
        logit_val  = 32'(logit_wide >>> 8);
    end

    logic               rm_clear, rm_valid;
    logic signed [31:0] rm_max;
    fa_row_max u_rowmax (
        .clk, .rst_n, .clear(rm_clear), .in_valid(rm_valid), .in_val(logit_val), .max_val(rm_max));

    // ------------------------------------------------------------------ online softmax + PV
    logic        sm_init, sm_tile_start, sm_alpha_valid, sm_p_start, sm_p_valid;
    logic [16:0] sm_alpha, sm_p;
    logic signed [63:0] sm_l;
    fa_online_softmax u_softmax (
        .clk, .rst_n, .init(sm_init),
        .tile_start(sm_tile_start), .row(r[RW-1:0]), .tile_max(rm_max),
        .alpha_valid(sm_alpha_valid), .alpha(sm_alpha),
        .p_start(sm_p_start), .logit(score_buf[jj[$clog2(BC)-1:0]]),
        .p_valid(sm_p_valid), .p(sm_p),
        .l_rd_row(nr[RW-1:0]), .l_out(sm_l));

    logic        pv_rescale, pv_acc, pv_done;
    logic signed [63:0] pv_o;
    fa_pv_accum u_pv (
        .clk, .rst_n, .init(sm_init),
        .rescale_en(pv_rescale), .row(r[RW-1:0]), .alpha(sm_alpha),
        .acc_en(pv_acc), .p(sm_p), .v_vec(v_rd), .d(hd[4:0]),
        .op_done(pv_done),
        .rd_row(nr[RW-1:0]), .rd_col(nc[CW-1:0]), .o_out(pv_o));

    // ------------------------------------------------------------------ normalizer
    logic               nrm_start, nrm_done;
    logic signed [31:0] nrm_out;
    fa_output_normalizer u_norm (
        .clk, .rst_n, .start(nrm_start), .o(pv_o), .l(sm_l), .done(nrm_done), .out(nrm_out));

    // ------------------------------------------------------------------ control outputs
    always_comb begin
        dma_cmd_valid = 1'b0; dma_op = OP_RD; dma_addr = '0; dma_wdata = '0;
        dma_rows = '0; dma_cols = hd[4:0];
        mac_in_valid = 1'b0; rm_clear = 1'b0;
        rm_valid = 1'b0;
        sm_init = 1'b0; sm_tile_start = 1'b0; sm_p_start = 1'b0;
        pv_rescale = 1'b0; pv_acc = 1'b0; nrm_start = 1'b0;

        case (state)
            S_DESC_REQ: begin
                dma_cmd_valid = 1'b1; dma_op = OP_RD; dma_addr = desc_base + 32'(dcnt);
            end
            S_Q_REQ: begin
                dma_cmd_valid = 1'b1; dma_op = OP_LOAD; dma_addr = q_ptr + i0 * hd;
                dma_rows = rows_q[3:0];
            end
            S_K_REQ: begin
                dma_cmd_valid = 1'b1; dma_op = OP_LOAD; dma_addr = k_ptr + j0 * hd;
                dma_rows = keys_k[3:0];
            end
            S_V_REQ: begin
                dma_cmd_valid = 1'b1; dma_op = OP_LOAD; dma_addr = v_ptr + j0 * hd;
                dma_rows = keys_k[3:0];
            end
            S_NRM_STORE: begin
                dma_cmd_valid = 1'b1; dma_op = OP_WR;
                dma_addr  = o_ptr + (i0 + nr) * hd + nc;
                dma_wdata = nrm_out_q;
            end
            S_INIT:      sm_init = 1'b1;
            S_SC_CLR:    rm_clear = 1'b1;
            S_SC_FEED:   mac_in_valid = 1'b1;
            S_SC_WAIT:   rm_valid = mac_out_valid;
            S_TILE_ST:   sm_tile_start = 1'b1;
            S_RESC:      pv_rescale = 1'b1;
            S_PR_RD:     sm_p_start = 1'b1;
            S_PR_ACC:    pv_acc = 1'b1;
            S_NRM_START: nrm_start = 1'b1;
            default: ;
        endcase
    end

    // ------------------------------------------------------------------ FSM
    logic [31:0] i0_next, j0_next;
    always_comb begin
        i0_next = i0 + 32'(BR);
        j0_next = j0 + 32'(BC);
    end

    always_ff @(posedge clk or negedge rst_n) begin
        if (!rst_n) begin
            state <= S_IDLE; done <= 1'b0; err_q <= 1'b0; tgt <= TGT_Q;
            desc_base <= '0; dcnt <= '0;
            q_ptr <= '0; k_ptr <= '0; v_ptr <= '0; o_ptr <= '0; seq_n <= '0; hd <= '0;
            i0 <= '0; j0 <= '0; rows_q <= '0; keys_k <= '0; r <= '0; jj <= '0; nr <= '0; nc <= '0;
            nrm_out_q <= '0;
            for (int i = 0; i < BC; i++) score_buf[i] <= '0;
        end else begin
            done <= 1'b0;
            case (state)
                S_IDLE: if (start) begin
                    desc_base <= desc_addr; dcnt <= '0; err_q <= 1'b0; state <= S_DESC_REQ;
                end

                S_DESC_REQ: state <= S_DESC_WAIT;
                S_DESC_WAIT: if (dma_done) begin
                    case (dcnt)
                        3'd0: q_ptr <= dma_rd_data;
                        3'd1: k_ptr <= dma_rd_data;
                        3'd2: v_ptr <= dma_rd_data;
                        3'd3: o_ptr <= dma_rd_data;
                        3'd4: seq_n <= dma_rd_data;
                        default: hd <= dma_rd_data;
                    endcase
                    if (dcnt == 3'd5) state <= S_CHECK;
                    else begin dcnt <= dcnt + 3'd1; state <= S_DESC_REQ; end
                end

                S_CHECK: begin
                    if (seq_n == 0 || seq_n > MAX_N || hd == 0 || hd > 32'(D_MAX)) state <= S_ERR;
                    else begin
                        i0 <= '0; rows_q <= min32(32'(BR), seq_n); tgt <= TGT_Q; state <= S_Q_REQ;
                    end
                end

                S_Q_REQ:  state <= S_Q_WAIT;
                S_Q_WAIT: if (dma_done) state <= S_INIT;
                S_INIT: begin
                    j0 <= '0; keys_k <= min32(32'(BC), seq_n); tgt <= TGT_K; state <= S_K_REQ;
                end

                S_K_REQ:  state <= S_K_WAIT;
                S_K_WAIT: if (dma_done) begin tgt <= TGT_V; state <= S_V_REQ; end
                S_V_REQ:  state <= S_V_WAIT;
                S_V_WAIT: if (dma_done) begin r <= '0; state <= S_SC_CLR; end

                S_SC_CLR:  begin jj <= '0; state <= S_SC_RD; end
                S_SC_RD:   state <= S_SC_FEED;
                S_SC_FEED: state <= S_SC_WAIT;
                S_SC_WAIT: if (mac_out_valid) begin
                    score_buf[jj[$clog2(BC)-1:0]] <= logit_val;
                    if (jj + 32'd1 == keys_k) state <= S_TILE_ST;
                    else begin jj <= jj + 32'd1; state <= S_SC_RD; end
                end

                S_TILE_ST:    state <= S_ALPHA_WAIT;
                S_ALPHA_WAIT: if (sm_alpha_valid) state <= S_RESC;
                S_RESC:       state <= S_RESC_WAIT;
                S_RESC_WAIT:  if (pv_done) begin jj <= '0; state <= S_PR_RD; end

                S_PR_RD:   state <= S_PR_WAIT;
                S_PR_WAIT: if (sm_p_valid) state <= S_PR_ACC;
                S_PR_ACC:  state <= S_PR_ACCW;
                S_PR_ACCW: if (pv_done) begin
                    if (jj + 32'd1 != keys_k) begin
                        jj <= jj + 32'd1; state <= S_PR_RD;
                    end else if (r + 32'd1 != rows_q) begin
                        r <= r + 32'd1; state <= S_SC_CLR;
                    end else if (j0_next < seq_n) begin
                        j0 <= j0_next; keys_k <= min32(32'(BC), seq_n - j0_next);
                        tgt <= TGT_K; state <= S_K_REQ;
                    end else begin
                        nr <= '0; nc <= '0; state <= S_NRM_START;
                    end
                end

                S_NRM_START: state <= S_NRM_WAIT;
                S_NRM_WAIT:  if (nrm_done) begin nrm_out_q <= nrm_out; state <= S_NRM_STORE; end
                S_NRM_STORE: state <= S_NRM_STWAIT;
                S_NRM_STWAIT: if (dma_done) begin
                    if (nc + 32'd1 != hd) begin
                        nc <= nc + 32'd1; state <= S_NRM_START;
                    end else if (nr + 32'd1 != rows_q) begin
                        nc <= '0; nr <= nr + 32'd1; state <= S_NRM_START;
                    end else if (i0_next < seq_n) begin
                        i0 <= i0_next; rows_q <= min32(32'(BR), seq_n - i0_next);
                        tgt <= TGT_Q; state <= S_Q_REQ;
                    end else begin
                        state <= S_FIN;
                    end
                end

                S_FIN: begin done <= 1'b1; state <= S_IDLE; end
                S_ERR: begin err_q <= 1'b1; done <= 1'b1; state <= S_IDLE; end
                default: state <= S_IDLE;
            endcase
        end
    end

endmodule : fa_engine