File size: 33,679 Bytes
dc7d8aa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24191a8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
dc7d8aa
 
 
 
 
24191a8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
dc7d8aa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24191a8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright (c) 2024, Tri Dao.
# Copyright 2026 Cloudflare, Inc.

"""CUDA kernels for attention and causal convolution."""

from typing import Any

import torch
import triton
import triton.language as tl


@triton.jit(do_not_specialize=["XB", "XD", "XT", "OB", "OD", "OT"])
def _prefill(X: tl.tensor, W: tl.tensor, Bias: tl.tensor, Out: tl.tensor,
             T: tl.int32, D: tl.constexpr,
             XB: tl.int64, XD: tl.int64, XT: tl.int64,
             OB: tl.int64, OD: tl.int64, OT: tl.int64,
             HAS_BIAS: tl.constexpr, SILU: tl.constexpr,
             BT: tl.constexpr, BD: tl.constexpr) -> None:
    batch = tl.program_id(1)
    token = tl.program_id(0) * BT + tl.arange(0, BT)
    channel = tl.program_id(2) * BD + tl.arange(0, BD)
    if HAS_BIAS:
        initial = tl.load(Bias + channel, channel < D, other=0).to(tl.float32)
    else:
        initial = tl.full((BD,), 0, tl.float32)
    acc = tl.broadcast_to(initial[None, :], (BT, BD))
    for tap in tl.static_range(4):
        position = token + tap - 3
        value = tl.load(X + batch * XB + channel[None, :] * XD + position[:, None] * XT,
                        (position[:, None] >= 0) & (position[:, None] < T)
                        & (channel[None, :] < D), other=0)
        weight = tl.load(W + channel * 4 + tap, channel < D, other=0)
        acc += value * weight[None, :]
    if SILU:
        acc = acc / (1 + tl.exp(-acc))
    tl.store(Out + batch * OB + channel[None, :] * OD + token[:, None] * OT,
             acc, (token[:, None] < T) & (channel[None, :] < D))


@triton.jit(do_not_specialize=["XB", "XD", "SB", "SD", "ST", "OB", "OD"])
def _update(X: tl.tensor, State: tl.tensor, W: tl.tensor, Bias: tl.tensor, Out: tl.tensor,
            D: tl.constexpr, XB: tl.int64, XD: tl.int64,
            SB: tl.int64, SD: tl.int64, ST: tl.int64,
            OB: tl.int64, OD: tl.int64,
            HAS_BIAS: tl.constexpr, SILU: tl.constexpr, BD: tl.constexpr) -> None:
    batch = tl.program_id(0)
    channel = tl.program_id(1) * BD + tl.arange(0, BD)
    mask = channel < D
    base = State + batch * SB + channel * SD
    x0 = tl.load(base + ST, mask, other=0)
    x1 = tl.load(base + 2 * ST, mask, other=0)
    x2 = tl.load(base + 3 * ST, mask, other=0)
    x3 = tl.load(X + batch * XB + channel * XD, mask, other=0)
    if HAS_BIAS:
        acc = tl.load(Bias + channel, mask, other=0).to(tl.float32)
    else:
        acc = tl.full((BD,), 0, tl.float32)
    for tap in tl.static_range(4):
        if tap == 0:
            value = x0
        elif tap == 1:
            value = x1
        elif tap == 2:
            value = x2
        else:
            value = x3
        weight = tl.load(W + channel * 4 + tap, mask, other=0)
        acc += value * weight
    if SILU:
        acc = acc / (1 + tl.exp(-acc))
    tl.store(Out + batch * OB + channel * OD, acc, mask)
    # Preserve the four-token cache owned by Transformers.
    tl.store(base, x0, mask)
    tl.store(base + ST, x1, mask)
    tl.store(base + 2 * ST, x2, mask)
    tl.store(base + 3 * ST, x3, mask)


def prefill(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None,
            activation: str | None) -> torch.Tensor:
    output = torch.empty_like(x)
    batch, channels, length = x.shape
    _prefill[(triton.cdiv(length, 8), batch, triton.cdiv(channels, 256))](
        x, weight, bias, output, length, channels, *x.stride(), *output.stride(),
        HAS_BIAS=bias is not None, SILU=activation is not None, BT=8, BD=256,
        num_warps=4, num_stages=2,
    )
    return output


def update(x: torch.Tensor, state: torch.Tensor, weight: torch.Tensor,
           bias: torch.Tensor | None, activation: str | None) -> torch.Tensor:
    output = torch.empty_like(x)
    _update[(x.shape[0], triton.cdiv(x.shape[1], 256))](
        x, state, weight, bias, output, x.shape[1], *x.stride()[:2],
        *state.stride(), *output.stride()[:2], HAS_BIAS=bias is not None,
        SILU=activation is not None, BD=256, num_warps=4,
    )
    return output


@triton.jit(do_not_specialize=["QGT", "KT", "QT", "KOT", "GT", "CT", "ST"])
def _dense_qk_kernel(QG: tl.tensor, K: tl.tensor, QO: tl.tensor, KO: tl.tensor, GO: tl.tensor,
            QW: tl.tensor, KW: tl.tensor, C: tl.tensor, S: tl.tensor,
            QGT: tl.int64, KT: tl.int64, QT: tl.int64, KOT: tl.int64, GT: tl.int64,
            CT: tl.int64, ST: tl.int64,
            NQ: tl.constexpr, NKV: tl.constexpr, D: tl.constexpr, R: tl.constexpr,
            HALF: tl.constexpr, HEAD_BLOCK: tl.constexpr, ROT_BLOCK: tl.constexpr,
            EPS: tl.constexpr) -> None:
    token, head = tl.program_id(0), tl.program_id(1)
    is_k = head >= NQ
    local_head = tl.where(is_k, head - NQ, head)
    if is_k:
        in_base = K + token * KT + local_head * D
        w_ptr = KW
        out_base = KO + token * KOT + local_head * D
    else:
        in_base = QG + token * QGT + local_head * 2 * D
        w_ptr = QW
        out_base = QO + token * QT + local_head * D
    head_offs = tl.arange(0, HEAD_BLOCK)
    head_mask = head_offs < D
    x = tl.load(in_base + head_offs, mask=head_mask, other=0.0).to(tl.float32)
    w = tl.load(w_ptr + head_offs, mask=head_mask, other=0.0).to(tl.float32)
    var = tl.sum(x * x, axis=0) / D
    inv_rms = tl.rsqrt(var + EPS)
    x_norm = (x * inv_rms * (w + 1.0)).to(tl.bfloat16).to(tl.float32)
    pass_mask = head_mask & (head_offs >= R)
    tl.store(out_base + head_offs, x_norm, mask=pass_mask)
    rot_offs = tl.arange(0, ROT_BLOCK)
    rot_mask = rot_offs < HALF
    xr1 = tl.load(in_base + rot_offs, mask=rot_mask, other=0.0).to(tl.float32)
    xr2 = tl.load(in_base + HALF + rot_offs, mask=rot_mask, other=0.0).to(tl.float32)
    wr1 = tl.load(w_ptr + rot_offs, mask=rot_mask, other=0.0).to(tl.float32)
    wr2 = tl.load(w_ptr + HALF + rot_offs, mask=rot_mask, other=0.0).to(tl.float32)
    xr1 = (xr1 * inv_rms * (wr1 + 1.0)).to(tl.bfloat16).to(tl.float32)
    xr2 = (xr2 * inv_rms * (wr2 + 1.0)).to(tl.bfloat16).to(tl.float32)
    cos = tl.load(C + token * CT + rot_offs, mask=rot_mask, other=0.0).to(tl.float32)
    sin = tl.load(S + token * ST + rot_offs, mask=rot_mask, other=0.0).to(tl.float32)
    tl.store(out_base + rot_offs, xr1 * cos - xr2 * sin, mask=rot_mask)
    tl.store(out_base + HALF + rot_offs, xr2 * cos + xr1 * sin, mask=rot_mask)
    gate_in = in_base + D
    if not is_k:
        gate_out = GO + token * GT + local_head * D
        gate = tl.load(gate_in + head_offs, mask=head_mask, other=0.0)
        tl.store(gate_out + head_offs, gate, mask=head_mask)


def dense_qk(q_gate: torch.Tensor, key: torch.Tensor, q_weight: torch.Tensor,
        k_weight: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor,
        eps: float) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
    batch, length, q_width = q_gate.shape
    dim, rotary_dim = q_weight.numel(), cos.shape[-1]
    q_heads, kv_heads = q_width // (2 * dim), key.shape[-1] // dim
    tokens = batch * length
    qg, k = q_gate.view(tokens, -1), key.view(tokens, -1)
    c, s = cos.view(tokens, rotary_dim), sin.view(tokens, rotary_dim)
    q_out = torch.empty(tokens, q_heads * dim, device=q_gate.device, dtype=q_gate.dtype)
    k_out = torch.empty(tokens, kv_heads * dim, device=key.device, dtype=key.dtype)
    gate = torch.empty(tokens, q_heads * dim, device=q_gate.device, dtype=q_gate.dtype)
    _dense_qk_kernel[(tokens, q_heads + kv_heads)](
        qg, k, q_out, k_out, gate, q_weight, k_weight, c, s,
        qg.stride(0), k.stride(0), q_out.stride(0), k_out.stride(0), gate.stride(0),
        c.stride(0), s.stride(0), NQ=q_heads, NKV=kv_heads, D=dim, R=rotary_dim,
        HALF=rotary_dim // 2, HEAD_BLOCK=triton.next_power_of_2(dim),
        ROT_BLOCK=triton.next_power_of_2(rotary_dim // 2), EPS=eps, num_warps=4)
    return (q_out.view(batch, length, q_heads, dim).transpose(1, 2),
            k_out.view(batch, length, kv_heads, dim).transpose(1, 2),
            gate.view(batch, length, q_heads * dim))


@triton.jit
def _layer_norm_fwd_1pass_kernel(
    X: tl.tensor, Y: tl.tensor, W: tl.tensor, B: tl.tensor, Z: tl.tensor,
    Mean: tl.tensor, Rstd: tl.tensor,
    stride_x_row: tl.int64, stride_y_row: tl.int64, stride_z_row: tl.int64,
    M: tl.int32, N: tl.constexpr, eps: tl.float32,
    BLOCK_N: tl.constexpr, ROWS_PER_BLOCK: tl.constexpr,
    HAS_BIAS: tl.constexpr, HAS_Z: tl.constexpr, NORM_BEFORE_GATE: tl.constexpr,
    IS_RMS_NORM: tl.constexpr, ACTIVATION: tl.constexpr,
    USE_GDC: tl.constexpr = False,
) -> None:
    if USE_GDC:
        tl.extra.cuda.gdc_wait()
    row_start = tl.program_id(0) * ROWS_PER_BLOCK
    group = tl.program_id(1)
    rows = row_start + tl.arange(0, ROWS_PER_BLOCK)
    cols = tl.arange(0, BLOCK_N)
    row_offsets = rows[:, None] * stride_x_row
    col_offsets = cols[None, :] + group * N
    X_base = X + row_offsets + col_offsets
    Y_base = Y + rows[:, None] * stride_y_row + col_offsets
    row_mask = rows[:, None] < M
    col_mask = cols[None, :] < N
    mask = row_mask & col_mask
    x = tl.load(X_base, mask=mask, other=0.0).to(tl.float32)
    if HAS_Z and (not NORM_BEFORE_GATE):
        Z_base = Z + rows[:, None] * stride_z_row + col_offsets
        z = tl.load(Z_base, mask=mask, other=0.0).to(tl.float32)
        if ACTIVATION == 'swish' or ACTIVATION == 'silu':
            x *= z * tl.sigmoid(z)
        elif ACTIVATION == 'sigmoid':
            x *= tl.sigmoid(z)
    if not IS_RMS_NORM:
        mean = tl.sum(x, axis=1) / N
        mean_offsets = group * M + rows
        mean_mask = rows < M
        tl.store(Mean + mean_offsets, mean, mask=mean_mask)
        xbar = tl.where(mask, x - mean[:, None], 0.0)
        var = tl.sum(xbar * xbar, axis=1) / N
    else:
        xbar = tl.where(mask, x, 0.0)
        var = tl.sum(xbar * xbar, axis=1) / N
        mean = 0.0
    rstd = tl.rsqrt(var + eps)
    rstd_offsets = group * M + rows
    rstd_mask = rows < M
    tl.store(Rstd + rstd_offsets, rstd, mask=rstd_mask)
    w_offsets = cols + group * N
    w_mask = cols < N
    w = tl.load(W + w_offsets, mask=w_mask, other=0.0).to(tl.float32)
    if HAS_BIAS:
        b = tl.load(B + w_offsets, mask=w_mask, other=0.0).to(tl.float32)
    if not IS_RMS_NORM:
        x_hat = (x - mean[:, None]) * rstd[:, None]
    else:
        x_hat = x * rstd[:, None]
    y = x_hat * w[None, :] + b[None, :] if HAS_BIAS else x_hat * w[None, :]
    if HAS_Z and NORM_BEFORE_GATE:
        Z_base = Z + rows[:, None] * stride_z_row + col_offsets
        z = tl.load(Z_base, mask=mask, other=0.0).to(tl.float32)
        if ACTIVATION == 'swish' or ACTIVATION == 'silu':
            y *= z * tl.sigmoid(z)
        elif ACTIVATION == 'sigmoid':
            y *= tl.sigmoid(z)
    tl.store(Y_base, y, mask=mask)
    if USE_GDC:
        tl.extra.cuda.gdc_launch_dependents()


def gated_norm(x: torch.Tensor, z: torch.Tensor, weight: torch.Tensor,
        eps: float) -> torch.Tensor:
    output = torch.empty_like(x)
    rstd = torch.empty((x.shape[0],), dtype=torch.float32, device=x.device)
    block_n = triton.next_power_of_2(x.shape[1])
    pdl = torch.cuda.get_device_capability(x.device)[0] >= 9
    launch = {"USE_GDC": True, "launch_pdl": True} if pdl else {}
    _layer_norm_fwd_1pass_kernel[(triton.cdiv(x.shape[0], 4), 1)](
        x, output, weight, None, z, None, rstd,
        x.stride(0), output.stride(0), z.stride(0), x.shape[0], x.shape[1], eps,
        BLOCK_N=block_n, ROWS_PER_BLOCK=4, HAS_BIAS=False, HAS_Z=True,
        NORM_BEFORE_GATE=True, IS_RMS_NORM=True, ACTIVATION="swish",
        num_warps=min(max(block_n // 256, 1), 8), **launch,
    )
    return output


@triton.jit
def _fused_sigmoid_mul_kernel(
    output_ptr: tl.tensor, attn_output_ptr: tl.tensor, gate_ptr: tl.tensor,
    gate_stride_row: tl.int64, gate_stride_head: tl.int64,
    hidden_dim: tl.constexpr, HEAD_DIM: tl.constexpr, BLOCK_H: tl.constexpr,
) -> None:
    """Fuse sigmoid(gate) * attn_output into a single kernel."""
    pid_row = tl.program_id(0).to(tl.int64)
    pid_block = tl.program_id(1)
    offsets = pid_block * BLOCK_H + tl.arange(0, BLOCK_H)
    mask = offsets < hidden_dim
    head = offsets // HEAD_DIM
    d = offsets - head * HEAD_DIM
    attn_off = pid_row * hidden_dim + offsets
    attn = tl.load(attn_output_ptr + attn_off, mask=mask, other=0.0).to(tl.float32)
    gate_off = pid_row * gate_stride_row + head * gate_stride_head + d
    g = tl.load(gate_ptr + gate_off, mask=mask, other=0.0).to(tl.float32)
    result = attn * tl.sigmoid(g)
    tl.store(output_ptr + attn_off, result, mask=mask)

def fused_sigmoid_mul(attn_output: torch.Tensor, gate: torch.Tensor, inplace: bool=False) -> torch.Tensor:
    """
    Fused sigmoid-mul for attention output gating.

    Equivalent to: attn_output * sigmoid(gate)

    The production Qwen3.5 path passes a 3D strided gate. A single hidden-block
    Triton kernel handles both that path and flat contiguous inputs.

    When inplace=True, writes result back to attn_output and returns it.

    Supports strided gate: if gate is 3D (num_tokens, num_heads, head_dim)
    and attn_output is 2D (num_tokens, hidden_dim), the kernel reads gate
    via explicit strides without requiring a contiguous copy.
    """
    if gate.ndim == 3 and attn_output.ndim == 2:
        num_tokens, num_heads, head_dim = gate.shape
        hidden_dim = num_heads * head_dim
        assert attn_output.shape == (num_tokens, hidden_dim)
        gate_stride_row = gate.stride(0)
        gate_stride_head = gate.stride(1)
    else:
        assert attn_output.shape == gate.shape, 'attn_output and gate must have the same shape'
        hidden_dim = attn_output.shape[-1]
        num_tokens = attn_output.numel() // hidden_dim
        head_dim = hidden_dim
        gate_stride_row = hidden_dim
        gate_stride_head = hidden_dim
    out = attn_output if inplace else torch.empty_like(attn_output)
    block_h = 1024 if num_tokens < 1024 else 2048
    grid = (num_tokens, triton.cdiv(hidden_dim, block_h))
    _fused_sigmoid_mul_kernel[grid](out, attn_output, gate, gate_stride_row, gate_stride_head, hidden_dim, HEAD_DIM=head_dim, BLOCK_H=block_h, num_warps=4)
    return out


@triton.jit
def _silu_mul_kernel(
    gate_ptr: tl.tensor, up_ptr: tl.tensor, output_ptr: tl.tensor,
    elements: tl.constexpr, BLOCK: tl.constexpr,
) -> None:
    offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
    mask = offsets < elements
    gate = tl.load(gate_ptr + offsets, mask=mask, other=0.0).to(tl.float32)
    up = tl.load(up_ptr + offsets, mask=mask, other=0.0).to(tl.float32)
    # Preserve the qualified CUDA --use_fast_math operations, including FTZ.
    output = tl.inline_asm_elementwise(
        "{ .reg .f32 exponent, denominator, activated; "
        "mul.ftz.f32 exponent, $1, 0fBFB8AA3B; "
        "ex2.approx.ftz.f32 exponent, exponent; "
        "add.ftz.f32 denominator, exponent, 0f3F800000; "
        "div.approx.ftz.f32 activated, $1, denominator; "
        "mul.ftz.f32 $0, $2, activated; }",
        constraints="=f,f,f", args=[gate, up], dtype=tl.float32,
        is_pure=True, pack=1,
    )
    tl.store(output_ptr + offsets, output, mask=mask)


def silu_mul(gate: torch.Tensor, up: torch.Tensor) -> torch.Tensor:
    """Match the qualified fused CUDA SiLU-mul before the BF16 output store."""
    if (gate.shape != up.shape or gate.device != up.device
            or gate.dtype != torch.bfloat16 or up.dtype != torch.bfloat16
            or not gate.is_cuda or not gate.is_contiguous() or not up.is_contiguous()):
        raise ValueError("Expected aligned contiguous CUDA BF16 gate and up tensors")
    output = torch.empty_like(gate)
    _silu_mul_kernel[(triton.cdiv(gate.numel(), 2048),)](
        gate, up, output, gate.numel(), BLOCK=2048, num_warps=8,
    )
    return output


# MIT License
#
# Copyright 2025 Thinking Machines Lab
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

def get_device_core_count() -> int:
    return torch.cuda.get_device_properties(torch.cuda.current_device()).multi_processor_count

@triton.jit
def _compute_pid(tile_id: tl.tensor, num_pid_in_group: tl.tensor, num_pid_m: tl.tensor, GROUP_SIZE_M: tl.tensor, NUM_SMS: tl.tensor) -> tuple[tl.tensor, tl.tensor]:
    group_id = tile_id // num_pid_in_group
    first_pid_m = group_id * GROUP_SIZE_M
    group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
    pid_m = first_pid_m + tile_id % group_size_m
    pid_n = tile_id % num_pid_in_group // group_size_m
    return (pid_m, pid_n)

@triton.jit
def matmul_kernel_persistent(a_ptr: tl.tensor, b_ptr: tl.tensor, c_ptr: tl.tensor, bias_ptr: tl.tensor, M: tl.int32, N: tl.int32, K: tl.int32, stride_am: tl.int32, stride_ak: tl.int32, stride_bk: tl.int32, stride_bn: tl.int32, stride_cm: tl.int32, stride_cn: tl.int32, BLOCK_SIZE_M: tl.constexpr, BLOCK_SIZE_N: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, GROUP_SIZE_M: tl.constexpr, NUM_SMS: tl.constexpr, A_LARGE: tl.constexpr, B_LARGE: tl.constexpr, C_LARGE: tl.constexpr, HAS_BIAS: tl.constexpr) -> None:
    start_pid = tl.program_id(axis=0)
    num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
    num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
    k_tiles = tl.cdiv(K, BLOCK_SIZE_K)
    num_tiles = num_pid_m * num_pid_n
    offs_k_for_mask = tl.arange(0, BLOCK_SIZE_K)
    num_pid_in_group = GROUP_SIZE_M * num_pid_n
    for tile_id in tl.range(start_pid, num_tiles, NUM_SMS, flatten=True):
        (pid_m, pid_n) = _compute_pid(tile_id, num_pid_in_group, num_pid_m, GROUP_SIZE_M, NUM_SMS)
        start_m = pid_m * BLOCK_SIZE_M
        start_n = pid_n * BLOCK_SIZE_N
        offs_am = start_m + tl.arange(0, BLOCK_SIZE_M)
        offs_bn = start_n + tl.arange(0, BLOCK_SIZE_N)
        if A_LARGE:
            offs_am = offs_am.to(tl.int64)
        if B_LARGE:
            offs_bn = offs_bn.to(tl.int64)
        offs_am = tl.where(offs_am < M, offs_am, 0)
        offs_bn = tl.where(offs_bn < N, offs_bn, 0)
        offs_am = tl.max_contiguous(tl.multiple_of(offs_am, BLOCK_SIZE_M), BLOCK_SIZE_M)
        offs_bn = tl.max_contiguous(tl.multiple_of(offs_bn, BLOCK_SIZE_N), BLOCK_SIZE_N)
        accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
        for ki in range(k_tiles):
            if A_LARGE or B_LARGE:
                offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K).to(tl.int64)
            else:
                offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K)
            a_ptrs = a_ptr + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak)
            b_ptrs = b_ptr + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn)
            a = tl.load(a_ptrs, mask=offs_k_for_mask[None, :] < K - ki * BLOCK_SIZE_K, other=0.0)
            b = tl.load(b_ptrs, mask=offs_k_for_mask[:, None] < K - ki * BLOCK_SIZE_K, other=0.0)
            accumulator = tl.dot(a, b, accumulator)
        offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
        offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
        if C_LARGE:
            offs_cm = offs_cm.to(tl.int64)
            offs_cn = offs_cn.to(tl.int64)
        c_ptrs = c_ptr + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :]
        c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N)
        if HAS_BIAS:
            bias_ptrs = bias_ptr + offs_cn
            bias = tl.load(bias_ptrs, mask=offs_cn < N, other=0.0).to(tl.float32)
            accumulator += bias
        if c_ptr.dtype.element_ty == tl.float8e4nv:
            c = accumulator.to(tl.float8e4nv)
        elif c_ptr.dtype.element_ty == tl.bfloat16:
            c = accumulator.to(tl.bfloat16)
        elif c_ptr.dtype.element_ty == tl.float32:
            c = accumulator.to(tl.float32)
        else:
            c = accumulator.to(tl.float16)
        tl.store(c_ptrs, c, mask=c_mask)

def _matmul_persistent_triton(a: torch.Tensor, b: torch.Tensor, bias: torch.Tensor | None=None) -> torch.Tensor:
    assert a.shape[1] == b.shape[0], 'Incompatible dimensions'
    assert a.dtype == b.dtype, 'Incompatible dtypes'
    assert bias is None or bias.dim() == 1, 'Expected a one-dimensional bias'
    NUM_SMS = get_device_core_count()
    (M, K) = a.shape
    (K, N) = b.shape
    dtype = a.dtype
    c = torch.empty((M, N), device=a.device, dtype=dtype)

    def grid(META: dict[str, Any]) -> tuple[int]:
        return (min(NUM_SMS, triton.cdiv(M, META['BLOCK_SIZE_M']) * triton.cdiv(N, META['BLOCK_SIZE_N'])),)
    configs = {torch.bfloat16: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}, torch.float16: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}, torch.float32: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}}
    matmul_kernel_persistent[grid](a, b, c, bias, M, N, K, a.stride(0), a.stride(1), b.stride(0), b.stride(1), c.stride(0), c.stride(1), NUM_SMS=NUM_SMS, A_LARGE=a.numel() > 2 ** 31, B_LARGE=b.numel() > 2 ** 31, C_LARGE=c.numel() > 2 ** 31, HAS_BIAS=bias is not None, **configs[dtype])
    return c

@triton.jit
def mean_kernel(input_ptr: tl.tensor, output_ptr: tl.tensor, input_stride0: tl.int32, input_stride1: tl.int32, input_stride2: tl.int32, output_stride0: tl.int32, output_stride1: tl.int32, M: tl.int32, N: tl.int32, K: tl.int32, BLOCK_SIZE: tl.constexpr) -> None:
    """
    Kernel for computing mean along a single dimension.
    Input is viewed as (M, N, K) where N is the dimension being reduced.
    """
    pid = tl.program_id(0)
    m_idx = pid // K
    k_idx = pid % K
    if m_idx >= M or k_idx >= K:
        return
    acc = 0.0
    for n_start in range(0, N, BLOCK_SIZE):
        n_offsets = n_start + tl.arange(0, BLOCK_SIZE)
        mask = n_offsets < N
        input_idx = m_idx * input_stride0 + n_offsets * input_stride1 + k_idx * input_stride2
        vals = tl.load(input_ptr + input_idx, mask=mask, other=0.0)
        acc += tl.sum(vals)
    mean_val = acc / N
    output_idx = m_idx * output_stride0 + k_idx * output_stride1
    tl.store(output_ptr + output_idx, mean_val)

def mean_dim(input: torch.Tensor, dim: int, keepdim: bool=False, dtype: torch.dtype | None=None) -> torch.Tensor:
    """
    Triton implementation of torch.mean with single dimension reduction.

    Args:
        input: Input tensor
        dim: Single dimension along which to compute mean
        keepdim: Whether to keep the reduced dimension
        dtype: Output dtype. If None, uses input dtype (or float32 for integer inputs)

    Returns:
        Tensor with mean values along specified dimension
    """
    assert input.is_cuda or input.is_xpu, 'Input must be a CUDA or XPU tensor'
    assert -input.ndim <= dim < input.ndim, f'Invalid dimension {dim} for tensor with {input.ndim} dimensions'
    if dim < 0:
        dim = dim + input.ndim
    if dtype is None:
        if input.dtype in [torch.int8, torch.int16, torch.int32, torch.int64]:
            dtype = torch.float32
        else:
            dtype = input.dtype
    if input.dtype != dtype:
        input = input.to(dtype)
    shape = list(input.shape)
    M = 1
    for i in range(dim):
        M *= shape[i]
    N = shape[dim]
    K = 1
    for i in range(dim + 1, len(shape)):
        K *= shape[i]
    input_3d = input.reshape(M, N, K)
    if keepdim:
        output_shape = shape.copy()
        output_shape[dim] = 1
    else:
        output_shape = shape[:dim] + shape[dim + 1:]
    output = torch.empty(output_shape, dtype=dtype, device=input.device)
    if keepdim:
        output_2d = output.reshape(M, 1, K).squeeze(1)
    else:
        output_2d = output.reshape(M, K)
    grid = (M * K,)
    BLOCK_SIZE = 1024
    mean_kernel[grid](input_3d, output_2d, input_3d.stride(0), input_3d.stride(1), input_3d.stride(2), output_2d.stride(0), output_2d.stride(1) if output_2d.ndim > 1 else 0, M, N, K, BLOCK_SIZE)
    return output

def mean_batch_invariant(input: torch.Tensor, dim: list[int], keepdim: bool=False, dtype: torch.dtype | None=None) -> torch.Tensor:
    assert dtype is None or dtype == torch.float32, f'unsupported dtype: {dtype}'
    if len(dim) == 1:
        return mean_dim(input, dim[0], keepdim=keepdim)
    else:
        assert input.dtype in {torch.float16, torch.bfloat16, torch.float32}, 'only float types supported for now'
        n_elems = 1
        for d in dim:
            n_elems *= input.shape[d]
        return torch.sum(input, dim=dim, keepdim=keepdim, dtype=torch.float32) / n_elems

@triton.jit
def bmm_kernel_persistent(a_ptr: tl.tensor, b_ptr: tl.tensor, c_ptr: tl.tensor, B: tl.int32, M: tl.int32, N: tl.int32, K: tl.int32, stride_ab: tl.int32, stride_am: tl.int32, stride_ak: tl.int32, stride_bb: tl.int32, stride_bk: tl.int32, stride_bn: tl.int32, stride_cb: tl.int32, stride_cm: tl.int32, stride_cn: tl.int32, BLOCK_SIZE_M: tl.constexpr, BLOCK_SIZE_N: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, GROUP_SIZE_M: tl.constexpr, NUM_SMS: tl.constexpr, A_LARGE: tl.constexpr, B_LARGE: tl.constexpr, C_LARGE: tl.constexpr) -> None:
    """
    Batched matrix multiplication kernel that processes batches in parallel.
    Each tile processes a (BLOCK_SIZE_M, BLOCK_SIZE_N) output block for a specific batch.
    """
    start_pid = tl.program_id(axis=0)
    num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
    num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
    k_tiles = tl.cdiv(K, BLOCK_SIZE_K)
    num_tiles_per_batch = num_pid_m * num_pid_n
    num_tiles_total = B * num_tiles_per_batch
    offs_k_for_mask = tl.arange(0, BLOCK_SIZE_K)
    num_pid_in_group = GROUP_SIZE_M * num_pid_n
    for tile_id in tl.range(start_pid, num_tiles_total, NUM_SMS, flatten=True):
        batch_idx = tile_id // num_tiles_per_batch
        tile_in_batch = tile_id % num_tiles_per_batch
        (pid_m, pid_n) = _compute_pid(tile_in_batch, num_pid_in_group, num_pid_m, GROUP_SIZE_M, NUM_SMS)
        start_m = pid_m * BLOCK_SIZE_M
        start_n = pid_n * BLOCK_SIZE_N
        offs_am = start_m + tl.arange(0, BLOCK_SIZE_M)
        offs_bn = start_n + tl.arange(0, BLOCK_SIZE_N)
        if A_LARGE:
            offs_am = offs_am.to(tl.int64)
        if B_LARGE:
            offs_bn = offs_bn.to(tl.int64)
        offs_am = tl.where(offs_am < M, offs_am, 0)
        offs_bn = tl.where(offs_bn < N, offs_bn, 0)
        offs_am = tl.max_contiguous(tl.multiple_of(offs_am, BLOCK_SIZE_M), BLOCK_SIZE_M)
        offs_bn = tl.max_contiguous(tl.multiple_of(offs_bn, BLOCK_SIZE_N), BLOCK_SIZE_N)
        if A_LARGE or B_LARGE:
            batch_idx_typed = batch_idx.to(tl.int64)
        else:
            batch_idx_typed = batch_idx
        accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
        for ki in range(k_tiles):
            if A_LARGE or B_LARGE:
                offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K).to(tl.int64)
            else:
                offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K)
            a_ptrs = a_ptr + (batch_idx_typed * stride_ab + offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak)
            b_ptrs = b_ptr + (batch_idx_typed * stride_bb + offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn)
            a = tl.load(a_ptrs, mask=offs_k_for_mask[None, :] < K - ki * BLOCK_SIZE_K, other=0.0)
            b = tl.load(b_ptrs, mask=offs_k_for_mask[:, None] < K - ki * BLOCK_SIZE_K, other=0.0)
            accumulator = tl.dot(a, b, accumulator)
        offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
        offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
        if C_LARGE:
            offs_cm = offs_cm.to(tl.int64)
            offs_cn = offs_cn.to(tl.int64)
        c_ptrs = c_ptr + batch_idx_typed * stride_cb + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :]
        c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N)
        if c_ptr.dtype.element_ty == tl.float8e4nv:
            c = accumulator.to(tl.float8e4nv)
        elif c_ptr.dtype.element_ty == tl.bfloat16:
            c = accumulator.to(tl.bfloat16)
        elif c_ptr.dtype.element_ty == tl.float32:
            c = accumulator.to(tl.float32)
        else:
            c = accumulator.to(tl.float16)
        tl.store(c_ptrs, c, mask=c_mask)

def bmm_batch_invariant(a: torch.Tensor, b: torch.Tensor, *, out: torch.Tensor | None=None) -> torch.Tensor:
    if a.ndim == 3 and b.ndim == 3:
        assert a.shape[0] == b.shape[0], 'Batch sizes must match'
        assert a.shape[2] == b.shape[1], 'Incompatible dimensions'
        assert a.dtype == b.dtype, 'Incompatible dtypes'
        B = a.shape[0]
        M = a.shape[1]
        K = a.shape[2]
        N = b.shape[2]
        dtype = a.dtype
        if out is None:
            c = torch.empty((B, M, N), device=a.device, dtype=dtype)
        else:
            c = out
        NUM_SMS = get_device_core_count()
        configs = {torch.bfloat16: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}, torch.float16: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}, torch.float32: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}}
        config = configs.get(dtype)
        if config is None:
            raise ValueError(f'Unsupported dtype {dtype} for bmm_batch_invariant. Supported dtypes are: {list(configs.keys())}')
        num_tiles_per_batch = triton.cdiv(M, config['BLOCK_SIZE_M']) * triton.cdiv(N, config['BLOCK_SIZE_N'])
        num_tiles_total = B * num_tiles_per_batch
        grid = (min(NUM_SMS, num_tiles_total),)
        bmm_kernel_persistent[grid](a, b, c, B, M, N, K, a.stride(0), a.stride(1), a.stride(2), b.stride(0), b.stride(1), b.stride(2), c.stride(0), c.stride(1), c.stride(2), NUM_SMS=NUM_SMS, A_LARGE=a.numel() > 2 ** 31, B_LARGE=b.numel() > 2 ** 31, C_LARGE=c.numel() > 2 ** 31, **config)
        return c
    else:
        raise ValueError(f'bmm_batch_invariant expects 3D tensors, got shapes {a.shape} and {b.shape}')



@triton.jit(do_not_specialize=["count"])
def _bf16_pair_kernel(value: tl.tensor, high: tl.tensor, low: tl.tensor,
                 count: tl.int32, limit: tl.constexpr, block: tl.constexpr) -> None:
    offsets = tl.program_id(0) * block + tl.arange(0, block)
    current = tl.load(value + offsets, offsets < count, other=0)
    clipped = tl.minimum(tl.maximum(current, -limit, propagate_nan=tl.PropagateNan.ALL),
                         limit, propagate_nan=tl.PropagateNan.ALL)
    upper = clipped.to(tl.bfloat16)
    lower = (current - upper.to(tl.float32)).to(tl.bfloat16)
    lower = tl.where(current == 0, upper, lower)
    tl.store(high + offsets, upper, offsets < count)
    tl.store(low + offsets, lower, offsets < count)



def bf16_pair_sum(value: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
    if not value.is_cuda or value.dtype != torch.float32 or not value.is_contiguous():
        raise ValueError("Expected contiguous CUDA FP32 residuals")
    high = torch.empty_like(value, dtype=torch.bfloat16)
    low = torch.empty_like(value, dtype=torch.bfloat16)
    count = value.numel()
    if count:
        _bf16_pair_kernel[(triton.cdiv(count, 1024),)](
            value, high, low, count, torch.finfo(torch.bfloat16).max, 1024,
            num_warps=4, enable_fp_fusion=False,
        )
    return high, low



@triton.jit(do_not_specialize=["count"])
def _residual_kernel(first: tl.tensor, second: tl.tensor, output: tl.tensor,
                     count: tl.int32, block: tl.constexpr) -> None:
    offsets = tl.program_id(0) * block + tl.arange(0, block)
    upper = tl.load(first + offsets, offsets < count, other=0).to(tl.bfloat16).to(tl.float32)
    lower = tl.load(second + offsets, offsets < count, other=0).to(tl.float32)
    tl.store(output + offsets, upper + lower, offsets < count)



def residual_sum(first: torch.Tensor, second: torch.Tensor) -> torch.Tensor:
    if (not first.is_cuda or second.device != first.device or second.shape != first.shape
            or first.dtype not in (torch.bfloat16, torch.float32)
            or second.dtype != torch.bfloat16
            or not first.is_contiguous() or not second.is_contiguous()):
        raise ValueError("Expected aligned contiguous CUDA residuals")
    output = torch.empty_like(first, dtype=torch.float32)
    count = first.numel()
    if count:
        _residual_kernel[(triton.cdiv(count, 1024),)](
            first, second, output, count, 1024, num_warps=4, enable_fp_fusion=False,
        )
    return output