File size: 2,532 Bytes
f5d6a7c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
"""Compare fused causal convolution outputs/gradients with PyTorch's reference."""

import json
from pathlib import Path

import torch
import torch.nn.functional as F
from causal_conv1d import causal_conv1d_fn

torch.backends.cudnn.allow_tf32 = False
torch.backends.cuda.matmul.allow_tf32 = False
results = []
for dtype, channel_last, with_bias, length in (
    (dtype, layout, bias, length)
    for dtype in [torch.float32, torch.bfloat16]
    for layout in [False, True]
    for bias in [False, True]
    for length in [511, 512, 513, 1025, 1537, 1730]
):
    torch.manual_seed(3407)
    if channel_last:
        x = torch.randn(2, length, 128, device="cuda", dtype=dtype).transpose(1, 2).requires_grad_()
    else:
        x = torch.randn(2, 128, length, device="cuda", dtype=dtype, requires_grad=True)
    w = torch.randn(128, 4, device="cuda", dtype=dtype, requires_grad=True)
    b = torch.randn(128, device="cuda", dtype=dtype, requires_grad=True) if with_bias else None
    inputs = (x, w, b) if with_bias else (x, w)
    grad = torch.randn_like(x)
    ref = F.silu(F.conv1d(x, w.unsqueeze(1), b, padding=3, groups=128)[..., :x.shape[-1]])
    g_ref = torch.autograd.grad(ref, inputs, grad)
    # Match production's padding to a multiple of 8. The reference stays
    # unpadded, so this also verifies that padding does not change active tokens.
    padded = F.pad(x, (0, (-x.shape[-1]) % 8))
    if channel_last:
        padded = padded.transpose(1, 2).contiguous().transpose(1, 2)
    out = causal_conv1d_fn(padded, w, b, activation="silu")[..., :x.shape[-1]]
    g_out = torch.autograd.grad(out, inputs, grad)
    errors = []
    for name, actual, expected in zip(["output", "input_gradient", "weight_gradient", "bias_gradient"], [out, *g_out], [ref, *g_ref]):
        relative = ((actual.float() - expected.float()).norm() / expected.float().norm()).item()
        errors.append(relative)
        assert relative < (0.012 if dtype == torch.bfloat16 else 1e-4), (dtype, channel_last, with_bias, length, name, relative)
    results.append({"dtype": str(dtype), "channel_last": channel_last, "bias": with_bias, "sequence_length": length, "relative_l2_output_and_gradients": errors})
Path("kernel_correctness.json").write_text(json.dumps(results, indent=2))
summary = {dtype: max(max(r["relative_l2_output_and_gradients"]) for r in results if r["dtype"] == dtype) for dtype in ["torch.float32", "torch.bfloat16"]}
print(json.dumps({"passed_cases": len(results), "maximum_relative_l2_error": summary}, indent=2), flush=True)