temp / project /verify_kernels.py
PriyaMishra's picture
Copy model from AjayRangoji/temp
f5d6a7c verified
Raw History Blame Contribute Delete
2.53 kB
"""Compare fused causal convolution outputs/gradients with PyTorch's reference."""
import json
from pathlib import Path
import torch
import torch.nn.functional as F
from causal_conv1d import causal_conv1d_fn
torch.backends.cudnn.allow_tf32 = False
torch.backends.cuda.matmul.allow_tf32 = False
results = []
for dtype, channel_last, with_bias, length in (
(dtype, layout, bias, length)
for dtype in [torch.float32, torch.bfloat16]
for layout in [False, True]
for bias in [False, True]
for length in [511, 512, 513, 1025, 1537, 1730]
):
torch.manual_seed(3407)
if channel_last:
x = torch.randn(2, length, 128, device="cuda", dtype=dtype).transpose(1, 2).requires_grad_()
else:
x = torch.randn(2, 128, length, device="cuda", dtype=dtype, requires_grad=True)
w = torch.randn(128, 4, device="cuda", dtype=dtype, requires_grad=True)
b = torch.randn(128, device="cuda", dtype=dtype, requires_grad=True) if with_bias else None
inputs = (x, w, b) if with_bias else (x, w)
grad = torch.randn_like(x)
ref = F.silu(F.conv1d(x, w.unsqueeze(1), b, padding=3, groups=128)[..., :x.shape[-1]])
g_ref = torch.autograd.grad(ref, inputs, grad)
# Match production's padding to a multiple of 8. The reference stays
# unpadded, so this also verifies that padding does not change active tokens.
padded = F.pad(x, (0, (-x.shape[-1]) % 8))
if channel_last:
padded = padded.transpose(1, 2).contiguous().transpose(1, 2)
out = causal_conv1d_fn(padded, w, b, activation="silu")[..., :x.shape[-1]]
g_out = torch.autograd.grad(out, inputs, grad)
errors = []
for name, actual, expected in zip(["output", "input_gradient", "weight_gradient", "bias_gradient"], [out, *g_out], [ref, *g_ref]):
relative = ((actual.float() - expected.float()).norm() / expected.float().norm()).item()
errors.append(relative)
assert relative < (0.012 if dtype == torch.bfloat16 else 1e-4), (dtype, channel_last, with_bias, length, name, relative)
results.append({"dtype": str(dtype), "channel_last": channel_last, "bias": with_bias, "sequence_length": length, "relative_l2_output_and_gradients": errors})
Path("kernel_correctness.json").write_text(json.dumps(results, indent=2))
summary = {dtype: max(max(r["relative_l2_output_and_gradients"]) for r in results if r["dtype"] == dtype) for dtype in ["torch.float32", "torch.bfloat16"]}
print(json.dumps({"passed_cases": len(results), "maximum_relative_l2_error": summary}, indent=2), flush=True)