Instructions to use PriyaMishra/temp with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use PriyaMishra/temp with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- Unsloth Desktop
Download project/verify_kernels.py from PriyaMishra/temp: direct link, hf CLI and curl.
- Browser
- Download file 2.53 kB
-
https://huggingface.co/PriyaMishra/temp/resolve/main/project/verify_kernels.py
- Command line
-
hf download hf://PriyaMishra/temp/project/verify_kernels.py
-
curl -L -o verify_kernels.py https://huggingface.co/PriyaMishra/temp/resolve/main/project/verify_kernels.py
2.53 kB
| """Compare fused causal convolution outputs/gradients with PyTorch's reference.""" | |
| import json | |
| from pathlib import Path | |
| import torch | |
| import torch.nn.functional as F | |
| from causal_conv1d import causal_conv1d_fn | |
| torch.backends.cudnn.allow_tf32 = False | |
| torch.backends.cuda.matmul.allow_tf32 = False | |
| results = [] | |
| for dtype, channel_last, with_bias, length in ( | |
| (dtype, layout, bias, length) | |
| for dtype in [torch.float32, torch.bfloat16] | |
| for layout in [False, True] | |
| for bias in [False, True] | |
| for length in [511, 512, 513, 1025, 1537, 1730] | |
| ): | |
| torch.manual_seed(3407) | |
| if channel_last: | |
| x = torch.randn(2, length, 128, device="cuda", dtype=dtype).transpose(1, 2).requires_grad_() | |
| else: | |
| x = torch.randn(2, 128, length, device="cuda", dtype=dtype, requires_grad=True) | |
| w = torch.randn(128, 4, device="cuda", dtype=dtype, requires_grad=True) | |
| b = torch.randn(128, device="cuda", dtype=dtype, requires_grad=True) if with_bias else None | |
| inputs = (x, w, b) if with_bias else (x, w) | |
| grad = torch.randn_like(x) | |
| ref = F.silu(F.conv1d(x, w.unsqueeze(1), b, padding=3, groups=128)[..., :x.shape[-1]]) | |
| g_ref = torch.autograd.grad(ref, inputs, grad) | |
| # Match production's padding to a multiple of 8. The reference stays | |
| # unpadded, so this also verifies that padding does not change active tokens. | |
| padded = F.pad(x, (0, (-x.shape[-1]) % 8)) | |
| if channel_last: | |
| padded = padded.transpose(1, 2).contiguous().transpose(1, 2) | |
| out = causal_conv1d_fn(padded, w, b, activation="silu")[..., :x.shape[-1]] | |
| g_out = torch.autograd.grad(out, inputs, grad) | |
| errors = [] | |
| for name, actual, expected in zip(["output", "input_gradient", "weight_gradient", "bias_gradient"], [out, *g_out], [ref, *g_ref]): | |
| relative = ((actual.float() - expected.float()).norm() / expected.float().norm()).item() | |
| errors.append(relative) | |
| assert relative < (0.012 if dtype == torch.bfloat16 else 1e-4), (dtype, channel_last, with_bias, length, name, relative) | |
| results.append({"dtype": str(dtype), "channel_last": channel_last, "bias": with_bias, "sequence_length": length, "relative_l2_output_and_gradients": errors}) | |
| Path("kernel_correctness.json").write_text(json.dumps(results, indent=2)) | |
| summary = {dtype: max(max(r["relative_l2_output_and_gradients"]) for r in results if r["dtype"] == dtype) for dtype in ["torch.float32", "torch.bfloat16"]} | |
| print(json.dumps({"passed_cases": len(results), "maximum_relative_l2_error": summary}, indent=2), flush=True) | |