Check that torch has flash attention before taking the flash path (fixes CUDA on Windows)

#6
by YAKA001 - opened
Files changed (1) hide show
  1. attention.py +2 -0
attention.py CHANGED
@@ -42,6 +42,8 @@ def _fused(q: torch.Tensor) -> bool:
42
  """Whether the flash kernels take these queries: a GPU in half precision, ROCm or CUDA from Ampere on."""
43
  if not q.is_cuda or q.dtype not in (torch.float16, torch.bfloat16):
44
  return False
 
 
45
  return bool(torch.version.hip) or torch.cuda.get_device_capability(q.device)[0] >= 8
46
 
47
 
 
42
  """Whether the flash kernels take these queries: a GPU in half precision, ROCm or CUDA from Ampere on."""
43
  if not q.is_cuda or q.dtype not in (torch.float16, torch.bfloat16):
44
  return False
45
+ if not torch.backends.cuda.is_flash_attention_available():
46
+ return False
47
  return bool(torch.version.hip) or torch.cuda.get_device_capability(q.device)[0] >= 8
48
 
49