Raphael Glon commited on
Commit
6da9c81
·
unverified ·
1 Parent(s): 2989a24

Add debug logging for Blackwell worker_init CUDA failure

Browse files
Files changed (1) hide show
  1. app.py +24 -0
app.py CHANGED
@@ -1,5 +1,22 @@
1
  import spaces
2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  import torch
4
  import torchvision.transforms.functional as TF
5
  import numpy as np
@@ -7,6 +24,13 @@ import random
7
  import os
8
  import sys
9
 
 
 
 
 
 
 
 
10
  from diffusers.utils import load_image
11
  from diffusers import EulerDiscreteScheduler, T2IAdapter
12
 
 
1
  import spaces
2
 
3
+ # Patch spaces torch init to log debug info before the failing cuda call
4
+ try:
5
+ import spaces.zero.torch.patching as _sp
6
+ _orig_init = _sp.init
7
+ def _debug_init(nvidia_uuid):
8
+ import os, torch
9
+ print(f"[DEBUG worker_init] nvidia_uuid={nvidia_uuid!r}")
10
+ print(f"[DEBUG worker_init] CUDA_VISIBLE_DEVICES={os.environ.get('CUDA_VISIBLE_DEVICES')!r}")
11
+ try:
12
+ print(f"[DEBUG worker_init] torch.cuda.device_count() before init={torch.cuda.device_count()}")
13
+ except Exception as e:
14
+ print(f"[DEBUG worker_init] device_count() raised: {e}")
15
+ return _orig_init(nvidia_uuid)
16
+ _sp.init = _debug_init
17
+ except Exception as e:
18
+ print(f"[DEBUG] Could not patch spaces.zero.torch.patching: {e}")
19
+
20
  import torch
21
  import torchvision.transforms.functional as TF
22
  import numpy as np
 
24
  import os
25
  import sys
26
 
27
+ # Log CUDA state at startup (no GPU expected here in ZeroGPU)
28
+ print(f"[DEBUG startup] CUDA_VISIBLE_DEVICES={os.environ.get('CUDA_VISIBLE_DEVICES')!r}")
29
+ try:
30
+ print(f"[DEBUG startup] torch.cuda.device_count()={torch.cuda.device_count()}")
31
+ except Exception as e:
32
+ print(f"[DEBUG startup] device_count() raised: {e}")
33
+
34
  from diffusers.utils import load_image
35
  from diffusers import EulerDiscreteScheduler, T2IAdapter
36