harp-dev Claude Opus 5.5 commited on
Commit
4d3ab93
·
verified ·
1 Parent(s): a8c158e

Build the model inside the GPU call again

Browse files

Building the default size in the worker failed at startup with No CUDA GPUs are available: muscriptor loads its safetensors straight onto CUDA, which ZeroGPU emulation does not answer outside a call, and its conditioners keep the device they were built for. The model is built inside the call again, as before, from weights the worker downloads.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Files changed (1) hide show
  1. app.py +7 -9
app.py CHANGED
@@ -48,11 +48,13 @@ DEFAULT_SIZE = "medium"
48
 
49
 
50
  def _get_model(variant: str) -> TranscriptionModel:
51
- """Builds the TranscriptionModel of one size, once.
52
 
53
- The default size is built in the worker at startup. Holding all three would keep about 7 GB
54
- there, so another size is built when a request picks it, inside the call, where ZeroGPU builds
55
- it again on every call, which takes seconds from local weights.
 
 
56
  """
57
  if variant not in _models:
58
  _models[variant] = TranscriptionModel.load_model(_weights[variant], device=DEVICE)
@@ -60,9 +62,7 @@ def _get_model(variant: str) -> TranscriptionModel:
60
 
61
 
62
  # Fetched in the worker, which runs process_fn and starts as the server comes up, so that no
63
- # request downloads inside its GPU reservation, and the default size is built and placed there, on
64
- # its main thread, where ZeroGPU's emulation records the placement. The server never runs
65
- # process_fn.
66
  if __name__ != "__main__":
67
  for _size, _revision in _WEIGHT_REVISIONS.items():
68
  _weights[_size] = Path(snapshot_download(
@@ -81,8 +81,6 @@ if __name__ != "__main__":
81
  revision="4afbe376535aaab71b76ae2ca2d7c16b08877321",
82
  ))
83
 
84
- _get_model(DEFAULT_SIZE)
85
-
86
 
87
  def _resolve_instruments(text: str) -> tuple[list[str] | None, str]:
88
  """Resolve the comma-separated instruments box into exact group names.
 
48
 
49
 
50
  def _get_model(variant: str) -> TranscriptionModel:
51
+ """Builds the TranscriptionModel of one size, on the GPU the call holds.
52
 
53
+ Built inside the call, so ZeroGPU builds it again on every call, which takes seconds from local
54
+ weights. It cannot be built in the worker at startup, since muscriptor loads its safetensors
55
+ straight onto CUDA, which ZeroGPU's emulation does not answer outside a call, and its
56
+ conditioners keep the device they were built for, so building on the CPU and moving it is not
57
+ enough either.
58
  """
59
  if variant not in _models:
60
  _models[variant] = TranscriptionModel.load_model(_weights[variant], device=DEVICE)
 
62
 
63
 
64
  # Fetched in the worker, which runs process_fn and starts as the server comes up, so that no
65
+ # request downloads inside its GPU reservation. The server never runs process_fn.
 
 
66
  if __name__ != "__main__":
67
  for _size, _revision in _WEIGHT_REVISIONS.items():
68
  _weights[_size] = Path(snapshot_download(
 
81
  revision="4afbe376535aaab71b76ae2ca2d7c16b08877321",
82
  ))
83
 
 
 
84
 
85
  def _resolve_instruments(text: str) -> tuple[list[str] | None, str]:
86
  """Resolve the comma-separated instruments box into exact group names.