raylim commited on
Commit
4d7da11
·
unverified ·
1 Parent(s): 8d94a99

Reduce GPU durations to fit 300s total limit per request

Browse files

- CTransPath: 180s b†’ 60s
- Optimus: 300s → 120s
- Aeon: 90s → 60s
- Paladin: 90s → 60s
- TOTAL: 300s (was 660s)

ZeroGPU likely has 300s TOTAL limit per request, not per call.
Duration reserves GPU time, so total must fit within limit.

Files changed (1) hide show
  1. src/mosaic/analysis.py +25 -125
src/mosaic/analysis.py CHANGED
@@ -40,36 +40,8 @@ from mosaic.inference import run_aeon, run_paladin
40
 
41
 
42
  @spaces.GPU(duration=180)
43
- def _extract_ctranspath_features_single_chunk(coords_chunk, slide_path, attrs, num_workers, batch_size):
44
- """Extract CTransPath features for ONE chunk with its own GPU allocation.
45
-
46
- This function gets its own GPU token for up to 180 seconds.
47
-
48
- Args:
49
- coords_chunk: Chunk of tissue tile coordinates
50
- slide_path: Path to the whole slide image file
51
- attrs: Slide attributes
52
- num_workers: Number of worker processes
53
- batch_size: Batch size for inference
54
-
55
- Returns:
56
- CTransPath features for this chunk
57
- """
58
- features, _ = get_features(
59
- coords_chunk,
60
- slide_path,
61
- attrs,
62
- model_type=ModelType.CTRANSPATH,
63
- model_path="data/ctranspath.pth",
64
- num_workers=num_workers,
65
- batch_size=batch_size,
66
- use_gpu=True,
67
- )
68
- return features
69
-
70
-
71
  def _extract_ctranspath_features(coords, slide_path, attrs, num_workers):
72
- """Extract CTransPath features, splitting into multiple GPU calls if needed.
73
 
74
  Args:
75
  coords: Tissue tile coordinates
@@ -82,85 +54,34 @@ def _extract_ctranspath_features(coords, slide_path, attrs, num_workers):
82
  """
83
  if IS_ZEROGPU:
84
  num_workers = 0
85
- # Split into chunks - each chunk gets its own GPU call/token
86
- chunk_size = 10000 # Increased from 2000 - each chunk gets full 180s
87
- logger.info(f"Running CTransPath on ZeroGPU: splitting {len(coords)} tiles into chunks of {chunk_size}")
88
  else:
89
  num_workers = max(num_workers, 8)
90
- chunk_size = len(coords) # Process all at once
91
  logger.info(f"Running CTransPath with {num_workers} workers")
92
 
93
  batch_size = 128 if IS_ZEROGPU else 64
94
-
95
  start_time = pd.Timestamp.now()
96
 
97
- # Process each chunk with separate GPU allocation
98
- all_features = []
99
- for i in range(0, len(coords), chunk_size):
100
- chunk_coords = coords[i:i+chunk_size]
101
- chunk_num = i // chunk_size + 1
102
- total_chunks = (len(coords) + chunk_size - 1) // chunk_size
103
-
104
- logger.info(f"Processing CTransPath chunk {chunk_num}/{total_chunks} ({len(chunk_coords)} tiles)")
105
-
106
- if IS_ZEROGPU:
107
- # Each call gets fresh GPU token
108
- chunk_features = _extract_ctranspath_features_single_chunk(
109
- chunk_coords, slide_path, attrs, num_workers, batch_size
110
- )
111
- else:
112
- # Non-ZeroGPU: direct call without decorator overhead
113
- chunk_features, _ = get_features(
114
- chunk_coords, slide_path, attrs,
115
- model_type=ModelType.CTRANSPATH,
116
- model_path="data/ctranspath.pth",
117
- num_workers=num_workers,
118
- batch_size=batch_size,
119
- use_gpu=True,
120
- )
121
-
122
- all_features.append(chunk_features)
123
- logger.info(f"Chunk {chunk_num}/{total_chunks} completed")
124
-
125
- # Concatenate all features
126
- import numpy as np
127
- ctranspath_features = np.concatenate(all_features, axis=0) if len(all_features) > 1 else all_features[0]
128
-
129
- end_time = pd.Timestamp.now()
130
- logger.info(f"CTransPath extraction took {end_time - start_time} total")
131
-
132
- return ctranspath_features, coords
133
-
134
-
135
- @spaces.GPU(duration=300)
136
- def _extract_optimus_features_single_chunk(coords_chunk, slide_path, attrs, num_workers, batch_size):
137
- """Extract Optimus features for ONE chunk with its own GPU allocation.
138
-
139
- Args:
140
- coords_chunk: Chunk of tissue tile coordinates
141
- slide_path: Path to the whole slide image file
142
- attrs: Slide attributes
143
- num_workers: Number of worker processes
144
- batch_size: Batch size for inference
145
-
146
- Returns:
147
- Optimus features for this chunk
148
- """
149
- features, _ = get_features(
150
- coords_chunk,
151
  slide_path,
152
  attrs,
153
- model_type=ModelType.OPTIMUS,
154
- model_path="data/optimus.pkl",
155
  num_workers=num_workers,
156
  batch_size=batch_size,
157
  use_gpu=True,
158
  )
159
- return features
 
 
 
 
160
 
161
 
 
162
  def _extract_optimus_features(filtered_coords, slide_path, attrs, num_workers):
163
- """Extract Optimus features, potentially splitting into multiple GPU calls.
164
 
165
  Args:
166
  filtered_coords: Filtered tissue tile coordinates
@@ -173,48 +94,27 @@ def _extract_optimus_features(filtered_coords, slide_path, attrs, num_workers):
173
  """
174
  if IS_ZEROGPU:
175
  num_workers = 0
176
- # Conservative chunk size - each chunk should complete in ~60-90s
177
- # Leaving lots of buffer in the 300s allocation
178
- chunk_size = 3000
179
- logger.info(f"Running Optimus on ZeroGPU: splitting {len(filtered_coords)} tiles into chunks of {chunk_size}")
180
  else:
181
  num_workers = max(num_workers, 8)
182
- chunk_size = len(filtered_coords)
183
  logger.info(f"Running Optimus with {num_workers} workers")
184
 
185
  batch_size = 128 if IS_ZEROGPU else 64
186
  start_time = pd.Timestamp.now()
187
 
188
- all_features = []
189
- for i in range(0, len(filtered_coords), chunk_size):
190
- chunk_coords = filtered_coords[i:i+chunk_size]
191
- chunk_num = i // chunk_size + 1
192
- total_chunks = (len(filtered_coords) + chunk_size - 1) // chunk_size
193
-
194
- logger.info(f"Processing Optimus chunk {chunk_num}/{total_chunks} ({len(chunk_coords)} tiles)")
195
-
196
- if IS_ZEROGPU:
197
- chunk_features = _extract_optimus_features_single_chunk(
198
- chunk_coords, slide_path, attrs, num_workers, batch_size
199
- )
200
- else:
201
- chunk_features, _ = get_features(
202
- chunk_coords, slide_path, attrs,
203
- model_type=ModelType.OPTIMUS,
204
- model_path="data/optimus.pkl",
205
- num_workers=num_workers,
206
- batch_size=batch_size,
207
- use_gpu=True,
208
- )
209
-
210
- all_features.append(chunk_features)
211
- logger.info(f"Chunk {chunk_num}/{total_chunks} completed")
212
-
213
- import numpy as np
214
- features = np.concatenate(all_features, axis=0) if len(all_features) > 1 else all_features[0]
215
 
216
  end_time = pd.Timestamp.now()
217
- logger.info(f"Optimus total time: {end_time - start_time}")
218
 
219
  return features
220
 
 
40
 
41
 
42
  @spaces.GPU(duration=180)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
43
  def _extract_ctranspath_features(coords, slide_path, attrs, num_workers):
44
+ """Extract CTransPath features on GPU.
45
 
46
  Args:
47
  coords: Tissue tile coordinates
 
54
  """
55
  if IS_ZEROGPU:
56
  num_workers = 0
57
+ logger.info(f"Running CTransPath on ZeroGPU: processing {len(coords)} tiles")
 
 
58
  else:
59
  num_workers = max(num_workers, 8)
 
60
  logger.info(f"Running CTransPath with {num_workers} workers")
61
 
62
  batch_size = 128 if IS_ZEROGPU else 64
 
63
  start_time = pd.Timestamp.now()
64
 
65
+ ctranspath_features, _ = get_features(
66
+ coords,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67
  slide_path,
68
  attrs,
69
+ model_type=ModelType.CTRANSPATH,
70
+ model_path="data/ctranspath.pth",
71
  num_workers=num_workers,
72
  batch_size=batch_size,
73
  use_gpu=True,
74
  )
75
+
76
+ end_time = pd.Timestamp.now()
77
+ logger.info(f"CTransPath extraction took {end_time - start_time}")
78
+
79
+ return ctranspath_features, coords
80
 
81
 
82
+ @spaces.GPU(duration=300)
83
  def _extract_optimus_features(filtered_coords, slide_path, attrs, num_workers):
84
+ """Extract Optimus features on GPU.
85
 
86
  Args:
87
  filtered_coords: Filtered tissue tile coordinates
 
94
  """
95
  if IS_ZEROGPU:
96
  num_workers = 0
97
+ logger.info(f"Running Optimus on ZeroGPU: processing {len(filtered_coords)} tiles")
 
 
 
98
  else:
99
  num_workers = max(num_workers, 8)
 
100
  logger.info(f"Running Optimus with {num_workers} workers")
101
 
102
  batch_size = 128 if IS_ZEROGPU else 64
103
  start_time = pd.Timestamp.now()
104
 
105
+ features, _ = get_features(
106
+ filtered_coords,
107
+ slide_path,
108
+ attrs,
109
+ model_type=ModelType.OPTIMUS,
110
+ model_path="data/optimus.pkl",
111
+ num_workers=num_workers,
112
+ batch_size=batch_size,
113
+ use_gpu=True,
114
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
115
 
116
  end_time = pd.Timestamp.now()
117
+ logger.info(f"Optimus extraction took {end_time - start_time}")
118
 
119
  return features
120