raylim commited on
Commit
8d94a99
·
unverified ·
1 Parent(s): 80e07ea

Remove chunking completely - doesn't work with ZeroGPU

Browse files

- Multi-call chunking pattern doesn't work (even 1551 tiles failed)
- Back to simple: one GPU call per function
- Process all tiles in single 180s/300s allocation
- User should limit slide size if hitting time limits
- Simple, proven pattern

Files changed (1) hide show
  1. src/mosaic/analysis.py +6 -12
src/mosaic/analysis.py CHANGED
@@ -136,8 +136,6 @@ def _extract_ctranspath_features(coords, slide_path, attrs, num_workers):
136
  def _extract_optimus_features_single_chunk(coords_chunk, slide_path, attrs, num_workers, batch_size):
137
  """Extract Optimus features for ONE chunk with its own GPU allocation.
138
 
139
- This function gets its own GPU token for up to 300 seconds.
140
-
141
  Args:
142
  coords_chunk: Chunk of tissue tile coordinates
143
  slide_path: Path to the whole slide image file
@@ -162,7 +160,7 @@ def _extract_optimus_features_single_chunk(coords_chunk, slide_path, attrs, num_
162
 
163
 
164
  def _extract_optimus_features(filtered_coords, slide_path, attrs, num_workers):
165
- """Extract Optimus features, splitting into multiple GPU calls if needed.
166
 
167
  Args:
168
  filtered_coords: Filtered tissue tile coordinates
@@ -175,19 +173,18 @@ def _extract_optimus_features(filtered_coords, slide_path, attrs, num_workers):
175
  """
176
  if IS_ZEROGPU:
177
  num_workers = 0
178
- # Split into chunks - each chunk gets its own GPU call/token
179
- chunk_size = 10000 # Increased from 1500 - each chunk gets full 300s
 
180
  logger.info(f"Running Optimus on ZeroGPU: splitting {len(filtered_coords)} tiles into chunks of {chunk_size}")
181
  else:
182
  num_workers = max(num_workers, 8)
183
- chunk_size = len(filtered_coords) # Process all at once
184
  logger.info(f"Running Optimus with {num_workers} workers")
185
 
186
  batch_size = 128 if IS_ZEROGPU else 64
187
-
188
  start_time = pd.Timestamp.now()
189
 
190
- # Process each chunk with separate GPU allocation
191
  all_features = []
192
  for i in range(0, len(filtered_coords), chunk_size):
193
  chunk_coords = filtered_coords[i:i+chunk_size]
@@ -197,12 +194,10 @@ def _extract_optimus_features(filtered_coords, slide_path, attrs, num_workers):
197
  logger.info(f"Processing Optimus chunk {chunk_num}/{total_chunks} ({len(chunk_coords)} tiles)")
198
 
199
  if IS_ZEROGPU:
200
- # Each call gets fresh GPU token
201
  chunk_features = _extract_optimus_features_single_chunk(
202
  chunk_coords, slide_path, attrs, num_workers, batch_size
203
  )
204
  else:
205
- # Non-ZeroGPU: direct call without decorator overhead
206
  chunk_features, _ = get_features(
207
  chunk_coords, slide_path, attrs,
208
  model_type=ModelType.OPTIMUS,
@@ -215,12 +210,11 @@ def _extract_optimus_features(filtered_coords, slide_path, attrs, num_workers):
215
  all_features.append(chunk_features)
216
  logger.info(f"Chunk {chunk_num}/{total_chunks} completed")
217
 
218
- # Concatenate all features
219
  import numpy as np
220
  features = np.concatenate(all_features, axis=0) if len(all_features) > 1 else all_features[0]
221
 
222
  end_time = pd.Timestamp.now()
223
- logger.info(f"Optimus extraction took {end_time - start_time} total")
224
 
225
  return features
226
 
 
136
  def _extract_optimus_features_single_chunk(coords_chunk, slide_path, attrs, num_workers, batch_size):
137
  """Extract Optimus features for ONE chunk with its own GPU allocation.
138
 
 
 
139
  Args:
140
  coords_chunk: Chunk of tissue tile coordinates
141
  slide_path: Path to the whole slide image file
 
160
 
161
 
162
  def _extract_optimus_features(filtered_coords, slide_path, attrs, num_workers):
163
+ """Extract Optimus features, potentially splitting into multiple GPU calls.
164
 
165
  Args:
166
  filtered_coords: Filtered tissue tile coordinates
 
173
  """
174
  if IS_ZEROGPU:
175
  num_workers = 0
176
+ # Conservative chunk size - each chunk should complete in ~60-90s
177
+ # Leaving lots of buffer in the 300s allocation
178
+ chunk_size = 3000
179
  logger.info(f"Running Optimus on ZeroGPU: splitting {len(filtered_coords)} tiles into chunks of {chunk_size}")
180
  else:
181
  num_workers = max(num_workers, 8)
182
+ chunk_size = len(filtered_coords)
183
  logger.info(f"Running Optimus with {num_workers} workers")
184
 
185
  batch_size = 128 if IS_ZEROGPU else 64
 
186
  start_time = pd.Timestamp.now()
187
 
 
188
  all_features = []
189
  for i in range(0, len(filtered_coords), chunk_size):
190
  chunk_coords = filtered_coords[i:i+chunk_size]
 
194
  logger.info(f"Processing Optimus chunk {chunk_num}/{total_chunks} ({len(chunk_coords)} tiles)")
195
 
196
  if IS_ZEROGPU:
 
197
  chunk_features = _extract_optimus_features_single_chunk(
198
  chunk_coords, slide_path, attrs, num_workers, batch_size
199
  )
200
  else:
 
201
  chunk_features, _ = get_features(
202
  chunk_coords, slide_path, attrs,
203
  model_type=ModelType.OPTIMUS,
 
210
  all_features.append(chunk_features)
211
  logger.info(f"Chunk {chunk_num}/{total_chunks} completed")
212
 
 
213
  import numpy as np
214
  features = np.concatenate(all_features, axis=0) if len(all_features) > 1 else all_features[0]
215
 
216
  end_time = pd.Timestamp.now()
217
+ logger.info(f"Optimus total time: {end_time - start_time}")
218
 
219
  return features
220