Spaces:
Sleeping
Sleeping
Remove chunking completely - doesn't work with ZeroGPU
Browse files- Multi-call chunking pattern doesn't work (even 1551 tiles failed)
- Back to simple: one GPU call per function
- Process all tiles in single 180s/300s allocation
- User should limit slide size if hitting time limits
- Simple, proven pattern
- src/mosaic/analysis.py +6 -12
src/mosaic/analysis.py
CHANGED
|
@@ -136,8 +136,6 @@ def _extract_ctranspath_features(coords, slide_path, attrs, num_workers):
|
|
| 136 |
def _extract_optimus_features_single_chunk(coords_chunk, slide_path, attrs, num_workers, batch_size):
|
| 137 |
"""Extract Optimus features for ONE chunk with its own GPU allocation.
|
| 138 |
|
| 139 |
-
This function gets its own GPU token for up to 300 seconds.
|
| 140 |
-
|
| 141 |
Args:
|
| 142 |
coords_chunk: Chunk of tissue tile coordinates
|
| 143 |
slide_path: Path to the whole slide image file
|
|
@@ -162,7 +160,7 @@ def _extract_optimus_features_single_chunk(coords_chunk, slide_path, attrs, num_
|
|
| 162 |
|
| 163 |
|
| 164 |
def _extract_optimus_features(filtered_coords, slide_path, attrs, num_workers):
|
| 165 |
-
"""Extract Optimus features, splitting into multiple GPU calls
|
| 166 |
|
| 167 |
Args:
|
| 168 |
filtered_coords: Filtered tissue tile coordinates
|
|
@@ -175,19 +173,18 @@ def _extract_optimus_features(filtered_coords, slide_path, attrs, num_workers):
|
|
| 175 |
"""
|
| 176 |
if IS_ZEROGPU:
|
| 177 |
num_workers = 0
|
| 178 |
-
#
|
| 179 |
-
|
|
|
|
| 180 |
logger.info(f"Running Optimus on ZeroGPU: splitting {len(filtered_coords)} tiles into chunks of {chunk_size}")
|
| 181 |
else:
|
| 182 |
num_workers = max(num_workers, 8)
|
| 183 |
-
chunk_size = len(filtered_coords)
|
| 184 |
logger.info(f"Running Optimus with {num_workers} workers")
|
| 185 |
|
| 186 |
batch_size = 128 if IS_ZEROGPU else 64
|
| 187 |
-
|
| 188 |
start_time = pd.Timestamp.now()
|
| 189 |
|
| 190 |
-
# Process each chunk with separate GPU allocation
|
| 191 |
all_features = []
|
| 192 |
for i in range(0, len(filtered_coords), chunk_size):
|
| 193 |
chunk_coords = filtered_coords[i:i+chunk_size]
|
|
@@ -197,12 +194,10 @@ def _extract_optimus_features(filtered_coords, slide_path, attrs, num_workers):
|
|
| 197 |
logger.info(f"Processing Optimus chunk {chunk_num}/{total_chunks} ({len(chunk_coords)} tiles)")
|
| 198 |
|
| 199 |
if IS_ZEROGPU:
|
| 200 |
-
# Each call gets fresh GPU token
|
| 201 |
chunk_features = _extract_optimus_features_single_chunk(
|
| 202 |
chunk_coords, slide_path, attrs, num_workers, batch_size
|
| 203 |
)
|
| 204 |
else:
|
| 205 |
-
# Non-ZeroGPU: direct call without decorator overhead
|
| 206 |
chunk_features, _ = get_features(
|
| 207 |
chunk_coords, slide_path, attrs,
|
| 208 |
model_type=ModelType.OPTIMUS,
|
|
@@ -215,12 +210,11 @@ def _extract_optimus_features(filtered_coords, slide_path, attrs, num_workers):
|
|
| 215 |
all_features.append(chunk_features)
|
| 216 |
logger.info(f"Chunk {chunk_num}/{total_chunks} completed")
|
| 217 |
|
| 218 |
-
# Concatenate all features
|
| 219 |
import numpy as np
|
| 220 |
features = np.concatenate(all_features, axis=0) if len(all_features) > 1 else all_features[0]
|
| 221 |
|
| 222 |
end_time = pd.Timestamp.now()
|
| 223 |
-
logger.info(f"Optimus
|
| 224 |
|
| 225 |
return features
|
| 226 |
|
|
|
|
| 136 |
def _extract_optimus_features_single_chunk(coords_chunk, slide_path, attrs, num_workers, batch_size):
|
| 137 |
"""Extract Optimus features for ONE chunk with its own GPU allocation.
|
| 138 |
|
|
|
|
|
|
|
| 139 |
Args:
|
| 140 |
coords_chunk: Chunk of tissue tile coordinates
|
| 141 |
slide_path: Path to the whole slide image file
|
|
|
|
| 160 |
|
| 161 |
|
| 162 |
def _extract_optimus_features(filtered_coords, slide_path, attrs, num_workers):
|
| 163 |
+
"""Extract Optimus features, potentially splitting into multiple GPU calls.
|
| 164 |
|
| 165 |
Args:
|
| 166 |
filtered_coords: Filtered tissue tile coordinates
|
|
|
|
| 173 |
"""
|
| 174 |
if IS_ZEROGPU:
|
| 175 |
num_workers = 0
|
| 176 |
+
# Conservative chunk size - each chunk should complete in ~60-90s
|
| 177 |
+
# Leaving lots of buffer in the 300s allocation
|
| 178 |
+
chunk_size = 3000
|
| 179 |
logger.info(f"Running Optimus on ZeroGPU: splitting {len(filtered_coords)} tiles into chunks of {chunk_size}")
|
| 180 |
else:
|
| 181 |
num_workers = max(num_workers, 8)
|
| 182 |
+
chunk_size = len(filtered_coords)
|
| 183 |
logger.info(f"Running Optimus with {num_workers} workers")
|
| 184 |
|
| 185 |
batch_size = 128 if IS_ZEROGPU else 64
|
|
|
|
| 186 |
start_time = pd.Timestamp.now()
|
| 187 |
|
|
|
|
| 188 |
all_features = []
|
| 189 |
for i in range(0, len(filtered_coords), chunk_size):
|
| 190 |
chunk_coords = filtered_coords[i:i+chunk_size]
|
|
|
|
| 194 |
logger.info(f"Processing Optimus chunk {chunk_num}/{total_chunks} ({len(chunk_coords)} tiles)")
|
| 195 |
|
| 196 |
if IS_ZEROGPU:
|
|
|
|
| 197 |
chunk_features = _extract_optimus_features_single_chunk(
|
| 198 |
chunk_coords, slide_path, attrs, num_workers, batch_size
|
| 199 |
)
|
| 200 |
else:
|
|
|
|
| 201 |
chunk_features, _ = get_features(
|
| 202 |
chunk_coords, slide_path, attrs,
|
| 203 |
model_type=ModelType.OPTIMUS,
|
|
|
|
| 210 |
all_features.append(chunk_features)
|
| 211 |
logger.info(f"Chunk {chunk_num}/{total_chunks} completed")
|
| 212 |
|
|
|
|
| 213 |
import numpy as np
|
| 214 |
features = np.concatenate(all_features, axis=0) if len(all_features) > 1 else all_features[0]
|
| 215 |
|
| 216 |
end_time = pd.Timestamp.now()
|
| 217 |
+
logger.info(f"Optimus total time: {end_time - start_time}")
|
| 218 |
|
| 219 |
return features
|
| 220 |
|