Spaces:
Sleeping
Sleeping
Download src/inference/writer.py from farrell236/OCT_Segmentation_a1: direct link, hf CLI and curl.
- Browser
- Download file 1.77 kB
-
https://huggingface.co/spaces/farrell236/OCT_Segmentation_a1/resolve/main/src/inference/writer.py
- Command line
-
hf download hf://spaces/farrell236/OCT_Segmentation_a1/src/inference/writer.py
-
curl -L -o writer.py https://huggingface.co/spaces/farrell236/OCT_Segmentation_a1/resolve/main/src/inference/writer.py
1.77 kB
| """Bounded asynchronous NPZ serialization; values and dtypes are preserved.""" | |
| from collections import deque | |
| from concurrent.futures import ThreadPoolExecutor | |
| from pathlib import Path | |
| import zipfile | |
| import numpy as np | |
| def save_npz(path, arrays, level=1): | |
| path = Path(path) | |
| temporary = path.with_suffix(".tmp.npz") | |
| with zipfile.ZipFile( | |
| temporary, | |
| "w", | |
| compression=zipfile.ZIP_DEFLATED, | |
| compresslevel=level, | |
| allowZip64=True, | |
| ) as archive: | |
| for name, value in arrays.items(): | |
| with archive.open(name + ".npy", "w", force_zip64=True) as handle: | |
| np.lib.format.write_array(handle, np.asarray(value), allow_pickle=False) | |
| temporary.replace(path) | |
| class PredictionWriter: | |
| def __init__(self, output_dir, workers=2, maximum_pending_cases=2): | |
| self.output_dir = Path(output_dir) | |
| self.queue = deque() | |
| self.maximum_pending_cases = maximum_pending_cases | |
| self.executor = ThreadPoolExecutor(max_workers=workers) | |
| def submit(self, case_id, items): | |
| # Keep at most this many completed-volume arrays alive behind inference. | |
| while len(self.queue) >= self.maximum_pending_cases: | |
| self.queue.popleft().result() | |
| def write_case(): | |
| for name, positions, keys in items: | |
| save_npz( | |
| self.output_dir / name / (case_id + ".npz"), | |
| {"positions": positions, "boundary_keys": np.asarray(keys)}, | |
| ) | |
| self.queue.append(self.executor.submit(write_case)) | |
| def close(self): | |
| try: | |
| while self.queue: | |
| self.queue.popleft().result() | |
| finally: | |
| self.executor.shutdown(wait=True, cancel_futures=True) | |