Spaces:
Running on Zero
Running on Zero
Download mmdet/engine/hooks/batch_trace_hook.py from hugging-apps/levirdetnet: direct link, hf CLI and curl.
- Browser
- Download file 4.57 kB
-
https://huggingface.co/spaces/hugging-apps/levirdetnet/resolve/main/mmdet/engine/hooks/batch_trace_hook.py
- Command line
-
hf download hf://spaces/hugging-apps/levirdetnet/mmdet/engine/hooks/batch_trace_hook.py
-
curl -L -o batch_trace_hook.py https://huggingface.co/spaces/hugging-apps/levirdetnet/resolve/main/mmdet/engine/hooks/batch_trace_hook.py
4.57 kB
| # Copyright (c) OpenMMLab. All rights reserved. | |
| import json | |
| import os | |
| import socket | |
| import tempfile | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| from typing import Any, Dict, List, Optional | |
| from mmengine.dist import get_rank | |
| from mmengine.hooks import Hook | |
| from mmengine.runner import Runner | |
| from mmdet.registry import HOOKS | |
| class BatchTraceHook(Hook): | |
| """Persist each rank's latest batch for distributed-hang diagnosis. | |
| The trace is written outside ``work_dir`` by default so this hook does not | |
| add per-iteration traffic to shared checkpoint storage. Each write uses an | |
| atomic replacement; files remain readable after NCCL aborts the process. | |
| """ | |
| priority = 'VERY_HIGH' | |
| def __init__(self, output_dir: Optional[str] = None) -> None: | |
| self.output_dir = output_dir | |
| self._trace_dir: Optional[Path] = None | |
| def _sample_info(sample: Any) -> Dict[str, Any]: | |
| metainfo = getattr(sample, 'metainfo', {}) or {} | |
| info = { | |
| 'img_id': metainfo.get('img_id'), | |
| 'img_path': metainfo.get('img_path', ''), | |
| 'ori_shape': metainfo.get('ori_shape'), | |
| 'img_shape': metainfo.get('img_shape'), | |
| } | |
| gt_instances = getattr(sample, 'gt_instances', None) | |
| if gt_instances is not None: | |
| bboxes = getattr(gt_instances, 'bboxes', None) | |
| labels = getattr(gt_instances, 'labels', None) | |
| if bboxes is not None: | |
| info['num_gt_bboxes'] = int(len(bboxes)) | |
| if labels is not None: | |
| try: | |
| info['gt_labels'] = labels.detach().cpu().tolist() | |
| except Exception: | |
| info['gt_labels'] = '<unavailable>' | |
| return info | |
| def _batch_info(cls, data_batch: Any) -> List[Dict[str, Any]]: | |
| if not isinstance(data_batch, dict): | |
| return [] | |
| samples = data_batch.get('data_samples') | |
| if not isinstance(samples, (list, tuple)): | |
| return [] | |
| return [cls._sample_info(sample) for sample in samples] | |
| def _resolve_trace_dir(self, runner: Runner) -> Path: | |
| if self._trace_dir is not None: | |
| return self._trace_dir | |
| configured = self.output_dir or os.environ.get( | |
| 'TRAIN_BATCH_TRACE_DIR') | |
| if configured: | |
| trace_dir = Path(configured) | |
| else: | |
| job_tag = os.environ.get('TRAIN_JOB_TAG') | |
| if not job_tag: | |
| job_tag = Path(str(runner.work_dir)).name | |
| trace_dir = Path(tempfile.gettempdir()) / 'levirdet-batch-trace' \ | |
| / job_tag | |
| trace_dir.mkdir(parents=True, exist_ok=True) | |
| self._trace_dir = trace_dir | |
| return trace_dir | |
| def _atomic_dump(path: Path, payload: Dict[str, Any]) -> None: | |
| temporary = path.with_name(f'.{path.name}.{os.getpid()}.tmp') | |
| with temporary.open('w', encoding='utf-8') as file: | |
| json.dump(payload, file, ensure_ascii=False, indent=2) | |
| file.flush() | |
| os.fsync(file.fileno()) | |
| os.replace(temporary, path) | |
| def _write(self, runner: Runner, batch_idx: int, data_batch: Any, | |
| phase: str) -> None: | |
| rank = get_rank() | |
| payload = { | |
| 'phase': phase, | |
| 'timestamp_utc': datetime.now(timezone.utc).isoformat(), | |
| 'hostname': socket.gethostname(), | |
| 'pid': os.getpid(), | |
| 'rank': rank, | |
| 'local_rank': int(os.environ.get('LOCAL_RANK', rank)), | |
| 'epoch_zero_based': int(runner.epoch), | |
| 'epoch_one_based': int(runner.epoch) + 1, | |
| 'batch_idx_zero_based': int(batch_idx), | |
| 'runner_iter_zero_based': int(runner.iter), | |
| 'samples': self._batch_info(data_batch), | |
| } | |
| path = self._resolve_trace_dir(runner) / f'rank_{rank}.json' | |
| self._atomic_dump(path, payload) | |
| def before_train_iter(self, | |
| runner: Runner, | |
| batch_idx: int, | |
| data_batch: Optional[dict] = None) -> None: | |
| self._write(runner, batch_idx, data_batch, 'train_step_started') | |
| def after_train_iter(self, | |
| runner: Runner, | |
| batch_idx: int, | |
| data_batch: Optional[dict] = None, | |
| outputs: Optional[dict] = None) -> None: | |
| self._write(runner, batch_idx, data_batch, 'train_step_completed') | |