fix the wrong GPU index issue of multi-node
Browse files
configs/metadata.json
CHANGED
|
@@ -1,14 +1,15 @@
|
|
| 1 |
{
|
| 2 |
"schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_generator_ldm_20230507.json",
|
| 3 |
-
"version": "1.0.
|
| 4 |
"changelog": {
|
|
|
|
| 5 |
"1.0.4": "update with new lr scheduler api",
|
| 6 |
"1.0.3": "update required packages",
|
| 7 |
"1.0.2": "unify dataset dir in different configs",
|
| 8 |
"1.0.1": "update dependency, update trained model weights",
|
| 9 |
"1.0.0": "Initial release"
|
| 10 |
},
|
| 11 |
-
"monai_version": "1.2.
|
| 12 |
"pytorch_version": "1.13.1",
|
| 13 |
"numpy_version": "1.22.2",
|
| 14 |
"optional_packages_version": {
|
|
|
|
| 1 |
{
|
| 2 |
"schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_generator_ldm_20230507.json",
|
| 3 |
+
"version": "1.0.5",
|
| 4 |
"changelog": {
|
| 5 |
+
"1.0.5": "fix the wrong GPU index issue of multi-node",
|
| 6 |
"1.0.4": "update with new lr scheduler api",
|
| 7 |
"1.0.3": "update required packages",
|
| 8 |
"1.0.2": "unify dataset dir in different configs",
|
| 9 |
"1.0.1": "update dependency, update trained model weights",
|
| 10 |
"1.0.0": "Initial release"
|
| 11 |
},
|
| 12 |
+
"monai_version": "1.2.0",
|
| 13 |
"pytorch_version": "1.13.1",
|
| 14 |
"numpy_version": "1.22.2",
|
| 15 |
"optional_packages_version": {
|
configs/multi_gpu_train_autoencoder.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"device": "$torch.device(
|
| 3 |
"gnetwork": {
|
| 4 |
"_target_": "torch.nn.parallel.DistributedDataParallel",
|
| 5 |
"module": "$@autoencoder_def.to(@device)",
|
|
@@ -25,6 +25,7 @@
|
|
| 25 |
"train#trainer#train_handlers": "$@train#handlers[: -2 if dist.get_rank() > 0 else None]",
|
| 26 |
"initialize": [
|
| 27 |
"$import torch.distributed as dist",
|
|
|
|
| 28 |
"$dist.is_initialized() or dist.init_process_group(backend='nccl')",
|
| 29 |
"$torch.cuda.set_device(@device)",
|
| 30 |
"$monai.utils.set_determinism(seed=123)",
|
|
|
|
| 1 |
{
|
| 2 |
+
"device": "$torch.device('cuda:' + os.environ['LOCAL_RANK'])",
|
| 3 |
"gnetwork": {
|
| 4 |
"_target_": "torch.nn.parallel.DistributedDataParallel",
|
| 5 |
"module": "$@autoencoder_def.to(@device)",
|
|
|
|
| 25 |
"train#trainer#train_handlers": "$@train#handlers[: -2 if dist.get_rank() > 0 else None]",
|
| 26 |
"initialize": [
|
| 27 |
"$import torch.distributed as dist",
|
| 28 |
+
"$import os",
|
| 29 |
"$dist.is_initialized() or dist.init_process_group(backend='nccl')",
|
| 30 |
"$torch.cuda.set_device(@device)",
|
| 31 |
"$monai.utils.set_determinism(seed=123)",
|