artifacts/model_device_summary.json

{
  "cuda_memory_allocated_gib_by_device": {
    "0": 13.119282722473145,
    "1": 15.742644309997559,
    "2": 15.726958274841309,
    "3": 17.702864170074463
  },
  "cuda_memory_reserved_gib_by_device": {
    "0": 13.390625,
    "1": 16.185546875,
    "2": 16.009765625,
    "3": 18.021484375
  },
  "cuda_peak_memory_allocated_gib_by_device": {
    "0": 14.064411163330078,
    "1": 16.74264430999756,
    "2": 16.72695827484131,
    "3": 18.70286226272583
  },
  "cuda_peak_memory_reserved_gib_by_device": {
    "0": 15.6953125,
    "1": 18.666015625,
    "2": 18.646484375,
    "3": 20.560546875
  },
  "hf_device_map": {
    "lm_head": 3,
    "model.embed_tokens": 0,
    "model.layers.0": 0,
    "model.layers.1": 0,
    "model.layers.10": 1,
    "model.layers.11": 1,
    "model.layers.12": 1,
    "model.layers.13": 1,
    "model.layers.14": 1,
    "model.layers.15": 1,
    "model.layers.16": 1,
    "model.layers.17": 1,
    "model.layers.18": 1,
    "model.layers.19": 2,
    "model.layers.2": 0,
    "model.layers.20": 2,
    "model.layers.21": 2,
    "model.layers.22": 2,
    "model.layers.23": 2,
    "model.layers.24": 2,
    "model.layers.25": 2,
    "model.layers.26": 2,
    "model.layers.27": 2,
    "model.layers.28": 2,
    "model.layers.29": 3,
    "model.layers.3": 0,
    "model.layers.30": 3,
    "model.layers.31": 3,
    "model.layers.32": 3,
    "model.layers.33": 3,
    "model.layers.34": 3,
    "model.layers.35": 3,
    "model.layers.36": 3,
    "model.layers.37": 3,
    "model.layers.38": 3,
    "model.layers.39": 3,
    "model.layers.4": 0,
    "model.layers.5": 0,
    "model.layers.6": 0,
    "model.layers.7": 0,
    "model.layers.8": 0,
    "model.layers.9": 1,
    "model.norm": 3,
    "model.rotary_emb": 3
  },
  "parameter_counts_by_device": {
    "cuda:0": 7043358976,
    "cuda:1": 8451765248,
    "cuda:2": 8443343872,
    "cuda:3": 9504148992
  },
  "parameter_gib_by_device": {
    "cuda:0": 13.11927843093872,
    "cuda:1": 15.742639541625977,
    "cuda:2": 15.726953506469727,
    "cuda:3": 17.70285701751709
  }
}