v1

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+test_model_v1/debug/attention/batch0_sample0_attention_all_layers.pdf filter=lfs diff=lfs merge=lfs -text

test_model_v1/checkpoints/best/checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e35dd1a13a10efae348d9c1acdada6b8055a25318e5b6ad0d6f30994aad2bc4
+size 302889894

test_model_v1/configs/config_inference.yaml

@@ -0,0 +1,58 @@
+model:
+  name: test_model
+  best_checkpoint_path: trained_models/test_model_v1/checkpoints/best/checkpoint.pt
+  latest_checkpoint_path: trained_models/test_model_v1/checkpoints/best/checkpoint.pt
+  tokenizer: bpe_8k_old
+  d_model: 512
+  num_heads: 8
+  num_layers: 4
+  d_ff: 1024
+  dropout_rate: 0.1
+  vocab_size: 8192
+  max_seq_len: 128
+  pad_id: 0
+  bos_id: 1
+  eos_id: 2
+tokenizer:
+  name: bpe_8k_old
+  path: trained_models/test_model_v1/tokenizer/bpe_8k_old.json
+  corpus: ./data/corpus/corpus.txt
+  dataset: concatenated_dataset
+  vocab_size: 8192
+  max_seq_len: 128
+  pad_token: <pad>
+  bos_token: <bos>
+  eos_token: <eos>
+  unk_token: <unk>
+  special_tokens:
+  - <pad>
+  - <bos>
+  - <eos>
+  - <unk>
+  pad_id: 0
+  bos_id: 1
+  eos_id: 2
+  unk_id: 3
+runtime:
+  seed: 42
+  device: cuda
+  output_dir: ./outputs/
+  data_dir: ./data/
+  tokenizer_dir: ./tokenizer/
+  cache_dir: ./data/raw
+  checkpoint_path: ./outputs/checkpoints/
+generation:
+  max_new_tokens: 50
+  temperature: 1
+  top_k: 10
+  top_p: 0.6
+  do_sample: false
+  presence_penalty: 0.6
+  frequency_penalty: 0.2
+  no_repeat_ngram: 4
+  min_steps_before_eos: 1
+input_text: i am sorry to hear that.
+hydra:
+  run:
+    dir: .
+  output_subdir: null

test_model_v1/configs/config_train.yaml

@@ -0,0 +1,81 @@
+model:
+  name: test_model
+  best_checkpoint_path: trained_models/test_model_v1/checkpoints/best/checkpoint.pt
+  latest_checkpoint_path: trained_models/test_model_v1/checkpoints/best/checkpoint.pt
+  tokenizer: bpe_8k_old
+  d_model: 512
+  num_heads: 8
+  num_layers: 4
+  d_ff: 1024
+  dropout_rate: 0.1
+  vocab_size: 8192
+  max_seq_len: 128
+  pad_id: 0
+  bos_id: 1
+  eos_id: 2
+tokenizer:
+  name: bpe_8k_old
+  path: trained_models/test_model_v1/tokenizer/bpe_8k_old.json
+  corpus: ./data/corpus/corpus.txt
+  dataset: concatenated_dataset
+  vocab_size: 8192
+  max_seq_len: 128
+  pad_token: <pad>
+  bos_token: <bos>
+  eos_token: <eos>
+  unk_token: <unk>
+  special_tokens:
+  - <pad>
+  - <bos>
+  - <eos>
+  - <unk>
+  pad_id: 0
+  bos_id: 1
+  eos_id: 2
+  unk_id: 3
+dataset:
+  name: medium_dataset
+  path: ./data/processed/processed_medium_dataset
+optim:
+  name: AdamW
+  lr: 0.0008
+  betas:
+  - 0.9
+  - 0.98
+  eps: 1.0e-08
+  weight_decay: 0.08
+  no_decay_on_bias_norm_embed: true
+sched:
+  name: cosine_warmup
+  warmup_ratio: 0.08
+  min_lr_ratio: 0.05
+  max_lr_ratio: 1
+  hold_min: true
+runtime:
+  seed: 42
+  device: cuda
+  output_dir: ./outputs/
+  data_dir: ./data/
+  tokenizer_dir: ./tokenizer/
+  cache_dir: ./data/raw
+  checkpoint_path: ./outputs/checkpoints/
+trainer:
+  epochs: 10
+  batch_size: 128
+  shuffle: true
+  precision: bf16
+  gradient_accumulation: 4
+  clip_grad_norm: 0.5
+  eval_interval: 1
+  save_interval: 1
+  debug_interval: 1000
+  mixed_precision: true
+  gradient_checkpointing: true
+  label_smoothing: 0.05
+  export_best_to_model_path: false
+  resume: best
+hydra:
+  run:
+    dir: .
+  output_subdir: null
+

test_model_v1/debug/attention/attention_context_0.json

@@ -0,0 +1,71 @@
+{
+  "src": [
+    "i have decided to go to the park. i will take my dog with me. we will have a great time together."
+  ],
+  "tgt": [
+    "j'ai d\u00e9cid\u00e9 d'aller au parc. je vais prendre mon chien avec moi. nous allons passer un bon moment ensemble."
+  ],
+  "src_padd_mask": [
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false
+  ],
+  "tgt_padd_mask": [
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false,
+    false
+  ],
+  "pred_text": "j ' ai décidé d ' aller au par c , je vais me mon do ien avec moi , aurons avoir un grand temps ensemble .",
+  "batch_accuracy": 0.6551724137931034
+}

test_model_v1/debug/attention/batch0_sample0_attention_all_layers.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8687aa82dcc82aaa5da58bb317496c2b1969cd7387d80744bf1df84db7176c8b
+size 165785

test_model_v1/debug/attention/diagnostics/attention_report.json

@@ -0,0 +1,1280 @@
+{
+  "config": {
+    "model": {
+      "name": "test_model",
+      "best_checkpoint_path": "test_model_v1/checkpoints/best/checkpoint.pt",
+      "latest_checkpoint_path": "test_model_v1/checkpoints/best/checkpoint.pt",
+      "tokenizer": "bpe_8k_old",
+      "d_model": 512,
+      "num_layers": 4,
+      "num_heads": 8,
+      "d_ff": 1024,
+      "dropout_rate": 0.1,
+      "max_seq_len": 128,
+      "vocab_size": 8192,
+      "pad_id": 0,
+      "bos_id": 1,
+      "eos_id": 2,
+      "layer_norm_style": null
+    },
+    "dataset": {
+      "name": "medium_dataset",
+      "path": "./data/processed/processed_medium_dataset"
+    },
+    "optim": {
+      "name": "AdamW",
+      "lr": 0.0008,
+      "betas": [
+        0.9,
+        0.98
+      ],
+      "weight_decay": 0.08,
+      "eps": 1e-08,
+      "no_decay_on_bias_norm_embed": true
+    },
+    "sched": {
+      "name": "cosine_warmup",
+      "warmup_ratio": 0.08,
+      "min_lr_ratio": 0.05,
+      "max_lr_ratio": 1,
+      "hold_min": true
+    },
+    "trainer": {
+      "epochs": 10,
+      "batch_size": 8,
+      "shuffle": true,
+      "precision": "bf16",
+      "gradient_accumulation": 4,
+      "clip_grad_norm": 0.5,
+      "eval_interval": 1,
+      "save_interval": 1,
+      "debug_interval": 1000,
+      "mixed_precision": true,
+      "gradient_checkpointing": true,
+      "label_smoothing": 0.05,
+      "export_best_to_model_path": false,
+      "resume": "best"
+    },
+    "runtime": {
+      "seed": 42,
+      "device": "cuda",
+      "output_dir": "./outputs/",
+      "data_dir": "./data/",
+      "cache_dir": "./data/raw",
+      "tokenizer_dir": "./tokenizer/",
+      "checkpoint_path": "./outputs/checkpoints/"
+    },
+    "tokenizer": {
+      "name": "bpe_8k_old",
+      "path": "test_model_v1/tokenizer/bpe_8k_old.json",
+      "corpus": "./data/corpus/corpus.txt",
+      "dataset": "concatenated_dataset",
+      "vocab_size": 8192,
+      "max_seq_len": 128,
+      "pad_token": "<pad>",
+      "bos_token": "<bos>",
+      "eos_token": "<eos>",
+      "unk_token": "<unk>",
+      "special_tokens": [
+        "<pad>",
+        "<bos>",
+        "<eos>",
+        "<unk>"
+      ],
+      "pad_id": 0,
+      "bos_id": 1,
+      "eos_id": 2,
+      "unk_id": 3
+    }
+  },
+  "device": "cuda",
+  "reports": [
+    {
+      "loss": 1.7494174242019653,
+      "gradients": {
+        "per_parameter": {
+          "embed.token_embed.weight": {
+            "grad_norm": 1.3044230937957764,
+            "param_norm": 108.40128326416016,
+            "grad_to_weight_ratio": 0.01203328092174934
+          },
+          "encoder.layers.0.attention_layer.query_linear.weight": {
+            "grad_norm": 0.21280944347381592,
+            "param_norm": 24.398685455322266,
+            "grad_to_weight_ratio": 0.008722168407945334
+          },
+          "encoder.layers.0.attention_layer.key_linear.weight": {
+            "grad_norm": 0.22556814551353455,
+            "param_norm": 24.673677444458008,
+            "grad_to_weight_ratio": 0.009142056186042531
+          },
+          "encoder.layers.0.attention_layer.value_linear.weight": {
+            "grad_norm": 0.3686007559299469,
+            "param_norm": 19.429746627807617,
+            "grad_to_weight_ratio": 0.018970950213133043
+          },
+          "encoder.layers.0.attention_layer.output_linear.weight": {
+            "grad_norm": 0.37529751658439636,
+            "param_norm": 18.044166564941406,
+            "grad_to_weight_ratio": 0.02079882798873921
+          },
+          "encoder.layers.0.attention_layer.output_linear.bias": {
+            "grad_norm": 0.027566326782107353,
+            "param_norm": 1.4551786184310913,
+            "grad_to_weight_ratio": 0.01894360350883192
+          },
+          "encoder.layers.0.feed_forward.fc1.weight": {
+            "grad_norm": 0.2714630365371704,
+            "param_norm": 40.89385986328125,
+            "grad_to_weight_ratio": 0.006638234626047404
+          },
+          "encoder.layers.0.feed_forward.fc1.bias": {
+            "grad_norm": 0.013187435455620289,
+            "param_norm": 2.7436025142669678,
+            "grad_to_weight_ratio": 0.004806612979482155
+          },
+          "encoder.layers.0.feed_forward.fc2.weight": {
+            "grad_norm": 0.2072729468345642,
+            "param_norm": 38.14583969116211,
+            "grad_to_weight_ratio": 0.005433697318310211
+          },
+          "encoder.layers.0.feed_forward.fc2.bias": {
+            "grad_norm": 0.03970116749405861,
+            "param_norm": 1.5145164728164673,
+            "grad_to_weight_ratio": 0.026213757464256696
+          },
+          "encoder.layers.0.norm1.weight": {
+            "grad_norm": 0.036068517714738846,
+            "param_norm": 20.266429901123047,
+            "grad_to_weight_ratio": 0.001779717389333499
+          },
+          "encoder.layers.0.norm1.bias": {
+            "grad_norm": 0.04475417360663414,
+            "param_norm": 2.927133560180664,
+            "grad_to_weight_ratio": 0.015289419729742909
+          },
+          "encoder.layers.0.norm2.weight": {
+            "grad_norm": 0.04418084770441055,
+            "param_norm": 17.73204231262207,
+            "grad_to_weight_ratio": 0.002491582578333864
+          },
+          "encoder.layers.0.norm2.bias": {
+            "grad_norm": 0.05286872386932373,
+            "param_norm": 1.6395295858383179,
+            "grad_to_weight_ratio": 0.032246276203828826
+          },
+          "encoder.layers.1.attention_layer.query_linear.weight": {
+            "grad_norm": 0.12787117063999176,
+            "param_norm": 31.757823944091797,
+            "grad_to_weight_ratio": 0.004026446234638088
+          },
+          "encoder.layers.1.attention_layer.key_linear.weight": {
+            "grad_norm": 0.1463305950164795,
+            "param_norm": 32.2238883972168,
+            "grad_to_weight_ratio": 0.0045410595149998605
+          },
+          "encoder.layers.1.attention_layer.value_linear.weight": {
+            "grad_norm": 0.4261511266231537,
+            "param_norm": 19.251535415649414,
+            "grad_to_weight_ratio": 0.0221359552587539
+          },
+          "encoder.layers.1.attention_layer.output_linear.weight": {
+            "grad_norm": 0.41516169905662537,
+            "param_norm": 18.684219360351562,
+            "grad_to_weight_ratio": 0.022219911415597482
+          },
+          "encoder.layers.1.attention_layer.output_linear.bias": {
+            "grad_norm": 0.03609948605298996,
+            "param_norm": 1.0489388704299927,
+            "grad_to_weight_ratio": 0.03441524293799622
+          },
+          "encoder.layers.1.feed_forward.fc1.weight": {
+            "grad_norm": 0.2787674069404602,
+            "param_norm": 42.02549362182617,
+            "grad_to_weight_ratio": 0.006633292863826689
+          },
+          "encoder.layers.1.feed_forward.fc1.bias": {
+            "grad_norm": 0.013572012074291706,
+            "param_norm": 2.7236733436584473,
+            "grad_to_weight_ratio": 0.004982980835747634
+          },
+          "encoder.layers.1.feed_forward.fc2.weight": {
+            "grad_norm": 0.22472964227199554,
+            "param_norm": 39.663970947265625,
+            "grad_to_weight_ratio": 0.005665838212991188
+          },
+          "encoder.layers.1.feed_forward.fc2.bias": {
+            "grad_norm": 0.03622201085090637,
+            "param_norm": 1.5143930912017822,
+            "grad_to_weight_ratio": 0.02391849980122243
+          },
+          "encoder.layers.1.norm1.weight": {
+            "grad_norm": 0.03555092588067055,
+            "param_norm": 20.03695297241211,
+            "grad_to_weight_ratio": 0.001774268070081169
+          },
+          "encoder.layers.1.norm1.bias": {
+            "grad_norm": 0.04127884283661842,
+            "param_norm": 3.8686933517456055,
+            "grad_to_weight_ratio": 0.010669970215649735
+          },
+          "encoder.layers.1.norm2.weight": {
+            "grad_norm": 0.04897938296198845,
+            "param_norm": 16.989957809448242,
+            "grad_to_weight_ratio": 0.0028828431189362796
+          },
+          "encoder.layers.1.norm2.bias": {
+            "grad_norm": 0.05488703399896622,
+            "param_norm": 1.776807188987732,
+            "grad_to_weight_ratio": 0.030890821659836438
+          },
+          "encoder.layers.2.attention_layer.query_linear.weight": {
+            "grad_norm": 0.1271914839744568,
+            "param_norm": 30.094524383544922,
+            "grad_to_weight_ratio": 0.004226399538781158
+          },
+          "encoder.layers.2.attention_layer.key_linear.weight": {
+            "grad_norm": 0.13454079627990723,
+            "param_norm": 30.392127990722656,
+            "grad_to_weight_ratio": 0.004426830405589633
+          },
+          "encoder.layers.2.attention_layer.value_linear.weight": {
+            "grad_norm": 0.39241737127304077,
+            "param_norm": 18.98895835876465,
+            "grad_to_weight_ratio": 0.020665555416940172
+          },
+          "encoder.layers.2.attention_layer.output_linear.weight": {
+            "grad_norm": 0.3735000491142273,
+            "param_norm": 17.741331100463867,
+            "grad_to_weight_ratio": 0.021052538109975337
+          },
+          "encoder.layers.2.attention_layer.output_linear.bias": {
+            "grad_norm": 0.04274407774209976,
+            "param_norm": 1.0758063793182373,
+            "grad_to_weight_ratio": 0.03973212890701384
+          },
+          "encoder.layers.2.feed_forward.fc1.weight": {
+            "grad_norm": 0.3147575855255127,
+            "param_norm": 42.78855895996094,
+            "grad_to_weight_ratio": 0.0073561155873474796
+          },
+          "encoder.layers.2.feed_forward.fc1.bias": {
+            "grad_norm": 0.014651515521109104,
+            "param_norm": 2.453015089035034,
+            "grad_to_weight_ratio": 0.005972859925157141
+          },
+          "encoder.layers.2.feed_forward.fc2.weight": {
+            "grad_norm": 0.25934258103370667,
+            "param_norm": 40.715431213378906,
+            "grad_to_weight_ratio": 0.006369638569577067
+          },
+          "encoder.layers.2.feed_forward.fc2.bias": {
+            "grad_norm": 0.03659326210618019,
+            "param_norm": 1.8184162378311157,
+            "grad_to_weight_ratio": 0.020123699593557327
+          },
+          "encoder.layers.2.norm1.weight": {
+            "grad_norm": 0.03604165092110634,
+            "param_norm": 19.945579528808594,
+            "grad_to_weight_ratio": 0.001806999434087509
+          },
+          "encoder.layers.2.norm1.bias": {
+            "grad_norm": 0.04253522679209709,
+            "param_norm": 4.029062271118164,
+            "grad_to_weight_ratio": 0.010557103348090463
+          },
+          "encoder.layers.2.norm2.weight": {
+            "grad_norm": 0.04806182160973549,
+            "param_norm": 17.368209838867188,
+            "grad_to_weight_ratio": 0.002767229441354301
+          },
+          "encoder.layers.2.norm2.bias": {
+            "grad_norm": 0.06715068221092224,
+            "param_norm": 1.7655342817306519,
+            "grad_to_weight_ratio": 0.03803419899899097
+          },
+          "encoder.layers.3.attention_layer.query_linear.weight": {
+            "grad_norm": 0.14690318703651428,
+            "param_norm": 29.45256805419922,
+            "grad_to_weight_ratio": 0.004987788730890123
+          },
+          "encoder.layers.3.attention_layer.key_linear.weight": {
+            "grad_norm": 0.13842861354351044,
+            "param_norm": 29.43021011352539,
+            "grad_to_weight_ratio": 0.00470362301218799
+          },
+          "encoder.layers.3.attention_layer.value_linear.weight": {
+            "grad_norm": 0.4884301722049713,
+            "param_norm": 19.081523895263672,
+            "grad_to_weight_ratio": 0.025597021227753286
+          },
+          "encoder.layers.3.attention_layer.output_linear.weight": {
+            "grad_norm": 0.37156692147254944,
+            "param_norm": 17.314102172851562,
+            "grad_to_weight_ratio": 0.02146036322086301
+          },
+          "encoder.layers.3.attention_layer.output_linear.bias": {
+            "grad_norm": 0.057290252298116684,
+            "param_norm": 1.1777573823928833,
+            "grad_to_weight_ratio": 0.04864350939721541
+          },
+          "encoder.layers.3.feed_forward.fc1.weight": {
+            "grad_norm": 0.3993653655052185,
+            "param_norm": 43.03874969482422,
+            "grad_to_weight_ratio": 0.009279204631570335
+          },
+          "encoder.layers.3.feed_forward.fc1.bias": {
+            "grad_norm": 0.01603385992348194,
+            "param_norm": 1.9775564670562744,
+            "grad_to_weight_ratio": 0.008107915091467051
+          },
+          "encoder.layers.3.feed_forward.fc2.weight": {
+            "grad_norm": 0.25355541706085205,
+            "param_norm": 40.18751907348633,
+            "grad_to_weight_ratio": 0.006309307538920178
+          },
+          "encoder.layers.3.feed_forward.fc2.bias": {
+            "grad_norm": 0.050069790333509445,
+            "param_norm": 1.1833248138427734,
+            "grad_to_weight_ratio": 0.042312803507321556
+          },
+          "encoder.layers.3.norm1.weight": {
+            "grad_norm": 0.040389370173215866,
+            "param_norm": 19.57526206970215,
+            "grad_to_weight_ratio": 0.0020632863064309597
+          },
+          "encoder.layers.3.norm1.bias": {
+            "grad_norm": 0.056045182049274445,
+            "param_norm": 3.980025291442871,
+            "grad_to_weight_ratio": 0.014081614548972478
+          },
+          "encoder.layers.3.norm2.weight": {
+            "grad_norm": 0.18994024395942688,
+            "param_norm": 11.451133728027344,
+            "grad_to_weight_ratio": 0.01658702522131237
+          },
+          "encoder.layers.3.norm2.bias": {
+            "grad_norm": 0.15254758298397064,
+            "param_norm": 2.4655158519744873,
+            "grad_to_weight_ratio": 0.06187248111251945
+          },
+          "decoder.layers.0.self_attention_layer.query_linear.weight": {
+            "grad_norm": 0.2506406009197235,
+            "param_norm": 25.307010650634766,
+            "grad_to_weight_ratio": 0.009903998713235097
+          },
+          "decoder.layers.0.self_attention_layer.key_linear.weight": {
+            "grad_norm": 0.26672789454460144,
+            "param_norm": 25.357986450195312,
+            "grad_to_weight_ratio": 0.010518496611253475
+          },
+          "decoder.layers.0.self_attention_layer.value_linear.weight": {
+            "grad_norm": 0.2856094539165497,
+            "param_norm": 21.773256301879883,
+            "grad_to_weight_ratio": 0.013117443250409784
+          },
+          "decoder.layers.0.self_attention_layer.output_linear.weight": {
+            "grad_norm": 0.30432453751564026,
+            "param_norm": 21.417236328125,
+            "grad_to_weight_ratio": 0.014209328078244564
+          },
+          "decoder.layers.0.self_attention_layer.output_linear.bias": {
+            "grad_norm": 0.01952493004500866,
+            "param_norm": 1.893370509147644,
+            "grad_to_weight_ratio": 0.010312260569532196
+          },
+          "decoder.layers.0.cross_attention_layer.query_linear.weight": {
+            "grad_norm": 0.09001103788614273,
+            "param_norm": 30.34445571899414,
+            "grad_to_weight_ratio": 0.0029663091907032387
+          },
+          "decoder.layers.0.cross_attention_layer.key_linear.weight": {
+            "grad_norm": 0.08555619418621063,
+            "param_norm": 30.103914260864258,
+            "grad_to_weight_ratio": 0.0028420288951404803
+          },
+          "decoder.layers.0.cross_attention_layer.value_linear.weight": {
+            "grad_norm": 0.13360156118869781,
+            "param_norm": 25.03641128540039,
+            "grad_to_weight_ratio": 0.005336290399838584
+          },
+          "decoder.layers.0.cross_attention_layer.output_linear.weight": {
+            "grad_norm": 0.14034530520439148,
+            "param_norm": 23.730072021484375,
+            "grad_to_weight_ratio": 0.005914238485130676
+          },
+          "decoder.layers.0.cross_attention_layer.output_linear.bias": {
+            "grad_norm": 0.02972015179693699,
+            "param_norm": 0.9056278467178345,
+            "grad_to_weight_ratio": 0.03281717971086643
+          },
+          "decoder.layers.0.feed_forward.fc1.weight": {
+            "grad_norm": 0.2960856854915619,
+            "param_norm": 43.673343658447266,
+            "grad_to_weight_ratio": 0.006779551568277654
+          },
+          "decoder.layers.0.feed_forward.fc1.bias": {
+            "grad_norm": 0.01381844561547041,
+            "param_norm": 2.0766494274139404,
+            "grad_to_weight_ratio": 0.00665420240558943
+          },
+          "decoder.layers.0.feed_forward.fc2.weight": {
+            "grad_norm": 0.2579648792743683,
+            "param_norm": 41.12253952026367,
+            "grad_to_weight_ratio": 0.006273077545399316
+          },
+          "decoder.layers.0.feed_forward.fc2.bias": {
+            "grad_norm": 0.02883533388376236,
+            "param_norm": 1.8753615617752075,
+            "grad_to_weight_ratio": 0.01537587976179463
+          },
+          "decoder.layers.0.norm1.weight": {
+            "grad_norm": 0.025097748264670372,
+            "param_norm": 22.931339263916016,
+            "grad_to_weight_ratio": 0.0010944737233102757
+          },
+          "decoder.layers.0.norm1.bias": {
+            "grad_norm": 0.029502904042601585,
+            "param_norm": 2.445063352584839,
+            "grad_to_weight_ratio": 0.012066314769063157
+          },
+          "decoder.layers.0.norm2.weight": {
+            "grad_norm": 0.030590059235692024,
+            "param_norm": 20.280179977416992,
+            "grad_to_weight_ratio": 0.0015083721776509922
+          },
+          "decoder.layers.0.norm2.bias": {
+            "grad_norm": 0.03403147682547569,
+            "param_norm": 2.343634605407715,
+            "grad_to_weight_ratio": 0.01452081171140619
+          },
+          "decoder.layers.0.norm3.weight": {
+            "grad_norm": 0.05133355036377907,
+            "param_norm": 16.039201736450195,
+            "grad_to_weight_ratio": 0.0032005053123757917
+          },
+          "decoder.layers.0.norm3.bias": {
+            "grad_norm": 0.06990908831357956,
+            "param_norm": 1.2945226430892944,
+            "grad_to_weight_ratio": 0.05400375859528578
+          },
+          "decoder.layers.1.self_attention_layer.query_linear.weight": {
+            "grad_norm": 0.1352008730173111,
+            "param_norm": 29.16192054748535,
+            "grad_to_weight_ratio": 0.0046362129269625525
+          },
+          "decoder.layers.1.self_attention_layer.key_linear.weight": {
+            "grad_norm": 0.12783995270729065,
+            "param_norm": 29.16992950439453,
+            "grad_to_weight_ratio": 0.004382593817651387
+          },
+          "decoder.layers.1.self_attention_layer.value_linear.weight": {
+            "grad_norm": 0.2795604467391968,
+            "param_norm": 23.656097412109375,
+            "grad_to_weight_ratio": 0.011817690884045825
+          },
+          "decoder.layers.1.self_attention_layer.output_linear.weight": {
+            "grad_norm": 0.2718953788280487,
+            "param_norm": 22.97449493408203,
+            "grad_to_weight_ratio": 0.011834661854728635
+          },
+          "decoder.layers.1.self_attention_layer.output_linear.bias": {
+            "grad_norm": 0.0436825305223465,
+            "param_norm": 0.9426340460777283,
+            "grad_to_weight_ratio": 0.04634092170133451
+          },
+          "decoder.layers.1.cross_attention_layer.query_linear.weight": {
+            "grad_norm": 0.1602097749710083,
+            "param_norm": 33.56377410888672,
+            "grad_to_weight_ratio": 0.004773294399231003
+          },
+          "decoder.layers.1.cross_attention_layer.key_linear.weight": {
+            "grad_norm": 0.14686958491802216,
+            "param_norm": 33.025421142578125,
+            "grad_to_weight_ratio": 0.004447167661661266
+          },
+          "decoder.layers.1.cross_attention_layer.value_linear.weight": {
+            "grad_norm": 0.20628413558006287,
+            "param_norm": 28.058584213256836,
+            "grad_to_weight_ratio": 0.007351908207919931
+          },
+          "decoder.layers.1.cross_attention_layer.output_linear.weight": {
+            "grad_norm": 0.19977344572544098,
+            "param_norm": 27.102577209472656,
+            "grad_to_weight_ratio": 0.007371012881225574
+          },
+          "decoder.layers.1.cross_attention_layer.output_linear.bias": {
+            "grad_norm": 0.03229990974068642,
+            "param_norm": 1.3018608093261719,
+            "grad_to_weight_ratio": 0.02481057076860595
+          },
+          "decoder.layers.1.feed_forward.fc1.weight": {
+            "grad_norm": 0.33991485834121704,
+            "param_norm": 43.57281494140625,
+            "grad_to_weight_ratio": 0.007801076400464454
+          },
+          "decoder.layers.1.feed_forward.fc1.bias": {
+            "grad_norm": 0.01567891240119934,
+            "param_norm": 1.973999261856079,
+            "grad_to_weight_ratio": 0.007942714419482149
+          },
+          "decoder.layers.1.feed_forward.fc2.weight": {
+            "grad_norm": 0.29842454195022583,
+            "param_norm": 41.970638275146484,
+            "grad_to_weight_ratio": 0.00711031697907094
+          },
+          "decoder.layers.1.feed_forward.fc2.bias": {
+            "grad_norm": 0.03726514056324959,
+            "param_norm": 2.216982364654541,
+            "grad_to_weight_ratio": 0.01680894767470989
+          },
+          "decoder.layers.1.norm1.weight": {
+            "grad_norm": 0.027726951986551285,
+            "param_norm": 24.06261444091797,
+            "grad_to_weight_ratio": 0.0011522834334826491
+          },
+          "decoder.layers.1.norm1.bias": {
+            "grad_norm": 0.03225488215684891,
+            "param_norm": 2.7702667713165283,
+            "grad_to_weight_ratio": 0.011643240460018443
+          },
+          "decoder.layers.1.norm2.weight": {
+            "grad_norm": 0.03624362498521805,
+            "param_norm": 20.215980529785156,
+            "grad_to_weight_ratio": 0.0017928205328362292
+          },
+          "decoder.layers.1.norm2.bias": {
+            "grad_norm": 0.04329323396086693,
+            "param_norm": 2.4272446632385254,
+            "grad_to_weight_ratio": 0.017836370027521475
+          },
+          "decoder.layers.1.norm3.weight": {
+            "grad_norm": 0.0731615498661995,
+            "param_norm": 15.708695411682129,
+            "grad_to_weight_ratio": 0.0046573918424687635
+          },
+          "decoder.layers.1.norm3.bias": {
+            "grad_norm": 0.103321373462677,
+            "param_norm": 1.133062720298767,
+            "grad_to_weight_ratio": 0.09118769121213513
+          },
+          "decoder.layers.2.self_attention_layer.query_linear.weight": {
+            "grad_norm": 0.1640961915254593,
+            "param_norm": 28.673992156982422,
+            "grad_to_weight_ratio": 0.005722823338552612
+          },
+          "decoder.layers.2.self_attention_layer.key_linear.weight": {
+            "grad_norm": 0.15322871506214142,
+            "param_norm": 28.582561492919922,
+            "grad_to_weight_ratio": 0.005360916134129258
+          },
+          "decoder.layers.2.self_attention_layer.value_linear.weight": {
+            "grad_norm": 0.3782458007335663,
+            "param_norm": 24.852397918701172,
+            "grad_to_weight_ratio": 0.015219690348226922
+          },
+          "decoder.layers.2.self_attention_layer.output_linear.weight": {
+            "grad_norm": 0.2884204685688019,
+            "param_norm": 23.534637451171875,
+            "grad_to_weight_ratio": 0.012255148147796437
+          },
+          "decoder.layers.2.self_attention_layer.output_linear.bias": {
+            "grad_norm": 0.05033516138792038,
+            "param_norm": 0.7505745887756348,
+            "grad_to_weight_ratio": 0.06706217095620291
+          },
+          "decoder.layers.2.cross_attention_layer.query_linear.weight": {
+            "grad_norm": 0.15913651883602142,
+            "param_norm": 35.03413772583008,
+            "grad_to_weight_ratio": 0.004542327260382042
+          },
+          "decoder.layers.2.cross_attention_layer.key_linear.weight": {
+            "grad_norm": 0.14329305291175842,
+            "param_norm": 34.63029861450195,
+            "grad_to_weight_ratio": 0.004137794320137574
+          },
+          "decoder.layers.2.cross_attention_layer.value_linear.weight": {
+            "grad_norm": 0.21875277161598206,
+            "param_norm": 30.684757232666016,
+            "grad_to_weight_ratio": 0.007129037064145246
+          },
+          "decoder.layers.2.cross_attention_layer.output_linear.weight": {
+            "grad_norm": 0.23608684539794922,
+            "param_norm": 29.766456604003906,
+            "grad_to_weight_ratio": 0.007931304976561608
+          },
+          "decoder.layers.2.cross_attention_layer.output_linear.bias": {
+            "grad_norm": 0.029830699786543846,
+            "param_norm": 1.054974913597107,
+            "grad_to_weight_ratio": 0.02827621718966093
+          },
+          "decoder.layers.2.feed_forward.fc1.weight": {
+            "grad_norm": 0.42240867018699646,
+            "param_norm": 43.39973068237305,
+            "grad_to_weight_ratio": 0.009732979065663868
+          },
+          "decoder.layers.2.feed_forward.fc1.bias": {
+            "grad_norm": 0.018086126074194908,
+            "param_norm": 2.058098316192627,
+            "grad_to_weight_ratio": 0.008787785273370466
+          },
+          "decoder.layers.2.feed_forward.fc2.weight": {
+            "grad_norm": 0.30871498584747314,
+            "param_norm": 41.78953170776367,
+            "grad_to_weight_ratio": 0.007387376054039691
+          },
+          "decoder.layers.2.feed_forward.fc2.bias": {
+            "grad_norm": 0.03709905594587326,
+            "param_norm": 2.4397153854370117,
+            "grad_to_weight_ratio": 0.015206304869538181
+          },
+          "decoder.layers.2.norm1.weight": {
+            "grad_norm": 0.029964420944452286,
+            "param_norm": 23.67780876159668,
+            "grad_to_weight_ratio": 0.0012655065021494165
+          },
+          "decoder.layers.2.norm1.bias": {
+            "grad_norm": 0.03235355392098427,
+            "param_norm": 2.7067246437072754,
+            "grad_to_weight_ratio": 0.011953027433429339
+          },
+          "decoder.layers.2.norm2.weight": {
+            "grad_norm": 0.038857363164424896,
+            "param_norm": 19.991548538208008,
+            "grad_to_weight_ratio": 0.0019436895091022315
+          },
+          "decoder.layers.2.norm2.bias": {
+            "grad_norm": 0.042886484414339066,
+            "param_norm": 2.711909055709839,
+            "grad_to_weight_ratio": 0.01581413075929929
+          },
+          "decoder.layers.2.norm3.weight": {
+            "grad_norm": 0.04816955327987671,
+            "param_norm": 17.58161735534668,
+            "grad_to_weight_ratio": 0.00273976803762171
+          },
+          "decoder.layers.2.norm3.bias": {
+            "grad_norm": 0.08085499703884125,
+            "param_norm": 1.0881868600845337,
+            "grad_to_weight_ratio": 0.07430249344536827
+          },
+          "decoder.layers.3.self_attention_layer.query_linear.weight": {
+            "grad_norm": 0.17428232729434967,
+            "param_norm": 31.422555923461914,
+            "grad_to_weight_ratio": 0.0055464083736172076
+          },
+          "decoder.layers.3.self_attention_layer.key_linear.weight": {
+            "grad_norm": 0.15520748496055603,
+            "param_norm": 31.094215393066406,
+            "grad_to_weight_ratio": 0.004991522796074804
+          },
+          "decoder.layers.3.self_attention_layer.value_linear.weight": {
+            "grad_norm": 0.44218549132347107,
+            "param_norm": 23.444210052490234,
+            "grad_to_weight_ratio": 0.018861181090487775
+          },
+          "decoder.layers.3.self_attention_layer.output_linear.weight": {
+            "grad_norm": 0.3175715208053589,
+            "param_norm": 21.424062728881836,
+            "grad_to_weight_ratio": 0.014823123178090076
+          },
+          "decoder.layers.3.self_attention_layer.output_linear.bias": {
+            "grad_norm": 0.04501575604081154,
+            "param_norm": 0.7656175494194031,
+            "grad_to_weight_ratio": 0.058796661694719386
+          },
+          "decoder.layers.3.cross_attention_layer.query_linear.weight": {
+            "grad_norm": 0.17294764518737793,
+            "param_norm": 35.09070587158203,
+            "grad_to_weight_ratio": 0.004928588379506878
+          },
+          "decoder.layers.3.cross_attention_layer.key_linear.weight": {
+            "grad_norm": 0.15117591619491577,
+            "param_norm": 34.5621223449707,
+            "grad_to_weight_ratio": 0.004374034519234601
+          },
+          "decoder.layers.3.cross_attention_layer.value_linear.weight": {
+            "grad_norm": 0.27421149611473083,
+            "param_norm": 32.42921447753906,
+            "grad_to_weight_ratio": 0.008455693439773114
+          },
+          "decoder.layers.3.cross_attention_layer.output_linear.weight": {
+            "grad_norm": 0.2784635126590729,
+            "param_norm": 31.322601318359375,
+            "grad_to_weight_ratio": 0.00889017836765192
+          },
+          "decoder.layers.3.cross_attention_layer.output_linear.bias": {
+            "grad_norm": 0.030515627935528755,
+            "param_norm": 2.5462377071380615,
+            "grad_to_weight_ratio": 0.011984595094939484
+          },
+          "decoder.layers.3.feed_forward.fc1.weight": {
+            "grad_norm": 0.5449442863464355,
+            "param_norm": 40.63393020629883,
+            "grad_to_weight_ratio": 0.013411065175820677
+          },
+          "decoder.layers.3.feed_forward.fc1.bias": {
+            "grad_norm": 0.02406179904937744,
+            "param_norm": 3.274101734161377,
+            "grad_to_weight_ratio": 0.0073491299302992615
+          },
+          "decoder.layers.3.feed_forward.fc2.weight": {
+            "grad_norm": 0.38977277278900146,
+            "param_norm": 38.438724517822266,
+            "grad_to_weight_ratio": 0.010140106823999106
+          },
+          "decoder.layers.3.feed_forward.fc2.bias": {
+            "grad_norm": 0.04156862571835518,
+            "param_norm": 1.3945589065551758,
+            "grad_to_weight_ratio": 0.029807723089308385
+          },
+          "decoder.layers.3.norm1.weight": {
+            "grad_norm": 0.033089529722929,
+            "param_norm": 22.777145385742188,
+            "grad_to_weight_ratio": 0.001452751394546597
+          },
+          "decoder.layers.3.norm1.bias": {
+            "grad_norm": 0.033036645501852036,
+            "param_norm": 3.332117795944214,
+            "grad_to_weight_ratio": 0.009914609123979247
+          },
+          "decoder.layers.3.norm2.weight": {
+            "grad_norm": 0.04525093361735344,
+            "param_norm": 19.743022918701172,
+            "grad_to_weight_ratio": 0.0022919962056311112
+          },
+          "decoder.layers.3.norm2.bias": {
+            "grad_norm": 0.041699592024087906,
+            "param_norm": 4.823736190795898,
+            "grad_to_weight_ratio": 0.008644666784150769
+          },
+          "decoder.layers.3.norm3.weight": {
+            "grad_norm": 0.034289825707674026,
+            "param_norm": 31.38285255432129,
+            "grad_to_weight_ratio": 0.0010926293474539926
+          },
+          "decoder.layers.3.norm3.bias": {
+            "grad_norm": 0.033960916101932526,
+            "param_norm": 3.2170767784118652,
+            "grad_to_weight_ratio": 0.010556451847781836
+          }
+        },
+        "grouped": {
+          "embedding": {
+            "sum": 1.3044230937957764,
+            "count": 1.0,
+            "max": 1.3044230937957764,
+            "avg": 1.3044230937957764
+          },
+          "encoder.layer0.self_attn": {
+            "sum": 1.209842188283801,
+            "count": 5.0,
+            "max": 0.37529751658439636,
+            "avg": 0.2419684376567602
+          },
+          "encoder.layer0.ff": {
+            "sum": 0.5316245863214135,
+            "count": 4.0,
+            "max": 0.2714630365371704,
+            "avg": 0.13290614658035338
+          },
+          "encoder.layer0.norm": {
+            "sum": 0.17787226289510727,
+            "count": 4.0,
+            "max": 0.05286872386932373,
+            "avg": 0.04446806572377682
+          },
+          "encoder.layer1.self_attn": {
+            "sum": 1.1516140773892403,
+            "count": 5.0,
+            "max": 0.4261511266231537,
+            "avg": 0.23032281547784805
+          },
+          "encoder.layer1.ff": {
+            "sum": 0.5532910721376538,
+            "count": 4.0,
+            "max": 0.2787674069404602,
+            "avg": 0.13832276803441346
+          },
+          "encoder.layer1.norm": {
+            "sum": 0.18069618567824364,
+            "count": 4.0,
+            "max": 0.05488703399896622,
+            "avg": 0.04517404641956091
+          },
+          "encoder.layer2.self_attn": {
+            "sum": 1.0703937783837318,
+            "count": 5.0,
+            "max": 0.39241737127304077,
+            "avg": 0.21407875567674636
+          },
+          "encoder.layer2.ff": {
+            "sum": 0.6253449441865087,
+            "count": 4.0,
+            "max": 0.3147575855255127,
+            "avg": 0.15633623604662716
+          },
+          "encoder.layer2.norm": {
+            "sum": 0.19378938153386116,
+            "count": 4.0,
+            "max": 0.06715068221092224,
+            "avg": 0.04844734538346529
+          },
+          "encoder.layer3.self_attn": {
+            "sum": 1.2026191465556622,
+            "count": 5.0,
+            "max": 0.4884301722049713,
+            "avg": 0.24052382931113242
+          },
+          "encoder.layer3.ff": {
+            "sum": 0.7190244328230619,
+            "count": 4.0,
+            "max": 0.3993653655052185,
+            "avg": 0.17975610820576549
+          },
+          "encoder.layer3.norm": {
+            "sum": 0.43892237916588783,
+            "count": 4.0,
+            "max": 0.18994024395942688,
+            "avg": 0.10973059479147196
+          },
+          "decoder.layer0.self_attn": {
+            "sum": 1.1268274169415236,
+            "count": 5.0,
+            "max": 0.30432453751564026,
+            "avg": 0.2253654833883047
+          },
+          "decoder.layer0.cross_attn": {
+            "sum": 0.47923425026237965,
+            "count": 5.0,
+            "max": 0.14034530520439148,
+            "avg": 0.09584685005247592
+          },
+          "decoder.layer0.ff": {
+            "sum": 0.596704344265163,
+            "count": 4.0,
+            "max": 0.2960856854915619,
+            "avg": 0.14917608606629074
+          },
+          "decoder.layer0.norm": {
+            "sum": 0.2404648270457983,
+            "count": 6.0,
+            "max": 0.06990908831357956,
+            "avg": 0.04007747117429972
+          },
+          "decoder.layer1.self_attn": {
+            "sum": 0.8581791818141937,
+            "count": 5.0,
+            "max": 0.2795604467391968,
+            "avg": 0.17163583636283875
+          },
+          "decoder.layer1.cross_attn": {
+            "sum": 0.7454368509352207,
+            "count": 5.0,
+            "max": 0.20628413558006287,
+            "avg": 0.14908737018704415
+          },
+          "decoder.layer1.ff": {
+            "sum": 0.6912834532558918,
+            "count": 4.0,
+            "max": 0.33991485834121704,
+            "avg": 0.17282086331397295
+          },
+          "decoder.layer1.norm": {
+            "sum": 0.31600161641836166,
+            "count": 6.0,
+            "max": 0.103321373462677,
+            "avg": 0.052666936069726944
+          },
+          "decoder.layer2.self_attn": {
+            "sum": 1.0343263372778893,
+            "count": 5.0,
+            "max": 0.3782458007335663,
+            "avg": 0.20686526745557784
+          },
+          "decoder.layer2.cross_attn": {
+            "sum": 0.787099888548255,
+            "count": 5.0,
+            "max": 0.23608684539794922,
+            "avg": 0.157419977709651
+          },
+          "decoder.layer2.ff": {
+            "sum": 0.7863088380545378,
+            "count": 4.0,
+            "max": 0.42240867018699646,
+            "avg": 0.19657720951363444
+          },
+          "decoder.layer2.norm": {
+            "sum": 0.2730863727629185,
+            "count": 6.0,
+            "max": 0.08085499703884125,
+            "avg": 0.04551439546048641
+          },
+          "decoder.layer3.self_attn": {
+            "sum": 1.1342625804245472,
+            "count": 5.0,
+            "max": 0.44218549132347107,
+            "avg": 0.22685251608490944
+          },
+          "decoder.layer3.cross_attn": {
+            "sum": 0.9073141980916262,
+            "count": 5.0,
+            "max": 0.2784635126590729,
+            "avg": 0.18146283961832524
+          },
+          "decoder.layer3.ff": {
+            "sum": 1.0003474839031696,
+            "count": 4.0,
+            "max": 0.5449442863464355,
+            "avg": 0.2500868709757924
+          },
+          "decoder.layer3.norm": {
+            "sum": 0.22132744267582893,
+            "count": 6.0,
+            "max": 0.04525093361735344,
+            "avg": 0.03688790711263815
+          }
+        },
+        "missing": []
+      },
+      "attention": {
+        "summary": {
+          "encoder": [
+            {
+              "layer": 0,
+              "mean_entropy": 1.3633646965026855,
+              "mean_normalised_entropy": 0.38838931918144226
+            },
+            {
+              "layer": 1,
+              "mean_entropy": 0.6433506011962891,
+              "mean_normalised_entropy": 0.18644404411315918
+            },
+            {
+              "layer": 2,
+              "mean_entropy": 1.0018328428268433,
+              "mean_normalised_entropy": 0.2915217876434326
+            },
+            {
+              "layer": 3,
+              "mean_entropy": 1.1001973152160645,
+              "mean_normalised_entropy": 0.3180953860282898
+            }
+          ],
+          "decoder_self": [
+            {
+              "layer": 0,
+              "mean_entropy": 0.3746384084224701,
+              "mean_normalised_entropy": 0.10358868539333344
+            },
+            {
+              "layer": 1,
+              "mean_entropy": 0.7695723176002502,
+              "mean_normalised_entropy": 0.21214483678340912
+            },
+            {
+              "layer": 2,
+              "mean_entropy": 0.7980881333351135,
+              "mean_normalised_entropy": 0.22028915584087372
+            },
+            {
+              "layer": 3,
+              "mean_entropy": 0.6095401048660278,
+              "mean_normalised_entropy": 0.16934041678905487
+            }
+          ],
+          "decoder_cross": [
+            {
+              "layer": 0,
+              "mean_entropy": 0.9926103949546814,
+              "mean_normalised_entropy": 0.2929244041442871
+            },
+            {
+              "layer": 1,
+              "mean_entropy": 1.1003278493881226,
+              "mean_normalised_entropy": 0.3237256407737732
+            },
+            {
+              "layer": 2,
+              "mean_entropy": 0.7252426147460938,
+              "mean_normalised_entropy": 0.21367527544498444
+            },
+            {
+              "layer": 3,
+              "mean_entropy": 0.626167356967926,
+              "mean_normalised_entropy": 0.18314598500728607
+            }
+          ]
+        },
+        "flagged_layers": {}
+      },
+      "activations": {
+        "encoder_layers": {
+          "0": {
+            "mean": -0.006741917226463556,
+            "std": 0.7567456960678101,
+            "max_abs": 6.33781623840332
+          },
+          "1": {
+            "mean": -0.0020844819955527782,
+            "std": 0.7163907885551453,
+            "max_abs": 7.352891445159912
+          },
+          "2": {
+            "mean": -0.0032328872475773096,
+            "std": 0.7347663044929504,
+            "max_abs": 9.826601028442383
+          },
+          "3": {
+            "mean": -0.0013982581440359354,
+            "std": 0.3635603189468384,
+            "max_abs": 2.6102705001831055
+          }
+        },
+        "decoder_cross_attention": {
+          "0": {
+            "query_std": 1.0075185298919678,
+            "key_std": 0.3680627942085266,
+            "value_std": 0.3680627942085266,
+            "output_std": 0.2537617087364197
+          },
+          "1": {
+            "query_std": 1.0873469114303589,
+            "key_std": 0.3680627942085266,
+            "value_std": 0.3680627942085266,
+            "output_std": 0.31841492652893066
+          },
+          "2": {
+            "query_std": 1.1087111234664917,
+            "key_std": 0.3680627942085266,
+            "value_std": 0.3680627942085266,
+            "output_std": 0.39632153511047363
+          },
+          "3": {
+            "query_std": 1.0010381937026978,
+            "key_std": 0.3680627942085266,
+            "value_std": 0.3680627942085266,
+            "output_std": 0.40245166420936584
+          }
+        },
+        "decoder_self_attention": {
+          "0": {
+            "mean": 0.17315351963043213,
+            "std": 1.3055107593536377,
+            "max_abs": 7.235065937042236
+          },
+          "1": {
+            "mean": -0.011658968403935432,
+            "std": 0.6399785280227661,
+            "max_abs": 3.6515018939971924
+          },
+          "2": {
+            "mean": -0.007804906461387873,
+            "std": 0.6082821488380432,
+            "max_abs": 5.065135478973389
+          },
+          "3": {
+            "mean": -0.006232936400920153,
+            "std": 0.6916216015815735,
+            "max_abs": 11.009607315063477
+          }
+        }
+      },
+      "mask_usage": {
+        "src_pad_fraction": 0.590625,
+        "tgt_pad_fraction": 0.5855263157894737
+      },
+      "batch_index": 0,
+      "missing_from_optimizer": []
+    }
+  ],
+  "summary": {
+    "gradient_groups": {
+      "embedding": {
+        "mean_grad_norm": 1.3044230937957764,
+        "max_grad_norm": 1.3044230937957764
+      },
+      "encoder.layer0.self_attn": {
+        "mean_grad_norm": 0.24196843802928925,
+        "max_grad_norm": 0.24196843802928925
+      },
+      "encoder.layer0.ff": {
+        "mean_grad_norm": 0.13290615379810333,
+        "max_grad_norm": 0.13290615379810333
+      },
+      "encoder.layer0.norm": {
+        "mean_grad_norm": 0.04446806758642197,
+        "max_grad_norm": 0.04446806758642197
+      },
+      "encoder.layer1.self_attn": {
+        "mean_grad_norm": 0.23032280802726746,
+        "max_grad_norm": 0.23032280802726746
+      },
+      "encoder.layer1.ff": {
+        "mean_grad_norm": 0.13832277059555054,
+        "max_grad_norm": 0.13832277059555054
+      },
+      "encoder.layer1.norm": {
+        "mean_grad_norm": 0.045174047350883484,
+        "max_grad_norm": 0.045174047350883484
+      },
+      "encoder.layer2.self_attn": {
+        "mean_grad_norm": 0.21407875418663025,
+        "max_grad_norm": 0.21407875418663025
+      },
+      "encoder.layer2.ff": {
+        "mean_grad_norm": 0.1563362330198288,
+        "max_grad_norm": 0.1563362330198288
+      },
+      "encoder.layer2.norm": {
+        "mean_grad_norm": 0.048447344452142715,
+        "max_grad_norm": 0.048447344452142715
+      },
+      "encoder.layer3.self_attn": {
+        "mean_grad_norm": 0.2405238300561905,
+        "max_grad_norm": 0.2405238300561905
+      },
+      "encoder.layer3.ff": {
+        "mean_grad_norm": 0.17975610494613647,
+        "max_grad_norm": 0.17975610494613647
+      },
+      "encoder.layer3.norm": {
+        "mean_grad_norm": 0.10973059386014938,
+        "max_grad_norm": 0.10973059386014938
+      },
+      "decoder.layer0.self_attn": {
+        "mean_grad_norm": 0.22536548972129822,
+        "max_grad_norm": 0.22536548972129822
+      },
+      "decoder.layer0.cross_attn": {
+        "mean_grad_norm": 0.09584684669971466,
+        "max_grad_norm": 0.09584684669971466
+      },
+      "decoder.layer0.ff": {
+        "mean_grad_norm": 0.14917609095573425,
+        "max_grad_norm": 0.14917609095573425
+      },
+      "decoder.layer0.norm": {
+        "mean_grad_norm": 0.04007747024297714,
+        "max_grad_norm": 0.04007747024297714
+      },
+      "decoder.layer1.self_attn": {
+        "mean_grad_norm": 0.17163583636283875,
+        "max_grad_norm": 0.17163583636283875
+      },
+      "decoder.layer1.cross_attn": {
+        "mean_grad_norm": 0.14908736944198608,
+        "max_grad_norm": 0.14908736944198608
+      },
+      "decoder.layer1.ff": {
+        "mean_grad_norm": 0.17282086610794067,
+        "max_grad_norm": 0.17282086610794067
+      },
+      "decoder.layer1.norm": {
+        "mean_grad_norm": 0.052666936069726944,
+        "max_grad_norm": 0.052666936069726944
+      },
+      "decoder.layer2.self_attn": {
+        "mean_grad_norm": 0.20686526596546173,
+        "max_grad_norm": 0.20686526596546173
+      },
+      "decoder.layer2.cross_attn": {
+        "mean_grad_norm": 0.15741997957229614,
+        "max_grad_norm": 0.15741997957229614
+      },
+      "decoder.layer2.ff": {
+        "mean_grad_norm": 0.19657720625400543,
+        "max_grad_norm": 0.19657720625400543
+      },
+      "decoder.layer2.norm": {
+        "mean_grad_norm": 0.04551439732313156,
+        "max_grad_norm": 0.04551439732313156
+      },
+      "decoder.layer3.self_attn": {
+        "mean_grad_norm": 0.22685252130031586,
+        "max_grad_norm": 0.22685252130031586
+      },
+      "decoder.layer3.cross_attn": {
+        "mean_grad_norm": 0.1814628392457962,
+        "max_grad_norm": 0.1814628392457962
+      },
+      "decoder.layer3.ff": {
+        "mean_grad_norm": 0.25008687376976013,
+        "max_grad_norm": 0.25008687376976013
+      },
+      "decoder.layer3.norm": {
+        "mean_grad_norm": 0.03688790649175644,
+        "max_grad_norm": 0.03688790649175644
+      }
+    },
+    "attention_normalised_entropy": {
+      "encoder": [
+        {
+          "layer": 0,
+          "mean_normalised_entropy": 0.38838931918144226,
+          "max_normalised_entropy": 0.38838931918144226
+        },
+        {
+          "layer": 1,
+          "mean_normalised_entropy": 0.18644404411315918,
+          "max_normalised_entropy": 0.18644404411315918
+        },
+        {
+          "layer": 2,
+          "mean_normalised_entropy": 0.2915217876434326,
+          "max_normalised_entropy": 0.2915217876434326
+        },
+        {
+          "layer": 3,
+          "mean_normalised_entropy": 0.3180953860282898,
+          "max_normalised_entropy": 0.3180953860282898
+        }
+      ],
+      "decoder_self": [
+        {
+          "layer": 0,
+          "mean_normalised_entropy": 0.10358868539333344,
+          "max_normalised_entropy": 0.10358868539333344
+        },
+        {
+          "layer": 1,
+          "mean_normalised_entropy": 0.21214483678340912,
+          "max_normalised_entropy": 0.21214483678340912
+        },
+        {
+          "layer": 2,
+          "mean_normalised_entropy": 0.22028915584087372,
+          "max_normalised_entropy": 0.22028915584087372
+        },
+        {
+          "layer": 3,
+          "mean_normalised_entropy": 0.16934041678905487,
+          "max_normalised_entropy": 0.16934041678905487
+        }
+      ],
+      "decoder_cross": [
+        {
+          "layer": 0,
+          "mean_normalised_entropy": 0.2929244041442871,
+          "max_normalised_entropy": 0.2929244041442871
+        },
+        {
+          "layer": 1,
+          "mean_normalised_entropy": 0.3237256407737732,
+          "max_normalised_entropy": 0.3237256407737732
+        },
+        {
+          "layer": 2,
+          "mean_normalised_entropy": 0.21367527544498444,
+          "max_normalised_entropy": 0.21367527544498444
+        },
+        {
+          "layer": 3,
+          "mean_normalised_entropy": 0.18314598500728607,
+          "max_normalised_entropy": 0.18314598500728607
+        }
+      ]
+    },
+    "flagged_attention_layers": {}
+  }
+}

test_model_v1/manifest.json

@@ -0,0 +1,48 @@
+{
+  "run_dir": "/mnt/shared/Local/Projets/Transformer/outputs/test_model_20250925-115005",
+  "model": "test_model",
+  "resume": {
+    "mode": "best",
+    "source": "/mnt/shared/Local/Projets/Transformer/outputs/test_model_20250924-175245/checkpoints/best/checkpoint.pt",
+    "status": "loaded"
+  },
+  "latest": {
+    "path": "/mnt/shared/Local/Projets/Transformer/outputs/test_model_20250925-115005/checkpoints/latest/checkpoint.pt",
+    "epoch": 10,
+    "optimizer_step": 35670,
+    "batch_idx": 0,
+    "running_loss": 0.0,
+    "metadata": {
+      "train_loss": 1.5890974594954101,
+      "val_loss": 1.5723653676981884
+    },
+    "timestamp": "2025-09-25T16:17:29.670333",
+    "previous_path": "/mnt/shared/Local/Projets/Transformer/outputs/test_model_20250925-115005/checkpoints/latest/checkpoint_prev.pt"
+  },
+  "latest_previous": {
+    "path": "/mnt/shared/Local/Projets/Transformer/outputs/test_model_20250925-115005/checkpoints/latest/checkpoint_prev.pt",
+    "epoch": 9,
+    "optimizer_step": 35670,
+    "batch_idx": 14266,
+    "running_loss": 22670.064357161522,
+    "metadata": {},
+    "timestamp": "2025-09-25T16:15:41.672243",
+    "previous_path": null
+  },
+  "best": {
+    "path": "/mnt/shared/Local/Projets/Transformer/outputs/test_model_20250925-115005/checkpoints/best/checkpoint.pt",
+    "epoch": 10,
+    "optimizer_step": 35670,
+    "train_loss": 1.5890974594954101,
+    "val_loss": 1.5723653676981884,
+    "metadata": {
+      "train_loss": 1.5890974594954101,
+      "val_loss": 1.5723653676981884
+    },
+    "timestamp": "2025-09-25T16:17:30.033226"
+  },
+  "test": {
+    "test_loss": 1.5692907183949012,
+    "timestamp": "2025-09-25T17:06:55.316845"
+  }
+}