Upload folder using huggingface_hub

README.md

@@ -1,7 +1,13 @@
-# Week 2 MoE (hash routing)
+# Week 2 MoE Seq2Seq (hash routing)
 
-* Best validation accuracy: 0.8050
+* Best validation loss: 5.6076
 * Top-k: 1
 * Aux loss coef: 0.0
 
-Artifacts include the trained state dict (`model.pt`), metrics (`metrics.json`), per-epoch history (`history.csv`), and tokenizer files.
+Artifacts include the trained state dict (`model.pt`), metrics (`metrics.json`), per-epoch history (`history.csv`), and tokenizer files.
+
+## Architecture
+- Encoder-Decoder Transformer with Sparse MoE layers
+- Hash-based routing (deterministic) or Token-choice top-k routing (learned)
+- Load balancing auxiliary loss for top-k routing
+- Trained from scratch on XSum for abstractive summarization

history.csv

@@ -1,4 +1,4 @@
-epoch,train_loss,train_aux_loss,train_accuracy,val_loss,val_aux_loss,val_accuracy
-1,0.9436405922412873,0.0,0.6104,0.6622284393310547,0.0,0.754
-2,0.5059148602962494,0.0,0.8156,0.5826493395864963,0.0,0.805
-3,0.3382958103120327,0.0,0.8844,0.6135995112359524,0.0,0.803
+epoch,train_loss,train_aux_loss,train_perplexity,val_loss,val_aux_loss,val_perplexity
+1,6.781526548633617,0.0,881.413217772467,6.13985468708606,0.0,463.9861427818742
+2,5.734937044167949,0.0,309.49348572595704,5.784560862128246,0.0,325.23918390511113
+3,5.265651165721379,0.0,193.57231541377683,5.607645124527238,0.0,272.5017740962985

metrics.json

@@ -2,128 +2,225 @@
   "history": [
     {
       "epoch": 1,
-      "train_loss": 0.9436405922412873,
+      "train_loss": 6.781526548633617,
       "train_aux_loss": 0.0,
-      "train_accuracy": 0.6104,
-      "val_loss": 0.6622284393310547,
+      "train_perplexity": 881.413217772467,
+      "val_loss": 6.13985468708606,
       "val_aux_loss": 0.0,
-      "val_accuracy": 0.754
+      "val_perplexity": 463.9861427818742
     },
     {
       "epoch": 2,
-      "train_loss": 0.5059148602962494,
+      "train_loss": 5.734937044167949,
       "train_aux_loss": 0.0,
-      "train_accuracy": 0.8156,
-      "val_loss": 0.5826493395864963,
+      "train_perplexity": 309.49348572595704,
+      "val_loss": 5.784560862128246,
       "val_aux_loss": 0.0,
-      "val_accuracy": 0.805
+      "val_perplexity": 325.23918390511113
     },
     {
       "epoch": 3,
-      "train_loss": 0.3382958103120327,
+      "train_loss": 5.265651165721379,
       "train_aux_loss": 0.0,
-      "train_accuracy": 0.8844,
-      "val_loss": 0.6135995112359524,
+      "train_perplexity": 193.57231541377683,
+      "val_loss": 5.607645124527238,
       "val_aux_loss": 0.0,
-      "val_accuracy": 0.803
+      "val_perplexity": 272.5017740962985
     }
   ],
   "train_expert_usage": [
-    [
-      [
-        22.28339958190918,
-        40.70140075683594,
-        14.662799835205078,
-        50.35239791870117
+    {
+      "encoder": [
+        [
+          1.1995620727539062,
+          1.0382475852966309,
+          1.1820646524429321,
+          1.1452935934066772
+        ],
+        [
+          1.0309076309204102,
+          1.3545637130737305,
+          1.255271315574646,
+          0.9244250655174255
+        ]
       ],
-      [
-        58.98899841308594,
-        38.73780059814453,
-        12.463199615478516,
-        17.809999465942383
+      "decoder": [
+        [
+          0.767426073551178,
+          0.17747089266777039,
+          0.3735591173171997,
+          0.1704733520746231
+        ],
+        [
+          0.13853856921195984,
+          0.5398667454719543,
+          0.7335297465324402,
+          0.0769944041967392
+        ]
       ]
-    ],
-    [
-      [
-        23.274999618530273,
-        41.05299758911133,
-        14.551199913024902,
-        49.12080001831055
+    },
+    {
+      "encoder": [
+        [
+          1.220947027206421,
+          1.0275551080703735,
+          1.2033497095108032,
+          1.1133160591125488
+        ],
+        [
+          1.0806751251220703,
+          1.291507363319397,
+          1.1101988554000854,
+          1.0827864408493042
+        ]
       ],
-      [
-        57.284000396728516,
-        32.5275993347168,
-        11.510799407958984,
-        26.67759895324707
+      "decoder": [
+        [
+          0.5099791884422302,
+          0.2548719048500061,
+          0.5015906691551208,
+          0.2213464379310608
+        ],
+        [
+          0.2474392205476761,
+          0.3361818194389343,
+          0.8026777505874634,
+          0.10148938745260239
+        ]
       ]
-    ],
-    [
-      [
-        22.86240005493164,
-        43.082000732421875,
-        14.503599166870117,
-        47.551998138427734
+    },
+    {
+      "encoder": [
+        [
+          1.1927927732467651,
+          1.0216560363769531,
+          1.242702841758728,
+          1.1080161333084106
+        ],
+        [
+          1.1292370557785034,
+          1.267861247062683,
+          1.047805905342102,
+          1.120263695716858
+        ]
       ],
-      [
-        55.36159896850586,
-        30.51259994506836,
-        11.39799976348877,
-        30.727798461914062
+      "decoder": [
+        [
+          0.48166799545288086,
+          0.27755507826805115,
+          0.4995720088481903,
+          0.2349848747253418
+        ],
+        [
+          0.246026873588562,
+          0.32668769359588623,
+          0.8019145131111145,
+          0.11915087699890137
+        ]
       ]
-    ]
+    }
   ],
   "val_expert_usage": [
-    [
-      [
-        13.781001091003418,
-        31.424001693725586,
-        14.86400032043457,
-        67.93099975585938
+    {
+      "encoder": [
+        [
+          1.3101885318756104,
+          1.0473449230194092,
+          1.1509929895401,
+          1.0964527130126953
+        ],
+        [
+          0.9937760829925537,
+          1.3266657590866089,
+          1.1915743350982666,
+          1.0929629802703857
+        ]
       ],
-      [
-        71.9800033569336,
-        18.58700180053711,
-        12.019001007080078,
-        25.41400146484375
+      "decoder": [
+        [
+          0.7106418013572693,
+          0.2959778308868408,
+          0.3988703489303589,
+          0.20999424159526825
+        ],
+        [
+          0.3808821439743042,
+          0.277557909488678,
+          0.8908476233482361,
+          0.06619657576084137
+        ]
       ]
-    ],
-    [
-      [
-        14.10200023651123,
-        34.134002685546875,
-        14.655000686645508,
-        65.10900115966797
+    },
+    {
+      "encoder": [
+        [
+          1.2734206914901733,
+          1.0350050926208496,
+          1.2253562211990356,
+          1.071197271347046
+        ],
+        [
+          1.047668695449829,
+          1.4553172588348389,
+          1.0208303928375244,
+          1.0811628103256226
+        ]
       ],
-      [
-        57.595001220703125,
-        23.998001098632812,
-        11.643000602722168,
-        34.763999938964844
+      "decoder": [
+        [
+          0.5223413705825806,
+          0.2563318610191345,
+          0.5943660736083984,
+          0.2424449622631073
+        ],
+        [
+          0.22053532302379608,
+          0.38325658440589905,
+          0.9159231781959534,
+          0.09576917439699173
+        ]
       ]
-    ],
-    [
-      [
-        13.87600040435791,
-        41.534000396728516,
-        15.024001121520996,
-        57.566001892089844
+    },
+    {
+      "encoder": [
+        [
+          1.2540652751922607,
+          1.0315872430801392,
+          1.253417730331421,
+          1.0659087896347046
+        ],
+        [
+          1.1852425336837769,
+          1.2326953411102295,
+          1.0134551525115967,
+          1.173586130142212
+        ]
       ],
-      [
-        43.11000061035156,
-        32.54600143432617,
-        10.670000076293945,
-        41.67400360107422
+      "decoder": [
+        [
+          0.4280831813812256,
+          0.3081738352775574,
+          0.660526692867279,
+          0.21870052814483643
+        ],
+        [
+          0.16326090693473816,
+          0.34843143820762634,
+          0.9616851210594177,
+          0.1421067714691162
+        ]
       ]
-    ]
+    }
   ],
-  "best_val_accuracy": 0.805,
+  "best_val_loss": 5.607645124527238,
   "config": {
     "tokenizer": "bert-base-uncased",
     "max_seq_len": 128,
     "hidden_dim": 256,
     "ffn_dim": 512,
     "num_heads": 4,
-    "num_layers": 2,
+    "num_encoder_layers": 2,
+    "num_decoder_layers": 2,
     "num_experts": 4,
     "router_type": "hash",
     "top_k": 1,

model.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e5bd57ce382a8aaccd173f2d58c001818432fe316cf0301d70bc367b53394d87
-size 41943990
+oid sha256:9d8ca2ca01ce3311dfab7a8fe4b42dc60cac000c7d46108f12115879ff15f2b4
+size 85979282

tokenizer/tokenizer.json

@@ -6,16 +6,7 @@
     "strategy": "LongestFirst",
     "stride": 0
   },
-  "padding": {
-    "strategy": {
-      "Fixed": 128
-    },
-    "direction": "Right",
-    "pad_to_multiple_of": null,
-    "pad_id": 0,
-    "pad_type_id": 0,
-    "pad_token": "[PAD]"
-  },
+  "padding": null,
   "added_tokens": [
     {
       "id": 0,