LoRA (ANLP A3)

results/lora_comparison/comparison_metrics.json

@@ -0,0 +1,43 @@
+{
+  "standard_moe": {
+    "total_params": 179944868,
+    "trainable_params": 179944868,
+    "train_losses": [
+      6.993803797149658,
+      5.954492663574219,
+      5.53921777420044,
+      5.24759983291626,
+      5.0148901168823246
+    ],
+    "val_losses": [
+      6.212501205444336,
+      5.760108711242676,
+      5.491593559265136,
+      5.284299610137939,
+      5.145402297973633
+    ],
+    "best_val_loss": 5.145402297973633,
+    "training_time": 1290.9204235076904
+  },
+  "lora_moe": {
+    "total_params": 182828452,
+    "trainable_params": 2621440,
+    "lora_percent": 1.43382497161875,
+    "train_losses": [
+      7.029633666229248,
+      5.872530696105957,
+      5.428599870300293,
+      5.109180227661133,
+      4.84611487121582
+    ],
+    "val_losses": [
+      6.160612037658692,
+      5.704808525085449,
+      5.452941730499267,
+      5.289849403381347,
+      5.177924713134765
+    ],
+    "best_val_loss": 5.177924713134765,
+    "training_time": 18967.75845336914
+  }
+}

results/lora_moe/best_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe830c30e4365e4cc873e86af27dc33b40e32e7967070df6276f1184957d27c7
+size 731567101

results/standard_moe/best_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6df29ee4a7a27f6f28ce9af56d8d6d9a5cfae6cafe129ce3d67d6b7cc9da2f9
+size 720972925