Compression metadata and results

compression_metadata.json

@@ -0,0 +1,59 @@
+{
+  "method": "downscaling_projection",
+  "source_model": "LisaMegaWatts/JuliaSLM",
+  "source_params": 5037312,
+  "source_loss": 3.552281068317482,
+  "best_config": "A-3M",
+  "finetune_steps": 2000,
+  "finetune_lr": 0.0006,
+  "configs": {
+    "A-3M": {
+      "config": {
+        "d_model": 192,
+        "n_layers": 6,
+        "n_heads": 3,
+        "head_dim": 64,
+        "ffn_inner": 480,
+        "context_length": 256,
+        "vocab_size": 2000
+      },
+      "params": 2930112,
+      "reduction": 0.41831834121055034,
+      "pre_finetune_loss": 6.9663840472007115,
+      "post_finetune_loss": 3.9081690383949335,
+      "post_finetune_ppl": 49.8076724983192
+    },
+    "B-2.5M": {
+      "config": {
+        "d_model": 192,
+        "n_layers": 5,
+        "n_heads": 3,
+        "head_dim": 64,
+        "ffn_inner": 480,
+        "context_length": 256,
+        "vocab_size": 2000
+      },
+      "params": 2505792,
+      "reduction": 0.5025537429486202,
+      "pre_finetune_loss": 7.5051396219378885,
+      "post_finetune_loss": 3.9149324556950593,
+      "post_finetune_ppl": 50.14568434125149
+    },
+    "C-2M": {
+      "config": {
+        "d_model": 192,
+        "n_layers": 4,
+        "n_heads": 3,
+        "head_dim": 64,
+        "ffn_inner": 480,
+        "context_length": 256,
+        "vocab_size": 2000
+      },
+      "params": 2081472,
+      "reduction": 0.58678914468669,
+      "pre_finetune_loss": 8.09140216928519,
+      "post_finetune_loss": 3.9662209947702864,
+      "post_finetune_ppl": 52.78467987729236
+    }
+  }
+}