Phase 3 eval: baseline + finetuned metrics

eval/baseline.json

@@ -0,0 +1,8 @@
+{
+  "exact_match": 0.0,
+  "bleu_4": 0.0,
+  "mean_edit_similarity": 0.0,
+  "num_samples": 200,
+  "syntax_valid_rate": 1.0,
+  "run_name": "baseline"
+}

eval/finetuned.json

@@ -0,0 +1,8 @@
+{
+  "exact_match": 0.0,
+  "bleu_4": 0.0,
+  "mean_edit_similarity": 0.0,
+  "num_samples": 200,
+  "syntax_valid_rate": 1.0,
+  "run_name": "finetuned"
+}

eval/summary.json

@@ -0,0 +1,22 @@
+{
+  "dataset": "cmndcntrlcyber/code-trainer-offsec-dataset@v2-multimodal",
+  "adapter": "cmndcntrlcyber/code-trainer-vision-adapter",
+  "split": "test",
+  "num_samples": 200,
+  "baseline": {
+    "exact_match": 0.0,
+    "bleu_4": 0.0,
+    "mean_edit_similarity": 0.0,
+    "num_samples": 200,
+    "syntax_valid_rate": 1.0,
+    "run_name": "baseline"
+  },
+  "finetuned": {
+    "exact_match": 0.0,
+    "bleu_4": 0.0,
+    "mean_edit_similarity": 0.0,
+    "num_samples": 200,
+    "syntax_valid_rate": 1.0,
+    "run_name": "finetuned"
+  }
+}