lol

glue-suite-v2.py

@@ -1,8 +1,9 @@
-
 from typing import Optional, Union, Callable
 from dataclasses import dataclass
 from datasets import Dataset
 
+from evaluate.evaluation_suite import EvaluationSuite
+
 @dataclass
 class SubTask:
     model_or_pipeline: Optional[Union[str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel"]] = None
@@ -13,35 +14,38 @@ class SubTask:
     args_for_task: Optional[dict] = None
 
 
-preprocessor = lambda x: x["text"].lower()
+class EvaluationSuite:
+
+    def __init__(self):
+        self.preprocessor = None #lambda x: x["text"].lower()
 
-suite = [
-    SubTask(
-        data="imdb",
-        split="test",
-        data_preprocessor=preprocessor,
-        args_for_task={
-            "metric": "accuracy",
-            "input_column": "text",
-            "label_column": "label",
-            "label_mapping": {
-                "LABEL_0": 0.0,
-                "LABEL_1": 1.0
-            }
-        }
-    ),
-    SubTask(
-        data="sst2",
-        split="test[:10]",
-        data_preprocessor=preprocessor,
-        args_for_task={
-            "metric": "accuracy",
-            "input_column": "sentence",
-            "label_column": "label",
-            "label_mapping": {
-                "LABEL_0": 0.0,
-                "LABEL_1": 1.0
-            }
-        }
-    )
-]
+        self.suite = [
+            SubTask(
+                data="imdb",
+                split="test",
+                data_preprocessor=self.preprocessor,
+                args_for_task={
+                    "metric": "accuracy",
+                    "input_column": "text",
+                    "label_column": "label",
+                    "label_mapping": {
+                        "LABEL_0": 0.0,
+                        "LABEL_1": 1.0
+                    }
+                }
+            ),
+            SubTask(
+                data="sst2",
+                split="test[:10]",
+                data_preprocessor=self.preprocessor,
+                args_for_task={
+                    "metric": "accuracy",
+                    "input_column": "sentence",
+                    "label_column": "label",
+                    "label_mapping": {
+                        "LABEL_0": 0.0,
+                        "LABEL_1": 1.0
+                    }
+                }
+            )
+        ]