ugh

glue-suite-v2.py

@@ -13,34 +13,35 @@ class Suite(evaluate.EvaluationSuite):
             citation="insert citation here",
             features=Features({"predictions": Value("int64"), "references": Value("int64")}))
 
-    preprocessor = None #lambda x: x["text"].lower()
-    suite = [
-        SubTask(
-            data="imdb",
-            split="test",
-            data_preprocessor=lambda x: x["text"].lower(),#self.preprocessor,
-            args_for_task={
-                "metric": "accuracy",
-                "input_column": "text",
-                "label_column": "label",
-                "label_mapping": {
-                    "LABEL_0": 0.0,
-                    "LABEL_1": 1.0
+    def setup(self):
+        self.preprocessor = None #lambda x: x["text"].lower()
+        self.suite = [
+            SubTask(
+                data="imdb",
+                split="test",
+                data_preprocessor=lambda x: x["text"].lower(),#self.preprocessor,
+                args_for_task={
+                    "metric": "accuracy",
+                    "input_column": "text",
+                    "label_column": "label",
+                    "label_mapping": {
+                        "LABEL_0": 0.0,
+                        "LABEL_1": 1.0
+                    }
                 }
-            }
-        ),
-        SubTask(
-            data="sst2",
-            split="test[:10]",
-            data_preprocessor=self.preprocessor,
-            # args_for_task={
-            #     "metric": "accuracy",
-            #     "input_column": "sentence",
-            #     "label_column": "label",
-            #     "label_mapping": {
-            #         "LABEL_0": 0.0,
-            #         "LABEL_1": 1.0
-            #     }
-            # }
-        )
-    ]
+            ),
+            SubTask(
+                data="sst2",
+                split="test[:10]",
+                data_preprocessor=self.preprocessor,
+                # args_for_task={
+                #     "metric": "accuracy",
+                #     "input_column": "sentence",
+                #     "label_column": "label",
+                #     "label_mapping": {
+                #         "LABEL_0": 0.0,
+                #         "LABEL_1": 1.0
+                #     }
+                # }
+            )
+        ]