test-suite.py

test-suite.py

@@ -23,5 +23,19 @@ class Suite(evaluate.EvaluationSuite):
                         "IDK": 1.0
                     }
                 }
+            ),
+            SubTask(
+                task_type="text-classification",
+                data="mteb/toxic_conversations_50k",
+                split="test[:10]",
+                args_for_task={
+                    "metric": "accuracy",
+                    "input_column": "text",
+                    "label_column": "label",
+                    "label_mapping": {
+                        "not toxic": 1.0,
+                        "toxic": 1.0
+                    }
+                }
             )
         ]