update

src/backend/tasks/cnndm/task.py

@@ -1,6 +1,6 @@
-from lm_eval.api.task import Task
+from lm_eval.api.task import ConfigurableTask
 from lm_eval.api.instance import Instance
-from lm_eval.api.registry import register_task
+# from lm_eval.api.registry import register_task
 from lm_eval.api.metrics import mean
 
 import torch
@@ -60,7 +60,7 @@ def rouge(refs, preds):
 
 
 # @register_task("cnndm")
-class CNNDM(Task):
+class CNNDM(ConfigurableTask):
     VERSION = 0
     DATASET_PATH = "cnn_dailymail"
     DATASET_NAME = "3.0.0"

src/backend/tasks/cnndm/task_v2.py

@@ -1,6 +1,6 @@
-from lm_eval.api.task import Task
+from lm_eval.api.task import ConfigurableTask
 from lm_eval.api.instance import Instance
-from lm_eval.api.registry import register_task
+# from lm_eval.api.registry import register_task
 from lm_eval.api.metrics import mean
 
 import torch
@@ -60,7 +60,7 @@ def rouge(refs, preds):
 
 
 # @register_task("cnndm_v2")
-class CNNDMv2(Task):
+class CNNDMv2(ConfigurableTask):
     VERSION = 0
     DATASET_PATH = "cnn_dailymail"
     DATASET_NAME = "3.0.0"

src/backend/tasks/selfcheckgpt/task.py

@@ -1,9 +1,9 @@
 import os
 from typing import Union, List
 
-from lm_eval.api.task import Task
+from lm_eval.api.task import ConfigurableTask
 from lm_eval.api.instance import Instance
-from lm_eval.api.registry import register_task
+# from lm_eval.api.registry import register_task
 from lm_eval.api.metrics import mean
 
 from src.backend.envs import DEVICE
@@ -13,7 +13,7 @@ from selfcheckgpt.modeling_selfcheck import SelfCheckMQAG, SelfCheckNLI, SelfChe
 
 
 # @register_task("selfcheckgpt")
-class SelfCheckGPT(Task):
+class SelfCheckGPT(ConfigurableTask):
     VERSION = 0.0
     DATASET_PATH = "potsawee/wiki_bio_gpt3_hallucination"
     DATASET_NAME = None

src/backend/tasks/xsum/task.py

@@ -1,5 +1,6 @@
-from lm_eval.api.task import Task, ConfigurableTask
+from lm_eval.api.task import ConfigurableTask
 from lm_eval.api.instance import Instance
+# from lm_eval.api.registry import register_task
 from lm_eval.api.metrics import mean
 
 import torch
@@ -51,7 +52,7 @@ def rouge(refs, preds):
 
 
 # @register_task("xsum")
-class XSum(Task):
+class XSum(ConfigurableTask):
     VERSION = 0
     DATASET_PATH = "EdinburghNLP/xsum"
     DATASET_NAME = None

src/backend/tasks/xsum/task_v2.py

@@ -1,6 +1,6 @@
-from lm_eval.api.task import ConfigurableTask, Task, TaskConfig
+from lm_eval.api.task import ConfigurableTask
 from lm_eval.api.instance import Instance
-from lm_eval.api.registry import register_task
+# from lm_eval.api.registry import register_task
 from lm_eval.api.metrics import mean
 
 import torch