update

src/backend/run_eval_suite.py

@@ -19,6 +19,8 @@ def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_siz
     # include_task_folder("src/backend/tasks/")
     # initialize_tasks('INFO')
 
+    print(f"Allocating task manager for: {task_names}")
+
     task_manager = TaskManager(include_path="./src/backend/tasks/")
     # task_manager.initialize_tasks('INFO')
 

src/backend/tasks/faithdial/faithdial.yaml

@@ -1,4 +1,3 @@
-group: faithdial
 task: faithdial_hallu
 dataset_path: McGill-NLP/FaithDial
 training_split: train

src/backend/tasks/faithdial/faithdial_v2.yaml

@@ -1,4 +1,3 @@
-group: faithdial
 task: faithdial_hallu_v2
 dataset_path: McGill-NLP/FaithDial
 training_split: train

src/backend/tasks/fever/fever10.yaml

@@ -1,4 +1,3 @@
-group: fever
 task: fever10
 dataset_path: fever
 dataset_name: v1.0

src/backend/tasks/fever/fever11.yaml

@@ -1,4 +1,3 @@
-group: fever
 task: fever11
 dataset_path: pminervini/hl-fever
 dataset_name: v1.0

src/backend/tasks/halueval/halueval_dialogue.yaml

@@ -1,5 +1,3 @@
-group:
-  - halueval
 task: halueval_dialogue
 dataset_path: pminervini/HaluEval
 dataset_name: dialogue_samples

src/backend/tasks/halueval/halueval_qa.yaml

@@ -1,5 +1,3 @@
-group:
-  - halueval
 task: halueval_qa
 dataset_path: pminervini/HaluEval
 dataset_name: qa_samples

src/backend/tasks/halueval/halueval_summarization.yaml

@@ -1,5 +1,3 @@
-group:
-  - halueval
 task: halueval_summarization
 dataset_path: pminervini/HaluEval
 dataset_name: summarization_samples

src/backend/tasks/truefalse/truefalse.yaml

@@ -1,4 +1,3 @@
-group: truefalse
 task: truefalse_cieacf
 dataset_path: pminervini/true-false
 dataset_name: default

src/backend/tasks/xsum/xsum.yaml

@@ -1,2 +0,0 @@
-task:
-  - xsum