Update evaluate_agent.py

evaluate_agent.py

@@ -8,22 +8,22 @@ def test_questions():
             ),
             "expected_keywords": ["clarify", "incomplete", "missing", "please provide", "need more information"]
         },
-        # {
-        #   "task_id": "q7",
-        #   "question": (
-        #     "Given this table defining * on the set S = {a, b, c, d, e}\n\n"
-        #     "|*|a|b|c|d|e|\n"
-        #     "|---|---|---|---|---|---|\n"
-        #     "|a|a|b|c|b|d|\n"
-        #     "|b|b|c|a|e|c|\n"
-        #     "|c|c|a|b|b|a|\n"
-        #     "|d|b|e|b|e|d|\n"
-        #     "|e|d|b|a|d|c|\n\n"
-        #     "Provide the subset of S involved in any possible counter-examples that prove * is not commutative. "
-        #     "Provide your answer as a comma-separated list of the elements in the set in alphabetical order."
-        #   ),
-        #   "expected_keywords": ["b, e"]
-        # },
+        {
+          "task_id": "q7",
+          "question": (
+            "Given this table defining * on the set S = {a, b, c, d, e}\n\n"
+            "|*|a|b|c|d|e|\n"
+            "|---|---|---|---|---|---|\n"
+            "|a|a|b|c|b|d|\n"
+            "|b|b|c|a|e|c|\n"
+            "|c|c|a|b|b|a|\n"
+            "|d|b|e|b|e|d|\n"
+            "|e|d|b|a|d|c|\n\n"
+            "Provide the subset of S involved in any possible counter-examples that prove * is not commutative. "
+            "Provide your answer as a comma-separated list of the elements in the set in alphabetical order."
+          ),
+          "expected_keywords": ["b, e"]
+        },
         {
             "task_id": "q3",
             "question": (
@@ -45,14 +45,14 @@ def test_questions():
         #         "broccoli", "celery", "green beans", "lettuce", "sweet potatoes", "zucchini"
         #     ]
         # },
-        # {
-        #     "task_id": "q2",
-        #     "question": (
-        #         "How many studio albums did Mercedes Sosa release between 2000 and 2009 (inclusive)? "
-        #         "Use Wikipedia to find the answer."
-        #     ),
-        #     "expected_keywords": ["3", "three"]
-        # },
+        {
+            "task_id": "q2",
+            "question": (
+                "How many studio albums did Mercedes Sosa release between 2000 and 2009 (inclusive)? "
+                "Use Wikipedia to find the answer."
+            ),
+            "expected_keywords": ["3", "three"]
+        },
         # {
         #     "task_id": "q3",
         #     "question": (
@@ -61,29 +61,13 @@ def test_questions():
         #     ),
         #     "expected_keywords": ["14", "fourteen"]
         # },
-        # {
-        #     "task_id": "q4",
-        #     "question": (
-        #         "Who nominated the only Featured Article on English Wikipedia about a dinosaur that was promoted in November 2016?"
-        #     ),
-        #     "expected_keywords": ["FunkMonk"]
-        # },
-        # {
-        #     "task_id": "q5",
-        #     "question": (
-        #         "Given this table defining * on the set S = {a, b, c, d, e}\n\n"
-        #         "|*|a|b|c|d|e|\n"
-        #         "|---|---|---|---|---|---|\n"
-        #         "|a|a|b|c|b|d|\n"
-        #         "|b|b|c|a|e|c|\n"
-        #         "|c|c|a|b|b|a|\n"
-        #         "|d|b|e|b|e|d|\n"
-        #         "|e|d|b|a|d|c|\n\n"
-        #         "provide the subset of S involved in any possible counter-examples that prove * is not commutative. "
-        #         "Provide your answer as a comma separated list of the elements in the set in alphabetical order."
-        #     ),
-        #     "expected_keywords": ["b,c,d,e", "b,c,d,e"]  # accepted formatted answer
-        # }
+        {
+            "task_id": "q4",
+            "question": (
+                "Who nominated the only Featured Article on English Wikipedia about a dinosaur that was promoted in November 2016?"
+            ),
+            "expected_keywords": ["FunkMonk"]
+        }
     ]