Update evaluate_agent.py

evaluate_agent.py

@@ -9,20 +9,20 @@ def test_questions():
             "expected_keywords": ["clarify", "incomplete", "missing", "please provide", "need more information"]
         },
         {
-          "task_id": "q7",
-          "question": (
-            "Given this table defining * on the set S = {a, b, c, d, e}\n\n"
-            "|*|a|b|c|d|e|\n"
-            "|---|---|---|---|---|---|\n"
-            "|a|a|b|c|b|d|\n"
-            "|b|b|c|a|e|c|\n"
-            "|c|c|a|b|b|a|\n"
-            "|d|b|e|b|e|d|\n"
-            "|e|d|b|a|d|c|\n\n"
-            "Provide the subset of S involved in any possible counter-examples that prove * is not commutative. "
-            "Provide your answer as a comma-separated list of the elements in the set in alphabetical order."
-          ),
-          "expected_keywords": ["b, e"]
+            "task_id": "q7",
+            "question": (
+                "Given this table defining * on the set S = {a, b, c, d, e}\n\n"
+                "|*|a|b|c|d|e|\n"
+                "|---|---|---|---|---|---|\n"
+                "|a|a|b|c|b|d|\n"
+                "|b|b|c|a|e|c|\n"
+                "|c|c|a|b|b|a|\n"
+                "|d|b|e|b|e|d|\n"
+                "|e|d|b|a|d|c|\n\n"
+                "Provide the subset of S involved in any possible counter-examples that prove * is not commutative. "
+                "Provide your answer as a comma-separated list of the elements in the set in alphabetical order."
+            ),
+            "expected_keywords": ["b, e"]
         },
         {
             "task_id": "q3",
@@ -31,20 +31,6 @@ def test_questions():
             ),
             "expected_keywords": ["right"]
         },
-        # {
-        #     "task_id": "q10",
-        #     "question": (
-        #         "I’m organizing a grocery list and only want to include true vegetables "
-        #         "(not fruits, even if they’re used as vegetables in cooking). "
-        #         "From the following list, which items are true vegetables?\n\n"
-        #         "milk, eggs, flour, whole bean coffee, Oreos, sweet potatoes, fresh basil, plums, "
-        #         "green beans, rice, corn, bell pepper, whole allspice, acorns, broccoli, celery, "
-        #         "zucchini, lettuce, peanuts"
-        #     ),
-        #     "expected_keywords": [
-        #         "broccoli", "celery", "green beans", "lettuce", "sweet potatoes", "zucchini"
-        #     ]
-        # },
         {
             "task_id": "q2",
             "question": (
@@ -53,20 +39,26 @@ def test_questions():
             ),
             "expected_keywords": ["3", "three"]
         },
-        # {
-        #     "task_id": "q3",
-        #     "question": (
-        #         "In the video https://www.youtube.com/watch?v=L1vXCYZAYYM, what is the highest number of bird species "
-        #         "to be on camera simultaneously?"
-        #     ),
-        #     "expected_keywords": ["14", "fourteen"]
-        # },
         {
-            "task_id": "q4",
+            "task_id": "q4b",
             "question": (
                 "Who nominated the only Featured Article on English Wikipedia about a dinosaur that was promoted in November 2016?"
             ),
             "expected_keywords": ["FunkMonk"]
+        },
+        {
+            "task_id": "q5",
+            "question": (
+                "Who is the CEO of OpenAI?"
+            ),
+            "expected_keywords": ["sam altman"]
+        },
+        {
+            "task_id": "q6",
+            "question": (
+                "When was the Eiffel Tower built?"
+            ),
+            "expected_keywords": ["1889"]
         }
     ]