Update app.py

app.py

@@ -1,169 +1,3 @@
-import os
-import gradio as gr
-import requests
-import string
-import warnings
-import pandas as pd
-from huggingface_hub import login
-import re
-import json
-from groq import Groq
-
-# --- Constants ---
-DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
-
-
-# --- Basic Agent Definition ---
-class BasicAgent:
-    def __init__(self):
-        print("BasicAgent initialized.")
-        self.client = Groq(api_key=os.environ["GROQ_API_KEY"])
-        self.agent_prompt = (
-            """You are a general AI assistant. I will ask you a question. Report your thoughts, and
-            finish your answer with the following template: FINAL ANSWER: [YOUR FINAL ANSWER].
-            YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated
-            list of numbers and/or strings.
-            If you are asked for a number, don't use comma to write your number neither use units such as $
-            or percent sign unless specified otherwise.
-            If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the
-            digits in plain text unless specified otherwise.
-            If you are asked for a comma separated list, apply the above rules depending of whether the element
-            to be put in the list is a number or a string."""
-        )
-
-    def format_final_answer(self, answer: str) -> str:
-        cleaned = " ".join(answer.split())
-        return f"FINAL ANSWER: {cleaned}"
-
-    def check_commutativity(self):
-        S = ['a', 'b', 'c', 'd', 'e']
-        counter_example_elements = set()
-        index = {'a': 0, 'b': 1, 'c': 2, 'd': 3, 'e': 4}
-        self.operation_table = [
-            ['a', 'b', 'c', 'b', 'd'],
-            ['b', 'c', 'a', 'e', 'c'],
-            ['c', 'a', 'b', 'b', 'a'],
-            ['b', 'e', 'b', 'e', 'd'],
-            ['d', 'b', 'a', 'd', 'c']
-        ]
-        for x in S:
-            for y in S:
-                x_idx = index[x]
-                y_idx = index[y]
-                if self.operation_table[x_idx][y_idx] != self.operation_table[y_idx][x_idx]:
-                    counter_example_elements.add(x)
-                    counter_example_elements.add(y)
-        return self.format_final_answer(", ".join(sorted(counter_example_elements)))
-
-    def maybe_reversed(self, text: str) -> bool:
-        words = text.split()
-        reversed_ratio = sum(
-            1 for word in words if word[::-1].lower() in {
-                "if", "you", "understand", "this", "sentence", "write",
-                "opposite", "of", "the", "word", "left", "answer"
-            }
-        ) / len(words)
-        return reversed_ratio > 0.3
-
-    def solve_riddle(self, question: str) -> str:
-        question = question[::-1]
-        if "opposite of the word" in question:
-            match = re.search(r"opposite of the word ['\"](\w+)['\"]", question)
-            if match:
-                word = match.group(1).lower()
-                opposites = {
-                    "left": "right", "up": "down", "hot": "cold",
-                    "true": "false", "yes": "no", "black": "white"
-                }
-                opposite = opposites.get(word, f"UNKNOWN_OPPOSITE_OF_{word}")
-                return "FINAL ANSWER: RIGHT"
-        return self.format_final_answer("COULD_NOT_SOLVE")
-
-    def query_groq(self, question: str) -> str:
-        full_prompt = f"{self.agent_prompt}\n\nQuestion: {question}"
-        try:
-            response = self.client.chat.completions.create(
-                model="llama3-8b-8192",
-                messages=[{"role": "user", "content": full_prompt}]
-            )
-            answer = response.choices[0].message.content
-            if "FINAL ANSWER: " in answer:
-                return answer.split("FINAL ANSWER: ")[-1].strip().upper()
-            else:
-                return self.format_final_answer(answer).upper()
-        except Exception as e:
-            print(f"[Groq ERROR]: {e}")
-            return self.format_final_answer("GROQ_ERROR")
-
-    def __call__(self, question: str) -> str:
-        print(f"Received question: {question[:50]}...")
-        if "commutative" in question.lower():
-            return self.check_commutativity()
-        if self.maybe_reversed(question):
-            print("Detected likely reversed riddle.")
-            return self.solve_riddle(question)
-        return self.query_groq(question)
-
-def question_scorer(model_answer: str, ground_truth: str) -> bool:
-    def normalize_str(input_str, remove_punct=True) -> str:
-        no_spaces = re.sub(r"\s", "", input_str)
-        if remove_punct:
-            translator = str.maketrans("", "", string.punctuation)
-            return no_spaces.lower().translate(translator)
-        else:
-            return no_spaces.lower()
-
-    def normalize_number_str(number_str: str) -> float | None:
-        for char in ["$", "%", ","]:
-            number_str = number_str.replace(char, "")
-        try:
-            return float(number_str)
-        except ValueError:
-            print(f"String '{number_str}' cannot be normalized to number.")
-            return None
-
-    def split_string(s: str, char_list: list[str] = [",", ";"]) -> list[str]:
-        pattern = f"[{''.join(map(re.escape, char_list))}]"
-        return [elem.strip() for elem in re.split(pattern, s)]
-
-    def is_float(val) -> bool:
-        try:
-            float(val)
-            return True
-        except ValueError:
-            return False
-
-    if model_answer is None:
-        model_answer = "None"
-
-    if is_float(ground_truth):
-        print(f"Evaluating '{model_answer}' as a number.")
-        normalized = normalize_number_str(model_answer)
-        return normalized == float(ground_truth) if normalized is not None else False
-
-    elif any(char in ground_truth for char in [",", ";"]):
-        print(f"Evaluating '{model_answer}' as a comma/semicolon-separated list.")
-        gt_elems = split_string(ground_truth)
-        ma_elems = split_string(model_answer)
-
-        if len(gt_elems) != len(ma_elems):
-            warnings.warn("Answer lists have different lengths, returning False.", UserWarning)
-            return False
-
-        for ma_elem, gt_elem in zip(ma_elems, gt_elems):
-            if is_float(gt_elem):
-                normalized = normalize_number_str(ma_elem)
-                if normalized != float(gt_elem):
-                    return False
-            else:
-                if normalize_str(ma_elem, remove_punct=False) != normalize_str(gt_elem, remove_punct=False):
-                    return False
-        return True
-
-    else:
-        print(f"Evaluating '{model_answer}' as a string.")
-        return normalize_str(model_answer) == normalize_str(ground_truth)
-
 def run_and_submit_all(profile: gr.OAuthProfile | None):
     space_id = os.getenv("SPACE_ID")
     if profile:
@@ -183,6 +17,7 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
         return f"Error initializing agent: {e}", None
 
     agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
+
     try:
         response = requests.get(questions_url, timeout=15)
         response.raise_for_status()
@@ -194,21 +29,50 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
 
     results_log = []
     answers_payload = []
+    correct_count = 0
+    total_with_gold = 0
+
     for item in questions_data:
         task_id = item.get("task_id")
         question_text = item.get("question")
+        gold_answer = item.get("gold_answer")
+
         if not task_id or question_text is None:
             continue
+
         try:
             submitted_answer = agent(question_text)
-            print(submitted_answer)
-            answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
-            results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
+            is_correct = question_scorer(submitted_answer, gold_answer) if gold_answer else None
+
+            if is_correct is not None:
+                total_with_gold += 1
+                if is_correct:
+                    correct_count += 1
+
+            answers_payload.append({
+                "task_id": task_id,
+                "submitted_answer": submitted_answer
+            })
+
+            results_log.append({
+                "Task ID": task_id,
+                "Question": question_text,
+                "Submitted Answer": submitted_answer,
+                "Gold Answer": gold_answer,
+                "Correct?": "✅" if is_correct else "❌" if is_correct is not None else "N/A"
+            })
         except Exception as e:
-            results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
+            results_log.append({
+                "Task ID": task_id,
+                "Question": question_text,
+                "Submitted Answer": f"AGENT ERROR: {e}",
+                "Gold Answer": gold_answer,
+                "Correct?": "❌"
+            })
 
     if not answers_payload:
         return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
+
     submission_data = {
         "username": username.strip(),
         "agent_code": agent_code,
@@ -220,30 +84,21 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
         response.raise_for_status()
         result_data = response.json()
         print(result_data)
+
+        accuracy_text = ""
+        if total_with_gold > 0:
+            accuracy = (correct_count / total_with_gold) * 100
+            accuracy_text = f"\nLocal Accuracy: {accuracy:.2f}% ({correct_count}/{total_with_gold} correct)"
+
         final_status = (
             f"Submission Successful!\n"
             f"User: {result_data.get('username')}\n"
-            f"Overall Score: {result_data.get('score', '?')}% "
+            f"Overall Score (from server): {result_data.get('score', '?')}% "
             f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
             f"Message: {result_data.get('message', 'No message received.')}"
+            f"{accuracy_text}"
         )
         return final_status, pd.DataFrame(results_log)
+
     except Exception as e:
         return f"Submission Failed: {e}", pd.DataFrame(results_log)
-
-# --- Build Gradio Interface ---
-with gr.Blocks() as demo:
-    gr.Markdown("# Basic Agent Evaluation Runner")
-    gr.LoginButton()
-    run_button = gr.Button("Run Evaluation & Submit All Answers")
-    status_output = gr.Textbox(label="Run Status / Submission Result", max_lines=5, interactive=False, max_length=200)
-    results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
-
-    run_button.click(
-        fn=run_and_submit_all,
-        outputs=[status_output, results_table]
-    )
-
-if __name__ == "__main__":
-    print("Launching Gradio Interface for Basic Agent Evaluation...")
-    demo.launch(debug=True, share=False)