src/managers/evaluator.py

import os
import pandas as pd

from agent.open_ai_agent import OpenAiAgent
from rest_clients.hs_evaluator_client import HsEvaluatorClient


class Evaluator:

    def __init__(self, profile):
        self.profile = profile
        self.username = profile.username if profile else None
        self.space_id = os.getenv("SPACE_ID", "tommaso1288/Final_Assignment_Template")
        self.agent = OpenAiAgent()
        self.hs_evaluator_client: HsEvaluatorClient | None = None

    def run_and_submit(self):
        if not self.username:
            return "Please Login to Hugging Face with the button.", None

        questions = self.get_hs_evaluator_client().fetch_questions()
        if not questions:
            return "Fetched questions list is empty or invalid format.", None

        results_log, answers_payload = self._run_agent(questions)
        if not answers_payload:
            return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)

        return self.get_hs_evaluator_client().submit_answers(answers_payload, results_log)

    def _run_agent(self, questions):
        results_log = []
        answers_payload = []
        print(f"Running agent on {len(questions)} questions...")
        for item in questions:
            task_id = item.get("task_id")
            question_text = item.get("question")

            # ----------fetch any attached file ----------
            try:
                file_path = self.get_hs_evaluator_client().download_file_if_any(task_id)
            except Exception as e:
                file_path = None
                print(f"[file fetch error] {task_id}: {e}")

            # ---------- Build the prompt sent to the agent ----------
            if file_path:
                q_for_agent = (
                    f"{question_text}\n\n"
                    f"---\n"
                    f"A file was downloaded for this task and saved locally at:\n"
                    f"{file_path}\n"
                    f"---\n\n"
                )
            else:
                q_for_agent = question_text

            if not task_id or question_text is None:
                print(f"Skipping item with missing task_id or question: {item}")
                continue
            try:
                submitted_answer = self.agent(q_for_agent)
                answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
                results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
            except Exception as e:
                print(f"Error running agent on task {task_id}: {e}")
                results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
        return results_log, answers_payload

    def get_hs_evaluator_client(self):
        if not self.hs_evaluator_client:
            self.hs_evaluator_client = HsEvaluatorClient(self.username, self.space_id)
        return self.hs_evaluator_client