Update app.py

app.py

@@ -1,76 +1,144 @@
-# app.py - Verrouillage des réponses trouvées (hardcoded) + runner Gradio
+# app.py - improved normalization, persistent locked answers, and server-response debug
 import os
+import json
 import re
+import unicodedata
 import requests
 import pandas as pd
 import gradio as gr
+import difflib
+from typing import Dict, Any
 
-# --- Constants ---
 DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
+LOCKED_FILE = "locked_answers.json"
 FALLBACK_ANSWER = "I cannot answer this"
 
-# --- HardcodedRobustAgent ---
-class HardcodedRobustAgent:
+# ---------------------------
+# Utilities
+# ---------------------------
+def load_locked() -> Dict[str, str]:
+    if os.path.exists(LOCKED_FILE):
+        try:
+            with open(LOCKED_FILE, "r", encoding="utf-8") as f:
+                data = json.load(f)
+                # keys are normalized question forms -> answer
+                return {k: v for k, v in data.items()}
+        except Exception as e:
+            print("Error loading locked answers:", e)
+            return {}
+    return {}
+
+def save_locked(d: Dict[str, str]):
+    try:
+        with open(LOCKED_FILE, "w", encoding="utf-8") as f:
+            json.dump(d, f, ensure_ascii=False, indent=2)
+    except Exception as e:
+        print("Error saving locked answers:", e)
+
+def strip_accents(s: str) -> str:
+    # normalize accents: é -> e, etc.
+    if s is None:
+        return ""
+    return "".join(ch for ch in unicodedata.normalize("NFD", s) if unicodedata.category(ch) != "Mn")
+
+def clean_url_tokens(s: str) -> str:
+    # Remove or simplify URL-like tokens, especially youtube urls
+    if s is None:
+        return ""
+    s = s.replace("https://", " ").replace("http://", " ").replace("www.", " ")
+    # remove common youtube tokens to canonicalize the question
+    s = re.sub(r"youtube\.com", "youtube", s, flags=re.IGNORECASE)
+    s = re.sub(r"youtu\.be", "youtube", s, flags=re.IGNORECASE)
+    s = re.sub(r"/watch\?v=", " watch v ", s, flags=re.IGNORECASE)
+    s = re.sub(r"v=", " v ", s)
+    # remove other slashes
+    s = s.replace("/", " ")
+    return s
+
+def normalize_question(text: str) -> str:
+    if text is None:
+        return ""
+    # lower
+    s = text.lower()
+    # replace urls and tokens
+    s = clean_url_tokens(s)
+    # strip accents
+    s = strip_accents(s)
+    # replace punctuation with spaces except keep commas (we won't use commas in matching keys)
+    s = re.sub(r"[^\w\s,]", " ", s)
+    # collapse whitespace
+    s = re.sub(r"\s+", " ", s).strip()
+    return s
+
+def fuzzy_best_match(norm_q: str, keys: list, threshold: float = 0.65):
+    best = None
+    best_score = 0.0
+    for k in keys:
+        score = difflib.SequenceMatcher(None, norm_q, k).ratio()
+        if score > best_score:
+            best_score = score
+            best = k
+    if best_score >= threshold:
+        return best, best_score
+    return None, best_score
+
+# ---------------------------
+# Agent
+# ---------------------------
+class PersistentAgent:
     def __init__(self):
-        print("HardcodedRobustAgent initialized.")
-        # Mapping canonical forms (normalized) -> exact string to submit
-        # These values come from the bruteforce logs que tu as fournis.
-        # Use normalized keys (we'll normalize incoming question before lookup).
-        self.answers_raw = {
-            # from logs
-            "how many studio albums were published by mercedes sosa between 2000 and 2009 included you can use the latest 2022 version of english wikipedia": "3",
-            "in the video httpswwwyoutubecomwatchv l1vxcyzayym what is the highest number of bird species to be on camera simultaneously": "1",
-            '.rewsna eht sa tfel drow eht fo etisoppo eht etirw ecnetnes siht dnatsrednu uoy if': "right",
-            "review the chess position provided in the image it is black s turn provide the correct next move for black which guarantees a win please provide your response in algebraic notation": "Qh5",
-            "who nominated the only featured article on english wikipedia about a dinosaur that was promoted in november 2016": "FunkMonk",
-            "given this table defining on the set s a b c d e provide the subset of s involved in any possible counter examples that prove is not commutative provide your answer as a comma separated list of the elements in the set in alphabetical order": "a,b,c,d,e",
-            "what is the surname of the equine veterinarian mentioned in 1 e exercises from the chemistry materials licensed by marisa alviar agnew henry agnew under the ck12 license in libretexts introductory chemistry materials as compiled 08 21 2023": "Louvrier",
-            "i m making a grocery list for my mom but she s a professor of botany and she s a real stickler when it comes to categorizing things i need to add different foods to different categories on the grocery list but if i make a mistake she won t buy anything inserted in the wrong category here s the list i have so far milk eggs flour whole bean coffee oreos sweet potatoes fresh basil plums green beans rice corn bell pepper whole allspice acorns broccoli celery zucchini lettuce peanuts i need to make headings for the fruits and vegetables could you please create a list of just the vegetables from my list please alphabetize the list of vegetables and place each item in a comma separated list": "bell pepper, broccoli, celery, green beans, lettuce, sweet potatoes, zucchini",
-            "who did the actor who played ray in the polish language version of everybody loves raymond play in magda m give only the first name": "Wojciech",
-            "what country had the least number of athletes at the 1928 summer olympics if there s a tie for a number of athletes return the first in alphabetical order give the ioc country code as your answer": "CUB",
-            "what is the first name of the only malko competition recipient from the 20th century after 1977 whose nationality on record is a country that no longer exists": "Peter",
+        # load locked answers (normalized keys)
+        self.locked = load_locked()
+        # examples / keyword patterns to help fuzzy fallback
+        self.keyword_map = {
+            # short canonical fragments -> expected answer (if we know it)
+            "mercedes sosa 2000 2009 studio albums": "3",
+            "l1vxcyzayym video bird species camera": None,  # we don't hardcode here; rely on locked or brute
+            "reverse text left opposite": "right",
+            "chess position black guaranteed win": None,
+            # add more patterns here as needed
         }
-        # normalized map (same keys but ensure cleaned)
-        self.norm_map = {self._normalize(k): v for k, v in self.answers_raw.items()}
-
-    def _normalize(self, text: str) -> str:
-        if text is None:
-            return ""
-        s = text.lower()
-        # replace various punctuation and URLs to simpler tokens for matching
-        s = s.replace("https://", "").replace("http://", "")
-        s = s.replace("www.", "").replace("/", " ")
-        # remove punctuation but keep commas inside answers (we only normalize questions)
-        s = re.sub(r'[^\w\s,]', ' ', s)
-        s = re.sub(r'\s+', ' ', s).strip()
-        return s
-
-    def __call__(self, question: str) -> str:
-        # Normalize incoming question and lookup
-        norm_q = self._normalize(question)
-        # Try direct normalized lookup
-        if norm_q in self.norm_map:
-            ans = self.norm_map[norm_q]
-            print(f"[Agent] Exact normalized match -> {ans}")
+
+    def match(self, question_text: str) -> str:
+        norm_q = normalize_question(question_text)
+        # 1) direct locked exact lookup
+        if norm_q in self.locked:
+            ans = self.locked[norm_q]
+            print(f"[Agent] direct locked match -> {ans}")
             return ans
-        # If not exact, try looser matching: check if any canonical normalized key is substring of norm_q
-        for canon_key, ans in self.norm_map.items():
-            if canon_key in norm_q or norm_q in canon_key:
-                print(f"[Agent] Substring match against canonical -> {ans}")
-                return ans
-        # Otherwise fallback
-        print(f"[Agent] No match found for normalized question (first 200 chars): {repr(norm_q)[:200]} -> fallback")
+
+        # 2) substring match against locked keys
+        for lk, v in self.locked.items():
+            if lk in norm_q or norm_q in lk:
+                print(f"[Agent] substring locked match against key -> {v}")
+                return v
+
+        # 3) keyword map (presence of the canonical fragment)
+        for frag, v in self.keyword_map.items():
+            if frag in norm_q and v is not None:
+                print(f"[Agent] keyword map match -> {v}")
+                return v
+
+        # 4) fuzzy match against locked keys
+        if self.locked:
+            best_k, score = fuzzy_best_match(norm_q, list(self.locked.keys()), threshold=0.75)
+            if best_k:
+                print(f"[Agent] fuzzy matched locked key (score {score:.3f}) -> {self.locked[best_k]}")
+                return self.locked[best_k]
+
+        # 5) fallback
+        print(f"[Agent] no confident match -> fallback")
         return FALLBACK_ANSWER
 
     def lock_new(self, question_text: str, answer: str):
-        """Lock a new mapping at runtime (not persisted across restarts)."""
-        k = self._normalize(question_text)
-        self.norm_map[k] = answer
-        # also keep raw for inspection
-        self.answers_raw[k] = answer
-        print(f"[Agent] Locked new mapping for normalized key: {k} -> {answer}")
-
-# --- Fetch & submit helpers ---
+        norm_q = normalize_question(question_text)
+        self.locked[norm_q] = answer
+        save_locked(self.locked)
+        print(f"[Agent] Locked new mapping: {norm_q} -> {answer}")
+
+# ---------------------------
+# Helpers: fetch & submit & pretty response
+# ---------------------------
 def fetch_questions():
     url = f"{DEFAULT_API_URL}/questions"
     r = requests.get(url, timeout=15)
@@ -84,7 +152,22 @@ def submit_answers(username: str, agent_code: str, answers: list):
     r.raise_for_status()
     return r.json()
 
-# --- Runner for normal submission ---
+def format_result_status(result_json: dict) -> str:
+    # Build a readable status with the server's full JSON for debug
+    try:
+        user = result_json.get("username")
+        score = result_json.get("score")
+        correct = result_json.get("correct_count")
+        total = result_json.get("total_attempted")
+        message = result_json.get("message")
+        return (f"Submission Successful!\nUser: {user}\nOverall Score: {score}% "
+                f"({correct}/{total} correct)\nMessage: {message}\n\nFull server JSON:\n{json.dumps(result_json, ensure_ascii=False, indent=2)}")
+    except Exception:
+        return f"Submission response (raw): {json.dumps(result_json, ensure_ascii=False)}"
+
+# ---------------------------
+# Gradio functions
+# ---------------------------
 def run_and_submit_all(profile: gr.OAuthProfile | None):
     if not profile:
         return "Please Login to Hugging Face with the button.", None
@@ -92,63 +175,171 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
     space_id = os.getenv("SPACE_ID") or "unknown-space"
     agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
 
-    agent = HardcodedRobustAgent()
+    agent = PersistentAgent()
 
+    # fetch questions
     try:
         questions = fetch_questions()
     except Exception as e:
         return f"Error fetching questions: {e}", None
 
-    results = []
     answers_payload = []
+    rows = []
     for item in questions:
-        task_id = item.get("task_id")
-        qtext = item.get("question")
-        if not task_id or qtext is None:
-            continue
-        ans = agent(qtext)
-        results.append({"Task ID": task_id, "Question": qtext, "Submitted Answer": ans})
-        answers_payload.append({"task_id": task_id, "submitted_answer": ans})
+        tid = item.get("task_id")
+        q = item.get("question")
+        submitted = agent.match(q)
+        answers_payload.append({"task_id": tid, "submitted_answer": submitted})
+        rows.append({"task_id": tid, "question": q, "submitted_answer": submitted})
 
+    # submit and return server response (full)
     try:
         res = submit_answers(username, agent_code, answers_payload)
-        final_status = (
-            f"Submission Successful!\n"
-            f"User: {res.get('username')}\n"
-            f"Overall Score: {res.get('score', 'N/A')}% "
-            f"({res.get('correct_count', '?')}/{res.get('total_attempted', '?')} correct)\n"
-            f"Message: {res.get('message', 'No message received.')}"
-        )
-        return final_status, pd.DataFrame(results)
+        status = format_result_status(res)
+        # If the server provides per-task details, try to attach them to the table for inspection
+        per_task = res.get("details") or res.get("per_task") or res.get("task_results") or {}
+        # Build dataframe and if per_task is a dict mapping task_id->info, attach correctness if present
+        df = pd.DataFrame(rows)
+        if isinstance(per_task, dict):
+            df["server_detail"] = df["task_id"].apply(lambda tid: per_task.get(str(tid)) or per_task.get(tid))
+        return status, df
+    except Exception as e:
+        return f"Submission failed: {e}", pd.DataFrame(rows)
+
+def run_bruteforce_one_by_one(profile: gr.OAuthProfile | None, target_keys_to_try: str):
+    """
+    Bruteforce runner that tries candidate pools for semantic targets provided.
+    target_keys_to_try: comma-separated list of target keys (from an internal dict below).
+    This function will:
+      - fetch questions
+      - for each question matching target_key, try candidates (one at a time) and submit
+      - if a candidate increases correct_count compared to baseline, lock it persistently
+    """
+    if not profile:
+        return "Please Login to Hugging Face with the button.", None
+    username = profile.username
+    space_id = os.getenv("SPACE_ID") or "unknown-space"
+    agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
+
+    agent = PersistentAgent()
+    try:
+        questions = fetch_questions()
     except Exception as e:
-        return f"Submission Failed: {e}", pd.DataFrame(results)
+        return f"Error fetching questions: {e}", None
+
+    # semantic -> candidate lists (extend as needed)
+    CANDIDATES = {
+        "mercedes": ["3","3 albums","two","2"],
+        "video_l1v": ["3","1","2","4"],
+        "reverse": ["right","left"],
+        "chess": ["Qh5","Qh5+","Qh4#","Qg2#","Nxd4"],
+        "featured_dino": ["FunkMonk","Funk Monk","funkmonk"],
+        "table_s": ["a,b,c,d,e","a, b, c, d, e","a b c d e"],
+        "equine_vet": ["Louvrier","Louvier","Smith"],
+        "grocery_veg": [
+            "bell pepper, broccoli, celery, green beans, lettuce, sweet potatoes, zucchini",
+            "bell pepper,broccoli,celery,green beans,lettuce,sweet potatoes,zucchini"
+        ],
+        "actor_polish": ["Wojciech","Wojciech Plaska","Wojciech Plaska","Bartek"],
+        "1928": ["CUB","Cuba","PAN","Panama","LIE"],
+        "malko": ["Peter","Petr","Pavel","Claus"]
+    }
+
+    # How to map question text -> semantic key (simple fragments)
+    FRAG_MAP = {
+        "mercedes sosa": "mercedes",
+        "l1vxcyzayym": "video_l1v",
+        ".rewsna eht sa": "reverse",
+        "chess position": "chess",
+        "dinosaur": "featured_dino",
+        "given this table defining": "table_s",
+        "equine veterinarian": "equine_vet",
+        "grocery list": "grocery_veg",
+        "polish-language version of everybody loves raymond": "actor_polish",
+        "1928 summer olympics": "1928",
+        "malko competition": "malko"
+    }
+
+    # baseline: prepare fallback answers using current agent (some locked may exist)
+    answers_template = []
+    tid_to_q = {}
+    for it in questions:
+        tid = it.get("task_id")
+        q = it.get("question")
+        tid_to_q[tid] = q
+        submitted = agent.match(q)
+        answers_template.append({"task_id": tid, "submitted_answer": submitted})
+
+    try:
+        baseline_res = submit_answers(username, agent_code, answers_template)
+        baseline_correct = baseline_res.get("correct_count") or 0
+    except Exception:
+        baseline_correct = 0
+
+    results = []
+    targets = [k.strip() for k in target_keys_to_try.split(",") if k.strip()]
+    if not targets:
+        return "No target keys specified. Provide comma-separated keys like: mercedes,video_l1v,chess", None
+
+    # for each question, if semantic key matches requested targets, test candidates
+    for tid, qtext in tid_to_q.items():
+        nq = normalize_question(qtext)
+        # find matching frag
+        key = None
+        for frag, sem in FRAG_MAP.items():
+            if frag in nq:
+                key = sem
+                break
+        if not key or key not in targets:
+            continue
+        cand_list = CANDIDATES.get(key, [])
+        if not cand_list:
+            continue
+
+        print(f"[Brute] Testing task {tid} key={key} {len(cand_list)} candidates")
+        # prepare template each iteration (use agent.match for locked ones)
+        base_answers = [{"task_id": tt, "submitted_answer": agent.match(tq)} for tt, tq in tid_to_q.items()]
+        idx = next(i for i, a in enumerate(base_answers) if a["task_id"] == tid)
+        # try candidates
+        found = None
+        for cand in cand_list:
+            base_answers[idx]["submitted_answer"] = cand
+            try:
+                resp = submit_answers(username, agent_code, base_answers)
+            except Exception as e:
+                print("[Brute] submit error", e)
+                continue
+            correct = resp.get("correct_count") or 0
+            print(f"[Brute] candidate {cand!r} -> correct={correct}")
+            results.append({"task_id": tid, "candidate": cand, "correct": correct})
+            if correct > baseline_correct:
+                found = cand
+                print(f"[Brute] FOUND: {cand!r} increases correct {baseline_correct} -> {correct}")
+                # lock it persistently
+                agent.lock_new(qtext, cand)
+                baseline_correct = correct
+                break
+        # polite pause
+    df = pd.DataFrame(results)
+    status_msg = f"Bruteforce finished. Baseline was {baseline_correct} (after any locks)."
+    return status_msg, df
 
-# --- Gradio UI ---
+# ---------------------------
+# Gradio UI
+# ---------------------------
 with gr.Blocks() as demo:
-    gr.Markdown("# Agent Hardcoded — Verrouillage des réponses trouvées")
-    gr.Markdown(
-        """
-        Réponses verrouillées (issues du bruteforce) :  
-        - Mercedes Sosa (2000-2009) → 3  
-        - Video L1vXCYZAYYM → 1  
-        - Reverse-text puzzle → right  
-        - Chess image → Qh5  
-        - Featured dinosaur nominator → FunkMonk  
-        - Table S counterexamples → a,b,c,d,e  
-        - Equine vet surname → Louvrier  
-        - Grocery vegetables → bell pepper, broccoli, celery, green beans, lettuce, sweet potatoes, zucchini  
-        - Actor (Polish) first name → Wojciech  
-        - 1928 least athletes IOC code → CUB  
-        - Malko Competition first name → Peter  
-        """
-    )
+    gr.Markdown("# Debuggable Agent Runner (robust normalization + persistence)")
+    gr.Markdown("Use the buttons below. Locked answers are persisted in `locked_answers.json`.")
     gr.LoginButton()
-    run_btn = gr.Button("Run Evaluation & Submit All Answers")
-    status = gr.Textbox(label="Run Status / Submission Result", lines=6, interactive=False)
-    out_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
+    submit_btn = gr.Button("Run Evaluation & Submit All Answers")
+    brute_input = gr.Textbox(label="Comma-separated target keys to brute-force (e.g. mercedes,video_l1v,chess)", lines=1)
+    brute_btn = gr.Button("Run Bruteforce Targets")
+    status = gr.Textbox(lines=10, label="Submission / Bruteforce Status", interactive=False)
+    table = gr.DataFrame(label="Questions / Submissions / Bruteforce attempts", wrap=True)
 
-    run_btn.click(fn=run_and_submit_all, outputs=[status, out_table])
+    submit_btn.click(fn=run_and_submit_all, inputs=[gr.State()], outputs=[status, table])
+    brute_btn.click(fn=run_bruteforce_one_by_one, inputs=[gr.State(), brute_input], outputs=[status, table])
 
 if __name__ == "__main__":
-    print("Launching Gradio app with locked answers...")
+    print("Launching debuggable Gradio app...")
     demo.launch(debug=True, share=False)