Spaces:
Running
Running
| """ | |
| FINAL Bench v4.2 — Baseline (Non-AGI) Evaluation System | |
| ========================================================= | |
| ★ Multi-Provider: OpenAI / Anthropic / Google (Gemini 3 Pro Preview) | |
| ★ Both Eval Model AND Judge Model support all 3 providers | |
| ★ 100 Tasks · 15 Domains · 8 TICOS Types · 5-Axis · 5-Stage AGI Grade | |
| ★ Dataset: HuggingFace FINAL-Bench/Metacognitive | |
| Author: Ginigen AI — Choi Sunyoung | License: Apache 2.0 | |
| """ | |
| import json, os, time, csv, io, re, html, hashlib, sqlite3, threading, random | |
| from datetime import datetime | |
| from dataclasses import dataclass, field | |
| from typing import List, Dict | |
| import requests | |
| import numpy as np | |
| import gradio as gr | |
| from concurrent.futures import ThreadPoolExecutor | |
| from datasets import load_dataset | |
| DOMAIN_INFO = { | |
| "Mathematics & Logic":{"icon":"🔢","color":"#FF6B35"},"Science":{"icon":"🔬","color":"#7B2FF7"}, | |
| "Philosophy":{"icon":"🤔","color":"#00B4D8"},"Medicine":{"icon":"🏥","color":"#2EC4B6"}, | |
| "Economics":{"icon":"📈","color":"#E63946"},"History":{"icon":"📜","color":"#F4A261"}, | |
| "War & Security":{"icon":"🛡️","color":"#264653"},"Space & Physics":{"icon":"🚀","color":"#6C63FF"}, | |
| "Chemistry & Biology":{"icon":"🧬","color":"#06D6A0"},"Language & Writing":{"icon":"✍️","color":"#EF476F"}, | |
| "Literature":{"icon":"📖","color":"#8338EC"},"Art":{"icon":"🎨","color":"#FF006E"}, | |
| "Religion & Mythology":{"icon":"🕊️","color":"#FFD166"},"Ethics":{"icon":"⚖️","color":"#118AB2"}, | |
| "AI & Technology":{"icon":"🤖","color":"#073B4C"}, | |
| } | |
| GRADE_WEIGHT={"A":1.5,"B":1.0,"C":0.7} | |
| RUBRIC={ | |
| "process_quality":{"weight":0.25,"desc":"Systematic reasoning transparency"}, | |
| "metacognitive_accuracy":{"weight":0.25,"desc":"Confidence calibration + uncertainty honesty"}, | |
| "error_recovery":{"weight":0.20,"desc":"Mid-analysis self-correction"}, | |
| "integration_depth":{"weight":0.15,"desc":"Multi-perspective synthesis"}, | |
| "final_correctness":{"weight":0.15,"desc":"Answer accuracy and completeness"}, | |
| } | |
| AXIS_MAP={ | |
| "generalization":{"rubrics":["process_quality","final_correctness"],"ticos":[]}, | |
| "reasoning":{"rubrics":["process_quality","error_recovery"],"ticos":["E_SelfCorrecting","C_ProgressiveDiscovery"]}, | |
| "planning":{"rubrics":["integration_depth","process_quality"],"ticos":["D_MultiConstraint","H_DecisionUnderUncertainty"]}, | |
| "reliability":{"rubrics":["metacognitive_accuracy"],"ticos":["E_SelfCorrecting","G_PivotDetection"]}, | |
| "safety":{"rubrics":["error_recovery","metacognitive_accuracy"],"ticos":["A_TrapEscape","G_PivotDetection"]}, | |
| } | |
| AGI_STAGES=[ | |
| {"stage":1,"name":"FINAL-Partial","label":"Partial Intelligence","min":0,"max":39,"color":"#f44336"}, | |
| {"stage":2,"name":"FINAL-Proto","label":"Proto Intelligence","min":40,"max":59,"color":"#ff9800"}, | |
| {"stage":3,"name":"FINAL-Pre","label":"Pre-AGI","min":60,"max":79,"color":"#2196f3"}, | |
| {"stage":4,"name":"FINAL-Pass","label":"AGI Achieved","min":80,"max":94,"color":"#4caf50"}, | |
| {"stage":5,"name":"FINAL-Post","label":"Operationally Mature AGI","min":95,"max":100,"color":"#9c27b0"}, | |
| ] | |
| class FinalTask: | |
| task_id:str;domain:str;grade:str;ticos_type:str | |
| difficulty:str;lens:str;title:str;prompt:str | |
| expected_behavior:str;hidden_trap:str | |
| ticos_required:List[str]=field(default_factory=list) | |
| metadata:Dict=field(default_factory=dict) | |
| def load_tasks(): | |
| print("📥 Loading FINAL-Bench/Metacognitive from HuggingFace...") | |
| try: | |
| ds=load_dataset("FINAL-Bench/Metacognitive",split="train") | |
| tasks=[] | |
| for row in ds: | |
| tr=row.get("ticos_required",[]) | |
| if isinstance(tr,str): | |
| try:tr=json.loads(tr) | |
| except:tr=[x.strip() for x in tr.split(",") if x.strip()] | |
| tasks.append(FinalTask(task_id=row["task_id"],domain=row["domain"],grade=row["grade"], | |
| ticos_type=row["ticos_type"],difficulty=row["difficulty"],lens=row.get("lens",""), | |
| title=row.get("title",row["task_id"]),prompt=row["prompt"], | |
| expected_behavior=row.get("expected_behavior",""),hidden_trap=row.get("hidden_trap",""), | |
| ticos_required=tr if isinstance(tr,list) else [],metadata={})) | |
| print(f" ✅ Loaded {len(tasks)} tasks from HuggingFace") | |
| return tasks | |
| except Exception as e: | |
| print(f" ⚠️ HF load failed: {e}") | |
| raise FileNotFoundError("Dataset not found!") | |
| ALL_TASKS=load_tasks() | |
| print(f"✅ FINAL Bench v4.2: {len(ALL_TASKS)} tasks loaded") | |
| # ═══ §3. Model Registry ═══ | |
| PROVIDER_MODELS={ | |
| "OpenAI":{ | |
| "gpt-5.2":"GPT-5.2 (flagship)","gpt-5-mini":"GPT-5 Mini", | |
| "gpt-4.1":"GPT-4.1","o4-mini":"o4-mini (reasoning)","gpt-4o":"GPT-4o", | |
| }, | |
| "Anthropic":{ | |
| "claude-opus-4-6":"Claude Opus 4.6", | |
| "claude-sonnet-4-5-20250929":"Claude Sonnet 4.5", | |
| "claude-haiku-4-5-20251001":"Claude Haiku 4.5", | |
| }, | |
| "Google":{ | |
| "gemini-3-pro-preview":"Gemini 3 Pro Preview", | |
| }, | |
| } | |
| ALL_MODELS={} | |
| for prov,models in PROVIDER_MODELS.items(): | |
| for mid,label in models.items(): | |
| ALL_MODELS[f"{label} [{prov}]"]={"id":mid,"provider":prov} | |
| MODEL_CHOICES=list(ALL_MODELS.keys()) | |
| DEFAULT_EVAL="GPT-5.2 (flagship) [OpenAI]" | |
| DEFAULT_JUDGE="GPT-5.2 (flagship) [OpenAI]" | |
| def _resolve_model(choice): | |
| info=ALL_MODELS.get(choice,{}) | |
| return info.get("id","gpt-5.2"),info.get("provider","OpenAI") | |
| # ═══ §4. API Clients ═══ | |
| def _strip_think(text): | |
| if not text:return text | |
| for tag in['think','thinking','reasoning','reflection']: | |
| text=re.sub(rf'<{tag}>.*?</{tag}>','',text,flags=re.DOTALL) | |
| return text.strip() | |
| def call_openai(prompt,system="",api_key="",model="gpt-5.2", | |
| max_tokens=8192,temperature=0.6,reasoning_effort=None, | |
| json_mode=False,json_schema=None): | |
| headers={"Content-Type":"application/json","Authorization":f"Bearer {api_key}"} | |
| messages=[] | |
| if system:messages.append({"role":"system","content":system}) | |
| messages.append({"role":"user","content":prompt}) | |
| payload={"model":model,"max_completion_tokens":max_tokens,"temperature":temperature,"messages":messages} | |
| if reasoning_effort:payload["reasoning_effort"]=reasoning_effort | |
| if json_schema: | |
| payload["reasoning_effort"]="none" | |
| payload["response_format"]={"type":"json_schema","json_schema":{"name":"FINALJudge","strict":True,"schema":json_schema}} | |
| elif json_mode: | |
| payload["response_format"]={"type":"json_object"} | |
| for attempt in range(3): | |
| try: | |
| r=requests.post("https://api.openai.com/v1/chat/completions",headers=headers,data=json.dumps(payload),timeout=300) | |
| r.raise_for_status();c=r.json()["choices"][0]["message"]["content"] | |
| return _strip_think(c) if c else "[EMPTY]" | |
| except requests.exceptions.HTTPError: | |
| if r.status_code==429:time.sleep(5*(attempt+1));continue | |
| try:err=r.json().get("error",{}).get("message","") | |
| except:err=str(r.status_code) | |
| if attempt<2:time.sleep(3*(attempt+1));continue | |
| return f"[API_ERROR] OpenAI {r.status_code}: {err}" | |
| except Exception as e: | |
| if attempt<2:time.sleep(3*(attempt+1)) | |
| else:return f"[API_ERROR] {e}" | |
| def call_anthropic(prompt,system="",api_key="",model="claude-opus-4-6", | |
| max_tokens=8192,temperature=0.6): | |
| headers={"Content-Type":"application/json","x-api-key":api_key,"anthropic-version":"2023-06-01"} | |
| messages=[{"role":"user","content":prompt}] | |
| payload={"model":model,"max_tokens":max_tokens,"temperature":temperature,"messages":messages} | |
| if system:payload["system"]=system | |
| for attempt in range(3): | |
| try: | |
| r=requests.post("https://api.anthropic.com/v1/messages",headers=headers,data=json.dumps(payload),timeout=300) | |
| r.raise_for_status();resp=r.json() | |
| text_parts=[] | |
| for block in resp.get("content",[]): | |
| if block.get("type")=="text":text_parts.append(block["text"]) | |
| c="\n".join(text_parts) | |
| return _strip_think(c) if c else "[EMPTY]" | |
| except requests.exceptions.HTTPError: | |
| if r.status_code==429:time.sleep(5*(attempt+1));continue | |
| if r.status_code==529:time.sleep(8*(attempt+1));continue | |
| try:err=r.json().get("error",{}).get("message","") | |
| except:err=str(r.status_code) | |
| return f"[API_ERROR] Claude {r.status_code}: {err}" | |
| except Exception as e: | |
| if attempt<2:time.sleep(3*(attempt+1)) | |
| else:return f"[API_ERROR] {e}" | |
| # ★ Gemini — x-goog-api-key header · data=json.dumps · thinking skip | |
| GEMINI_API_BASE="https://generativelanguage.googleapis.com/v1beta" | |
| def call_gemini(prompt,system="",api_key="",model="gemini-3-pro-preview", | |
| max_tokens=8192,temperature=1.0,json_mode=False): | |
| url=f"{GEMINI_API_BASE}/models/{model}:generateContent" | |
| headers={"Content-Type":"application/json","x-goog-api-key":api_key} | |
| contents=[{"role":"user","parts":[{"text":prompt}]}] | |
| gen_config={"maxOutputTokens":max_tokens,"temperature":temperature} | |
| payload={"contents":contents,"generationConfig":gen_config} | |
| if system:payload["systemInstruction"]={"parts":[{"text":system}]} | |
| if json_mode:gen_config["responseMimeType"]="application/json" | |
| for attempt in range(3): | |
| try: | |
| r=requests.post(url,headers=headers,data=json.dumps(payload),timeout=300) | |
| r.raise_for_status();data=r.json() | |
| candidates=data.get("candidates",[]) | |
| if not candidates: | |
| br=data.get("promptFeedback",{}).get("blockReason","UNKNOWN") | |
| return f"[API_ERROR] Gemini BLOCKED: {br}" | |
| parts=candidates[0].get("content",{}).get("parts",[]) | |
| result=[] | |
| for p in parts: | |
| if "text" in p: | |
| if p.get("thought",False):continue | |
| result.append(p["text"]) | |
| c="\n".join(result) if result else "" | |
| return _strip_think(c) if c else "[EMPTY]" | |
| except requests.exceptions.HTTPError: | |
| if r.status_code==429:time.sleep(5*(attempt+1)+random.uniform(0,2));continue | |
| if r.status_code==503:time.sleep(8*(attempt+1)+random.uniform(0,3));continue | |
| try:err=r.json().get("error",{}).get("message","") | |
| except:err=str(r.status_code) | |
| print(f" [Gemini] ERROR {r.status_code}: {err[:200]}") | |
| return f"[API_ERROR] Gemini {r.status_code}: {err}" | |
| except Exception as e: | |
| print(f" [Gemini] Exception: {e}") | |
| if attempt<2:time.sleep(3*(attempt+1)) | |
| else:return f"[API_ERROR] Gemini: {e}" | |
| def call_model(prompt,system="",api_key="",model_id="gpt-5.2", | |
| provider="OpenAI",max_tokens=8192,temperature=0.6): | |
| if provider=="OpenAI":return call_openai(prompt,system,api_key,model_id,max_tokens,temperature) | |
| elif provider=="Anthropic":return call_anthropic(prompt,system,api_key,model_id,max_tokens,temperature) | |
| elif provider=="Google":return call_gemini(prompt,system,api_key,model_id,max_tokens,temperature=1.0) | |
| return f"[API_ERROR] Unknown provider: {provider}" | |
| # ═══ §5. Judge ═══ | |
| JUDGE_SYSTEM="""You are a FINAL Bench judge for AGI-Level Verification. | |
| Score each rubric using ONLY: 0.0 / 0.25 / 0.5 / 0.75 / 1.0 | |
| RUBRIC: | |
| process_quality (25%): Systematic step-by-step reasoning. Complete answers score higher. | |
| metacognitive_accuracy (25%): Confidence calibration. Overconfidence=0.25 max. | |
| error_recovery (20%): EXPLICIT self-correction. Score 0.5+ if ANY self-corrections exist. | |
| integration_depth (15%): Multi-perspective synthesis + emergent insights | |
| final_correctness (15%): Answer accuracy and completeness. INCOMPLETE=0.25 max. | |
| STRICT: 1.0=AGI-worthy 0.75=expert 0.5=competent 0.25=gaps 0.0=failure | |
| Output ONLY valid JSON: {"scores":{"process_quality":X,"metacognitive_accuracy":X,"error_recovery":X,"integration_depth":X,"final_correctness":X},"comment":"<50 words>"}""" | |
| def _build_judge_schema(): | |
| sp={k:{"type":"number","enum":[0.0,0.25,0.5,0.75,1.0]} for k in RUBRIC} | |
| return {"type":"object","properties":{"scores":{"type":"object","properties":sp, | |
| "required":list(RUBRIC.keys()),"additionalProperties":False}, | |
| "comment":{"type":"string"}},"required":["scores","comment"],"additionalProperties":False} | |
| JUDGE_SCHEMA=_build_judge_schema() | |
| def build_judge_prompt(task,response): | |
| return f"""FINAL Bench Task Evaluation | |
| Task: {task.task_id} | {task.domain} | Grade {task.grade} | {task.difficulty} | |
| TICOS: {task.ticos_type} | Title: {task.title} | |
| PROMPT:\n{task.prompt[:2000]} | |
| EXPECTED:\n{task.expected_behavior[:600]} | |
| HIDDEN TRAPS: {task.hidden_trap or 'None'} | |
| RESPONSE TO JUDGE:\n{response[:17000]} | |
| Score all 5 rubrics. Apply {task.ticos_type} bonus criteria. | |
| Output ONLY JSON: {{"scores":{{...}},"comment":"..."}}""" | |
| def _parse_judge_json(text): | |
| if not text or text.startswith("[API_ERROR") or text=="[EMPTY]":return None | |
| cleaned=_strip_think(text);VALID={0.0,0.25,0.5,0.75,1.0};keys=list(RUBRIC.keys()) | |
| try: | |
| t=re.sub(r'^```(?:json)?\s*','',cleaned.strip());t=re.sub(r'\s*```$','',t.strip()) | |
| data=json.loads(t) | |
| if "scores" in data and isinstance(data["scores"],dict): | |
| scores={k:min(VALID,key=lambda x,v=float(data["scores"].get(k,0.5)):abs(x-v)) for k in keys} | |
| return {"scores":scores,"comment":data.get("comment","ok")} | |
| except:pass | |
| try: | |
| m=re.search(r'\{[^{}]*"scores"\s*:\s*\{[^{}]*\}[^{}]*\}',cleaned,re.DOTALL) | |
| if m: | |
| data=json.loads(m.group()) | |
| if "scores" in data: | |
| scores={k:min(VALID,key=lambda x,v=float(data["scores"].get(k,0.5)):abs(x-v)) for k in keys} | |
| return {"scores":scores,"comment":data.get("comment","parsed")} | |
| except:pass | |
| try: | |
| sc={} | |
| for k in keys: | |
| m2=re.search(rf'["\']?{re.escape(k)}["\']?\s*[:=]\s*([\d.]+)',cleaned,re.IGNORECASE) | |
| if m2: | |
| v=float(m2.group(1)) | |
| if 0<=v<=1:sc[k]=min(VALID,key=lambda x,v=v:abs(x-v)) | |
| if len(sc)>=3: | |
| for k in keys: | |
| if k not in sc:sc[k]=0.5 | |
| return {"scores":sc,"comment":"regex_parsed"} | |
| except:pass | |
| return None | |
| def call_judge(prompt,system,api_key,model_id,provider,temperature=0.1,max_tokens=2048): | |
| if provider=="OpenAI": | |
| raw=call_openai(prompt,system=system,api_key=api_key,model=model_id,max_tokens=max_tokens,temperature=temperature,json_schema=JUDGE_SCHEMA) | |
| result=_parse_judge_json(raw) | |
| if result:return result | |
| raw2=call_openai(prompt,system=system,api_key=api_key,model=model_id,max_tokens=max_tokens,temperature=temperature,json_mode=True) | |
| return _parse_judge_json(raw2) | |
| elif provider=="Anthropic": | |
| raw=call_anthropic(prompt,system=system,api_key=api_key,model=model_id,max_tokens=max_tokens,temperature=temperature) | |
| return _parse_judge_json(raw) | |
| elif provider=="Google": | |
| raw=call_gemini(prompt,system=system,api_key=api_key,model=model_id,max_tokens=max_tokens,temperature=1.0,json_mode=True) | |
| result=_parse_judge_json(raw) | |
| if result:return result | |
| raw2=call_gemini(prompt,system=system,api_key=api_key,model=model_id,max_tokens=max_tokens,temperature=1.0,json_mode=False) | |
| return _parse_judge_json(raw2) | |
| return None | |
| # ═══ §6. Scoring ═══ | |
| def compute_task_score(scores): | |
| return round(sum(scores.get(k,0.5)*v["weight"] for k,v in RUBRIC.items())*100,2) | |
| def compute_axis_scores(results,tasks): | |
| tm={t.task_id:t for t in tasks};ax={} | |
| for an,ai in AXIS_MAP.items(): | |
| vals=[] | |
| for tid,d in results.items(): | |
| if d["score"]<0:continue | |
| t=tm.get(tid) | |
| if not t:continue | |
| try:jd=json.loads(d["judge"]) if isinstance(d["judge"],str) else d["judge"];sc=jd.get("scores",{}) if isinstance(jd,dict) else {} | |
| except:sc={} | |
| rv=[float(sc.get(r,0.5)) for r in ai["rubrics"] if r in sc] | |
| w=1.5 if(ai["ticos"] and t.ticos_type in ai["ticos"]) else 1.0 | |
| if rv:vals.append(np.mean(rv)*w) | |
| ax[an]=round(min(np.mean(vals)*100,100),2) if vals else 0.0 | |
| return ax | |
| def compute_final_score(results,tasks): | |
| tm={t.task_id:t for t in tasks};ds={} | |
| for tid,d in results.items(): | |
| if d["score"]<0:continue | |
| t=tm.get(tid) | |
| if t:ds.setdefault(t.domain,[]).append(d["score"]) | |
| da={d:np.mean(v) for d,v in ds.items() if v} | |
| gd={} | |
| for t in tasks:gd.setdefault(t.grade,set()).add(t.domain) | |
| ws,wt=0,0 | |
| for g,doms in gd.items(): | |
| w=GRADE_WEIGHT.get(g,1.0) | |
| for d in doms: | |
| if d in da:ws+=da[d]*w;wt+=w | |
| base=ws/wt if wt>0 else 0 | |
| axis=compute_axis_scores(results,tasks) | |
| av=[max(v,0.01) for v in axis.values()] | |
| har=(len(av)/sum(1.0/v for v in av)) if av else 50 | |
| har_p=har/100.0 | |
| return round(base*har_p,2),round(base,2),round(har_p,3),axis,da | |
| def determine_agi_stage(score,axis): | |
| all60=all(v>=60 for v in axis.values()) if axis else False | |
| for s in reversed(AGI_STAGES): | |
| if score>=s["min"]: | |
| if s["stage"]>=4 and not all60:return AGI_STAGES[2] | |
| return s | |
| return AGI_STAGES[0] | |
| # ═══ §7. Checkpoint DB ═══ | |
| DB_PATH="final_bench_eval.db" | |
| def _init_db(): | |
| c=sqlite3.connect(DB_PATH);c.execute("CREATE TABLE IF NOT EXISTS eval_results(run_id TEXT,task_id TEXT,model_response TEXT,judge_response TEXT,weighted_score REAL,timestamp REAL,PRIMARY KEY(run_id,task_id))");c.commit();c.close() | |
| def _make_run_id(m):return hashlib.md5(f"FINALv42_BL_{m}".encode()).hexdigest()[:12] | |
| def _save_result(rid,tid,resp,jresp,sc): | |
| c=sqlite3.connect(DB_PATH);c.execute("INSERT OR REPLACE INTO eval_results VALUES(?,?,?,?,?,?)",(rid,tid,resp,jresp,sc,time.time()));c.commit();c.close() | |
| def _load_all(rid): | |
| c=sqlite3.connect(DB_PATH);cur=c.execute("SELECT task_id,model_response,judge_response,weighted_score FROM eval_results WHERE run_id=?",(rid,));rows=cur.fetchall();c.close() | |
| result={} | |
| for r in rows: | |
| resp=r[1] or "";score=r[3] | |
| if score<=0 and(resp.startswith("[API_ERROR") or resp.startswith("[BLOCKED") or resp=="[EMPTY]" or resp.startswith("[ERROR")):continue | |
| result[r[0]]={"response":resp,"judge":r[2],"score":score} | |
| return result | |
| def _clear_run(rid): | |
| c=sqlite3.connect(DB_PATH);c.execute("DELETE FROM eval_results WHERE run_id=?",(rid,));c.commit();c.close() | |
| _init_db() | |
| # ═══ §8. CSV Export ═══ | |
| def generate_csv(results,tasks,model_name,judge_name,mode="BASELINE"): | |
| out=io.StringIO();w=csv.writer(out) | |
| w.writerow(["task_id","domain","grade","ticos_type","difficulty","title","eval_model","judge_model","mode","weighted_score","process_quality","metacognitive_accuracy","error_recovery","integration_depth","final_correctness","judge_comment","response_preview","timestamp"]) | |
| tm={t.task_id:t for t in tasks} | |
| for tid,d in sorted(results.items()): | |
| t=tm.get(tid) | |
| if not t:continue | |
| jd={} | |
| try:jd=json.loads(d["judge"]) if isinstance(d["judge"],str) else(d["judge"] or {}) | |
| except:pass | |
| sc=jd.get("scores",{}) if isinstance(jd,dict) else {} | |
| cm=(jd.get("comment","") if isinstance(jd,dict) else "")[:200];s=d["score"] | |
| if s<0:s=-1;cm=f"JUDGE_FAILED:{cm}" | |
| w.writerow([tid,t.domain,t.grade,t.ticos_type,t.difficulty,t.title,model_name,judge_name,mode,s,sc.get("process_quality",""),sc.get("metacognitive_accuracy",""),sc.get("error_recovery",""),sc.get("integration_depth",""),sc.get("final_correctness",""),cm,(d.get("response","") or "")[:300].replace("\n"," "),datetime.now().isoformat()]) | |
| return out.getvalue() | |
| # ═══ §9. HTML Builders ═══ | |
| CSS="""<style> | |
| .eval-table{width:100%;border-collapse:collapse;font-size:0.82em} | |
| .eval-table th{background:#f0f4f8;padding:8px;text-align:left;border-bottom:2px solid #ccc;font-size:0.9em} | |
| .eval-table td{padding:5px 8px;border-bottom:1px solid #eee} | |
| .score-bar{background:#e0e0e0;border-radius:8px;height:16px;overflow:hidden;min-width:70px} | |
| .score-fill{height:100%;border-radius:8px;transition:width .4s} | |
| .summary-card{background:linear-gradient(135deg,#0a0a1a,#1a1a3e);border-radius:16px;padding:24px;color:#fff;margin:8px 0} | |
| .axis-row{display:flex;align-items:center;gap:10px;margin:5px 0} | |
| .axis-bar{flex:1;background:#333;border-radius:6px;height:14px;overflow:hidden} | |
| .axis-fill{height:100%;border-radius:6px} | |
| .stage-badge{display:inline-block;padding:6px 16px;border-radius:20px;font-weight:700;font-size:1.1em;margin:8px 0} | |
| .progress-bar{background:#e0e0e0;border-radius:8px;height:22px;margin:12px 0;overflow:hidden} | |
| .progress-fill{height:100%;border-radius:8px;transition:width .4s;background:linear-gradient(90deg,#1565c0,#00c853)} | |
| </style>""" | |
| def _sc(s): | |
| if s>=80:return "#4caf50" | |
| if s>=60:return "#ff9800" | |
| if s>=40:return "#ff5722" | |
| return "#f44336" | |
| def _build_progress_table(results,tasks): | |
| rows="" | |
| for t in tasks: | |
| info=DOMAIN_INFO.get(t.domain,{"icon":"?","color":"#999"}) | |
| gb=f'<span style="background:{"#c62828" if t.grade=="A" else "#1565c0" if t.grade=="B" else "#6a1b9a"};color:#fff;padding:1px 6px;border-radius:4px;font-size:0.8em">{t.grade}</span>' | |
| if t.task_id in results: | |
| d=results[t.task_id];s=d["score"];resp=d.get("response","") | |
| if s<0:rows+=f'<tr style="background:#fff3e0"><td>{t.task_id}</td><td>{info["icon"]} {t.domain[:15]}</td><td>{gb}</td><td>{t.ticos_type.split("_")[0]}</td><td>{t.difficulty}</td><td style="color:#ff9800">❌ JF</td><td>—</td></tr>' | |
| elif s==0 and resp and(resp.startswith("[API_ERROR") or resp.startswith("[BLOCKED") or resp=="[EMPTY]"): | |
| err_short=html.escape(resp[:60]) | |
| rows+=f'<tr style="background:#ffebee"><td>{t.task_id}</td><td>{info["icon"]} {t.domain[:15]}</td><td>{gb}</td><td>{t.ticos_type.split("_")[0]}</td><td>{t.difficulty}</td><td colspan="2" style="color:#c62828;font-size:0.75em">🚫 {err_short}</td></tr>' | |
| else: | |
| c=_sc(s);rows+=f'<tr><td>{t.task_id}</td><td>{info["icon"]} {t.domain[:15]}</td><td>{gb}</td><td>{t.ticos_type.split("_")[0]}</td><td>{t.difficulty}</td><td><div class="score-bar"><div class="score-fill" style="width:{min(s,100)}%;background:{c}"></div></div></td><td style="font-weight:700;color:{c}">{s:.1f}</td></tr>' | |
| else:rows+=f'<tr style="opacity:0.35"><td>{t.task_id}</td><td>{info["icon"]}</td><td>{gb}</td><td>{t.ticos_type.split("_")[0]}</td><td>{t.difficulty}</td><td>⏳</td><td>—</td></tr>' | |
| return f'{CSS}<table class="eval-table"><thead><tr><th>ID</th><th>Domain</th><th>G</th><th>TICOS</th><th>Diff</th><th>Score</th><th>Val</th></tr></thead><tbody>{rows}</tbody></table>' | |
| def _build_summary_card(results,tasks,eval_label,judge_label,hf_status): | |
| final,base,har_p,axis,dom_avgs=compute_final_score(results,tasks) | |
| stage=determine_agi_stage(final,axis) | |
| labels={"generalization":"🌐 Generalization","reasoning":"🧠 Reasoning","planning":"📋 Planning","reliability":"🎯 Reliability","safety":"🛡️ Safety"} | |
| ax_html="" | |
| for an,av in axis.items(): | |
| c=_sc(av);ax_html+=f'<div class="axis-row"><span style="width:120px;font-size:0.85em">{labels.get(an,an)}</span><div class="axis-bar"><div class="axis-fill" style="width:{min(av,100)}%;background:{c}"></div></div><span style="width:50px;text-align:right;font-weight:700;color:{c}">{av:.1f}</span></div>' | |
| gh="" | |
| for g in["A","B","C"]: | |
| gd=[t.domain for t in tasks if t.grade==g];gs=[dom_avgs[d] for d in set(gd) if d in dom_avgs] | |
| if gs:a=np.mean(gs);gh+=f'<span style="margin-right:14px">{g}×{GRADE_WEIGHT[g]}: <b style="color:{_sc(a)}">{a:.1f}</b></span>' | |
| done=sum(1 for t in tasks if t.task_id in results) | |
| jf=sum(1 for t in tasks if t.task_id in results and results[t.task_id]["score"]<0) | |
| api_errs=sum(1 for t in tasks if t.task_id in results and results[t.task_id]["score"]==0 and(results[t.task_id].get("response","") or "").startswith("[")) | |
| ma_vals,er_vals=[],[] | |
| for tid,d in results.items(): | |
| if d["score"]<0:continue | |
| try: | |
| jd=json.loads(d["judge"]) if isinstance(d["judge"],str) else d["judge"];sc=jd.get("scores",{}) if isinstance(jd,dict) else {} | |
| if "metacognitive_accuracy" in sc:ma_vals.append(float(sc["metacognitive_accuracy"])) | |
| if "error_recovery" in sc:er_vals.append(float(sc["error_recovery"])) | |
| except:pass | |
| avg_ma=np.mean(ma_vals) if ma_vals else 0;avg_er=np.mean(er_vals) if er_vals else 0 | |
| gap=avg_ma-avg_er;gc="#f44336" if gap>0.2 else "#ff9800" if gap>0.1 else "#4caf50" | |
| gl="Declaration-Action Gap" if gap>0.2 else "Moderate Gap" if gap>0.1 else "Balanced" | |
| ad=[t.domain for t in tasks if t.grade=="A"];asc_vals=[dom_avgs[d] for d in set(ad) if d in dom_avgs];aa=np.mean(asc_vals) if asc_vals else 0 | |
| checks=[("Score≥80",final>=80),("Axes≥60",all(v>=60 for v in axis.values())),(f"A-avg≥75({aa:.0f})",aa>=75)] | |
| ch="".join([f'<span style="margin-right:8px">{"✅" if ok else "❌"}{lb}</span>' for lb,ok in checks]) | |
| err_html=f'<div style="color:#ff5722;font-size:0.82em;margin-top:4px">⚠️ API Errors: {api_errs} tasks</div>' if api_errs else "" | |
| return f"""{CSS}<div class="summary-card"><div style="text-align:center"><div class="stage-badge" style="background:{stage['color']}">{stage['name']}</div><h2 style="margin:6px 0;font-size:1.6em">🤖 Baseline FINAL: {final:.1f}</h2><p style="color:#aaa;font-size:0.85em">{stage['label']} · Base {base:.1f} × HAR {har_p:.3f} · {done}/{len(tasks)}{f" · JF={jf}" if jf else ""}</p><p style="color:#8af;font-size:0.82em;margin:4px 0">Eval: {eval_label} · Judge: {judge_label}</p>{err_html}</div><hr style="border-color:#333;margin:12px 0"><h4 style="color:#aaa;margin:6px 0">🎯 5-Axis Scores</h4>{ax_html}<hr style="border-color:#333;margin:10px 0"><div style="font-size:0.88em">{gh}</div><div style="display:flex;align-items:center;gap:12px;margin:8px 0;padding:8px;background:rgba(255,255,255,0.05);border-radius:8px"><span style="font-size:0.85em">MA-ER Gap:</span><span style="font-weight:700;color:{gc}">{gap:.3f}</span><span style="font-size:0.8em;color:{gc}">({gl})</span><span style="font-size:0.78em;color:#888">MA={avg_ma:.3f} ER={avg_er:.3f}</span></div><div style="font-size:0.82em;margin-top:6px">{ch}</div><p style="font-size:0.78em;color:#666;margin-top:8px">{hf_status}</p><div style="background:rgba(233,69,96,0.15);border:1px solid #e94560;border-radius:8px;padding:10px;margin-top:12px"><p style="font-size:0.82em;color:#e94560;margin:0">🔒 <b>MetaCog (Self-Correction) evaluation: COMING SOON</b></p></div></div>""" | |
| def _build_detail_view(results,tasks): | |
| items="" | |
| for t in tasks: | |
| if t.task_id not in results:continue | |
| d=results[t.task_id];info=DOMAIN_INFO.get(t.domain,{"icon":"?"});s=d["score"];resp=html.escape((d.get("response","") or "")[:500]) | |
| jc="";ss="" | |
| try: | |
| jd=json.loads(d["judge"]) if isinstance(d["judge"],str) else(d["judge"] or {});jc=html.escape((jd.get("comment","") if isinstance(jd,dict) else "")[:200]);sc=jd.get("scores",{}) if isinstance(jd,dict) else {};ss=" · ".join([f"{k.split('_')[0]}={v}" for k,v in sc.items()]) | |
| except:pass | |
| c=_sc(s) if s>=0 else "#ff9800";badge=f'{s:.1f}' if s>=0 else "JF" | |
| items+=f'<details style="margin:3px 0;border:1px solid #ddd;border-radius:8px;padding:8px"><summary style="cursor:pointer;font-weight:600">{info["icon"]} {t.task_id} [{t.grade}] — <span style="color:{c}">{badge}</span></summary><div style="font-size:0.8em;margin-top:6px"><b>{t.title}</b><br>TICOS: {t.ticos_type} | Scores: {ss}<br>Judge: {jc}<br>Response: {resp}...</div></details>' | |
| return CSS+items | |
| # ═══ §10. Evaluation Engine ═══ | |
| def _eval_single(task,run_id,eval_api_key,eval_model_id,eval_provider,judge_api_key,judge_model_id,judge_provider,state): | |
| try: | |
| sys_p=(f"You are being evaluated on FINAL Bench.\nTask: {task.ticos_type}\n" | |
| f"State confidence (0-100%) for EVERY claim. If wrong, EXPLICITLY backtrack. If unsure, say so honestly.") | |
| print(f" ▶ {task.task_id} → {eval_provider}/{eval_model_id}") | |
| model_response=call_model(task.prompt,system=sys_p,api_key=eval_api_key,model_id=eval_model_id,provider=eval_provider,max_tokens=12288) | |
| if model_response.startswith("[API_ERROR") or model_response.startswith("[BLOCKED") or model_response=="[EMPTY]": | |
| print(f" ✗ {task.task_id}: {model_response[:100]}") | |
| _save_result(run_id,task.task_id,model_response,"{}",0) | |
| with state["lock"]:state["done"]+=1;state["errors"].append(f"{task.task_id}: {model_response[:80]}") | |
| return task.task_id,{"response":model_response,"judge":"{}","score":0} | |
| print(f" ✓ {task.task_id} len={len(model_response)}") | |
| jp=build_judge_prompt(task,model_response) | |
| jd=call_judge(jp,system=JUDGE_SYSTEM,api_key=judge_api_key,model_id=judge_model_id,provider=judge_provider) | |
| if jd is None:jd={"scores":{k:0.0 for k in RUBRIC},"comment":"JUDGE_PARSE_FAILED","failed":True} | |
| if jd.get("failed"):ws=-1.0;jd["comment"]=f"JF:{jd.get('comment','')}" | |
| else:ws=compute_task_score(jd["scores"]); | |
| with state["lock"]:state["parse_ok"]+=1 | |
| jj=json.dumps(jd,ensure_ascii=False) | |
| _save_result(run_id,task.task_id,model_response,jj,ws) | |
| with state["lock"]: | |
| state["done"]+=1;info=DOMAIN_INFO.get(task.domain,{"icon":"?"}) | |
| state["active"].append(f'{info["icon"]} {task.task_id}') | |
| if len(state["active"])>10:state["active"]=state["active"][-10:] | |
| return task.task_id,{"response":model_response,"judge":jj,"score":ws} | |
| except Exception as e: | |
| print(f" ✗ {task.task_id} EXCEPTION: {e}") | |
| with state["lock"]:state["done"]+=1;state["errors"].append(f"{task.task_id}: {str(e)[:60]}") | |
| _save_result(run_id,task.task_id,f"[ERROR] {e}","{}",0) | |
| return task.task_id,{"response":f"[ERROR] {e}","judge":"{}","score":0} | |
| # ═══ §11. State Machine ═══ | |
| _EVAL_STATE={"running":False,"stop_requested":False,"finished":False,"run_id":"","eval_label":"","judge_label":"","done":0,"total":0,"cached":0,"errors":[],"active":[],"parse_ok":0,"parse_fail":0,"start_time":0,"results":{},"tasks":[],"grade_done":{},"grade_total":{},"lock":threading.Lock(),"message":"","csv_path":None,"hf_status":"","n_workers":5} | |
| def _reset(): | |
| with _EVAL_STATE["lock"]:_EVAL_STATE.update({"running":False,"stop_requested":False,"finished":False,"done":0,"cached":0,"errors":[],"active":[],"parse_ok":0,"parse_fail":0,"start_time":0,"results":{},"tasks":[],"grade_done":{},"grade_total":{},"message":"","csv_path":None,"hf_status":""}) | |
| def _prog_html(state,pending): | |
| done=state["done"];pct=min(int(done/max(pending,1)*100),100);gb="" | |
| for g in["A","B","C"]: | |
| gt=state["grade_total"].get(g,0);gd=state["grade_done"].get(g,0) | |
| if gt==0:continue | |
| gp=min(int(gd/gt*100),100);c="#4caf50" if gp==100 else("#1976d2" if gp>0 else "#e0e0e0") | |
| emoji="🅰️" if g=="A" else "🅱️" if g=="B" else "🅾️" | |
| gb+=f'<div style="display:flex;align-items:center;gap:8px;margin:3px 0"><span style="width:100px;font-size:0.85em">{emoji} {g}×{GRADE_WEIGHT[g]}</span><div style="flex:1;background:#e0e0e0;border-radius:6px;height:14px;overflow:hidden"><div style="width:{gp}%;height:100%;background:{c};border-radius:6px"></div></div><span style="width:55px;font-size:0.82em;text-align:right;color:{c}">{gd}/{gt}</span></div>' | |
| o=f'<div style="margin:8px 0"><div style="display:flex;justify-content:space-between;font-size:0.95em;margin-bottom:6px"><span>⚡ <b>🤖 Baseline</b> — {done}/{pending}</span><span style="font-weight:700">{pct}%</span></div><div class="progress-bar"><div class="progress-fill" style="width:{pct}%"></div></div>{gb}' | |
| ac=state.get("active",[]) | |
| if ac:o+='<div style="margin-top:8px">🔄 '+" ".join([f'<span style="background:#e3f2fd;padding:2px 6px;border-radius:4px;font-size:0.78em">{a}</span>' for a in ac[-8:]])+'</div>' | |
| er=state.get("errors",[]) | |
| if er: | |
| o+='<div style="color:#c62828;margin-top:6px;font-size:0.8em;max-height:120px;overflow-y:auto">' | |
| for e in er[-6:]:o+=f'<div>⚠️ {html.escape(e[:100])}</div>' | |
| o+='</div>' | |
| return o+'</div>' | |
| def _bg_eval(eval_api_key,eval_model_id,eval_provider,eval_label,judge_api_key,judge_model_id,judge_provider,judge_label,tasks,run_id,n_workers): | |
| global _EVAL_STATE | |
| try: | |
| with _EVAL_STATE["lock"]:_EVAL_STATE["start_time"]=time.time();_EVAL_STATE["message"]=f"⚡ Eval: {eval_label} · Judge: {judge_label} · {len(tasks)} tasks" | |
| results=dict(_load_all(run_id));cached=sum(1 for t in tasks if t.task_id in results);pending=[t for t in tasks if t.task_id not in results] | |
| print(f" 📊 Cached: {cached} / Pending: {len(pending)} / Total: {len(tasks)}") | |
| gt={}; | |
| for t in pending:gt.setdefault(t.grade,[]).append(t) | |
| with _EVAL_STATE["lock"]:_EVAL_STATE["results"]=results;_EVAL_STATE["cached"]=cached;_EVAL_STATE["total"]=len(pending);_EVAL_STATE["grade_total"]={g:len(ts) for g,ts in gt.items()};_EVAL_STATE["grade_done"]={g:0 for g in gt};_EVAL_STATE["done"]=0;_EVAL_STATE["errors"]=[];_EVAL_STATE["active"]=[] | |
| if pending: | |
| with ThreadPoolExecutor(max_workers=n_workers) as ex: | |
| futs={} | |
| for t in pending: | |
| if _EVAL_STATE["stop_requested"]:break | |
| futs[ex.submit(_eval_single,t,run_id,eval_api_key,eval_model_id,eval_provider,judge_api_key,judge_model_id,judge_provider,_EVAL_STATE)]=t | |
| done_set=set() | |
| while len(done_set)<len(futs): | |
| if _EVAL_STATE["stop_requested"]:ex.shutdown(wait=False,cancel_futures=True);break | |
| for f in list(futs): | |
| if f in done_set:continue | |
| if f.done(): | |
| done_set.add(f) | |
| try: | |
| tid,data=f.result() | |
| with _EVAL_STATE["lock"]:_EVAL_STATE["results"][tid]=data;to=futs[f];_EVAL_STATE["grade_done"][to.grade]=_EVAL_STATE["grade_done"].get(to.grade,0)+1 | |
| except:pass | |
| time.sleep(0.5) | |
| with _EVAL_STATE["lock"]:results=dict(_EVAL_STATE["results"]) | |
| final,base,har,axis,_=compute_final_score(results,tasks);stage=determine_agi_stage(final,axis) | |
| csv_str=generate_csv(results,tasks,eval_label,judge_label,"BASELINE");cp=f"/tmp/final_{run_id}.csv" | |
| with open(cp,"w",encoding="utf-8") as f:f.write(csv_str) | |
| elapsed=int(time.time()-_EVAL_STATE["start_time"]) | |
| with _EVAL_STATE["lock"]:_EVAL_STATE["csv_path"]=cp;_EVAL_STATE["hf_status"]="";_EVAL_STATE["message"]=f"🏁 {stage['name']} — FINAL={final:.1f} · {elapsed}s";_EVAL_STATE["running"]=False;_EVAL_STATE["finished"]=True | |
| except Exception as e: | |
| print(f" ❌ Fatal: {e}");import traceback;traceback.print_exc() | |
| with _EVAL_STATE["lock"]:_EVAL_STATE["message"]=f"❌ Fatal: {str(e)[:100]}";_EVAL_STATE["running"]=False;_EVAL_STATE["finished"]=True | |
| def _start_eval(eval_api_key,judge_api_key,eval_model_choice,judge_model_choice,grade_f,diff_f,max_t,n_w,fresh): | |
| global _EVAL_STATE | |
| if _EVAL_STATE["running"]:return "⚠️ Already running" | |
| eval_api_key=(eval_api_key or "").strip();judge_api_key=(judge_api_key or "").strip() | |
| eval_model_id,eval_provider=_resolve_model(eval_model_choice);judge_model_id,judge_provider=_resolve_model(judge_model_choice) | |
| if not eval_api_key:return f"❌ {eval_provider} API Key required for Eval model" | |
| if not judge_api_key:return f"❌ {judge_provider} API Key required for Judge model" | |
| tasks=ALL_TASKS[:] | |
| if grade_f!="All":tasks=[t for t in tasks if t.grade==grade_f] | |
| if diff_f!="All":tasks=[t for t in tasks if t.difficulty==diff_f] | |
| tasks=tasks[:int(max_t)];rid=_make_run_id(eval_model_id) | |
| if fresh:_clear_run(rid) | |
| _reset() | |
| with _EVAL_STATE["lock"]:_EVAL_STATE.update({"running":True,"run_id":rid,"eval_label":eval_model_choice,"judge_label":judge_model_choice,"tasks":tasks,"total":len(tasks),"n_workers":int(n_w)}) | |
| threading.Thread(target=_bg_eval,daemon=True,args=(eval_api_key,eval_model_id,eval_provider,eval_model_choice,judge_api_key,judge_model_id,judge_provider,judge_model_choice,tasks,rid,int(n_w))).start() | |
| return f"⚡ Started — Eval: {eval_model_choice} · Judge: {judge_model_choice} ({len(tasks)} tasks)" | |
| def _stop(): | |
| if _EVAL_STATE["running"]:_EVAL_STATE["stop_requested"]=True;return "⏹️ Stopping..." | |
| return "ℹ️ Not running" | |
| def _poll(): | |
| with _EVAL_STATE["lock"]:running=_EVAL_STATE["running"];finished=_EVAL_STATE["finished"];tasks=_EVAL_STATE.get("tasks",[]);results=dict(_EVAL_STATE.get("results",{}));msg=_EVAL_STATE.get("message","");cp=_EVAL_STATE.get("csv_path") | |
| if not running and not finished and not results:return("ℹ️ Configure API keys, select models, then press ▶️ Start","","","",None) | |
| if running:pend=_EVAL_STATE.get("total",0)-_EVAL_STATE.get("cached",0);ph=CSS+_prog_html(_EVAL_STATE,pend) | |
| elif finished:ph=f'<div style="background:#e8f5e9;padding:12px;border-radius:8px;font-weight:600">{msg}</div>' | |
| else:ph=msg | |
| th=_build_progress_table(results,tasks) if tasks else "";sh,dh,co="","",None | |
| if finished and tasks: | |
| el=_EVAL_STATE.get("eval_label","?");jl=_EVAL_STATE.get("judge_label","?");hf_st=_EVAL_STATE.get("hf_status","") | |
| sh=_build_summary_card(results,tasks,el,jl,hf_st);dh=_build_detail_view(results,tasks);co=cp | |
| return(ph,th,sh,dh,co) | |
| # ═══ §12. Gradio App ═══ | |
| HEADER="""<div style="text-align:center;padding:16px 0"> | |
| <h1 style="margin:0;font-size:1.8em">🏆 FINAL Bench v4.2 — Baseline Evaluation</h1> | |
| <h2 style="margin:4px 0;color:#555;font-size:1.05em">Frontier Intelligence Nexus for AGI-Level Verification</h2> | |
| <p style="color:#888;font-size:0.88em;max-width:720px;margin:8px auto"><b>100 Tasks · 15 Domains · 8 TICOS · 5-Axis · 5-Stage AGI Grade</b><br> | |
| 🤖 Baseline (Non-AGI) — Single LLM Evaluation · Multi-Provider<br>Both <b>Eval</b> and <b>Judge</b> support OpenAI / Anthropic / Google</p> | |
| <div style="display:flex;justify-content:center;gap:6px;margin-top:8px;flex-wrap:wrap;font-size:0.82em"> | |
| <span style="background:#e3f2fd;padding:2px 10px;border-radius:12px">OpenAI · GPT-5.2 / 5-Mini / 4.1 / o4-mini / 4o</span> | |
| <span style="background:#fce4ec;padding:2px 10px;border-radius:12px">Anthropic · Opus 4.6 / Sonnet 4.5 / Haiku 4.5</span> | |
| <span style="background:#e8f5e9;padding:2px 10px;border-radius:12px">Google · Gemini 3 Pro Preview</span></div> | |
| <div style="background:rgba(233,69,96,0.1);border:1px solid #e94560;border-radius:10px;padding:10px;margin:12px auto;max-width:600px"> | |
| <p style="color:#e94560;font-size:0.85em;margin:0">🔒 <b>MetaCog (Self-Correction Protocol): COMING SOON</b></p></div> | |
| <div style="display:flex;justify-content:center;gap:8px;margin-top:8px;font-size:0.78em"> | |
| <a href="https://huggingface.co/datasets/FINAL-Bench/Metacognitive" target="_blank" style="background:#333;color:#fff;padding:3px 10px;border-radius:10px;text-decoration:none">📊 Dataset</a> | |
| <a href="https://huggingface.co/spaces/FINAL-Bench/Leaderboard" target="_blank" style="background:#333;color:#fff;padding:3px 10px;border-radius:10px;text-decoration:none">🏆 Leaderboard</a></div></div>""" | |
| def create_app(): | |
| with gr.Blocks(title="FINAL Bench v4.2",css=".gradio-container{max-width:1100px !important} header{display:none!important}") as app: | |
| gr.HTML(HEADER) | |
| gr.Markdown("### 🔑 API Keys") | |
| with gr.Row(): | |
| eval_api_key=gr.Textbox(label="🤖 Eval Model API Key",type="password",placeholder="sk-... / sk-ant-... / AIza...",info="OpenAI / Anthropic / Google key",scale=3) | |
| judge_api_key=gr.Textbox(label="⚖️ Judge Model API Key",type="password",placeholder="sk-... / sk-ant-... / AIza...",info="OpenAI / Anthropic / Google key",scale=3) | |
| gr.Markdown("### 🤖 Model Selection") | |
| with gr.Row(): | |
| eval_m=gr.Dropdown(label="🤖 Evaluation Target",choices=MODEL_CHOICES,value=DEFAULT_EVAL,scale=3) | |
| judge_m=gr.Dropdown(label="⚖️ Judge Model",choices=MODEL_CHOICES,value=DEFAULT_JUDGE,scale=3) | |
| gr.Markdown("### ⚙️ Settings") | |
| with gr.Row(): | |
| gf=gr.Dropdown(["All","A","B","C"],value="All",label="Grade Filter",scale=1) | |
| df=gr.Dropdown(["All","expert","frontier"],value="All",label="Difficulty",scale=1) | |
| mt=gr.Slider(1,100,value=100,step=1,label="Max Tasks",scale=1) | |
| nw=gr.Slider(1,10,value=5,step=1,label="Workers",scale=1) | |
| with gr.Row(): | |
| s_btn=gr.Button("▶️ Start (Resume)",variant="primary",size="lg",scale=2) | |
| f_btn=gr.Button("🚀 Fresh Start",variant="secondary",size="lg",scale=2) | |
| x_btn=gr.Button("⏹️ Stop",variant="stop",size="lg",scale=1) | |
| status=gr.Textbox(label="Status",interactive=False,max_lines=2) | |
| with gr.Tabs(): | |
| with gr.Tab("📊 Progress"):p_html=gr.HTML() | |
| with gr.Tab("📋 Results"):t_html=gr.HTML() | |
| with gr.Tab("🏆 FINAL Score"):s_html=gr.HTML() | |
| with gr.Tab("🔍 Details"):d_html=gr.HTML() | |
| with gr.Tab("💾 CSV"):c_file=gr.File(label="CSV") | |
| timer=gr.Timer(value=2,active=True) | |
| timer.tick(fn=_poll,outputs=[p_html,t_html,s_html,d_html,c_file]) | |
| eval_ins=[eval_api_key,judge_api_key,eval_m,judge_m,gf,df,mt,nw] | |
| s_btn.click(fn=lambda *a:_start_eval(*a,fresh=False),inputs=eval_ins,outputs=[status]) | |
| f_btn.click(fn=lambda *a:_start_eval(*a,fresh=True),inputs=eval_ins,outputs=[status]) | |
| x_btn.click(fn=_stop,outputs=[status]) | |
| gr.Markdown("---\n<center><b>FINAL Bench v4.2</b> · Baseline · OpenAI / Anthropic / Google · Apache 2.0 · <b>Ginigen AI</b></center>") | |
| return app | |
| if __name__=="__main__": | |
| sg,sd={},{} | |
| for t in ALL_TASKS:sg[t.grade]=sg.get(t.grade,0)+1;sd[t.domain]=sd.get(t.domain,0)+1 | |
| print(f"\n{'='*60}\n FINAL Bench v4.2 — Baseline (Non-AGI)\n Eval & Judge: OpenAI / Anthropic / Google\n{'='*60}") | |
| print(f" {len(ALL_TASKS)} tasks | {len(sd)} domains") | |
| for g in["A","B","C"]:print(f" Grade {g} (×{GRADE_WEIGHT[g]}): {sg.get(g,0)}") | |
| print(f" 🔒 MetaCog: COMING SOON\n{'='*60}\n") | |
| app=create_app();app.queue(default_concurrency_limit=2) | |
| app.launch(server_name="0.0.0.0",server_port=7860,ssr_mode=False) | |