inference.py

"""
Baseline inference agent for Voice Authenticity OpenEnv v2.

Uses the 5-action protocol:
  1. request_temporal_features  → get jitter, shimmer, HNR
  2. request_spectral_features  → get MFCC values
  3. request_comparison         → get similarity to real/fake centroids
  4. analyze_evidence           → synthesize gathered information
  5. final_classify             → submit label + confidence + reasoning
"""

from dotenv import load_dotenv
load_dotenv()

import asyncio
import os
import textwrap
import json
import requests
from typing import List
from openai import OpenAI

API_BASE_URL = os.getenv("API_BASE_URL", "https://router.huggingface.co/v1")
MODEL_NAME = os.getenv("MODEL_NAME", "Qwen/Qwen2.5-72B-Instruct")
HF_TOKEN = os.getenv("HF_TOKEN")

if HF_TOKEN is None:
    raise ValueError("HF_TOKEN environment variable is required")

BENCHMARK = "voice-authenticity"
SUCCESS_SCORE_THRESHOLD = 0.60


def _clamp_score(value: float) -> float:
    """Ensure score is strictly in (0, 1) — never 0.0 or 1.0."""
    return max(0.05, min(0.95, value))

# Environment server URL
ENV_SERVER_URL = os.getenv("ENV_SERVER_URL", "http://localhost:7860")

SYSTEM_PROMPT = textwrap.dedent("""
You are an expert audio forensics agent detecting synthetic (AI-generated) speech.
You operate in a multi-step environment where you must gather evidence before classifying.

REAL speech indicators:
- jitter > 0.025 (vocal cord irregularity)
- shimmer > 0.10 (amplitude variation)
- HNR < 12.0 (more noise in signal)
- Higher MFCC std deviations (natural variation)

SYNTHETIC speech indicators:
- jitter < 0.020 (too stable)
- shimmer < 0.09 (too uniform)
- HNR > 12.0 (too clean)
- Lower MFCC std deviations (artificial consistency)

COMPARISON interpretation:
- Higher cosine similarity to real centroid → likely real
- Higher cosine similarity to fake centroid → likely synthetic
- Closer euclidean distance to real → likely real

CONFIDENCE GUIDELINES:
- Easy tasks: confident predictions okay (0.7-0.9)
- Medium tasks: moderate confidence (0.6-0.8)
- Hard/extreme tasks: calibrate carefully, never exceed 0.85

Respond ONLY with valid JSON for the requested action type.
""").strip()


def log_start(task, env, model):
    print(f"[START] task={task} env={env} model={model}", flush=True)


def log_step(step, action, reward, done, error):
    error_val = error if error else "null"
    if isinstance(action, dict):
        action_str = action.get("action_type", "unknown")
    else:
        action_str = str(action)
    print(
        f"[STEP] step={step} action={action_str} "
        f"reward={reward:.2f} done={str(done).lower()} error={error_val}",
        flush=True,
    )


def log_end(success, steps, score, rewards):
    if not rewards:
        rewards = [0.05]
    rewards_str = ",".join(f"{r:.2f}" for r in rewards)
    score = max(0.05, min(0.95, float(score)))
    print(
        f"[END] success={str(success).lower()} steps={steps} "
        f"score={score:.3f} rewards={rewards_str}",
        flush=True,
    )


# ── Environment API calls ──────────────────────────────────────────────

def env_reset(task_name: str) -> dict:
    """Call /reset on the environment server."""
    response = requests.post(
        f"{ENV_SERVER_URL}/reset",
        json={"task_name": task_name, "seed": 7},
        timeout=30,
    )
    response.raise_for_status()
    return response.json()


def env_step(action: dict, task_name: str) -> dict:
    """Call /step on the environment server."""
    payload = {
        "action_type": action.get("action_type", "final_classify"),
        "label": action.get("label", 0),
        "confidence": action.get("confidence", 0.5),
        "reasoning": action.get("reasoning", ""),
        "task_name": task_name,
    }
    response = requests.post(
        f"{ENV_SERVER_URL}/step",
        json=payload,
        timeout=30,
    )
    response.raise_for_status()
    return response.json()


# ── LLM agent decision making ──────────────────────────────────────────

def get_classification(client, context: dict) -> dict:
    """Ask the LLM to make a final classification based on accumulated evidence."""
    user_prompt = f"""
Based on the following evidence gathered from an audio sample, classify it as
real (0) or synthetic (1). Task: {context['task_name']} (difficulty: {context['difficulty']})

EVIDENCE GATHERED:
{json.dumps(context.get('visible_features', {}), indent=2)}

COMPARISON RESULTS:
{json.dumps(context.get('comparison_result', {}), indent=2)}

ANALYSIS SUMMARY:
{context.get('evidence_summary', 'No analysis performed.')}

ACTIONS TAKEN: {', '.join(context.get('actions_taken', []))}

Respond with JSON only:
{{"label": 0 or 1, "confidence": 0.0-1.0, "reasoning": "brief explanation under 70 words"}}
"""
    try:
        completion = client.chat.completions.create(
            model=MODEL_NAME,
            messages=[
                {"role": "system", "content": SYSTEM_PROMPT},
                {"role": "user", "content": user_prompt.strip()},
            ],
            temperature=0.3,
            max_tokens=300,
            stream=False,
        )
        text = completion.choices[0].message.content.strip()
        text = text.replace("```json", "").replace("```", "").strip()
        last_brace = text.rfind("}")
        if last_brace != -1:
            text = text[: last_brace + 1]
        result = json.loads(text)
        result["label"] = int(result.get("label", 0))
        result["confidence"] = float(result.get("confidence", 0.5))
        result["label"] = result["label"] if result["label"] in [0, 1] else 0
        result["confidence"] = max(0.0, min(1.0, result["confidence"]))
        return result
    except Exception as e:
        print(f"[DEBUG] Model error: {e}", flush=True)
        return {"label": 0, "confidence": 0.5, "reasoning": "fallback"}


# ── Main task runner ────────────────────────────────────────────────────

async def run_task(client: OpenAI, task_name: str):
    """Run one episode of a task using the 5-action protocol."""
    rewards: List[float] = []
    steps_taken = 0
    success = False
    score = 0.05  # Default: strictly > 0 in case of error
    context = {}
    grader_breakdown = None

    log_start(task=task_name, env=BENCHMARK, model=MODEL_NAME)

    try:
        # ── Reset ───────────────────────────────────────────
        reset_response = env_reset(task_name)
        observation = reset_response.get("observation", {})
        context = {
            "task_name": observation.get("task_name", task_name),
            "difficulty": observation.get("difficulty", ""),
            "visible_features": {},
            "comparison_result": None,
            "evidence_summary": None,
            "actions_taken": [],
        }

        # ── Step 1: Request temporal features ───────────────
        action1 = {"action_type": "request_temporal_features"}
        step1 = env_step(action1, task_name)
        observation = step1.get("observation", {})
        reward1 = _clamp_score(float(step1.get("reward", 0.05)))
        rewards.append(reward1)
        steps_taken = 1
        context["visible_features"] = observation.get("visible_features", {})
        context["actions_taken"] = observation.get("actions_taken", [])

        log_step(step=1, action=action1, reward=reward1,
                 done=step1.get("done", False), error=None)

        if step1.get("done", False):
            raise RuntimeError("Episode ended prematurely at step 1")

        # ── Step 2: Request spectral features ───────────────
        action2 = {"action_type": "request_spectral_features"}
        step2 = env_step(action2, task_name)
        observation = step2.get("observation", {})
        reward2 = _clamp_score(float(step2.get("reward", 0.05)))
        rewards.append(reward2)
        steps_taken = 2
        context["visible_features"] = observation.get("visible_features", {})
        context["actions_taken"] = observation.get("actions_taken", [])

        log_step(step=2, action=action2, reward=reward2,
                 done=step2.get("done", False), error=None)

        if step2.get("done", False):
            raise RuntimeError("Episode ended prematurely at step 2")

        # ── Step 3: Request comparison ──────────────────────
        action3 = {"action_type": "request_comparison"}
        step3 = env_step(action3, task_name)
        observation = step3.get("observation", {})
        reward3 = _clamp_score(float(step3.get("reward", 0.05)))
        rewards.append(reward3)
        steps_taken = 3
        context["visible_features"] = observation.get("visible_features", {})
        context["comparison_result"] = observation.get("comparison_result", {})
        context["actions_taken"] = observation.get("actions_taken", [])

        log_step(step=3, action=action3, reward=reward3,
                 done=step3.get("done", False), error=None)

        if step3.get("done", False):
            raise RuntimeError("Episode ended prematurely at step 3")

        # ── Step 4: Analyze evidence ────────────────────────
        action4 = {"action_type": "analyze_evidence"}
        step4 = env_step(action4, task_name)
        observation = step4.get("observation", {})
        reward4 = _clamp_score(float(step4.get("reward", 0.05)))
        rewards.append(reward4)
        steps_taken = 4
        context["evidence_summary"] = observation.get("evidence_summary", "")
        context["actions_taken"] = observation.get("actions_taken", [])

        log_step(step=4, action=action4, reward=reward4,
                 done=step4.get("done", False), error=None)

        if step4.get("done", False):
            raise RuntimeError("Episode ended prematurely at step 4")

        # ── Step 5: Final classify (LLM decision) ──────────
        classification = get_classification(client, context)
        action5 = {
            "action_type": "final_classify",
            "label": classification["label"],
            "confidence": classification["confidence"],
            "reasoning": classification.get("reasoning", ""),
        }

        step5 = env_step(action5, task_name)
        reward5 = _clamp_score(float(step5.get("reward", 0.05)))
        rewards.append(reward5)
        steps_taken = 5
        done = step5.get("done", True)

        # Capture grader breakdown from environment info
        step5_info = step5.get("info", {})
        grader_breakdown = step5_info.get("grader_breakdown", None)

        log_step(step=5, action=action5, reward=reward5,
                 done=done, error=None)

        # Score is the final classify reward (main grader score)
        score = reward5
        success = score >= SUCCESS_SCORE_THRESHOLD

    except Exception as e:
        print(f"[DEBUG] Task error: {e}", flush=True)

    finally:
        log_end(
            success=success,
            steps=steps_taken,
            score=score,
            rewards=rewards,
        )


async def run_realtime_task(client: OpenAI, task_name: str):
    """Run one episode of realtime_detection.

    Strategy: gather 2 features (temporal + spectral) then classify
    immediately to minimize the time penalty (-0.03 per extra step).
    The agent only takes 3 steps total: 2 gathering + 1 classify.
    """
    rewards: List[float] = []
    steps_taken = 0
    success = False
    score = 0.05
    context = {}

    log_start(task=task_name, env=BENCHMARK, model=MODEL_NAME)

    try:
        # Reset
        reset_response = env_reset(task_name)
        observation = reset_response.get("observation", {})
        context = {
            "task_name": observation.get("task_name", task_name),
            "difficulty": observation.get("difficulty", ""),
            "visible_features": {},
            "comparison_result": None,
            "evidence_summary": None,
            "actions_taken": [],
        }

        # Step 1: Request temporal features
        action1 = {"action_type": "request_temporal_features"}
        step1 = env_step(action1, task_name)
        observation = step1.get("observation", {})
        reward1 = _clamp_score(float(step1.get("reward", 0.05)))
        rewards.append(reward1)
        steps_taken = 1
        context["visible_features"] = observation.get("visible_features", {})
        context["actions_taken"] = observation.get("actions_taken", [])
        log_step(step=1, action=action1, reward=reward1,
                 done=step1.get("done", False), error=None)

        # Step 2: Request spectral features
        action2 = {"action_type": "request_spectral_features"}
        step2 = env_step(action2, task_name)
        observation = step2.get("observation", {})
        reward2 = _clamp_score(float(step2.get("reward", 0.05)))
        rewards.append(reward2)
        steps_taken = 2
        context["visible_features"] = observation.get("visible_features", {})
        context["actions_taken"] = observation.get("actions_taken", [])
        log_step(step=2, action=action2, reward=reward2,
                 done=step2.get("done", False), error=None)

        # Step 3: Classify immediately (no extra steps = no time penalty)
        classification = get_classification(client, context)
        action3 = {
            "action_type": "final_classify",
            "label": classification["label"],
            "confidence": classification["confidence"],
            "reasoning": classification.get("reasoning", ""),
        }
        step3 = env_step(action3, task_name)
        reward3 = _clamp_score(float(step3.get("reward", 0.05)))
        rewards.append(reward3)
        steps_taken = 3
        log_step(step=3, action=action3, reward=reward3,
                 done=step3.get("done", True), error=None)

        score = reward3
        success = score >= SUCCESS_SCORE_THRESHOLD

    except Exception as e:
        print(f"[DEBUG] Task error: {e}", flush=True)

    finally:
        log_end(
            success=success,
            steps=steps_taken,
            score=score,
            rewards=rewards,
        )
async def main():
    client = OpenAI(base_url=API_BASE_URL, api_key=HF_TOKEN)
    tasks = [
        "clean_detection",
        "compressed_detection",
        "adversarial_detection",
        "streaming_detection",
        "phonecall_detection",
        "realtime_detection",
    ]
    for task in tasks:
        if task == "realtime_detection":
            await run_realtime_task(client, task)
        else:
            await run_task(client, task)


if __name__ == "__main__":
    asyncio.run(main())