app.py

"""
Python Code Evaluator — SE-Group1
COS60011 Technology Design Project

Implements the 5-module architecture from the design document:
  1. UI Module            — Gradio web interface
  2. Validation & Flow    — Input validation and routing
  3. Pre-processing       — 6-Element structured prompt builder
  4. Generation           — Gemma-4 LLM via Hugging Face
  5. Output               — Parse raw LLM output into structured results
"""

import re
import torch
import gradio as gr
from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline

# ---------------------------------------------------------------------------
# MODULE 4 — Generation Module (LLM setup)
# ---------------------------------------------------------------------------
MODEL_ID = "google/gemma-4-1b-it"   # swap for "google/gemma-4-9b-it" if VRAM allows

print(f"[Generation] Loading model: {MODEL_ID}")
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID,
    torch_dtype=torch.bfloat16,          # efficient on modern GPUs/CPUs
    device_map="auto",                    # auto-detect GPU/CPU
)
pipe = pipeline(
    "text-generation",
    model=model,
    tokenizer=tokenizer,
    max_new_tokens=512,
    do_sample=False,                      # deterministic output
)
print("[Generation] Model ready.")


# ---------------------------------------------------------------------------
# MODULE 3 — Pre-processing Module (6-Element Framework)
# ---------------------------------------------------------------------------
def build_prompt(description: str, code: str) -> str:
    """
    Constructs a structured prompt using the 6-Element Framework:
      1. Role       — Who the LLM is
      2. Context    — Background information
      3. Input Data — The user's description and code
      4. Task       — What the LLM must do
      5. Constraints— Boundaries for the response
      6. Output     — Expected format
    """
    prompt = f"""<start_of_turn>user
### ROLE
You are an expert Python code reviewer. Your sole task is to determine whether the
provided Python code correctly implements the behaviour described in the user's
requirements description.

### CONTEXT
Developers sometimes write code that does not fully satisfy the requirements they
were given. You will analyse the semantic relationship between a natural-language
description and a Python code snippet, then produce a structured evaluation report.

### INPUT DATA
**Requirements Description:**
{description.strip()}

**Python Code:**
```python
{code.strip()}
```

### TASK
1. Read the requirements description carefully.
2. Analyse the Python code line by line.
3. Determine whether the code fulfils ALL requirements stated in the description.
4. Estimate an accuracy percentage (0–100) reflecting how completely the code
   matches the description.
5. List any specific requirements that are missing or incorrectly implemented.

### CONSTRAINTS
- Do NOT execute the code.
- Base your evaluation solely on static code analysis and logical reasoning.
- Keep feedback concise, clear, and actionable.
- Your response MUST follow the output format exactly.

### OUTPUT FORMAT
Respond only with the following structure — no extra text before or after:

RESULT: <PASS or FAIL>
ACCURACY: <integer 0-100>%
SUMMARY: <one sentence overall assessment>
ISSUES:
- <issue 1, or "None" if code fully matches the description>
- <issue 2>
...
<end_of_turn>
<start_of_turn>model
"""
    return prompt


# ---------------------------------------------------------------------------
# MODULE 5 — Output Module
# ---------------------------------------------------------------------------
def parse_output(raw: str) -> dict:
    """
    Extracts structured fields from the LLM's raw text response.
    Returns a dict with keys: result, accuracy, summary, issues.
    Falls back gracefully if parsing fails.
    """
    # Strip any echoed prompt (model sometimes repeats <start_of_turn>)
    if "<start_of_turn>model" in raw:
        raw = raw.split("<start_of_turn>model")[-1]

    result_match   = re.search(r"RESULT:\s*(PASS|FAIL)", raw, re.IGNORECASE)
    accuracy_match = re.search(r"ACCURACY:\s*(\d{1,3})%?", raw, re.IGNORECASE)
    summary_match  = re.search(r"SUMMARY:\s*(.+)", raw, re.IGNORECASE)
    issues_match   = re.search(r"ISSUES:\s*([\s\S]+)", raw, re.IGNORECASE)

    result   = result_match.group(1).upper()  if result_match   else "UNKNOWN"
    accuracy = int(accuracy_match.group(1))   if accuracy_match else -1
    summary  = summary_match.group(1).strip() if summary_match  else "Could not extract summary."

    if issues_match:
        raw_issues = issues_match.group(1).strip()
        issues = [
            line.lstrip("-•* ").strip()
            for line in raw_issues.splitlines()
            if line.strip() and line.strip() not in ("-", "•")
        ]
    else:
        issues = ["Could not extract issues from model response."]

    return {
        "result":   result,
        "accuracy": accuracy,
        "summary":  summary,
        "issues":   issues,
        "raw":      raw.strip(),
    }


def format_for_display(parsed: dict) -> tuple[str, str, str]:
    """
    Converts the parsed dict into three Gradio-friendly strings:
      - verdict     (shown in a highlighted Textbox)
      - metrics     (accuracy + summary)
      - issues_text (bullet list)
    """
    emoji  = "✅" if parsed["result"] == "PASS" else ("❌" if parsed["result"] == "FAIL" else "⚠️")
    verdict = f"{emoji}  {parsed['result']}"

    acc_str = f"{parsed['accuracy']}%" if parsed["accuracy"] >= 0 else "N/A"
    metrics = f"Accuracy: {acc_str}\n\nSummary: {parsed['summary']}"

    issues_text = "\n".join(f"• {issue}" for issue in parsed["issues"])
    return verdict, metrics, issues_text


# ---------------------------------------------------------------------------
# MODULE 4 — Generation Module (inference call)
# ---------------------------------------------------------------------------
def generate(prompt: str) -> str:
    outputs = pipe(prompt, return_full_text=False)
    return outputs[0]["generated_text"]


# ---------------------------------------------------------------------------
# MODULE 2 — Validation & Flow Management Module
# ---------------------------------------------------------------------------
def validate_and_evaluate(description: str, code: str):
    """
    Entry point called by the UI module.
    Returns (verdict, metrics, issues, error_message).
    """
    # --- Validation ---
    if not description or not description.strip():
        return "", "", "", "⚠️ Please provide a requirements description."
    if not code or not code.strip():
        return "", "", "", "⚠️ Please provide Python code to evaluate."

    # --- Pre-processing ---
    prompt = build_prompt(description, code)

    # --- Generation ---
    try:
        raw_output = generate(prompt)
    except Exception as exc:
        return "", "", "", f"❌ Model error: {exc}"

    # --- Output ---
    parsed  = parse_output(raw_output)
    verdict, metrics, issues = format_for_display(parsed)

    return verdict, metrics, issues, ""   # empty error = success


# ---------------------------------------------------------------------------
# MODULE 1 — UI Module (Gradio)
# ---------------------------------------------------------------------------
with gr.Blocks(
    title="Python Code Evaluator — SE-Group1",
    theme=gr.themes.Soft(primary_hue="blue"),
) as demo:

    gr.Markdown(
        """
        # 🐍 Python Code Evaluator
        **COS60011 — SE-Group1** | Powered by Gemma-4 via Hugging Face

        Enter a **requirements description** and your **Python code**.
        The system will evaluate whether the code correctly implements the described behaviour.
        """
    )

    with gr.Row():
        with gr.Column(scale=1):
            description_input = gr.Textbox(
                label="📋 Requirements Description",
                placeholder="Describe what the Python code should do...",
                lines=8,
            )
            code_input = gr.Code(
                label="🐍 Python Code",
                language="python",
                lines=15,
                value='def add(a, b):\n    return a + b\n',
            )
            submit_btn = gr.Button("▶  Evaluate", variant="primary", size="lg")

        with gr.Column(scale=1):
            error_output  = gr.Textbox(label="⚠️ Validation Error", visible=True, interactive=False)
            verdict_output = gr.Textbox(label="🏁 Verdict",  interactive=False, lines=1)
            metrics_output = gr.Textbox(label="📊 Metrics & Summary", interactive=False, lines=4)
            issues_output  = gr.Textbox(label="🔍 Issues Found", interactive=False, lines=8)

    submit_btn.click(
        fn=validate_and_evaluate,
        inputs=[description_input, code_input],
        outputs=[verdict_output, metrics_output, issues_output, error_output],
    )

    gr.Markdown(
        """
        ---
        > **Note:** This tool performs *static analysis* only — it does not execute the code.
        > Results should be treated as supplementary feedback, not a replacement for unit testing.
        """
    )

if __name__ == "__main__":
    demo.launch(share=False)