app.py

"""
LLM-Generated Text Detector
University Project - AI Text Detection System
Uses ensemble of: RoBERTa classifier + GPT-2 Perplexity + Burstiness scoring
Hosted on Hugging Face Spaces — CPU compatible, no GPU required, no API keys
"""

import gradio as gr
import torch
import math
import re
import json
import numpy as np
from transformers import (
    AutoTokenizer,
    AutoModelForSequenceClassification,
    GPT2LMHeadModel,
    GPT2TokenizerFast,
)

# Optional: language detection (graceful fallback if package missing)
try:
    from langdetect import detect as _langdetect, LangDetectException
    LANGDETECT_AVAILABLE = True
except ImportError:
    LANGDETECT_AVAILABLE = False

# ── Constants ───────────────────────────────────────────────────────────────
MAX_ROBERTA_TOKENS = 512   # hard token limit for RoBERTa
CHUNK_SIZE         = 480   # tokens per chunk (headroom for special tokens)
CHUNK_OVERLAP      = 50    # overlap between consecutive chunks

# ── Static UI content (defined once, not regenerated per request) ────────────
HOW_IT_WORKS = """### How this system works

This detector uses an **ensemble of three independent methods**:

| Method | What it measures | Weight |
|---|---|---|
| RoBERTa classifier | Learned patterns from millions of AI/human text pairs | 60% |
| GPT-2 perplexity | How predictable the text is to a language model | 25% |
| Burstiness analysis | Variation in sentence length across the text | 15% |

The three scores are combined using a weighted average to produce the final AI probability.
RoBERTa is given the highest weight because it was specifically trained for this task
and consistently outperforms statistical methods in peer-reviewed benchmarks.

**Confidence** is calculated from the agreement between the three methods.
High confidence means all three signals agree. Low confidence means they diverge —
the verdict should be treated with more caution in that case.

**Long text handling:** texts longer than ~380 words (~480 tokens) are automatically
split into overlapping chunks, each chunk is scored independently, and the scores are
averaged. This ensures the full document is analysed rather than just the first 380 words.

**Language:** all three models are optimised for English. A warning is shown if
non-English input is detected.

**References:**
- Guo et al. (2023). *How Close is ChatGPT to Human Experts?* (HC3 dataset, RoBERTa)
- Mitchell et al. (2023). *DetectGPT: Zero-Shot Machine-Generated Text Detection*
- Hans et al. (2024). *Spotting LLMs With Binoculars* (ICML 2024)
- Bao et al. (2024). *Fast-DetectGPT* (ICLR 2024)
"""

# ── Model loading (cached after first run) ──────────────────────────────────

ROBERTA_MODEL = "Hello-SimpleAI/chatgpt-detector-roberta"
GPT2_MODEL    = "gpt2"

print("Loading models… (cached automatically by Hugging Face after first run)")

roberta_tokenizer = AutoTokenizer.from_pretrained(ROBERTA_MODEL)
roberta_model     = AutoModelForSequenceClassification.from_pretrained(ROBERTA_MODEL)
roberta_model.eval()

gpt2_tokenizer = GPT2TokenizerFast.from_pretrained(GPT2_MODEL)
gpt2_tokenizer.pad_token = gpt2_tokenizer.eos_token
gpt2_lm        = GPT2LMHeadModel.from_pretrained(GPT2_MODEL)
gpt2_lm.eval()

# ── BUG FIX: Resolve the correct AI-label index at startup ──────────────────
# The original code checked for ("fake","ai","generated","1") but this model's
# label is "ChatGPT" — so ai_idx always fell to 0 (Human), inverting the score.
# Now we build a reverse-lookup from the model's actual id2label config.
_label_lookup = {v.lower(): k for k, v in roberta_model.config.id2label.items()}
AI_LABEL_IDX = (
    _label_lookup.get("chatgpt")
    or _label_lookup.get("fake")
    or _label_lookup.get("ai")
    or _label_lookup.get("generated")
    or 1  # safe default
)
print(f"Models loaded ✓  |  RoBERTa AI label → [{AI_LABEL_IDX}] "
      f"'{roberta_model.config.id2label[AI_LABEL_IDX]}'")

# ── Detection functions ─────────────────────────────────────────────────────

def _roberta_score_single(text: str) -> float:
    """Score a single text chunk (must be within the token limit)."""
    inputs = roberta_tokenizer(
        text, return_tensors="pt", truncation=True, max_length=MAX_ROBERTA_TOKENS
    )
    with torch.no_grad():
        logits = roberta_model(**inputs).logits
    probs = torch.softmax(logits, dim=-1)
    return float(probs[0][AI_LABEL_IDX].item())


def roberta_score(text: str) -> tuple:
    """
    RoBERTa fine-tuned classifier — returns (P(AI-generated), was_chunked).

    Texts longer than CHUNK_SIZE tokens are automatically split into overlapping
    windows. Each window is scored independently and the mean is returned.
    This prevents the silent truncation that occurred before.
    """
    token_ids = roberta_tokenizer.encode(text, add_special_tokens=False)

    if len(token_ids) <= CHUNK_SIZE:
        return _roberta_score_single(text), False

    # Build overlapping chunks from raw token IDs, decode each back to text
    chunk_scores = []
    start = 0
    while start < len(token_ids):
        end   = min(start + CHUNK_SIZE, len(token_ids))
        chunk = roberta_tokenizer.decode(token_ids[start:end], skip_special_tokens=True)
        chunk_scores.append(_roberta_score_single(chunk))
        if end == len(token_ids):
            break
        start += CHUNK_SIZE - CHUNK_OVERLAP

    return float(np.mean(chunk_scores)), True


def gpt2_perplexity(text: str) -> float:
    """
    GPT-2 perplexity — LLM text tends to be LOW perplexity
    (the model finds it very predictable).
    Returns perplexity value; lower → more likely AI-generated.
    """
    encodings = gpt2_tokenizer(text, return_tensors="pt", truncation=True, max_length=1024)
    input_ids = encodings.input_ids
    if input_ids.shape[1] < 5:
        return 9999.0  # too short to score reliably
    with torch.no_grad():
        outputs = gpt2_lm(input_ids, labels=input_ids)
    loss = outputs.loss.item()
    return math.exp(loss)


def burstiness_score(text: str) -> float:
    """
    Sentence-length burstiness — humans vary sentence length more.
    AI text is uniformly structured → low burstiness.
    Returns 0-1 where LOWER = more AI-like.
    """
    sentences = re.split(r'[.!?]+', text)
    lengths = [len(s.split()) for s in sentences if s.strip()]
    if len(lengths) < 3:
        return 0.5  # can't determine from too few sentences
    mean_len = np.mean(lengths)
    std_len  = np.std(lengths)
    if mean_len == 0:
        return 0.5
    cv = std_len / mean_len  # coefficient of variation
    # Humans typically CV > 0.5, AI typically CV < 0.35
    burstiness = min(cv / 0.8, 1.0)
    return float(burstiness)


def detect_language_safe(text: str):
    """Returns ISO 639-1 language code or None on any failure."""
    if not LANGDETECT_AVAILABLE:
        return None
    try:
        return _langdetect(text)
    except Exception:
        return None


def compute_confidence(scores: list) -> tuple:
    """
    Confidence = 1 − (normalised std of the three component scores).
    std=0 → perfect agreement → high confidence.
    std=0.5 → maximum spread → low confidence.
    Returns (confidence_float, label_string).
    """
    std = float(np.std(scores))
    confidence = max(0.0, min(1.0, 1.0 - (std / 0.5)))
    if confidence >= 0.75:
        label = "High confidence"
    elif confidence >= 0.50:
        label = "Moderate confidence"
    else:
        label = "Low confidence — signals disagree"
    return round(confidence, 3), label


def ensemble_predict(text: str):
    """
    Combine all three signals into a final verdict.
    Returns: (ai_probability, verdict, confidence, details_dict)
    """
    text = text.strip()
    word_count = len(text.split())
    if word_count < 20:
        return None, "⚠️ Too short", None, {
            "error": "Please enter at least 20 words for reliable detection."
        }

    # --- Language check ---
    lang = detect_language_safe(text)
    lang_warning = None
    if lang and lang != "en":
        lang_warning = (
            f"⚠️ Detected language: **{lang}** — "
            "all models are optimised for English; results may be unreliable."
        )

    # --- Individual scores ---
    rb_score, was_chunked = roberta_score(text)
    perplexity            = gpt2_perplexity(text)
    burst                 = burstiness_score(text)

    # Normalise perplexity to 0-1 AI probability using log-scale
    # (perplexity is log-normally distributed; linear mapping is poorly calibrated)
    # PPL ~20 → ~1.0 (very AI-like)   PPL ~300 → ~0.0 (very human-like)
    _log_ppl     = math.log(max(perplexity, 1.0))
    _log_min     = math.log(20)
    _log_max     = math.log(300)
    ppl_ai_prob  = max(0.0, min(1.0, 1.0 - (_log_ppl - _log_min) / (_log_max - _log_min)))

    # Burstiness: low burstiness → higher AI probability
    burst_ai_prob = max(0.0, min(1.0, 1.0 - burst))

    # Weighted ensemble
    weights    = [0.60, 0.25, 0.15]
    scores     = [rb_score, ppl_ai_prob, burst_ai_prob]
    final_prob = round(sum(w * s for w, s in zip(weights, scores)), 4)

    # Confidence from signal agreement
    confidence, conf_label = compute_confidence(scores)

    # Verdict
    if final_prob >= 0.75:
        verdict = "🤖 Almost certainly AI-generated"
    elif final_prob >= 0.55:
        verdict = "⚠️ Likely AI-generated"
    elif final_prob >= 0.40:
        verdict = "🔍 Uncertain — mixed signals"
    elif final_prob >= 0.25:
        verdict = "📝 Likely human-written"
    else:
        verdict = "✅ Almost certainly human-written"

    details = {
        "word_count":          word_count,
        "was_chunked":         was_chunked,
        "language":            lang or "unknown",
        "lang_warning":        lang_warning,
        "roberta_ai_prob":     round(rb_score,      4),
        "perplexity":          round(perplexity,     2),
        "perplexity_ai_prob":  round(ppl_ai_prob,    4),
        "burstiness":          round(burst,          4),
        "burstiness_ai_prob":  round(burst_ai_prob,  4),
        "ensemble_ai_prob":    final_prob,
        "confidence":          confidence,
        "confidence_label":    conf_label,
    }
    return final_prob, verdict, confidence, details


# ── Gradio UI ───────────────────────────────────────────────────────────────

def format_bar(prob: float) -> str:
    filled = int(prob * 20)
    empty  = 20 - filled
    return "█" * filled + "░" * empty


def detect(text):
    if not text or not text.strip():
        return (
            "Please paste some text above.",
            "",
            "",
            "",
            "",
            "",
        )

    final_prob, verdict, d = ensemble_predict(text)

    if "error" in d:
        return d["error"], "", "", "", "", ""

    human_prob = round(1.0 - final_prob, 4)

    # Main result card
    pct = int(final_prob * 100)
    bar = format_bar(final_prob)

    summary = f"""## {verdict}

**AI Probability:    {pct}%**  `{bar}`
**Human Probability: {int(human_prob*100)}%**  `{format_bar(human_prob)}`
"""

    # Classifier detail
    rb_pct  = int(d["roberta_ai_prob"] * 100)
    rb_bar  = format_bar(d["roberta_ai_prob"])
    rb_info = f"""### 🧠 RoBERTa Classifier  *(weight: 60%)*
Fine-tuned on millions of human vs. AI text pairs. Most reliable signal.

**AI score: {rb_pct}%**  `{rb_bar}`

> RoBERTa is a transformer model trained specifically to distinguish human and
> AI-generated text. It was fine-tuned on the HC3 dataset covering ChatGPT,
> GPT-3, and GPT-4 outputs across multiple domains.
"""

    # Perplexity detail
    ppl = d["perplexity"]
    ppl_ai = int(d["perplexity_ai_prob"] * 100)
    if ppl < 50:
        ppl_label = "Very low — strong AI signal"
    elif ppl < 100:
        ppl_label = "Low — possible AI signal"
    elif ppl < 200:
        ppl_label = "Medium — ambiguous"
    else:
        ppl_label = "High — more human-like"

    ppl_info = f"""### 📊 GPT-2 Perplexity  *(weight: 25%)*
Measures how "surprising" the text is to a language model.
AI-generated text is typically very predictable (low perplexity).

**Perplexity: {ppl:.1f}**  → {ppl_label}
**AI signal: {ppl_ai}%**  `{format_bar(d["perplexity_ai_prob"])}`

> Typical ranges: AI text = 20–80 | Human text = 100–300+
> *(Note: technical/formal writing can have naturally low perplexity)*
"""

    # Burstiness detail
    burst = d["burstiness"]
    burst_ai = int(d["burstiness_ai_prob"] * 100)
    if burst < 0.3:
        burst_label = "Very uniform — strong AI signal"
    elif burst < 0.5:
        burst_label = "Somewhat uniform — possible AI signal"
    else:
        burst_label = "Variable — more human-like"

    burst_info = f"""### 📈 Sentence Burstiness  *(weight: 15%)*
Measures how much sentence lengths vary throughout the text.
Humans write with natural rhythm — short bursts, long explanations.
AI tends to produce uniform sentence lengths.

**Burstiness coefficient: {burst:.3f}**  → {burst_label}
**AI signal: {burst_ai}%**  `{format_bar(d["burstiness_ai_prob"])}`

> Low coefficient = uniform sentence lengths (AI-like)
> High coefficient = varied sentence lengths (human-like)
"""

    how_it_works = """### How This System Works

This detector uses an **ensemble of three independent methods**:

| Method | What it measures | Weight |
|---|---|---|
| RoBERTa classifier | Learned patterns from millions of AI/human text pairs | 60% |
| GPT-2 perplexity | How predictable the text is to a language model | 25% |
| Burstiness analysis | Variation in sentence length across the text | 15% |

The three scores are combined using a weighted average to produce the final AI probability.
A higher weight is given to RoBERTa because it was specifically trained for this task
and consistently outperforms statistical methods in peer-reviewed benchmarks.

**References:**
- Guo et al. (2023). *How Close is ChatGPT to Human Experts?* (HC3 dataset, RoBERTa)
- Mitchell et al. (2023). *DetectGPT: Zero-Shot Machine-Generated Text Detection*
- Hans et al. (2024). *Spotting LLMs With Binoculars* (ICML 2024)
- Bao et al. (2024). *Fast-DetectGPT* (ICLR 2024)
"""

def live_counter(text: str) -> str:
    """Live word + token count shown below the text box as the user types."""
    if not text or not text.strip():
        return "*0 words · 0 tokens*"
    words  = len(text.strip().split())
    tokens = len(roberta_tokenizer.encode(text, add_special_tokens=False))
    status = "✓" if words >= 20 else f"— need at least 20"
    chunk_note = f" · **will be chunked** ({math.ceil(tokens / CHUNK_SIZE)} chunks)" \
                 if tokens > CHUNK_SIZE else ""
    return f"*{words} words · {tokens} tokens {status}{chunk_note}*"


def detect(text):
    """Main detection handler — called when the user clicks Analyse."""
    if not text or not text.strip():
        return "*Results will appear here…*", "", "", "", ""

    final_prob, verdict, confidence, d = ensemble_predict(text)

    if "error" in d:
        return d["error"], "", "", "", ""

    human_prob = round(1.0 - final_prob, 4)
    pct        = int(final_prob * 100)
    conf_pct   = int(d["confidence"] * 100)

    # Optional banners
    lang_banner  = f"\n> {d['lang_warning']}\n"    if d.get("lang_warning") else ""
    chunk_notice = (
        f"\n> 📄 Long text — analysis averaged across "
        f"{math.ceil(len(roberta_tokenizer.encode(text, add_special_tokens=False)) / CHUNK_SIZE)} chunks.\n"
    ) if d["was_chunked"] else ""

    summary = f"""## {verdict}
{lang_banner}{chunk_notice}
**AI probability:**    {pct}%  `{format_bar(final_prob)}`  

**Human probability:** {int(human_prob*100)}%  `{format_bar(human_prob)}`

**{d['confidence_label']}** `{format_bar(d['confidence'])}` {conf_pct}%
"""

    # ── RoBERTa tab ──
    rb_pct = int(d["roberta_ai_prob"] * 100)
    chunk_tag = "\n📄 *Text exceeded 512 tokens — scored across multiple overlapping chunks.*\n" \
                if d["was_chunked"] else ""
    rb_info = f"""### 🧠 RoBERTa Classifier  *(weight: 60%)*
Fine-tuned on millions of human vs. AI text pairs. Most reliable signal.
{chunk_tag}
**AI score: {rb_pct}%**  `{format_bar(d['roberta_ai_prob'])}`

> Fine-tuned on the HC3 dataset (Guo et al., 2023) — 37,175 QA pairs comparing
> ChatGPT and human expert answers across multiple domains.
> Architecture: RoBERTa-base with a binary classification head.
"""

    # ── Perplexity tab ──
    ppl = d["perplexity"]
    ppl_ai = int(d["perplexity_ai_prob"] * 100)
    if ppl < 50:
        ppl_label = "Very low — strong AI signal"
    elif ppl < 100:
        ppl_label = "Low — possible AI signal"
    elif ppl < 200:
        ppl_label = "Medium — ambiguous"
    else:
        ppl_label = "High — more human-like"

    ppl_info = f"""### 📊 GPT-2 Perplexity  *(weight: 25%)*
Measures how "surprising" the text is to a language model.
AI-generated text is typically very predictable (low perplexity).

**Perplexity: {ppl:.1f}**  → {ppl_label}
**AI signal: {ppl_ai}%**  `{format_bar(d['perplexity_ai_prob'])}`

> Typical ranges: AI text = 20–80 | Human text = 100–300+
> Scoring uses log-scale normalisation (perplexity is log-normally distributed).
> *(Note: technical/formal human writing can also have low perplexity)*
"""

    # ── Burstiness tab ──
    burst = d["burstiness"]
    burst_ai = int(d["burstiness_ai_prob"] * 100)
    if burst < 0.3:
        burst_label = "Very uniform — strong AI signal"
    elif burst < 0.5:
        burst_label = "Somewhat uniform — possible AI signal"
    else:
        burst_label = "Variable — more human-like"

    burst_info = f"""### 📈 Sentence Burstiness  *(weight: 15%)*
Measures how much sentence lengths vary throughout the text.
Humans write with natural rhythm — short bursts mixed with long explanations.
AI tends to produce sentences of similar length.

**Burstiness coefficient: {burst:.3f}**  → {burst_label}
**AI signal: {burst_ai}%**  `{format_bar(d['burstiness_ai_prob'])}`

> Low coefficient = uniform sentence lengths (AI-like)
> High coefficient = varied sentence lengths (human-like)
"""

    # Clean JSON output (exclude internal lang_warning string from display)
    clean_details = {k: v for k, v in d.items() if k != "lang_warning"}
    raw_json = json.dumps(clean_details, indent=2)

    return summary, rb_info, ppl_info, burst_info, raw_json


# ── Gradio UI ───────────────────────────────────────────────────────────────

CSS = """
.gradio-container { max-width: 980px !important; margin: auto; }
footer { display: none !important; }
"""

# Four diverse examples: clear AI, clear human, ambiguous, formal human
EXAMPLES = [
    # 1. Clearly AI — structured ChatGPT-style answer
    ["Large language models (LLMs) have demonstrated remarkable capabilities across a wide range of natural language processing tasks. These models, trained on vast corpora of text data, leverage transformer architectures to generate coherent and contextually appropriate responses. The implications for various industries are significant and multifaceted. In healthcare, these models can assist with medical documentation and patient communication. In education, they enable personalised tutoring and instant feedback mechanisms."],
    # 2. Clearly human — personal, messy, emotional
    ["So I finally tried making sourdough bread today and honestly?? It went terribly lol. The starter looked fine I thought. But then the dough was basically soup and I didn't know what I was doing so I just kept adding flour. Ended up with this weird dense brick thing that my roommate ate anyway because she was hungry. Will try again next weekend I guess. Maybe watch some YouTube videos first this time omg"],
    # 3. Ambiguous — formal human (low perplexity naturally)
    ["The mitochondria is the powerhouse of the cell. This organelle is responsible for producing ATP through a process called cellular respiration. It has a double membrane structure with the inner membrane folded into cristae. The matrix inside contains enzymes needed for the Krebs cycle."],
    # 4. Clearly human — casual news writing with specific details
    ["The city council voted 5-2 last Tuesday to approve a rezoning proposal allowing three new apartment complexes near Riverside Park. Opponents argued the development would increase traffic and strain local schools. Supporters said it would help ease the housing shortage. Mayor Johnson, who abstained from the vote, said she would review the decision before signing off on the permits."],
]

with gr.Blocks(css=CSS, title="LLM Text Detector") as demo:
    gr.Markdown("""
# 🔍 LLM-Generated Text Detector

## Build BY:
               Muhammad Zain-(F2023266257) 
               Muhammad Umar-(F20232661022)
# Under Supervision of 
               MAM NITASHA AROOJ
## Contact:
               nitasha.janjua@umt.edu.pk

# Capabilities: 
## Ensemble detection · Hugging Face Spaces (For Deployment and Resources) · Independent Setup
Detects AI-generated text from GPT-4, GPT-4o, Claude, Gemini, Llama, DeepSeek, and more.
Three independent methods + confidence scoring + automatic long-text chunking
---
""")

    with gr.Row():
        with gr.Column(scale=3):
            text_input = gr.Textbox(
                label="Paste text to analyse",
                placeholder="Paste any text here (minimum 20 words)…",
                lines=10,
            )
            # Live word + token counter (Step 4: new feature)
            counter_md = gr.Markdown("*0 words · 0 tokens*")
            with gr.Row():
                submit_btn = gr.Button("🔍 Analyse Text", variant="primary", scale=3)
                clear_btn  = gr.Button("Clear", scale=1)

            gr.Examples(
                examples=EXAMPLES,
                inputs=text_input,
                label="Example texts to try",
            )

        with gr.Column(scale=2):
            result_summary = gr.Markdown(value="*Results will appear here…*")

    gr.Markdown("---")
    gr.Markdown("## Detailed breakdown")

    with gr.Tabs():
        with gr.Tab("🧠 RoBERTa Classifier"):
            result_roberta = gr.Markdown()
        with gr.Tab("📊 Perplexity Analysis"):
            result_ppl = gr.Markdown()
        with gr.Tab("📈 Burstiness Analysis"):
            result_burst = gr.Markdown()
        with gr.Tab("ℹ️ How It Works"):
            # Step 3: static content — no longer a function output
            gr.Markdown(HOW_IT_WORKS)
        with gr.Tab("🔢 Raw Scores (JSON)"):
            result_raw = gr.Code(language="json")

    # Step 4: live counter wired to text_input
    text_input.change(
        fn=live_counter,
        inputs=text_input,
        outputs=counter_md,
    )

    submit_btn.click(
        fn=detect,
        inputs=text_input,
        outputs=[result_summary, result_roberta, result_ppl, result_burst, result_raw],
    )

    # Step 3 fix: clear button outputs now match exactly — 6 items for 6 outputs
    clear_btn.click(
        fn=lambda: ("", "*0 words · 0 tokens*", "*Results will appear here…*", "", "", "", ""),
        inputs=None,
        outputs=[text_input, counter_md, result_summary, result_roberta, result_ppl, result_burst, result_raw],
    )

    gr.Markdown("""
---
**Models:** `Hello-SimpleAI/chatgpt-detector-roberta` · `gpt2`
**References:** HC3 (Guo et al. 2023) · DetectGPT (Mitchell et al. 2023) · Binoculars (Hans et al. 2024) · Fast-DetectGPT (Bao et al. 2024)
""")

demo.queue()  # Required on HF Spaces: handles concurrent users without blocking
if __name__ == "__main__":
    demo.launch()