detect.py

import gradio as gr
import spacy
import re
import numpy as np
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM

# Define some constants for error messages
MAX_TEXT_LENGTH = 2048
MIN_GEN_LENGTH = 1
MAX_GEN_LENGTH = 2048
MIN_AI_PERCENTAGE = 50

# Download the Spacy model for English
spacy.cli.download("en_core_web_sm")
nlp = spacy.load("en_core_web_sm")

# Define a function to detect AI-generated content and calculate the perplexity score,
# burstiness score, and average perplexity score for a given text input
def detect_ai_content(text, max_gen_length, temperature, model_name, model_size, num_return_sequences, min_ai_percentage):
    # Clean the text by removing extra spaces, line breaks, and special characters
    cleaned_text = re.sub(r'\s+', ' ', text).strip()
    cleaned_text = re.sub(r'[^\w\s]', '', cleaned_text)

    # If the cleaned text is empty or contains only one sentence, return an error message
    doc = nlp(cleaned_text)
    if not re.search('\S', cleaned_text) or len(list(doc.sents)) < 2:
        return {"error": "Input text must contain at least two sentences."}

    # Check if the cleaned text is longer than the maximum allowed by the GPT model
    if len(cleaned_text) > MAX_TEXT_LENGTH:
        return {"error": f"Input text must be no longer than {MAX_TEXT_LENGTH} characters."}

    # Check if the minimum threshold for the percentage of AI-generated content is within the allowed range
    if not (0 <= min_ai_percentage <= 100):
        return {"error": "Minimum threshold for AI percentage must be between 0 and 100."}

    # Load the specified GPT model and tokenizer
    model_name = f"{model_name}-{model_size}"
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModelForCausalLM.from_pretrained(model_name)

    # Set the device to run the model on (either "cuda" or "cpu")
    device = "cuda" if torch.cuda.is_available() else "cpu"
    model.to(device)

    # Set the end of sequence token ID for text generation
    eos_token_id = tokenizer.eos_token_id

    # Generate multiple sequences using the pre-trained GPT model
    input_ids = tokenizer.encode(cleaned_text, add_special_tokens=True, return_tensors='pt').to(device)
    
#    with gr.progress.Progress("Generating text...", max=num_return_sequences) as progress:
    output_sequences = []
    for i in range(num_return_sequences):
        output_sequence = model.generate(
        input_ids=input_ids,
        max_length=max_gen_length + len(input_ids[0]),
        temperature=temperature,
        top_k=50,
        top_p=0.95,
        repetition_penalty=1.5,
        do_sample=True,
        num_return_sequences=1
        )[0]
        output_sequences.append(output_sequence)
       #     progress.update(i)

    # Decode the generated sequences using the GPT tokenizer
    generated_texts = []
    perplexities = []
    ai_percentages = []
    
    for i, output_sequence in enumerate(output_sequences):
        generated_text = tokenizer.decode(output_sequence.tolist()[len(input_ids[0]):], skip_special_tokens=True)

        # Calculate the percentage of AI-generated content in the generated text
        ai_percentage = round(len(generated_text) / len(cleaned_text) * 100, 2)
        ai_percentages.append(ai_percentage)

        # Check if the AI percentage and perplexity score are above their respective thresholds
        generated_input_ids = tokenizer.encode(generated_text, add_special_tokens=False, return_tensors='pt').to(device)
        with torch.no_grad():
            loss = model(generated_input_ids, labels=generated_input_ids).loss.item()
        perplexity = np.exp(loss)
        perplexities.append(perplexity)
        generated_texts.append(generated_text)

    # Remove the generated sequences that are identical or highly similar to the cleaned text
    clean_doc = nlp(cleaned_text)
    unique_generated_texts = []

    for generated_text in generated_texts:
        gen_doc = nlp(generated_text)
        similarity = clean_doc.similarity(gen_doc)

        if similarity < 0.8:
            is_unique = True

            for unique_text in unique_generated_texts:
                unique_doc = nlp(unique_text)
                unique_similarity = unique_doc.similarity(gen_doc)

                if unique_similarity >= 0.8:
                    is_unique = False
                    break

            if is_unique:
                unique_generated_texts.append(generated_text)

    # Calculate the burstiness score for the input text, which measures the diversity of vocabulary in the input text
    doc = nlp(cleaned_text)
    tokens = [token.text.lower() for token in doc if not token.is_stop and token.is_alpha]
    unique_tokens = set(tokens)
    burstiness = len(unique_tokens) / len(tokens)

    # Check if the minimum threshold for AI percentage is met for any of the generated sequences,
    # and if the perplexity score is above the minimum threshold
    valid_generated_texts = []
    valid_perplexities = []
    valid_ai_percentages = []

    for i, generated_text in enumerate(unique_generated_texts):
        ai_percentage = ai_percentages[i]
        perplexity = perplexities[i]

        if ai_percentage >= min_ai_percentage and perplexity > 5:
            valid_generated_texts.append(generated_text)
            valid_perplexities.append(perplexity)
            valid_ai_percentages.append(ai_percentage)

    # Check if any valid generated sequences were found
    if not valid_generated_texts:
        return {"error": "No AI-generated content meeting the minimum threshold was found in the input text."}

    # Add up to num_return_sequences generated sequences with the highest AI percentages and perplexity scores to the result dictionary
    max_num_returned_sequences = min(num_return_sequences, len(valid_generated_texts))
    combined_scores = [(i, valid_ai_percentages[i] * valid_perplexities[i]) for i in range(len(valid_generated_texts))]
    sorted_indices = sorted(combined_scores, key=lambda x: x[1], reverse=True)

    result = {
        "cleaned_text": cleaned_text,
        "generated_texts": [],
        "perplexities": valid_perplexities,
        "ai_percentages": valid_ai_percentages,
        "burstiness": burstiness,
        "avg_perplexity": np.mean(valid_perplexities),
    }

    for i in range(max_num_returned_sequences):
        idx = sorted_indices[i][0]
        result["generated_texts"].append(valid_generated_texts[idx])

    # Format the output for better readability
    cleaned_text = re.sub(r'\n+', '\n', cleaned_text)
    generated_texts = [re.sub(r'\n+', '\n', text) for text in result["generated_texts"]]
    perplexities = [f"{perplexity:.2f}" for perplexity in result["perplexities"]]
    ai_percentages = [f"{ai_percentage:.2f}%" for ai_percentage in result["ai_percentages"]]
    avg_perplexity = f"{result['avg_perplexity']:.2f}"
    burstiness = f"{result['burstiness']:.2f}"

    return {"cleaned_text": cleaned_text, "generated_texts": generated_texts,
            "perplexities": perplexities, "ai_percentages": ai_percentages,
            "burstiness": burstiness, "avg_perplexity": avg_perplexity}

# Define the input and output fields for the Gradio interface
input_textbox = gr.Textbox(label="Input Text", placeholder="Enter some text here...")
max_gen_length_slider = gr.Slider(minimum=MIN_GEN_LENGTH, maximum=MAX_GEN_LENGTH, default=256, label="Max Length for Generated Text")
temperature_slider = gr.Slider(minimum=0.1, maximum=2.0, step=0.1, default=1.0, label="Temperature",
                                      info="Higher values make the model generate more random text.")
model_dropdown = gr.Dropdown(choices=["gpt2", "gpt2-medium", "gpt2-large", "gpt2-xl", "EleutherAI/gpt-neo-125M",
                                             "EleutherAI/gpt-neo-1.3B", "EleutherAI/gpt-neo-2.7B"],
                                    default="gpt2", label="GPT Model", tooltip="The pre-trained GPT model to use for text generation.")
model_size_dropdown = gr.Dropdown(choices=["small", "medium", "large", "xl"], default="medium", label="Model Size",
                                         tooltip="The size of the pre-trained GPT model to use for text generation.")
num_return_sequences_slider = gr.Slider(minimum=1, maximum=10, default=1, label="Number of Generated Sequences to Return")
min_ai_percentage_slider = gr.Slider(minimum=0, maximum=100, default=MIN_AI_PERCENTAGE, label="Minimum Threshold for AI Percentage",
                                            info="The minimum percentage of AI-generated content that a generated sequence must have.")

# Define custom CSS styling for the Gradio interface
css_style = """
    .gradio-input-wrapper {
        margin-bottom: 10px;
    }
    
    .gradio-input-wrapper label {
        font-size: 1.2rem;
        font-weight: bold;
        margin-bottom: 5px;
    }
    
    .gradio-slider {
        height: 30px;
        margin-bottom: 15px;
    }
    
    .gradio-slider .input-label-container {
        display: flex;
        justify-content: space-between;
        align-items: center;
    }
    
    .gradio-dropdown {
        margin-bottom: 20px;
    }
    
    .gradio-textbox textarea {
        height: 150px;
        font-size: 1.2rem;
    }
    
    .gradio-output {
        background-color: #f5f5f5;
        border-radius: 4px;
        padding: 15px;
        margin-top: 30px;
        margin-bottom: 30px;
    }
    
    .gradio-output label {
        font-size: 1.2rem;
        font-weight: bold;
        margin-bottom: 10px;
        display: block;
    }
    
    .gradio-output p {
        margin-bottom: 10px;
    }
    
    .gradio-error {
        color: red;
        font-weight: bold;
        margin-top: 10px;
    }
    
    .gradio-progress {
        margin-top: 20px;
    }
"""

# Define the Gradio interface
iface = gr.Interface(
    detect_ai_content,
    inputs=[input_textbox, max_gen_length_slider, temperature_slider, model_dropdown, model_size_dropdown,
            num_return_sequences_slider, min_ai_percentage_slider],
    outputs=[
        gr.Label(label="Input Text"),
        gr.Label(label="Generated Text"),
        gr.Label(label="Perplexity Scores"),
        gr.Label(label="AI Percentages"),
        gr.Label(label="Burstiness Score"),
        gr.Label(label="Average Perplexity Score"),
    ],
    title="AI Content Detection and Generation",
    description="This app detects and generates AI-generated content in input text using GPT-2 and GPT-Neo models.",
    theme="default",
    layout="vertical",
    css_style=css_style,
)

# Launch the Gradio interface
iface.launch()