app.py

import gradio as gr
import spaces
import os
import sys
import torch
import torch.nn.functional as F
import torchvision.transforms as transforms
from PIL import Image
import numpy as np

# Setup paths and download GloVe if missing
base_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(base_dir)

from core.predictor import SkribblPredictorModel, filter_words_by_hint
from core.embedding_utils import load_glove_embeddings

glove_path = os.path.join(base_dir, "data", "glove.6B.300d.txt")
if not os.path.exists(glove_path):
    print("Downloading GloVe embeddings... this might take a minute on Hugging Face Spaces.")
    # import download_glove script and run
    import subprocess
    subprocess.run([sys.executable, os.path.join(base_dir, "scripts", "download_glove.py")])

vocab_path = os.path.join(base_dir, "data", "vocab.txt")
with open(vocab_path, 'r') as f:
    vocab = [line.strip() for line in f if line.strip()]

# Load Model
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = SkribblPredictorModel(embedding_dim=300).to(device)

model_path = os.path.join(base_dir, "weights", "skribbl_model.pth")
if os.path.exists(model_path):
    model.load_state_dict(torch.load(model_path, map_location=device))
    print("Loaded model weights.")
else:
    print("WARNING: Model weights not found!")
model.eval()

# Load Embeddings
cpu_embeddings = load_glove_embeddings(vocab, glove_path)
word_embeddings = {word: vec.to(device) for word, vec in cpu_embeddings.items()}

# Transform
transform = transforms.Compose([
    transforms.ToPILImage(),
    transforms.Resize((224, 224)),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
])

@spaces.GPU
def predict(image, hint):
    if image is None:
        return "Please draw something."
        
    # In Gradio, 'sketch' tool returns a dict with 'image' and 'mask', or just an image depending on gradio version.
    # We will assume it's a PIL Image or numpy array representing the drawing.
    if isinstance(image, dict):
        # Gradio 4.x sketchpad returns a dict
        if 'composite' in image:
            image_array = np.array(image['composite'])
        else:
            # Fallback
            image_array = np.array(image.get('image', image))
    else:
        image_array = np.array(image)
        
    # Preprocess image to match our training data
    # Skribbl is black strokes on white background.
    # Gradio sketchpad usually has transparent background with black strokes (or white background).
    img_pil = Image.fromarray(image_array)
    # Convert RGBA to RGB with white background
    if img_pil.mode == 'RGBA':
        background = Image.new('RGBA', img_pil.size, (255,255,255))
        img_pil = Image.alpha_composite(background, img_pil).convert('RGB')
    else:
        img_pil = img_pil.convert('RGB')
        
    # Convert to grayscale and threshold to match Pygame logic
    gray_img = img_pil.convert('L')
    gray_array = np.array(gray_img)
    thresholded = np.where(gray_array < 240, 0, 255).astype(np.uint8)
    canvas_array = np.stack((thresholded,)*3, axis=-1)

    hint = str(hint).strip().lower()
    if not hint:
        hint = "_____" # default 5
        
    target_length = len(hint)

    tensor_img = transform(canvas_array).unsqueeze(0).to(device)
    tensor_len = torch.tensor([target_length]).to(device)
    
    with torch.no_grad():
        output_emb = model(tensor_img, tensor_len).squeeze(0)
        
        valid_words = filter_words_by_hint(vocab, hint)
        if not valid_words:
            return "No words match that hint pattern!"
            
        logits = []
        for word in valid_words:
            word_emb = word_embeddings[word]
            sim = F.cosine_similarity(output_emb, word_emb, dim=0)
            logits.append(sim)
            
        logits_tensor = torch.stack(logits)
        probabilities = torch.softmax(logits_tensor * 10.0, dim=0).tolist()
        
        similarities = [(valid_words[i], probabilities[i]) for i in range(len(valid_words))]
        similarities.sort(key=lambda x: x[1], reverse=True)
        predictions = similarities[:10]
        
    output_text = "### Top Predictions\n"
    for i, (word, prob) in enumerate(predictions):
        output_text += f"{i+1}. **{word}** ({prob*100:.1f}%)\n"
        
    return output_text

# Create Gradio Interface
with gr.Blocks(title="Skribbl AI Predictor") as demo:
    gr.Markdown("# 🎨 Skribbl AI Predictor")
    gr.Markdown("Draw your sketch below and type in the hint pattern (e.g. `__i_k__` or `_____`). The AI will filter its vocabulary and guess the word!")
    
    with gr.Row():
        with gr.Column():
            canvas = gr.Sketchpad(type="pil", label="Drawing Canvas")
            hint_input = gr.Textbox(label="Word Hint Pattern", placeholder="e.g. __i_k__ or 5 for five underscores")
            predict_btn = gr.Button("Predict", variant="primary")
            
        with gr.Column():
            output = gr.Markdown(label="Predictions")
            
    # Whenever the hint changes or the user clicks predict, update
    predict_btn.click(fn=predict, inputs=[canvas, hint_input], outputs=output)
    hint_input.change(fn=predict, inputs=[canvas, hint_input], outputs=output)

if __name__ == "__main__":
    demo.launch()