app.py

import gradio as gr
import torch
import torch.nn as nn
from PIL import Image
from transformers import BlipProcessor, BlipForConditionalGeneration, AutoTokenizer, AutoModel

# --- Load BLIP model from Hugging Face Hub ---
processor = BlipProcessor.from_pretrained("Mewish/blip_medical")
blip_model = BlipForConditionalGeneration.from_pretrained("Mewish/blip_medical").to("cpu")

# --- Load BioClinicalBERT backbone ---
tokenizer = AutoTokenizer.from_pretrained("emilyalsentzer/Bio_ClinicalBERT")
bert_model = AutoModel.from_pretrained("emilyalsentzer/Bio_ClinicalBERT")

# --- Define BiLSTM classifier ---
class BiLSTMClassifier(nn.Module):
    def __init__(self, bert_model, hidden_dim=256, num_classes=7, dropout=0.5):
        super().__init__()
        self.bert = bert_model
        self.lstm = nn.LSTM(768, hidden_dim, batch_first=True, bidirectional=True)
        self.dropout = nn.Dropout(dropout)
        self.fc = nn.Linear(hidden_dim*2, num_classes)

    def forward(self, input_ids, attention_mask):
        outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask)
        embeddings = outputs.last_hidden_state
        lstm_out, _ = self.lstm(embeddings)
        pooled = torch.mean(lstm_out, dim=1)
        pooled = self.dropout(pooled)
        return self.fc(pooled)

# --- Load classifier weights from Hugging Face Hub ---
classifier = BiLSTMClassifier(bert_model).to("cpu")
state_dict_url = "https://huggingface.co/Mewish/skin_cancer_classifier/resolve/main/classifier.pt"
classifier.load_state_dict(torch.hub.load_state_dict_from_url(state_dict_url, map_location="cpu"))
classifier.eval()

# --- Classes ---
classes = [
    "actinic keratosis",
    "basal cell carcinoma",
    "dermatofibroma",
    "nevus",
    "pigmented benign keratosis",
    "squamous cell carcinoma",
    "vascular lesion"
]

# --- Prediction function ---
def predict(image):
    # Step 1: BLIP caption
    inputs = processor(images=image, return_tensors="pt").to("cpu")
    generated_ids = blip_model.generate(**inputs, max_length=50)
    caption = processor.decode(generated_ids[0], skip_special_tokens=True)

    # Step 2: Classifier prediction
    text_inputs = tokenizer(caption, max_length=50, padding="max_length",
                            truncation=True, return_tensors="pt")
    input_ids = text_inputs["input_ids"].to("cpu")
    attention_mask = text_inputs["attention_mask"].to("cpu")

    with torch.no_grad():
        outputs = classifier(input_ids, attention_mask)
        probs = torch.softmax(outputs, dim=1).cpu().numpy()[0]
        pred_class = classes[probs.argmax()]
        confidence = probs.max()

    return caption, pred_class, float(confidence)

# --- Gradio Interface ---
iface = gr.Interface(
    fn=predict,
    inputs=gr.Image(type="pil"),
    outputs=[
        gr.Textbox(label="Generated Caption"),
        gr.Textbox(label="Predicted Class"),
        gr.Number(label="Confidence Score")
    ],
    title="Skin Cancer AI Agent",
    description="Upload a lesion image to generate a caption and predict the cancer type."
)

if __name__ == "__main__":
    iface.launch()