Deploy final RCM-Pet multimodal app

README.md

@@ -1,13 +1,32 @@
 ---
-title: RCM PET Version Two
-emoji: 😻
-colorFrom: green
-colorTo: pink
+title: RCM-Pet
+emoji: 🐾
+colorFrom: blue
+colorTo: blue
 sdk: gradio
-sdk_version: 6.18.0
-python_version: '3.13'
 app_file: app.py
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# RCM-Pet: Risk-Calibrated Multimodal Pet Triage System
+
+RCM-Pet supports image, text, and audio-based pet triage.
+
+## Input Modes
+- Image only
+- Text only
+- Audio only
+- Image + text
+- Image + audio
+- Image + text + audio
+
+## Models
+- Species classifier: MobileNetV2 with unknown class
+- Dog disease image ML: ResNet18 + EfficientNet-B0
+- Cat disease image ML: EfficientNet-B0
+- Fish disease image ML: EfficientNet-B0
+- Symptom text ML: TF-IDF + Logistic Regression
+- Audio-to-text: Whisper Tiny
+
+## Safety
+This is a triage assistant, not a final veterinary diagnosis.

app.py

@@ -0,0 +1,1119 @@
+
+import os
+import json
+import warnings
+warnings.filterwarnings("ignore")
+
+import numpy as np
+import joblib
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from PIL import Image
+from torchvision import models, transforms
+from transformers import pipeline
+import gradio as gr
+
+
+# ==========================================================
+# GLOBAL SETTINGS
+# ==========================================================
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+MODEL_DIR = os.path.join(BASE_DIR, "models")
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+IMAGE_MEAN = [0.485, 0.456, 0.406]
+IMAGE_STD = [0.229, 0.224, 0.225]
+
+print("Using device:", DEVICE)
+print("Model directory:", MODEL_DIR)
+
+
+# ==========================================================
+# UTILITY FUNCTIONS
+# ==========================================================
+
+def safe_torch_load(path):
+    return torch.load(path, map_location=DEVICE)
+
+
+def clean_label(label):
+    if label is None:
+        return "Unknown"
+    return str(label).replace("_", " ").title()
+
+
+def softmax_dict(probs, idx_to_class):
+    result = {}
+    for idx, prob in enumerate(probs):
+        label = idx_to_class.get(idx, str(idx))
+        result[label] = float(prob)
+    return result
+
+
+def format_prob(x):
+    if x is None:
+        return None
+    try:
+        return round(float(x), 4)
+    except Exception:
+        return x
+
+
+# ==========================================================
+# SPECIES CLASSIFIER: MobileNetV2
+# ==========================================================
+
+SPECIES_MODEL_PATH = os.path.join(MODEL_DIR, "best_species_mobilenetv2_7class_unknown.pth")
+SPECIES_LABEL_MAP_PATH = os.path.join(MODEL_DIR, "species_label_map_7class_unknown.json")
+
+species_checkpoint = safe_torch_load(SPECIES_MODEL_PATH)
+
+if isinstance(species_checkpoint, dict) and "class_names" in species_checkpoint:
+    species_class_names = species_checkpoint["class_names"]
+elif os.path.exists(SPECIES_LABEL_MAP_PATH):
+    with open(SPECIES_LABEL_MAP_PATH, "r") as f:
+        label_map = json.load(f)
+
+    if isinstance(label_map, dict):
+        if all(str(k).isdigit() for k in label_map.keys()):
+            species_class_names = [label_map[str(i)] for i in range(len(label_map))]
+        else:
+            species_class_names = sorted(label_map, key=label_map.get)
+    else:
+        species_class_names = label_map
+else:
+    species_class_names = ["bird", "cat", "dog", "goldfish", "parrot", "rabbit", "unknown"]
+
+species_class_to_idx = {c: i for i, c in enumerate(species_class_names)}
+species_idx_to_class = {i: c for c, i in species_class_to_idx.items()}
+
+SPECIES_IMG_SIZE = species_checkpoint.get("img_size", 224) if isinstance(species_checkpoint, dict) else 224
+
+species_model = models.mobilenet_v2(weights=None)
+in_features = species_model.classifier[1].in_features
+species_model.classifier = nn.Sequential(
+    nn.Dropout(p=0.3),
+    nn.Linear(in_features, len(species_class_names))
+)
+
+species_state = species_checkpoint["model_state_dict"] if isinstance(species_checkpoint, dict) and "model_state_dict" in species_checkpoint else species_checkpoint
+species_model.load_state_dict(species_state, strict=False)
+species_model = species_model.to(DEVICE)
+species_model.eval()
+
+species_transform = transforms.Compose([
+    transforms.Resize((SPECIES_IMG_SIZE, SPECIES_IMG_SIZE)),
+    transforms.ToTensor(),
+    transforms.Normalize(mean=IMAGE_MEAN, std=IMAGE_STD)
+])
+
+print("Species model loaded:", species_class_names)
+
+
+def predict_species_open_set_strict(image_path):
+    img = Image.open(image_path).convert("RGB")
+    x = species_transform(img).unsqueeze(0).to(DEVICE)
+
+    with torch.no_grad():
+        outputs = species_model(x)
+        probs = F.softmax(outputs, dim=1)[0].cpu().numpy()
+
+    top_indices = np.argsort(probs)[::-1]
+    top1_idx = int(top_indices[0])
+    top2_idx = int(top_indices[1])
+    top3_idx = int(top_indices[2])
+
+    top1_class = species_idx_to_class[top1_idx]
+    top2_class = species_idx_to_class[top2_idx]
+    top3_class = species_idx_to_class[top3_idx]
+
+    top1_prob = float(probs[top1_idx])
+    top2_prob = float(probs[top2_idx])
+    top3_prob = float(probs[top3_idx])
+    margin = top1_prob - top2_prob
+
+    top3 = [
+        {"class": top1_class, "confidence": top1_prob},
+        {"class": top2_class, "confidence": top2_prob},
+        {"class": top3_class, "confidence": top3_prob},
+    ]
+
+    # Open-set / safety logic
+    if top1_class == "unknown":
+        final_output = "unsupported_animal"
+    elif ("unknown" in [top2_class, top3_class]) and max(
+        top2_prob if top2_class == "unknown" else 0,
+        top3_prob if top3_class == "unknown" else 0
+    ) >= 0.10:
+        final_output = "uncertain_possible_unknown"
+    elif top1_prob < 0.90:
+        final_output = "unknown_or_unclear_image"
+    elif margin < 0.30:
+        final_output = "uncertain_species"
+    else:
+        final_output = top1_class
+
+    return {
+        "top1_class": top1_class,
+        "top1_confidence": top1_prob,
+        "top2_class": top2_class,
+        "top2_confidence": top2_prob,
+        "top3_class": top3_class,
+        "top3_confidence": top3_prob,
+        "margin": margin,
+        "top3": top3,
+        "final_output": final_output
+    }
+
+
+# ==========================================================
+# DOG IMAGE MODELS
+# ==========================================================
+
+DOG_BINARY_PATH = os.path.join(MODEL_DIR, "best_dog_disease_resnet18.pth")
+DOG_TYPE_PATH = os.path.join(MODEL_DIR, "best_dog_disease_type_efficientnet_b0.pth")
+
+DOG_BINARY_THRESHOLD = 0.35
+
+dog_binary_checkpoint = safe_torch_load(DOG_BINARY_PATH)
+
+dog_binary_model = models.resnet18(weights=None)
+
+# Try 2-output first, then 1-output if checkpoint needs sigmoid binary
+dog_binary_is_sigmoid = False
+
+try:
+    dog_binary_model.fc = nn.Linear(dog_binary_model.fc.in_features, 2)
+    state = dog_binary_checkpoint["model_state_dict"] if isinstance(dog_binary_checkpoint, dict) and "model_state_dict" in dog_binary_checkpoint else dog_binary_checkpoint
+    dog_binary_model.load_state_dict(state, strict=True)
+except Exception:
+    dog_binary_model = models.resnet18(weights=None)
+    dog_binary_model.fc = nn.Linear(dog_binary_model.fc.in_features, 1)
+    state = dog_binary_checkpoint["model_state_dict"] if isinstance(dog_binary_checkpoint, dict) and "model_state_dict" in dog_binary_checkpoint else dog_binary_checkpoint
+    dog_binary_model.load_state_dict(state, strict=False)
+    dog_binary_is_sigmoid = True
+
+dog_binary_model = dog_binary_model.to(DEVICE)
+dog_binary_model.eval()
+
+DOG_BINARY_IMG_SIZE = dog_binary_checkpoint.get("img_size", 224) if isinstance(dog_binary_checkpoint, dict) else 224
+
+dog_binary_classes = dog_binary_checkpoint.get("class_names", ["diseased", "healthy"]) if isinstance(dog_binary_checkpoint, dict) else ["diseased", "healthy"]
+dog_binary_idx_to_class = {i: c for i, c in enumerate(dog_binary_classes)}
+
+dog_type_checkpoint = safe_torch_load(DOG_TYPE_PATH)
+
+dog_type_classes = dog_type_checkpoint.get(
+    "class_names",
+    [
+        "bacterial_dermatosis",
+        "fungal_infections",
+        "hypersensitivity_allergic_dermatosis"
+    ]
+) if isinstance(dog_type_checkpoint, dict) else [
+    "bacterial_dermatosis",
+    "fungal_infections",
+    "hypersensitivity_allergic_dermatosis"
+]
+
+dog_type_idx_to_class = {i: c for i, c in enumerate(dog_type_classes)}
+
+DOG_TYPE_IMG_SIZE = dog_type_checkpoint.get("img_size", 224) if isinstance(dog_type_checkpoint, dict) else 224
+
+dog_type_model = models.efficientnet_b0(weights=None)
+in_features = dog_type_model.classifier[1].in_features
+dog_type_model.classifier = nn.Sequential(
+    nn.Dropout(p=0.4),
+    nn.Linear(in_features, len(dog_type_classes))
+)
+
+dog_type_state = dog_type_checkpoint["model_state_dict"] if isinstance(dog_type_checkpoint, dict) and "model_state_dict" in dog_type_checkpoint else dog_type_checkpoint
+dog_type_model.load_state_dict(dog_type_state, strict=False)
+dog_type_model = dog_type_model.to(DEVICE)
+dog_type_model.eval()
+
+dog_transform = transforms.Compose([
+    transforms.Resize((224, 224)),
+    transforms.ToTensor(),
+    transforms.Normalize(mean=IMAGE_MEAN, std=IMAGE_STD)
+])
+
+DOG_TYPE_DISPLAY = {
+    "bacterial_dermatosis": "Bacterial dermatosis",
+    "fungal_infections": "Fungal infection",
+    "fungal_infection": "Fungal infection",
+    "hypersensitivity_allergic_dermatosis": "Hypersensitivity / allergic dermatosis",
+    "allergic_dermatosis": "Hypersensitivity / allergic dermatosis"
+}
+
+
+def predict_dog_image(image_path):
+    img = Image.open(image_path).convert("RGB")
+    x = dog_transform(img).unsqueeze(0).to(DEVICE)
+
+    with torch.no_grad():
+        binary_out = dog_binary_model(x)
+
+    if dog_binary_is_sigmoid or binary_out.shape[1] == 1:
+        disease_prob = float(torch.sigmoid(binary_out)[0][0].cpu().item())
+        is_diseased = disease_prob >= DOG_BINARY_THRESHOLD
+        binary_conf = disease_prob if is_diseased else 1.0 - disease_prob
+    else:
+        probs = F.softmax(binary_out, dim=1)[0].cpu().numpy()
+        label_probs = {dog_binary_idx_to_class[i].lower(): float(p) for i, p in enumerate(probs)}
+
+        disease_prob = 0.0
+        for k, v in label_probs.items():
+            if "disease" in k or "diseased" in k or "infect" in k:
+                disease_prob = max(disease_prob, v)
+
+        if disease_prob == 0.0 and len(probs) == 2:
+            disease_prob = float(probs[0])
+
+        is_diseased = disease_prob >= DOG_BINARY_THRESHOLD
+        binary_conf = disease_prob if is_diseased else 1.0 - disease_prob
+
+    if not is_diseased:
+        return {
+            "species": "dog",
+            "health_status": "healthy",
+            "image_prediction": "Healthy dog",
+            "image_disease_type": "Not detected",
+            "image_confidence": binary_conf,
+            "risk_level": "Normal",
+            "mode": "dog_image_ml"
+        }
+
+    with torch.no_grad():
+        type_out = dog_type_model(x)
+        type_probs = F.softmax(type_out, dim=1)[0].cpu().numpy()
+
+    top_idx = int(np.argmax(type_probs))
+    raw_type = dog_type_idx_to_class[top_idx]
+    type_conf = float(type_probs[top_idx])
+
+    return {
+        "species": "dog",
+        "health_status": "diseased",
+        "image_prediction": DOG_TYPE_DISPLAY.get(raw_type, clean_label(raw_type)),
+        "image_disease_type": DOG_TYPE_DISPLAY.get(raw_type, clean_label(raw_type)),
+        "image_confidence": type_conf,
+        "risk_level": "Monitor / See Vet",
+        "mode": "dog_image_ml"
+    }
+
+
+# ==========================================================
+# CAT IMAGE MODEL
+# ==========================================================
+
+CAT_IMAGE_PATH = os.path.join(MODEL_DIR, "best_cat_disease_efficientnet_b0.pth")
+cat_checkpoint = safe_torch_load(CAT_IMAGE_PATH)
+
+cat_class_names = cat_checkpoint.get("class_names", ["flea_allergy", "healthy", "ringworm", "scabies"]) if isinstance(cat_checkpoint, dict) else ["flea_allergy", "healthy", "ringworm", "scabies"]
+cat_idx_to_class = {i: c for i, c in enumerate(cat_class_names)}
+CAT_IMG_SIZE = cat_checkpoint.get("img_size", 224) if isinstance(cat_checkpoint, dict) else 224
+
+cat_model = models.efficientnet_b0(weights=None)
+in_features = cat_model.classifier[1].in_features
+cat_model.classifier = nn.Sequential(
+    nn.Dropout(p=0.4),
+    nn.Linear(in_features, len(cat_class_names))
+)
+
+cat_state = cat_checkpoint["model_state_dict"] if isinstance(cat_checkpoint, dict) and "model_state_dict" in cat_checkpoint else cat_checkpoint
+cat_model.load_state_dict(cat_state, strict=False)
+cat_model = cat_model.to(DEVICE)
+cat_model.eval()
+
+cat_transform = transforms.Compose([
+    transforms.Resize((CAT_IMG_SIZE, CAT_IMG_SIZE)),
+    transforms.ToTensor(),
+    transforms.Normalize(mean=IMAGE_MEAN, std=IMAGE_STD)
+])
+
+CAT_DISPLAY = {
+    "healthy": "Healthy cat",
+    "ringworm": "Ringworm / fungal infection",
+    "scabies": "Scabies",
+    "flea_allergy": "Flea allergy"
+}
+
+CAT_RISK = {
+    "healthy": "Normal",
+    "ringworm": "Monitor / See Vet",
+    "scabies": "See Vet",
+    "flea_allergy": "Monitor / See Vet"
+}
+
+
+def predict_cat_disease_image(image_path):
+    img = Image.open(image_path).convert("RGB")
+    x = cat_transform(img).unsqueeze(0).to(DEVICE)
+
+    with torch.no_grad():
+        outputs = cat_model(x)
+        probs = F.softmax(outputs, dim=1)[0].cpu().numpy()
+
+    top_idx = int(np.argmax(probs))
+    raw_label = cat_idx_to_class[top_idx]
+    conf = float(probs[top_idx])
+
+    health = "healthy" if raw_label == "healthy" else "diseased"
+
+    return {
+        "species": "cat",
+        "health_status": health,
+        "raw_label": raw_label,
+        "image_prediction": CAT_DISPLAY.get(raw_label, clean_label(raw_label)),
+        "image_confidence": conf,
+        "risk_level": CAT_RISK.get(raw_label, "Monitor / See Vet"),
+        "mode": "cat_image_ml"
+    }
+
+
+# ==========================================================
+# FISH IMAGE MODEL
+# ==========================================================
+
+FISH_IMAGE_PATH = os.path.join(MODEL_DIR, "best_fish_disease_efficientnet_b0.pth")
+fish_checkpoint = safe_torch_load(FISH_IMAGE_PATH)
+
+fish_class_names = fish_checkpoint.get("class_names", ["healthy", "infected"]) if isinstance(fish_checkpoint, dict) else ["healthy", "infected"]
+fish_idx_to_class = {i: c for i, c in enumerate(fish_class_names)}
+FISH_IMG_SIZE = fish_checkpoint.get("img_size", 224) if isinstance(fish_checkpoint, dict) else 224
+
+fish_model = models.efficientnet_b0(weights=None)
+in_features = fish_model.classifier[1].in_features
+fish_model.classifier = nn.Sequential(
+    nn.Dropout(p=0.4),
+    nn.Linear(in_features, len(fish_class_names))
+)
+
+fish_state = fish_checkpoint["model_state_dict"] if isinstance(fish_checkpoint, dict) and "model_state_dict" in fish_checkpoint else fish_checkpoint
+fish_model.load_state_dict(fish_state, strict=False)
+fish_model = fish_model.to(DEVICE)
+fish_model.eval()
+
+fish_transform = transforms.Compose([
+    transforms.Resize((FISH_IMG_SIZE, FISH_IMG_SIZE)),
+    transforms.ToTensor(),
+    transforms.Normalize(mean=IMAGE_MEAN, std=IMAGE_STD)
+])
+
+FISH_DISPLAY = {
+    "healthy": "Healthy fish",
+    "infected": "Infected fish"
+}
+
+
+def predict_fish_disease_image(image_path):
+    img = Image.open(image_path).convert("RGB")
+    x = fish_transform(img).unsqueeze(0).to(DEVICE)
+
+    with torch.no_grad():
+        outputs = fish_model(x)
+        probs = F.softmax(outputs, dim=1)[0].cpu().numpy()
+
+    top_idx = int(np.argmax(probs))
+    raw_label = fish_idx_to_class[top_idx]
+    conf = float(probs[top_idx])
+
+    health = "healthy" if raw_label == "healthy" else "diseased"
+    risk = "Normal" if raw_label == "healthy" else "Check water / Treat quickly"
+
+    return {
+        "species": "goldfish",
+        "health_status": health,
+        "raw_label": raw_label,
+        "image_prediction": FISH_DISPLAY.get(raw_label, clean_label(raw_label)),
+        "image_confidence": conf,
+        "risk_level": risk,
+        "mode": "fish_image_ml"
+    }
+
+
+# ==========================================================
+# TEXT MODELS
+# ==========================================================
+
+DOG_TEXT_PATH = os.path.join(MODEL_DIR, "dog_symptom_tfidf_logreg.pkl")
+CAT_TEXT_PATH = os.path.join(MODEL_DIR, "cat_symptom_tfidf_logreg.pkl")
+MULTI_TEXT_PATH = os.path.join(MODEL_DIR, "multispecies_symptom_tfidf_logreg.pkl")
+
+dog_text_model = joblib.load(DOG_TEXT_PATH)
+cat_text_model = joblib.load(CAT_TEXT_PATH)
+multi_text_model = joblib.load(MULTI_TEXT_PATH)
+
+DOG_TEXT_DISPLAY = {
+    "bacterial_dermatosis": "Bacterial dermatosis",
+    "fungal_infections": "Fungal infection",
+    "fungal_infection": "Fungal infection",
+    "hypersensitivity_allergic_dermatosis": "Hypersensitivity / allergic dermatosis",
+    "allergic_dermatosis": "Hypersensitivity / allergic dermatosis"
+}
+
+CAT_TEXT_DISPLAY = {
+    "cat_upper_respiratory": "Upper respiratory infection symptoms possible",
+    "cat_ringworm_skin": "Ringworm / fungal skin problem possible",
+    "cat_flea_allergy": "Flea allergy or itchy skin problem possible",
+    "cat_scabies_mites": "Scabies / mite problem possible",
+    "cat_worm_digestive": "Worm or digestive parasite problem possible",
+    "cat_diabetes_possible": "Diabetes symptoms possible",
+    "cat_emergency": "Emergency condition possible",
+    "cat_general_issue": "General health issue possible"
+}
+
+CAT_TEXT_RISK = {
+    "cat_upper_respiratory": "See Vet",
+    "cat_ringworm_skin": "Monitor / See Vet",
+    "cat_flea_allergy": "Monitor / See Vet",
+    "cat_scabies_mites": "See Vet",
+    "cat_worm_digestive": "See Vet",
+    "cat_diabetes_possible": "See Vet",
+    "cat_emergency": "Emergency Vet",
+    "cat_general_issue": "Monitor / See Vet"
+}
+
+MULTI_TEXT_DISPLAY = {
+    "rabbit_eye_problem": "Eye infection or eye irritation possible",
+    "rabbit_digestive_problem": "Digestive problem possible",
+    "rabbit_respiratory_problem": "Respiratory problem possible",
+    "rabbit_skin_problem": "Skin problem possible",
+    "bird_feather_skin_problem": "Feather or skin problem possible",
+    "bird_respiratory_problem": "Respiratory problem possible",
+    "bird_digestive_problem": "Digestive problem possible",
+    "bird_injury_possible": "Injury possible",
+    "fish_white_spot": "White spot disease possible",
+    "fish_fin_rot": "Fin rot possible",
+    "fish_swim_bladder": "Swim bladder problem possible",
+    "fish_fungal_problem": "Fungal problem possible",
+    "general_issue": "General health issue possible"
+}
+
+MULTI_TEXT_RISK = {
+    "rabbit_eye_problem": "See Vet",
+    "rabbit_digestive_problem": "See Vet",
+    "rabbit_respiratory_problem": "See Vet",
+    "rabbit_skin_problem": "Monitor / See Vet",
+    "bird_feather_skin_problem": "Monitor / See Vet",
+    "bird_respiratory_problem": "See Vet",
+    "bird_digestive_problem": "See Vet",
+    "bird_injury_possible": "See Vet",
+    "fish_white_spot": "Check water / Treat quickly",
+    "fish_fin_rot": "Check water / Treat quickly",
+    "fish_swim_bladder": "Monitor water and feeding",
+    "fish_fungal_problem": "Check water / Treat quickly",
+    "general_issue": "Monitor / See Vet"
+}
+
+
+def predict_text_model(model, text, display_map=None, risk_map=None):
+    label = model.predict([text])[0]
+    confidence = None
+
+    if hasattr(model, "predict_proba"):
+        probs = model.predict_proba([text])[0]
+        confidence = float(np.max(probs))
+
+    display = display_map.get(label, clean_label(label)) if display_map else clean_label(label)
+    risk = risk_map.get(label, "Monitor / See Vet") if risk_map else "Monitor / See Vet"
+
+    return {
+        "text_model_label": label,
+        "text_prediction": display,
+        "text_model_confidence": confidence,
+        "risk_level": risk
+    }
+
+
+def predict_dog_symptom_text(text):
+    return predict_text_model(dog_text_model, text, DOG_TEXT_DISPLAY, None)
+
+
+def predict_cat_symptom_text(text):
+    return predict_text_model(cat_text_model, text, CAT_TEXT_DISPLAY, CAT_TEXT_RISK)
+
+
+def fallback_symptom_triage(species, symptom_text):
+    result = predict_text_model(multi_text_model, symptom_text, MULTI_TEXT_DISPLAY, MULTI_TEXT_RISK)
+    return {
+        "species": species,
+        "health_status": "possible_issue",
+        "final_prediction": result["text_prediction"],
+        "text_model_label": result["text_model_label"],
+        "text_model_confidence": result["text_model_confidence"],
+        "risk_level": result["risk_level"],
+        "mode": "multispecies_text_ml"
+    }
+
+
+# ==========================================================
+# AUDIO MODEL: Whisper ASR
+# ==========================================================
+
+try:
+    asr_pipe = pipeline(
+        task="automatic-speech-recognition",
+        model="openai/whisper-tiny",
+        device=0 if torch.cuda.is_available() else -1
+    )
+    ASR_READY = True
+    print("Whisper loaded")
+except Exception as e:
+    print("Whisper failed to load:", e)
+    asr_pipe = None
+    ASR_READY = False
+
+
+def transcribe_audio(audio_path):
+    if audio_path is None:
+        return ""
+
+    if not ASR_READY:
+        return ""
+
+    try:
+        result = asr_pipe(audio_path)
+        text = result.get("text", "")
+        return text.strip()
+    except Exception:
+        return ""
+
+
+# ==========================================================
+# SPECIES FROM TEXT + MISMATCH CHECK
+# ==========================================================
+
+def detect_species_from_text(text):
+    if text is None:
+        return None
+
+    t = text.lower()
+
+    if any(w in t for w in ["dog", "puppy"]):
+        return "dog"
+    if any(w in t for w in ["cat", "kitten"]):
+        return "cat"
+    if any(w in t for w in ["rabbit", "bunny"]):
+        return "rabbit"
+    if "parrot" in t:
+        return "parrot"
+    if "bird" in t:
+        return "bird"
+    if any(w in t for w in ["goldfish", "fish"]):
+        return "goldfish"
+
+    return None
+
+
+def species_compatible(image_species, text_species):
+    if text_species is None:
+        return True
+
+    if image_species == text_species:
+        return True
+
+    if image_species in ["bird", "parrot"] and text_species in ["bird", "parrot"]:
+        return True
+
+    if image_species == "goldfish" and text_species == "goldfish":
+        return True
+
+    return False
+
+
+def check_species_text_mismatch(image_species, symptom_text):
+    text_species = detect_species_from_text(symptom_text)
+
+    if text_species is None:
+        return {
+            "mismatch": False,
+            "text_species": None,
+            "message": "No species mentioned in text."
+        }
+
+    compatible = species_compatible(image_species, text_species)
+
+    return {
+        "mismatch": not compatible,
+        "text_species": text_species,
+        "message": f"Image suggests {image_species}, but text suggests {text_species}."
+    }
+
+
+# ==========================================================
+# ROUTING SAFETY FUNCTIONS
+# ==========================================================
+
+def is_safe_dog_for_disease(species_result):
+    return (
+        species_result["top1_class"] == "dog"
+        and (
+            species_result["final_output"] == "dog"
+            or species_result["top1_confidence"] >= 0.75
+        )
+    )
+
+
+def is_safe_cat_for_disease(species_result):
+    top1 = species_result["top1_class"]
+    top1_prob = species_result["top1_confidence"]
+    margin = species_result["margin"]
+
+    if top1 == "cat" and species_result["final_output"] == "cat":
+        return True
+
+    if top1 == "cat" and top1_prob >= 0.45 and margin >= 0.10:
+        return True
+
+    # Rescue diseased cat close-ups where unknown is top1 but cat is in top predictions
+    for item in species_result.get("top3", []):
+        if item["class"] == "cat" and item["confidence"] >= 0.20 and top1_prob <= 0.70:
+            return True
+
+    return False
+
+
+def is_safe_fish_for_disease(species_result):
+    return (
+        species_result["top1_class"] == "goldfish"
+        and (
+            species_result["final_output"] == "goldfish"
+            or species_result["top1_confidence"] >= 0.80
+        )
+    )
+
+
+def is_safe_fallback_species(species_result, symptom_text):
+    top1 = species_result["top1_class"]
+    final_output = species_result["final_output"]
+
+    fallback_species = ["rabbit", "bird", "parrot"]
+
+    if top1 not in fallback_species:
+        return False
+
+    if final_output != top1 and species_result["top1_confidence"] < 0.85:
+        return False
+
+    if symptom_text is None or symptom_text.strip() == "":
+        return False
+
+    text_species = detect_species_from_text(symptom_text)
+
+    if text_species is None:
+        return False
+
+    return species_compatible(top1, text_species)
+
+
+# ==========================================================
+# FINAL PIPELINES
+# ==========================================================
+
+def final_text_only_triage(symptom_text):
+    if symptom_text is None or symptom_text.strip() == "":
+        return {
+            "input_mode": "text_only",
+            "status": "no_input",
+            "species": "unknown",
+            "health_status": "unknown",
+            "final_prediction": "Please provide symptoms or upload an image.",
+            "risk_level": "Unknown"
+        }
+
+    text_species = detect_species_from_text(symptom_text)
+
+    if text_species is None:
+        return {
+            "input_mode": "text_only",
+            "status": "species_not_found_in_text",
+            "species": "unknown",
+            "health_status": "unknown",
+            "final_prediction": "Species could not be identified from text. Please mention dog, cat, rabbit, parrot, bird, or fish.",
+            "risk_level": "Unknown"
+        }
+
+    if text_species == "dog":
+        r = predict_dog_symptom_text(symptom_text)
+        return {
+            "input_mode": "text_only",
+            "status": "dog_text_ml_used",
+            "species": "dog",
+            "health_status": "possible_issue",
+            "final_prediction": r["text_prediction"],
+            "text_prediction": r["text_prediction"],
+            "text_model_label": r["text_model_label"],
+            "text_model_confidence": r["text_model_confidence"],
+            "risk_level": "Monitor / See Vet"
+        }
+
+    if text_species == "cat":
+        r = predict_cat_symptom_text(symptom_text)
+        return {
+            "input_mode": "text_only",
+            "status": "cat_text_ml_used",
+            "species": "cat",
+            "health_status": "possible_issue",
+            "final_prediction": r["text_prediction"],
+            "text_prediction": r["text_prediction"],
+            "text_model_label": r["text_model_label"],
+            "text_model_confidence": r["text_model_confidence"],
+            "risk_level": r["risk_level"]
+        }
+
+    r = fallback_symptom_triage(text_species, symptom_text)
+    r["input_mode"] = "text_only"
+    r["status"] = "multispecies_text_ml_used"
+    return r
+
+
+def final_cat_disease_pipeline(image_path, symptom_text=None):
+    image_result = predict_cat_disease_image(image_path)
+
+    if symptom_text is None or symptom_text.strip() == "":
+        return {
+            **image_result,
+            "text_prediction": "Not provided",
+            "text_model_label": None,
+            "text_model_confidence": None,
+            "final_prediction": image_result["image_prediction"],
+            "mode": "cat_image_ml_only"
+        }
+
+    text_result = predict_cat_symptom_text(symptom_text)
+
+    text_conf = text_result["text_model_confidence"]
+    text_pred = text_result["text_prediction"]
+    text_risk = text_result["risk_level"]
+
+    if text_conf is not None and text_conf < 0.40:
+        mode = "cat_image_ml_plus_low_confidence_text_ml"
+        final_risk = image_result["risk_level"]
+        text_phrase = f"{text_pred} from symptoms, but text confidence is low"
+    else:
+        mode = "cat_image_ml_plus_text_ml"
+        text_phrase = f"{text_pred} from symptoms"
+
+        if text_risk == "Emergency Vet":
+            final_risk = "Emergency Vet"
+        elif image_result["health_status"] == "diseased":
+            final_risk = image_result["risk_level"]
+        else:
+            final_risk = text_risk
+
+    return {
+        **image_result,
+        "text_prediction": text_pred,
+        "text_model_label": text_result["text_model_label"],
+        "text_model_confidence": text_conf,
+        "final_prediction": f"{image_result['image_prediction']} from image; {text_phrase}",
+        "risk_level": final_risk,
+        "mode": mode
+    }
+
+
+def final_fish_disease_pipeline(image_path, symptom_text=None):
+    image_result = predict_fish_disease_image(image_path)
+
+    if symptom_text is None or symptom_text.strip() == "":
+        return {
+            **image_result,
+            "text_prediction": "Not provided",
+            "text_model_label": None,
+            "text_model_confidence": None,
+            "final_prediction": image_result["image_prediction"],
+            "mode": "fish_image_ml_only"
+        }
+
+    text_result = fallback_symptom_triage("goldfish", symptom_text)
+
+    return {
+        **image_result,
+        "text_prediction": text_result["final_prediction"],
+        "text_model_label": text_result.get("text_model_label"),
+        "text_model_confidence": text_result.get("text_model_confidence"),
+        "final_prediction": f"{image_result['image_prediction']} from image; {text_result['final_prediction']} from symptoms",
+        "risk_level": image_result["risk_level"] if image_result["health_status"] == "diseased" else text_result["risk_level"],
+        "mode": "fish_image_ml_plus_text_ml"
+    }
+
+
+def final_dog_disease_pipeline(image_path, symptom_text=None):
+    image_result = predict_dog_image(image_path)
+
+    if symptom_text is None or symptom_text.strip() == "":
+        return {
+            **image_result,
+            "text_prediction": "Not provided",
+            "text_model_label": None,
+            "text_model_confidence": None,
+            "final_prediction": image_result["image_prediction"],
+            "mode": "dog_image_ml_only"
+        }
+
+    text_result = predict_dog_symptom_text(symptom_text)
+
+    return {
+        **image_result,
+        "text_prediction": text_result["text_prediction"],
+        "text_model_label": text_result["text_model_label"],
+        "text_model_confidence": text_result["text_model_confidence"],
+        "final_prediction": f"{image_result['image_prediction']} from image; {text_result['text_prediction']} from symptoms",
+        "risk_level": image_result["risk_level"] if image_result["health_status"] == "diseased" else "Monitor / See Vet",
+        "mode": "dog_image_ml_plus_text_ml"
+    }
+
+
+def final_unified_pet_triage_pipeline(image_path, symptom_text=None):
+    species_result = predict_species_open_set_strict(image_path)
+    top_species = species_result["top1_class"]
+
+    if is_safe_dog_for_disease(species_result):
+        mismatch = check_species_text_mismatch("dog", symptom_text)
+        if mismatch["mismatch"]:
+            return {
+                "species_result": species_result,
+                "final_species": "unsupported_or_mismatch",
+                "triage_result": None,
+                "status": "species_text_mismatch",
+                "message": mismatch["message"]
+            }
+
+        return {
+            "species_result": species_result,
+            "final_species": "dog",
+            "triage_result": final_dog_disease_pipeline(image_path, symptom_text),
+            "status": "dog_image_disease_pipeline_used"
+        }
+
+    if is_safe_cat_for_disease(species_result):
+        mismatch = check_species_text_mismatch("cat", symptom_text)
+        if mismatch["mismatch"]:
+            return {
+                "species_result": species_result,
+                "final_species": "unsupported_or_mismatch",
+                "triage_result": None,
+                "status": "species_text_mismatch",
+                "message": mismatch["message"]
+            }
+
+        cat_result = final_cat_disease_pipeline(image_path, symptom_text)
+        if species_result["top1_class"] == "unknown":
+            cat_result["species_note"] = "Species gate was uncertain, but cat was rescued from top predictions."
+
+        return {
+            "species_result": species_result,
+            "final_species": "cat",
+            "triage_result": cat_result,
+            "status": "cat_smart_pipeline_used"
+        }
+
+    if is_safe_fish_for_disease(species_result):
+        mismatch = check_species_text_mismatch("goldfish", symptom_text)
+        if mismatch["mismatch"]:
+            return {
+                "species_result": species_result,
+                "final_species": "unsupported_or_mismatch",
+                "triage_result": None,
+                "status": "species_text_mismatch",
+                "message": mismatch["message"]
+            }
+
+        return {
+            "species_result": species_result,
+            "final_species": "goldfish",
+            "triage_result": final_fish_disease_pipeline(image_path, symptom_text),
+            "status": "fish_image_disease_pipeline_used"
+        }
+
+    if is_safe_fallback_species(species_result, symptom_text):
+        mismatch = check_species_text_mismatch(top_species, symptom_text)
+        if mismatch["mismatch"]:
+            return {
+                "species_result": species_result,
+                "final_species": "unsupported_or_mismatch",
+                "triage_result": None,
+                "status": "species_text_mismatch",
+                "message": mismatch["message"]
+            }
+
+        return {
+            "species_result": species_result,
+            "final_species": top_species,
+            "triage_result": fallback_symptom_triage(top_species, symptom_text),
+            "status": "ml_symptom_triage_used"
+        }
+
+    return {
+        "species_result": species_result,
+        "final_species": "unsupported_or_unclear_image",
+        "triage_result": None,
+        "status": "disease_prediction_stopped",
+        "message": "Disease prediction stopped because the image is unsupported, unclear, or not a trained pet species."
+    }
+
+
+# ==========================================================
+# GRADIO APP FUNCTION
+# ==========================================================
+
+def build_markdown_output(result, transcript=""):
+    lines = []
+
+    lines.append("## 🐾 RCM-Pet Triage Result")
+
+    if transcript:
+        lines.append(f"**Audio transcript:** {transcript}")
+
+    if "species_result" in result:
+        sr = result["species_result"]
+        tr = result.get("triage_result")
+
+        lines.append(f"**Top species guess:** {sr.get('top1_class')} ({format_prob(sr.get('top1_confidence'))})")
+        lines.append(f"**Final species / decision:** {result.get('final_species')}")
+        lines.append(f"**Status:** {result.get('status')}")
+
+        if tr is None:
+            lines.append(f"**Message:** {result.get('message')}")
+            lines.append("")
+            lines.append("⚠️ Disease prediction was stopped for safety.")
+        else:
+            lines.append(f"**Health status:** {clean_label(tr.get('health_status'))}")
+            lines.append(f"**Final prediction:** {tr.get('final_prediction')}")
+            lines.append(f"**Risk level:** {tr.get('risk_level')}")
+            lines.append(f"**Mode:** {clean_label(tr.get('mode'))}")
+
+            if tr.get("image_prediction") is not None:
+                lines.append(f"**Image finding:** {tr.get('image_prediction')}")
+
+            if tr.get("text_prediction") is not None:
+                lines.append(f"**Text finding:** {tr.get('text_prediction')}")
+
+            if tr.get("text_model_confidence") is not None:
+                lines.append(f"**Text ML confidence:** {format_prob(tr.get('text_model_confidence'))}")
+
+            if tr.get("image_confidence") is not None:
+                lines.append(f"**Image ML confidence:** {format_prob(tr.get('image_confidence'))}")
+
+    else:
+        lines.append(f"**Input mode:** {result.get('input_mode')}")
+        lines.append(f"**Species:** {result.get('species')}")
+        lines.append(f"**Health status:** {clean_label(result.get('health_status'))}")
+        lines.append(f"**Final prediction:** {result.get('final_prediction')}")
+        lines.append(f"**Risk level:** {result.get('risk_level')}")
+        lines.append(f"**Status:** {result.get('status')}")
+
+        if result.get("text_model_confidence") is not None:
+            lines.append(f"**Text ML confidence:** {format_prob(result.get('text_model_confidence'))}")
+
+    lines.append("")
+    lines.append("---")
+    lines.append("⚠️ **Safety note:** RCM-Pet is a triage assistant, not a final veterinary diagnosis. Please consult a veterinarian for serious symptoms.")
+
+    return "\n\n".join(lines)
+
+
+def run_rcm_pet(image_path, symptom_text, audio_path):
+    symptom_text = symptom_text or ""
+    audio_transcript = ""
+
+    if audio_path is not None:
+        audio_transcript = transcribe_audio(audio_path)
+
+    combined_text = " ".join([symptom_text.strip(), audio_transcript.strip()]).strip()
+
+    if image_path is None and combined_text == "":
+        result = {
+            "input_mode": "none",
+            "status": "no_input",
+            "species": "unknown",
+            "health_status": "unknown",
+            "final_prediction": "Please provide an image, text symptoms, or voice symptoms.",
+            "risk_level": "Unknown"
+        }
+        return build_markdown_output(result), result
+
+    if image_path is None:
+        result = final_text_only_triage(combined_text)
+        if audio_transcript:
+            result["input_mode"] = "audio_to_text" if symptom_text.strip() == "" else "text_plus_audio"
+        return build_markdown_output(result, audio_transcript), result
+
+    result = final_unified_pet_triage_pipeline(image_path, combined_text)
+
+    if audio_transcript:
+        result["audio_transcript"] = audio_transcript
+
+    return build_markdown_output(result, audio_transcript), result
+
+
+# ==========================================================
+# GRADIO UI
+# ==========================================================
+
+description = """
+RCM-Pet is a risk-calibrated multimodal pet triage assistant.
+
+You can use:
+- Image only
+- Text only
+- Audio only
+- Image + text
+- Image + audio
+- Image + text + audio
+
+Supported species: dog, cat, fish/goldfish, rabbit, bird/parrot, and unknown safety stop.
+"""
+
+with gr.Blocks(theme=gr.themes.Soft()) as demo:
+    gr.Markdown("# 🐾 RCM-Pet: Multimodal Pet Disease Triage System")
+    gr.Markdown(description)
+
+    with gr.Row():
+        with gr.Column():
+            image_input = gr.Image(
+                type="filepath",
+                label="Upload pet image",
+                sources=["upload", "webcam"]
+            )
+
+            text_input = gr.Textbox(
+                label="Write symptoms",
+                placeholder="Example: my cat is sneezing and has runny nose",
+                lines=4
+            )
+
+            audio_input = gr.Audio(
+                sources=["microphone", "upload"],
+                type="filepath",
+                label="Speak or upload symptoms"
+            )
+
+            submit_btn = gr.Button("Run RCM-Pet Triage", variant="primary")
+
+        with gr.Column():
+            markdown_output = gr.Markdown(label="Triage Summary")
+            json_output = gr.JSON(label="Raw Model Output")
+
+    submit_btn.click(
+        fn=run_rcm_pet,
+        inputs=[image_input, text_input, audio_input],
+        outputs=[markdown_output, json_output]
+    )
+
+    gr.Markdown(
+        """
+        ### Important
+        This system provides preliminary triage support only. It does not replace professional veterinary diagnosis.
+        """
+    )
+
+
+if __name__ == "__main__":
+    demo.launch()

model_coverage_summary.json

@@ -0,0 +1,30 @@
+{
+    "project_name": "RCM-Pet",
+    "deployment": "Hugging Face Gradio Space",
+    "input_modes": [
+        "image_only",
+        "text_only",
+        "audio_only",
+        "image_plus_text",
+        "image_plus_audio",
+        "image_plus_text_plus_audio"
+    ],
+    "species_model": "MobileNetV2 7-class with unknown",
+    "image_disease_models": {
+        "dog": "ResNet18 binary + EfficientNet-B0 disease type",
+        "cat": "EfficientNet-B0 disease classifier",
+        "fish": "EfficientNet-B0 healthy/infected classifier"
+    },
+    "text_models": {
+        "dog": "TF-IDF + Logistic Regression",
+        "cat": "TF-IDF + Logistic Regression",
+        "rabbit_bird_parrot_fish": "TF-IDF + Logistic Regression"
+    },
+    "audio": "Whisper Tiny converts speech to text",
+    "safety": [
+        "unknown class",
+        "confidence gate",
+        "species-text mismatch check",
+        "low-confidence text guard"
+    ]
+}

models/best_cat_disease_efficientnet_b0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59ce33edf5e5075d42b993664ae5b6a4b3c1ce5c5c179d2bd6d738633a4cbf94
+size 16354029

models/best_dog_disease_resnet18.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c68bda7594ed4a9d27b7eabaa5d4a936435b2eede98247ff55b63199caffe4de
+size 44790347

models/best_dog_disease_type_efficientnet_b0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db684667c0fca3fc6831295098ac5107aa7f8279ea6b5b04995497921d050bfd
+size 16350611

models/best_fish_disease_efficientnet_b0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d190b04a54113c2effab41c03578cf0bdc5b300576ddafdc8946142181100d4
+size 16344091

models/best_species_mobilenetv2_7class_unknown.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d1f0dd5fd5364b7a2fbf6a9c9bcfd926111dd8856b47d127c6eb8009161ef88
+size 9187147

models/cat_disease_config.json

@@ -0,0 +1,20 @@
+{
+    "model_name": "efficientnet_b0",
+    "task": "cat_skin_disease_4class",
+    "class_names": [
+        "flea_allergy",
+        "healthy",
+        "ringworm",
+        "scabies"
+    ],
+    "class_to_idx": {
+        "flea_allergy": 0,
+        "healthy": 1,
+        "ringworm": 2,
+        "scabies": 3
+    },
+    "img_size": 224,
+    "best_model_path": "/content/drive/MyDrive/DIP_DATASET/Models/cat_disease_classifier/best_cat_disease_efficientnet_b0.pth",
+    "best_val_acc": 0.9583333333333334,
+    "test_acc": 0.9530201342281879
+}

models/cat_symptom_tfidf_logreg.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2bc8c93459b9ba3453a6bef9b48bdbfe3a2086e71430f2f463145df176bb3e1
+size 78940

models/dog_symptom_tfidf_logreg.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf94cabd45ab380975162556e226b8fdd67c1b9cd15f52c956ed99bceda6861f
+size 27220

models/fish_disease_config.json

@@ -0,0 +1,16 @@
+{
+    "model_name": "efficientnet_b0",
+    "task": "fish_disease_binary_healthy_infected",
+    "class_names": [
+        "healthy",
+        "infected"
+    ],
+    "class_to_idx": {
+        "healthy": 0,
+        "infected": 1
+    },
+    "img_size": 224,
+    "best_model_path": "/content/drive/MyDrive/DIP_DATASET/Models/fish_disease_classifier/best_fish_disease_efficientnet_b0.pth",
+    "best_val_acc": 0.9777777777777777,
+    "test_acc": 0.9787234042553191
+}

models/multispecies_symptom_tfidf_logreg.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ff7b90c7d656cf2ed14fab3a232a34ecd67b386f594715b2c7ed1278ba58fab
+size 45236

models/species_label_map_7class_unknown.json

@@ -0,0 +1,20 @@
+{
+    "class_to_idx": {
+        "bird": 0,
+        "cat": 1,
+        "dog": 2,
+        "goldfish": 3,
+        "parrot": 4,
+        "rabbit": 5,
+        "unknown": 6
+    },
+    "idx_to_class": {
+        "0": "bird",
+        "1": "cat",
+        "2": "dog",
+        "3": "goldfish",
+        "4": "parrot",
+        "5": "rabbit",
+        "6": "unknown"
+    }
+}

requirements.txt

@@ -0,0 +1,11 @@
+gradio
+torch
+torchvision
+pillow
+numpy
+scikit-learn
+joblib
+transformers
+accelerate
+soundfile
+librosa