Update app.py

app.py

@@ -11,11 +11,10 @@ from transformers import (
 
 import open_clip
 
-
 st.set_page_config(page_title="Multi-Domain Zero Shot AI", layout="wide")
 
 st.title("Multi-Domain Zero Shot Image Classification")
-st.write("BioMedCLIP + RemoteCLIP + CLIP + BLIP Captioning")
+st.write("BiomedCLIP + RemoteCLIP + CLIP + BLIP Captioning")
 
 device = "cpu"
 
@@ -27,22 +26,19 @@ device = "cpu"
 @st.cache_resource
 def load_models():
 
-    # -------- BIO MED CLIP --------
-    biomed_model, _, biomed_preprocess = open_clip.create_model_and_transforms(
-        "ViT-B-16",
-        pretrained="hf-hub:microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224"
-    )
+    # -------- BIOMED CLIP (via Transformers CLIP) --------
+    biomed_model = CLIPModel.from_pretrained(
+        "microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224"
+    ).to(device).eval()
 
-    biomed_tokenizer = open_clip.get_tokenizer(
-        "hf-hub:microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224"
+    biomed_processor = CLIPProcessor.from_pretrained(
+        "microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224"
     )
 
-    biomed_model = biomed_model.to(device).eval()
-
     # -------- REMOTE CLIP --------
     remote_model, _, remote_preprocess = open_clip.create_model_and_transforms(
         "ViT-B-32",
-        pretrained="hf-hub:microsoft/remoteclip-vit-base"
+        pretrained="laion2b_s34b_b79k"
     )
 
     remote_tokenizer = open_clip.get_tokenizer("ViT-B-32")
@@ -69,8 +65,7 @@ def load_models():
 
     return (
         biomed_model,
-        biomed_preprocess,
-        biomed_tokenizer,
+        biomed_processor,
         remote_model,
         remote_preprocess,
         remote_tokenizer,
@@ -83,8 +78,7 @@ def load_models():
 
 (
     biomed_model,
-    biomed_preprocess,
-    biomed_tokenizer,
+    biomed_processor,
     remote_model,
     remote_preprocess,
     remote_tokenizer,
@@ -144,110 +138,6 @@ templates = {
 }
 
 
-# --------------------------------------------------
-# EXPLANATIONS
-# --------------------------------------------------
-
-EXPLANATIONS = {
-
-"pneumonia": """
-Cloudy or opaque regions may appear in lung areas.
-These patterns indicate possible infection or inflammation.
-Fluid buildup inside air sacs reduces oxygen exchange.
-Symptoms may include cough and breathing difficulty.
-Medical confirmation is required.
-""",
-
-"Normal": """
-Lungs appear clear without visible opacities.
-Both lung regions look symmetrical.
-No fluid accumulation is visible.
-The diaphragm boundaries appear normal.
-These features indicate healthy lungs.
-""",
-
-"Melanoma": """
-A dark pigmented lesion may appear on skin.
-Irregular borders and uneven color can occur.
-Melanoma is a serious form of skin cancer.
-Early detection improves treatment success.
-Consult a dermatologist for confirmation.
-""",
-
-"eczema": """
-Skin may appear red and inflamed.
-Dry and flaky patches are visible.
-Eczema causes itching and irritation.
-Often triggered by allergies or skin sensitivity.
-Treatment helps control symptoms.
-""",
-
-"psoriasis": """
-Thick red patches with white scales may appear.
-Skin cell growth becomes abnormally rapid.
-Affected regions can become itchy or cracked.
-It is an autoimmune skin condition.
-Dermatology treatment is recommended.
-""",
-
-"Normal Skin": """
-Skin tone appears even and smooth.
-No visible lesions or inflammation.
-Texture looks consistent across the surface.
-No abnormal pigmentation is visible.
-These features indicate healthy skin.
-""",
-
-"HIGHWAY": """
-A long linear road structure is visible.
-The road may extend across large areas.
-Vehicles typically use these routes for travel.
-Nearby regions may include urban infrastructure.
-Such patterns indicate highways.
-""",
-
-"RIVER": """
-A long winding water body is visible.
-Rivers often appear curved across terrain.
-They transport water through landscapes.
-Vegetation or farmland may surround them.
-This pattern indicates a natural river.
-""",
-
-"FOREST": """
-The region appears densely covered with trees.
-Vegetation creates textured green patterns.
-Different shades indicate varying tree heights.
-Forests support diverse ecosystems.
-This pattern indicates forest land.
-""",
-
-"INDUSTRIAL": """
-Large rectangular buildings are visible.
-Industrial zones contain factories and warehouses.
-Road networks connect production facilities.
-Structures appear dense and organized.
-Such patterns indicate industrial areas.
-""",
-
-"CROP": """
-Fields appear in organized rectangular patterns.
-Different shades indicate crop growth stages.
-Irrigation channels may be visible.
-Agricultural land is clearly structured.
-This pattern indicates cultivated crops.
-""",
-
-"HEALTHY": """
-Leaf surface appears green and smooth.
-No visible fungal spots are present.
-Leaf veins look healthy and intact.
-Edges appear natural and undamaged.
-These features indicate a healthy plant.
-"""
-}
-
-
 # --------------------------------------------------
 # SIDEBAR
 # --------------------------------------------------
@@ -284,20 +174,27 @@ if uploaded_file:
 
 
     # --------------------------------------------------
-    # MODEL SELECTION
+    # MEDICAL / SKIN (BIOMEDCLIP)
     # --------------------------------------------------
 
     if dataset_key in ["medical", "skin_disease"]:
 
-        img = biomed_preprocess(image).unsqueeze(0).to(device)
-        text = biomed_tokenizer(text_queries)
+        inputs = biomed_processor(
+            text=text_queries,
+            images=image,
+            return_tensors="pt",
+            padding=True
+        ).to(device)
 
         with torch.no_grad():
-            image_features = biomed_model.encode_image(img)
-            text_features = biomed_model.encode_text(text)
+            outputs = biomed_model(**inputs)
+
+        similarity = outputs.logits_per_image.softmax(dim=1)
 
-        similarity = (image_features @ text_features.T).softmax(dim=-1)
 
+    # --------------------------------------------------
+    # SATELLITE (REMOTE CLIP)
+    # --------------------------------------------------
 
     elif dataset_key == "satellite":
 
@@ -311,6 +208,10 @@ if uploaded_file:
         similarity = (image_features @ text_features.T).softmax(dim=-1)
 
 
+    # --------------------------------------------------
+    # AGRICULTURE (CLIP)
+    # --------------------------------------------------
+
     else:
 
         inputs = clip_processor(
@@ -349,20 +250,4 @@ if uploaded_file:
     )
 
     st.subheader("Image Description (BLIP)")
-    st.write(caption)
-
-
-    # --------------------------------------------------
-    # EXPLANATION
-    # --------------------------------------------------
-
-    explanation = EXPLANATIONS.get(
-        predicted_class,
-        "No explanation available."
-    )
-
-    st.subheader("Detailed Explanation")
-
-    for line in explanation.strip().split("\n"):
-        if line.strip():
-            st.write(line.strip())
+    st.write(caption)