Update app.py

app.py

@@ -11,6 +11,7 @@ from transformers import (
 
 import open_clip
 
+
 st.set_page_config(page_title="Multi-Domain Zero Shot AI", layout="wide")
 
 st.title("Multi-Domain Zero Shot Image Classification")
@@ -26,15 +27,17 @@ device = "cpu"
 @st.cache_resource
 def load_models():
 
-    # -------- BIOMED CLIP (via Transformers CLIP) --------
-    biomed_model = CLIPModel.from_pretrained(
-        "microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224"
-    ).to(device).eval()
+    # -------- BIOMED CLIP --------
+    biomed_model, _, biomed_preprocess = open_clip.create_model_and_transforms(
+        "hf-hub:microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224"
+    )
 
-    biomed_processor = CLIPProcessor.from_pretrained(
-        "microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224"
+    biomed_tokenizer = open_clip.get_tokenizer(
+        "hf-hub:microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224"
     )
 
+    biomed_model = biomed_model.to(device).eval()
+
     # -------- REMOTE CLIP --------
     remote_model, _, remote_preprocess = open_clip.create_model_and_transforms(
         "ViT-B-32",
@@ -65,7 +68,8 @@ def load_models():
 
     return (
         biomed_model,
-        biomed_processor,
+        biomed_preprocess,
+        biomed_tokenizer,
         remote_model,
         remote_preprocess,
         remote_tokenizer,
@@ -78,7 +82,8 @@ def load_models():
 
 (
     biomed_model,
-    biomed_processor,
+    biomed_preprocess,
+    biomed_tokenizer,
     remote_model,
     remote_preprocess,
     remote_tokenizer,
@@ -174,22 +179,19 @@ if uploaded_file:
 
 
     # --------------------------------------------------
-    # MEDICAL / SKIN (BIOMEDCLIP)
+    # MEDICAL + SKIN (BIOMEDCLIP)
     # --------------------------------------------------
 
     if dataset_key in ["medical", "skin_disease"]:
 
-        inputs = biomed_processor(
-            text=text_queries,
-            images=image,
-            return_tensors="pt",
-            padding=True
-        ).to(device)
+        img = biomed_preprocess(image).unsqueeze(0).to(device)
+        text = biomed_tokenizer(text_queries)
 
         with torch.no_grad():
-            outputs = biomed_model(**inputs)
+            image_features = biomed_model.encode_image(img)
+            text_features = biomed_model.encode_text(text)
 
-        similarity = outputs.logits_per_image.softmax(dim=1)
+        similarity = (image_features @ text_features.T).softmax(dim=-1)
 
 
     # --------------------------------------------------