Upload 8 files

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+example_images/cat.jpg filter=lfs diff=lfs merge=lfs -text
+example_images/dog2.jpeg filter=lfs diff=lfs merge=lfs -text
+example_images/leonberger.jpg filter=lfs diff=lfs merge=lfs -text
+example_images/snow_leopard.jpeg filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,42 @@
+import gradio as gr
+from transformers import pipeline
+
+# Load models
+vit_classifier = pipeline("image-classification", model="chrisis2/vit-base-oxford-iiit-pets")
+clip_detector = pipeline(model="openai/clip-vit-large-patch14", task="zero-shot-image-classification")
+
+labels_oxford_pets = [
+    'Siamese', 'Birman', 'shiba inu', 'staffordshire bull terrier', 'basset hound', 'Bombay', 'japanese chin',
+    'chihuahua', 'german shorthaired', 'pomeranian', 'beagle', 'english cocker spaniel', 'american pit bull terrier',
+    'Ragdoll', 'Persian', 'Egyptian Mau', 'miniature pinscher', 'Sphynx', 'Maine Coon', 'keeshond', 'yorkshire terrier',
+    'havanese', 'leonberger', 'wheaten terrier', 'american bulldog', 'english setter', 'boxer', 'newfoundland', 'Bengal',
+    'samoyed', 'British Shorthair', 'great pyrenees', 'Abyssinian', 'pug', 'saint bernard', 'Russian Blue', 'scottish terrier'
+]
+
+def classify_pet(image):
+    vit_results = vit_classifier(image)
+    vit_output = {result['label']: result['score'] for result in vit_results}
+    
+    clip_results = clip_detector(image, candidate_labels=labels_oxford_pets)
+    clip_output = {result['label']: result['score'] for result in clip_results}
+    
+    return {"ViT Classification": vit_output, "CLIP Zero-Shot Classification": clip_output}
+
+example_images = [
+    ["example_images/dog1.jpeg"],
+    ["example_images/dog2.jpeg"],
+    ["example_images/leonberger.jpg"],
+    ["example_images/snow_leopard.jpeg"],
+    ["example_images/cat.jpg"]
+]
+
+iface = gr.Interface(
+    fn=classify_pet,
+    inputs=gr.Image(type="filepath"),
+    outputs=gr.JSON(),
+    title="Pet Classification Comparison",
+    description="Upload an image of a pet, and compare results from a trained ViT model and a zero-shot CLIP model.",
+    examples=example_images
+)
+
+iface.launch()

oxford_pets_zero_shot.ipynb

@@ -0,0 +1,182 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import pipeline\n",
+    "\n",
+    "checkpoint = \"openai/clip-vit-large-patch14\"\n",
+    "detector = pipeline(model=checkpoint, task=\"zero-shot-image-classification\")\n",
+    "#checkpoint = \"google/siglip-so400m-patch14-384\"\n",
+    "#detector = pipeline(task=\"zero-shot-image-classification\", model=\"google/siglip-so400m-patch14-384\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from datasets import load_dataset\n",
+    "\n",
+    "dataset = load_dataset('pcuenq/oxford-pets')\n",
+    "dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset['train'][0]['image']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from PIL import Image\n",
+    "import io\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "labels_oxford_pets = ['Siamese', 'Birman', 'shiba inu', 'staffordshire bull terrier', 'basset hound', 'Bombay', 'japanese chin', 'chihuahua', 'german shorthaired', 'pomeranian', 'beagle', 'english cocker spaniel', 'american pit bull terrier', 'Ragdoll', 'Persian', 'Egyptian Mau', 'miniature pinscher', 'Sphynx', 'Maine Coon', 'keeshond', 'yorkshire terrier', 'havanese', 'leonberger', 'wheaten terrier', 'american bulldog', 'english setter', 'boxer', 'newfoundland', 'Bengal', 'samoyed', 'British Shorthair', 'great pyrenees', 'Abyssinian', 'pug', 'saint bernard', 'Russian Blue', 'scottish terrier']\n",
+    "\n",
+    "# List to store true labels and predicted labels\n",
+    "true_labels = []\n",
+    "predicted_labels = []"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "for i in tqdm(range(len(dataset['train']))):\n",
+    "    # Get the image bytes from the dataset\n",
+    "    image_bytes = dataset['train'][i]['image']['bytes']\n",
+    "    \n",
+    "    # Convert the bytes to a PIL image\n",
+    "    image = Image.open(io.BytesIO(image_bytes))\n",
+    "    \n",
+    "    # Run the detector on the image with the provided labels\n",
+    "    results = detector(image, candidate_labels=labels_oxford_pets)\n",
+    "    # Sort the results by score in descending order\n",
+    "    sorted_results = sorted(results, key=lambda x: x['score'], reverse=True)\n",
+    "    \n",
+    "    # Get the top predicted label\n",
+    "    predicted_label = sorted_results[0]['label']\n",
+    "    \n",
+    "    # Append the true and predicted labels to the respective lists\n",
+    "    true_labels.append(dataset['train'][i]['label'])\n",
+    "    predicted_labels.append(predicted_label)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.metrics import accuracy_score, precision_score, recall_score\n",
+    "\n",
+    "# Calculate accuracy\n",
+    "accuracy = accuracy_score(true_labels, predicted_labels)\n",
+    "\n",
+    "# Calculate precision and recall\n",
+    "precision = precision_score(true_labels, predicted_labels, average='weighted', labels=labels_oxford_pets)\n",
+    "recall = recall_score(true_labels, predicted_labels, average='weighted', labels=labels_oxford_pets)\n",
+    "\n",
+    "# Print the results\n",
+    "print(f\"Accuracy: {accuracy:.4f}\")\n",
+    "print(f\"Precision: {precision:.4f}\")\n",
+    "print(f\"Recall: {recall:.4f}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Gradio example"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import gradio as gr\n",
+    "from transformers import pipeline\n",
+    "\n",
+    "# Load models\n",
+    "vit_classifier = pipeline(\"image-classification\", model=\"kuhs/vit-base-oxford-iiit-pets\")\n",
+    "clip_detector = pipeline(model=\"openai/clip-vit-large-patch14\", task=\"zero-shot-image-classification\")\n",
+    "\n",
+    "labels_oxford_pets = [\n",
+    "    'Siamese', 'Birman', 'shiba inu', 'staffordshire bull terrier', 'basset hound', 'Bombay', 'japanese chin',\n",
+    "    'chihuahua', 'german shorthaired', 'pomeranian', 'beagle', 'english cocker spaniel', 'american pit bull terrier',\n",
+    "    'Ragdoll', 'Persian', 'Egyptian Mau', 'miniature pinscher', 'Sphynx', 'Maine Coon', 'keeshond', 'yorkshire terrier',\n",
+    "    'havanese', 'leonberger', 'wheaten terrier', 'american bulldog', 'english setter', 'boxer', 'newfoundland', 'Bengal',\n",
+    "    'samoyed', 'British Shorthair', 'great pyrenees', 'Abyssinian', 'pug', 'saint bernard', 'Russian Blue', 'scottish terrier'\n",
+    "]\n",
+    "\n",
+    "def classify_pet(image):\n",
+    "    vit_results = vit_classifier(image)\n",
+    "    vit_output = {result['label']: result['score'] for result in vit_results}\n",
+    "    \n",
+    "    clip_results = clip_detector(image, candidate_labels=labels_oxford_pets)\n",
+    "    clip_output = {result['label']: result['score'] for result in clip_results}\n",
+    "    \n",
+    "    return {\"ViT Classification\": vit_output, \"CLIP Zero-Shot Classification\": clip_output}\n",
+    "\n",
+    "example_images = [\n",
+    "    [\"example_images/dog1.jpeg\"],\n",
+    "    [\"example_images/dog2.jpeg\"],\n",
+    "    [\"example_images/leonberger.jpg\"],\n",
+    "    [\"example_images/snow_leopard.jpeg\"],\n",
+    "    [\"example_images/cat.jpg\"]\n",
+    "]\n",
+    "\n",
+    "iface = gr.Interface(\n",
+    "    fn=classify_pet,\n",
+    "    inputs=gr.Image(type=\"filepath\"),\n",
+    "    outputs=gr.JSON(),\n",
+    "    title=\"Pet Classification Comparison\",\n",
+    "    description=\"Upload an image of a pet, and compare results from a trained ViT model and a zero-shot CLIP model.\",\n",
+    "    examples=example_images\n",
+    ")\n",
+    "\n",
+    "iface.launch()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

requirements.txt

@@ -0,0 +1,2 @@
+transformers
+torch