{ "cells": [ { "cell_type": "code", "execution_count": 60, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: transformers in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (5.5.0)\n", "Requirement already satisfied: torch in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (2.11.0)\n", "Requirement already satisfied: pillow in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (12.1.1)\n", "Requirement already satisfied: openai in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (2.30.0)\n", "Requirement already satisfied: huggingface-hub<2.0,>=1.5.0 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from transformers) (1.6.0)\n", "Requirement already satisfied: numpy>=1.17 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from transformers) (2.4.2)\n", "Requirement already satisfied: packaging>=20.0 in c:\\users\\kathe\\appdata\\roaming\\python\\python314\\site-packages (from transformers) (26.0)\n", "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from transformers) (6.0.3)\n", "Requirement already satisfied: regex>=2025.10.22 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from transformers) (2026.4.4)\n", "Requirement already satisfied: tokenizers<=0.23.0,>=0.22.0 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from transformers) (0.22.2)\n", "Requirement already satisfied: typer in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from transformers) (0.24.1)\n", "Requirement already satisfied: safetensors>=0.4.3 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from transformers) (0.7.0)\n", "Requirement already satisfied: tqdm>=4.27 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from transformers) (4.67.3)\n", "Requirement already satisfied: filelock>=3.10.0 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from huggingface-hub<2.0,>=1.5.0->transformers) (3.25.0)\n", "Requirement already satisfied: fsspec>=2023.5.0 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from huggingface-hub<2.0,>=1.5.0->transformers) (2026.2.0)\n", "Requirement already satisfied: hf-xet<2.0.0,>=1.3.2 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from huggingface-hub<2.0,>=1.5.0->transformers) (1.3.2)\n", "Requirement already satisfied: httpx<1,>=0.23.0 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from huggingface-hub<2.0,>=1.5.0->transformers) (0.28.1)\n", "Requirement already satisfied: typing-extensions>=4.1.0 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from huggingface-hub<2.0,>=1.5.0->transformers) (4.15.0)\n", "Requirement already satisfied: anyio in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.5.0->transformers) (4.12.1)\n", "Requirement already satisfied: certifi in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.5.0->transformers) (2026.2.25)\n", "Requirement already satisfied: httpcore==1.* in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.5.0->transformers) (1.0.9)\n", "Requirement already satisfied: idna in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.5.0->transformers) (3.11)\n", "Requirement already satisfied: h11>=0.16 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from httpcore==1.*->httpx<1,>=0.23.0->huggingface-hub<2.0,>=1.5.0->transformers) (0.16.0)\n", "Requirement already satisfied: setuptools<82 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from torch) (81.0.0)\n", "Requirement already satisfied: sympy>=1.13.3 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from torch) (1.14.0)\n", "Requirement already satisfied: networkx>=2.5.1 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from torch) (3.6.1)\n", "Requirement already satisfied: jinja2 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from torch) (3.1.6)\n", "Requirement already satisfied: distro<2,>=1.7.0 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from openai) (1.9.0)\n", "Requirement already satisfied: jiter<1,>=0.10.0 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from openai) (0.13.0)\n", "Requirement already satisfied: pydantic<3,>=1.9.0 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from openai) (2.12.5)\n", "Requirement already satisfied: sniffio in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from openai) (1.3.1)\n", "Requirement already satisfied: annotated-types>=0.6.0 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from pydantic<3,>=1.9.0->openai) (0.7.0)\n", "Requirement already satisfied: pydantic-core==2.41.5 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from pydantic<3,>=1.9.0->openai) (2.41.5)\n", "Requirement already satisfied: typing-inspection>=0.4.2 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from pydantic<3,>=1.9.0->openai) (0.4.2)\n", "Requirement already satisfied: mpmath<1.4,>=1.1.0 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from sympy>=1.13.3->torch) (1.3.0)\n", "Requirement already satisfied: colorama in c:\\users\\kathe\\appdata\\roaming\\python\\python314\\site-packages (from tqdm>=4.27->transformers) (0.4.6)\n", "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from jinja2->torch) (3.0.3)\n", "Requirement already satisfied: click>=8.2.1 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from typer->transformers) (8.3.1)\n", "Requirement already satisfied: shellingham>=1.3.0 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from typer->transformers) (1.5.4)\n", "Requirement already satisfied: rich>=12.3.0 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from typer->transformers) (14.3.3)\n", "Requirement already satisfied: annotated-doc>=0.0.2 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from typer->transformers) (0.0.4)\n", "Requirement already satisfied: markdown-it-py>=2.2.0 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from rich>=12.3.0->typer->transformers) (4.0.0)\n", "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in c:\\users\\kathe\\appdata\\roaming\\python\\python314\\site-packages (from rich>=12.3.0->typer->transformers) (2.19.2)\n", "Requirement already satisfied: mdurl~=0.1 in c:\\users\\kathe\\appdata\\local\\python\\pythoncore-3.14-64\\lib\\site-packages (from markdown-it-py>=2.2.0->rich>=12.3.0->typer->transformers) (0.1.2)\n", "Note: you may need to restart the kernel to use updated packages.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", "[notice] A new release of pip is available: 25.3 -> 26.0.1\n", "[notice] To update, run: python.exe -m pip install --upgrade pip\n" ] } ], "source": [ "%pip install transformers torch pillow openai\n", "from transformers import pipeline\n", "from PIL import Image\n", "import os\n", "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 63, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Model path exists: True\n", "Image folder exists: True\n", "Images: ['Cheetah_032.jpg', 'Leopard_001.jpg', 'Lion_003.jpg', 'Puma_001.jpg', 'Tiger_001.jpg']\n" ] } ], "source": [ "MODEL_PATH = \"./cat-vit\"\n", "IMAGE_FOLDER = \"./Cats-classification-app/example_images\"\n", "\n", "labels = [\"cheetah\", \"leopard\", \"lion\", \"puma\", \"tiger\"]\n", "clip_labels = [f\"a photo of a {label}\" for label in labels]\n", "\n", "print(\"Model path exists:\", os.path.exists(MODEL_PATH))\n", "print(\"Image folder exists:\", os.path.exists(IMAGE_FOLDER))\n", "print(\"Images:\", [f for f in os.listdir(IMAGE_FOLDER) if f.lower().endswith((\".jpg\", \".jpeg\", \".png\"))])" ] }, { "cell_type": "code", "execution_count": 64, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "1bf87a05bbc346c9b3f30eb950c1f3a5", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Loading weights: 0%| | 0/200 [00:00\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
imagetrue_labelcustom_predcustom_scoreclip_predclip_scoreopenai_predcustom_correctclip_correctopenai_correct
0Cheetah_032.jpgcheetahcheetah0.5264cheetah0.8319ERROR: name 'base64' is not definedTrueTrueFalse
1Leopard_001.jpgleopardleopard0.5127leopard0.9232ERROR: name 'base64' is not definedTrueTrueFalse
2Lion_003.jpglionlion0.5408lion0.9949ERROR: name 'base64' is not definedTrueTrueFalse
3Puma_001.jpgpumapuma0.6112puma0.9986ERROR: name 'base64' is not definedTrueTrueFalse
4Tiger_001.jpgtigertiger0.6976tiger0.9892ERROR: name 'base64' is not definedTrueTrueFalse
\n", "" ], "text/plain": [ " image true_label custom_pred custom_score clip_pred clip_score \\\n", "0 Cheetah_032.jpg cheetah cheetah 0.5264 cheetah 0.8319 \n", "1 Leopard_001.jpg leopard leopard 0.5127 leopard 0.9232 \n", "2 Lion_003.jpg lion lion 0.5408 lion 0.9949 \n", "3 Puma_001.jpg puma puma 0.6112 puma 0.9986 \n", "4 Tiger_001.jpg tiger tiger 0.6976 tiger 0.9892 \n", "\n", " openai_pred custom_correct clip_correct \\\n", "0 ERROR: name 'base64' is not defined True True \n", "1 ERROR: name 'base64' is not defined True True \n", "2 ERROR: name 'base64' is not defined True True \n", "3 ERROR: name 'base64' is not defined True True \n", "4 ERROR: name 'base64' is not defined True True \n", "\n", " openai_correct \n", "0 False \n", "1 False \n", "2 False \n", "3 False \n", "4 False " ] }, "execution_count": 73, "metadata": {}, "output_type": "execute_result" } ], "source": [ "results = []\n", "\n", "image_files = sorted([\n", " f for f in os.listdir(IMAGE_FOLDER)\n", " if f.lower().endswith((\".jpg\", \".jpeg\", \".png\"))\n", "])\n", "\n", "for img_file in image_files:\n", " image_path = os.path.join(IMAGE_FOLDER, img_file)\n", " image = Image.open(image_path).convert(\"RGB\")\n", " true_label = get_true_label(img_file)\n", "\n", " # Custom model\n", " custom_result = custom_model(image)[0]\n", " custom_pred = custom_result[\"label\"].lower()\n", " custom_score = float(custom_result[\"score\"])\n", "\n", " # CLIP model\n", " clip_result = clip_model(image, candidate_labels=clip_labels)[0]\n", " clip_pred = clip_result[\"label\"].replace(\"a photo of a \", \"\").lower()\n", " clip_score = float(clip_result[\"score\"])\n", "\n", " # OpenAI model\n", " try:\n", " openai_pred = predict_openai_label(image_path)\n", " openai_correct = openai_pred == true_label\n", " except Exception as e:\n", " openai_pred = f\"ERROR: {e}\"\n", " openai_correct = False\n", "\n", " results.append({\n", " \"image\": img_file,\n", " \"true_label\": true_label,\n", " \"custom_pred\": custom_pred,\n", " \"custom_score\": round(custom_score, 4),\n", " \"clip_pred\": clip_pred,\n", " \"clip_score\": round(clip_score, 4),\n", " \"openai_pred\": openai_pred,\n", " \"custom_correct\": custom_pred == true_label,\n", " \"clip_correct\": clip_pred == true_label,\n", " \"openai_correct\": openai_correct,\n", " })\n", "\n", "df = pd.DataFrame(results)\n", "df" ] }, { "cell_type": "code", "execution_count": 74, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Custom accuracy: 1.0\n", "CLIP accuracy: 1.0\n", "OpenAI accuracy: 0.0\n" ] } ], "source": [ "custom_accuracy = df[\"custom_correct\"].mean()\n", "clip_accuracy = df[\"clip_correct\"].mean()\n", "openai_accuracy = df[\"openai_correct\"].mean()\n", "\n", "print(\"Custom accuracy:\", round(custom_accuracy, 4))\n", "print(\"CLIP accuracy:\", round(clip_accuracy, 4))\n", "print(\"OpenAI accuracy:\", round(openai_accuracy, 4))" ] }, { "cell_type": "code", "execution_count": 75, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Saved to ../comparison_results_with_openai.csv\n" ] } ], "source": [ "df.to_csv(\"../comparison_results_with_openai.csv\", index=False)\n", "print(\"Saved to ../comparison_results_with_openai.csv\")" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.14.3" } }, "nbformat": 4, "nbformat_minor": 2 }