Upload trained bird captioning model, tokenizer, image processor, species mapping, and captions

README.md

@@ -0,0 +1,94 @@
+
+# Bird Captioning and Classification Model (CUB-200-2011)
+
+This is a fine-tuned VisionEncoderDecoderModel based on `nlpconnect/vit-gpt2-image-captioning`, trained on the CUB-200-2011 dataset for bird species classification and image captioning.
+
+## Model Description
+- **Base Model**: ViT-GPT2 (`nlpconnect/vit-gpt2-image-captioning`)
+- **Tasks**:
+  - Generates descriptive captions for bird images, including species and attributes.
+  - Classifies images into one of 200 bird species.
+- **Dataset**: CUB-200-2011 (11,788 images, 200 bird species)
+- **Training**: 10 epochs, batch size 16, mixed precision, AdamW optimizer (lr=3e-5), combined loss (caption + 0.5 * classification).
+- **Best Validation Loss**: 0.0690 (Epoch 3)
+
+## Files
+- `pytorch_model.bin`: Trained model weights
+- `config.json`: Model configuration
+- `preprocessor_config.json`: ViTImageProcessor settings
+- `tokenizer_config.json`, `vocab.json`: GPT2 tokenizer files
+- `species_mapping.txt`: Mapping of class indices to bird species names
+- `cub200_captions.csv`: Generated captions for the dataset
+- `model.py`: Custom `BirdCaptioningModel` class definition
+
+## Usage
+### Prerequisites
+```bash
+pip install transformers torch huggingface_hub
+```
+
+### Load Model and Dependencies
+```python
+from transformers import ViTImageProcessor, AutoTokenizer
+from huggingface_hub import PyTorchModelHubMixin
+import torch
+from model import BirdCaptioningModel  # Save model.py locally
+
+# Load model
+model = BirdCaptioningModel.from_pretrained("INVERTO/bird-captioning-cub200")
+image_processor = ViTImageProcessor.from_pretrained("INVERTO/bird-captioning-cub200")
+tokenizer = AutoTokenizer.from_pretrained("INVERTO/bird-captioning-cub200")
+model.eval()
+
+# Load species mapping
+species_mapping = {}
+with open("species_mapping.txt", "r") as f:
+    for line in f:
+        idx, name = line.strip().split(",", 1)
+        species_mapping[int(idx)] = name
+```
+
+### Inference
+```python
+from PIL import Image
+
+def predict_bird_image(image_path):
+    image = Image.open(image_path).convert("RGB")
+    pixel_values = image_processor(image, return_tensors="pt").pixel_values
+    with torch.no_grad():
+        output_ids = model.base_model.generate(pixel_values, max_length=75, num_beams=4)
+        _, class_logits = model(pixel_values)
+        predicted_class_idx = torch.argmax(class_logits, dim=1).item()
+        confidence = torch.nn.functional.softmax(class_logits, dim=1)[0, predicted_class_idx].item() * 100
+        caption = tokenizer.decode(output_ids[0], skip_special_tokens=True)
+        species = species_mapping.get(predicted_class_idx, "Unknown")
+    return caption, species, confidence
+
+# Example
+caption, species, confidence = predict_bird_image("/kaggle/input/cub2002011/CUB_200_2011/images/006.Least_Auklet/Least_Auklet_0007_795123.jpg")
+print(f"Caption: {caption}")
+print(f"Species: {species}")
+print(f"Confidence: {confidence:.2f}%")
+```
+
+## Dataset
+- **CUB-200-2011**: 11,788 images of 200 bird species with attribute annotations.
+- Captions were generated based on species names and attributes (e.g., bill shape, wing color).
+
+## Training Details
+- **Loss**: Combined captioning (CrossEntropy) and classification (CrossEntropy) loss.
+- **Optimizer**: AdamW (lr=3e-5)
+- **Scheduler**: CosineAnnealingLR
+- **Hardware**: GPU (CUDA)
+- **Training Time**: ~5 min/epoch
+
+## Limitations
+- May overfit after Epoch 3 (validation loss increases).
+- Captions are limited to species and up to 5 attributes.
+- Classification accuracy not explicitly reported.
+
+## License
+MIT License
+
+## Contact
+For issues, contact INVERTO on Hugging Face.

config.json

@@ -0,0 +1,5 @@
+{
+  "base_model": "nlpconnect/vit-gpt2-image-captioning",
+  "hidden_size": 768,
+  "num_classes": 200
+}

model.py

@@ -0,0 +1,24 @@
+
+from huggingface_hub import PyTorchModelHubMixin
+import torch
+import torch.nn as nn
+from transformers import VisionEncoderDecoderModel
+
+class BirdCaptioningModel(nn.Module, PyTorchModelHubMixin):
+    def __init__(self, num_classes=200):
+        super().__init__()
+        self.base_model = VisionEncoderDecoderModel.from_pretrained("nlpconnect/vit-gpt2-image-captioning")
+        self.hidden_size = self.base_model.decoder.config.hidden_size
+        self.classifier = nn.Linear(self.hidden_size, num_classes)
+    
+    def forward(self, pixel_values, input_ids=None, attention_mask=None):
+        outputs = self.base_model(
+            pixel_values=pixel_values,
+            decoder_input_ids=input_ids,
+            decoder_attention_mask=attention_mask,
+            output_hidden_states=True,
+            return_dict=True
+        )
+        hidden_states = outputs.decoder_hidden_states[-1][:, 0, :]
+        class_logits = self.classifier(hidden_states)
+        return outputs.logits, class_logits

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa8335ff5c550fcc24f4847f774e9e0b7e832880adaf246e6b0ec81f60b69b06
+size 957455968

preprocessor_config.json

@@ -0,0 +1,23 @@
+{
+  "do_convert_rgb": null,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "ViTImageProcessor",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "resample": 2,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 224,
+    "width": 224
+  }
+}

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

species_mapping.txt

@@ -0,0 +1,200 @@
+0,Black footed Albatross
+1,Laysan Albatross
+2,Sooty Albatross
+3,Groove billed Ani
+4,Crested Auklet
+5,Least Auklet
+6,Parakeet Auklet
+7,Rhinoceros Auklet
+8,Brewer Blackbird
+9,Red winged Blackbird
+10,Rusty Blackbird
+11,Yellow headed Blackbird
+12,Bobolink
+13,Indigo Bunting
+14,Lazuli Bunting
+15,Painted Bunting
+16,Cardinal
+17,Spotted Catbird
+18,Gray Catbird
+19,Yellow breasted Chat
+20,Eastern Towhee
+21,Chuck will Widow
+22,Brandt Cormorant
+23,Red faced Cormorant
+24,Pelagic Cormorant
+25,Bronzed Cowbird
+26,Shiny Cowbird
+27,Brown Creeper
+28,American Crow
+29,Fish Crow
+30,Black billed Cuckoo
+31,Mangrove Cuckoo
+32,Yellow billed Cuckoo
+33,Gray crowned Rosy Finch
+34,Purple Finch
+35,Northern Flicker
+36,Acadian Flycatcher
+37,Great Crested Flycatcher
+38,Least Flycatcher
+39,Olive sided Flycatcher
+40,Scissor tailed Flycatcher
+41,Vermilion Flycatcher
+42,Yellow bellied Flycatcher
+43,Frigatebird
+44,Northern Fulmar
+45,Gadwall
+46,American Goldfinch
+47,European Goldfinch
+48,Boat tailed Grackle
+49,Eared Grebe
+50,Horned Grebe
+51,Pied billed Grebe
+52,Western Grebe
+53,Blue Grosbeak
+54,Evening Grosbeak
+55,Pine Grosbeak
+56,Rose breasted Grosbeak
+57,Pigeon Guillemot
+58,California Gull
+59,Glaucous winged Gull
+60,Heermann Gull
+61,Herring Gull
+62,Ivory Gull
+63,Ring billed Gull
+64,Slaty backed Gull
+65,Western Gull
+66,Anna Hummingbird
+67,Ruby throated Hummingbird
+68,Rufous Hummingbird
+69,Green Violetear
+70,Long tailed Jaeger
+71,Pomarine Jaeger
+72,Blue Jay
+73,Florida Jay
+74,Green Jay
+75,Dark eyed Junco
+76,Tropical Kingbird
+77,Gray Kingbird
+78,Belted Kingfisher
+79,Green Kingfisher
+80,Pied Kingfisher
+81,Ringed Kingfisher
+82,White breasted Kingfisher
+83,Red legged Kittiwake
+84,Horned Lark
+85,Pacific Loon
+86,Mallard
+87,Western Meadowlark
+88,Hooded Merganser
+89,Red breasted Merganser
+90,Mockingbird
+91,Nighthawk
+92,Clark Nutcracker
+93,White breasted Nuthatch
+94,Baltimore Oriole
+95,Hooded Oriole
+96,Orchard Oriole
+97,Scott Oriole
+98,Ovenbird
+99,Brown Pelican
+100,White Pelican
+101,Western Wood Pewee
+102,Sayornis
+103,American Pipit
+104,Whip poor Will
+105,Horned Puffin
+106,Common Raven
+107,White necked Raven
+108,American Redstart
+109,Geococcyx
+110,Loggerhead Shrike
+111,Great Grey Shrike
+112,Baird Sparrow
+113,Black throated Sparrow
+114,Brewer Sparrow
+115,Chipping Sparrow
+116,Clay colored Sparrow
+117,House Sparrow
+118,Field Sparrow
+119,Fox Sparrow
+120,Grasshopper Sparrow
+121,Harris Sparrow
+122,Henslow Sparrow
+123,Le Conte Sparrow
+124,Lincoln Sparrow
+125,Nelson Sharp tailed Sparrow
+126,Savannah Sparrow
+127,Seaside Sparrow
+128,Song Sparrow
+129,Tree Sparrow
+130,Vesper Sparrow
+131,White crowned Sparrow
+132,White throated Sparrow
+133,Cape Glossy Starling
+134,Bank Swallow
+135,Barn Swallow
+136,Cliff Swallow
+137,Tree Swallow
+138,Scarlet Tanager
+139,Summer Tanager
+140,Artic Tern
+141,Black Tern
+142,Caspian Tern
+143,Common Tern
+144,Elegant Tern
+145,Forsters Tern
+146,Least Tern
+147,Green tailed Towhee
+148,Brown Thrasher
+149,Sage Thrasher
+150,Black capped Vireo
+151,Blue headed Vireo
+152,Philadelphia Vireo
+153,Red eyed Vireo
+154,Warbling Vireo
+155,White eyed Vireo
+156,Yellow throated Vireo
+157,Bay breasted Warbler
+158,Black and white Warbler
+159,Black throated Blue Warbler
+160,Blue winged Warbler
+161,Canada Warbler
+162,Cape May Warbler
+163,Cerulean Warbler
+164,Chestnut sided Warbler
+165,Golden winged Warbler
+166,Hooded Warbler
+167,Kentucky Warbler
+168,Magnolia Warbler
+169,Mourning Warbler
+170,Myrtle Warbler
+171,Nashville Warbler
+172,Orange crowned Warbler
+173,Palm Warbler
+174,Pine Warbler
+175,Prairie Warbler
+176,Prothonotary Warbler
+177,Swainson Warbler
+178,Tennessee Warbler
+179,Wilson Warbler
+180,Worm eating Warbler
+181,Yellow Warbler
+182,Northern Waterthrush
+183,Louisiana Waterthrush
+184,Bohemian Waxwing
+185,Cedar Waxwing
+186,American Three toed Woodpecker
+187,Pileated Woodpecker
+188,Red bellied Woodpecker
+189,Red cockaded Woodpecker
+190,Red headed Woodpecker
+191,Downy Woodpecker
+192,Bewick Wren
+193,Cactus Wren
+194,Carolina Wren
+195,House Wren
+196,Marsh Wren
+197,Rock Wren
+198,Winter Wren
+199,Common Yellowthroat

tokenizer_config.json

@@ -0,0 +1,28 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "max_length": 32,
+  "model_max_length": 1024,
+  "pad_to_multiple_of": null,
+  "pad_token": "<|endoftext|>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "stride": 0,
+  "tokenizer_class": "GPT2Tokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "<|endoftext|>"
+}