Upload folder using huggingface_hub

.gitattributes

@@ -1,35 +1,35 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -1,24 +1,24 @@
-FROM python:3.13.5-slim
-
-WORKDIR /app
-
-RUN apt-get update && apt-get install -y \
-    build-essential \
-    curl \
-    git \
-    && rm -rf /var/lib/apt/lists/*
-
-COPY requirements.txt ./
-COPY src/ ./src/
-COPY resnet18_cifar10_finetuned.pth ./
-
-RUN pip3 install -r requirements.txt
-
-# Change EXPOSE to 7860
-EXPOSE 7860
-
-# Update HEALTHCHECK to use 7860
-HEALTHCHECK CMD curl --fail http://localhost:7860/_stcore/health
-
-# Add the XSRF and CORS disable flags to ENTRYPOINT
+FROM python:3.13.5-slim
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt ./
+COPY src/ ./src/
+COPY resnet18_cifar10_finetuned.pth ./
+
+RUN pip3 install -r requirements.txt
+
+# Change EXPOSE to 7860
+EXPOSE 7860
+
+# Update HEALTHCHECK to use 7860
+HEALTHCHECK CMD curl --fail http://localhost:7860/_stcore/health
+
+# Add the XSRF and CORS disable flags to ENTRYPOINT
 ENTRYPOINT ["streamlit", "run", "src/image_classifier_app.py", "--server.port=7860", "--server.address=0.0.0.0", "--server.enableCORS=false", "--server.enableXsrfProtection=false"]

README.md

@@ -0,0 +1,202 @@
+Here is a clean, professional `README.md` suitable for your HF Space:
+
+
+
+---
+
+
+
+\# AI Image Caption Generator
+
+
+
+A deep learning–based image captioning system built using a \*\*ResNet50 encoder\*\* and an \*\*LSTM decoder\*\*. The model generates natural language descriptions for uploaded images.
+
+
+
+\## Architecture
+
+
+
+\* \*\*Encoder:\*\* ResNet50 (frozen backbone)
+
+\* \*\*Decoder:\*\* LSTM-based sequence generator
+
+\* \*\*Training Dataset:\*\* Flickr8k
+
+\* \*\*Inference Framework:\*\* Streamlit
+
+\* \*\*Evaluation Metric:\*\* SacreBLEU
+
+
+
+The encoder extracts high-level visual features, which are then passed to the decoder to generate captions word by word.
+
+
+
+---
+
+
+
+\## How It Works
+
+
+
+1\. User uploads an image.
+
+2\. Image is preprocessed and passed through the ResNet50 encoder.
+
+3\. Extracted feature vector is fed into the LSTM decoder.
+
+4\. Caption is generated using temperature-based sampling.
+
+5\. If the image belongs to the Flickr8k dataset, BLEU metrics are displayed.
+
+
+
+---
+
+
+
+\## Features
+
+
+
+\* Temperature-controlled caption generation
+
+\* SacreBLEU evaluation
+
+\* N-gram precision breakdown (1–4 gram)
+
+\* Clean Streamlit interface
+
+\* Fully CPU-compatible deployment
+
+
+
+---
+
+
+
+\## Project Structure
+
+
+
+```
+
+app.py
+
+models/
+
+    encoder.pth
+
+    decoder.pth
+
+models/
+
+    encoder.py
+
+    decoder.py
+
+utils/
+
+    transforms.py
+
+    vocab.py
+
+    helpers.py
+
+vocabulary.json
+
+requirements.txt
+
+```
+
+
+
+---
+
+
+
+\## Model Details
+
+
+
+\* Encoder weights size: ~92 MB
+
+\* Decoder weights size: ~32 MB
+
+\* Full encoder backbone included in state\_dict
+
+\* Inference runs on CPU
+
+
+
+---
+
+
+
+\## Limitations
+
+
+
+\* Trained on Flickr8k (8,000 images)
+
+\* Performs best on outdoor scenes, people, and animals
+
+\* May generalize poorly to unseen domains
+
+\* CPU inference can be slow (2–5 seconds per image)
+
+
+
+---
+
+
+
+\## Setup (Local)
+
+
+
+```bash
+
+pip install -r requirements.txt
+
+streamlit run app.py
+
+```
+
+
+
+---
+
+
+
+\## Deployment
+
+
+
+This project is deployed on \*\*Hugging Face Spaces\*\* using Streamlit.
+
+
+
+---
+
+
+
+\## License
+
+
+
+MIT License
+
+
+
+---
+
+
+
+If you want, I can also write a \*\*short portfolio-style README\*\* optimized specifically for recruiters.
+
+
+

app.py

@@ -0,0 +1,105 @@
+import torch
+import pandas as pd
+import streamlit as st
+from PIL import Image
+from models.encoder import EncoderCNN
+from models.decoder import DecoderRNN
+from utils.vocab import Vocabulary
+from torchvision import transforms as T
+from utils.helpers import ENCODER_MODEL_PATH, DECODER_MODEL_PATH, VOCAB_FILE_PATH, CAPTIONS_TKN_PATH
+from inference import sample_with_temp, sample
+from utils.transforms import transforms
+import sacrebleu
+# ... (your other imports)
+
+@st.cache_resource
+def load_models():
+    captions = pd.read_csv(CAPTIONS_TKN_PATH).drop('tokens', axis=1)
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    
+    # Load Vocabulary
+    vocab = Vocabulary(load_path=VOCAB_FILE_PATH)
+    
+    # Initialize Models
+    encoder = EncoderCNN(256).to(device)
+    decoder = DecoderRNN(len(vocab), 256, 512).to(device)
+    
+    # Load Weights
+    encoder.load_state_dict(torch.load(ENCODER_MODEL_PATH, map_location=device))
+    decoder.load_state_dict(torch.load(DECODER_MODEL_PATH, map_location=device))
+    
+    encoder.eval()
+    decoder.eval()
+    
+    return encoder, decoder, vocab, device, captions
+
+
+# --- Main App Logic ---
+encoder, decoder, vocab, device, captions = load_models()
+act_caps = []
+caption = ''
+st.title("📸 AI Image Captioner")
+
+temp = st.slider("Sampling Temperature", min_value=0.1, max_value=2.0, value=0.8, step=0.1)
+st.info("Higher temperature = more creative/random. Lower = more predictable.")
+
+uploaded_file = st.file_uploader("Choose an image...", type=["jpg", "png", "jpeg"])
+if uploaded_file is not None:
+    img = Image.open(uploaded_file).convert('RGB')
+    st.image(img, caption='Uploaded Image', width=300)
+    
+    # Process
+    # Assuming transforms is defined or returned from load_models
+    img_tensor = transforms(img).unsqueeze(0).to(device)
+    
+    st.subheader("Actual Captions:")
+    act_caps = captions[captions['image'] == uploaded_file.name]['caption'].tolist()
+    st.success("  \n".join(act_caps))
+
+    with torch.no_grad():
+        encoder_out = encoder(img_tensor)
+        # Pass the 'temp' variable from the slider here
+        caption = sample_with_temp(encoder_out, decoder, vocab, temp=temp)
+    
+    st.subheader("Generated Caption:")
+    st.success(caption)
+
+if act_caps:
+    # sacrebleu expects a list of strings for hypothesis 
+    # and a list of lists of strings for references
+    refs = [act_caps] 
+    sys = [caption]
+    
+    bleu = sacrebleu.corpus_bleu(sys, refs)
+    
+    st.subheader("Evaluation Metrics:")
+    st.metric(label="SacreBLEU Score", value=f"{bleu.score:.2f}")
+    st.progress(min(bleu.score / 50, 1.0))
+
+    # N-gram Precision breakdown
+    # bleu.precisions is a list: [p1, p2, p3, p4]
+    cols = st.columns(4)
+    for i, p in enumerate(bleu.precisions):
+        cols[i].markdown(f"{i+1}-gram")
+        cols[i].write(f"{p:.1f}%")
+
+    # Brief explanation
+    with st.expander("What do these mean?"):
+        st.write("""
+        - **1-gram**: Individual word accuracy (Vocabulary).
+        - **2-gram**: Fluency of word pairs.
+        - **4-gram**: Capturing longer phrases/sentence structure.
+        """)
+else:
+    st.info("Upload an image from the Flickr8k set to see BLEU metrics.")
+
+st.header('About this Project')
+st.markdown("""
+This AI model generates descriptive captions for uploaded images using a **ResNet50 + LSTM** architecture.
+
+* **Encoder:** Pre-trained ResNet50 (Frozen) extracts high-level visual features.
+* **Decoder:** A Long Short-Term Memory (LSTM) network trained for 10 epochs.
+* **Dataset:** Trained on the **Flickr8k dataset** (8,000 images).
+
+⚠️ **Note:** Because the model was trained on a specific, small-scale dataset with a frozen backbone, it performs satisfactory on outdoor scenes, people, and animals. It may produce unexpected results for images significantly different from the Flickr8k distribution.
+""")

inference.py

@@ -0,0 +1,111 @@
+import torch
+import torch.nn.functional as F
+
+from utils.transforms import transforms
+from utils.vocab import Vocabulary
+from utils.helpers import VOCAB_FILE_PATH, ENCODER_MODEL_PATH, DECODER_MODEL_PATH
+from models.encoder import EncoderCNN
+from models.decoder import DecoderRNN
+import PIL.Image as Image
+
+
+def sample(features, decoder, vocab, max_len=20):
+    device = features.device
+    result_caption = []
+    word_idx = torch.tensor([vocab['<SOS>']]).unsqueeze(0).to(device) # Shape (1, 1)
+    outputs, hidden = decoder(features, word_idx) # outputs (1, 1, vocab_size)
+    for _ in range(max_len):
+        predicted = outputs.argmax(2) 
+        word = vocab[predicted.item()] # .item() to get numerical value from tensor
+        if word == '<EOS>':
+            break
+        result_caption.append(word)    
+        # Pass features=None and previous hidden state
+        outputs, hidden = decoder(None, predicted, hidden)
+    return ' '.join(result_caption)
+
+
+
+def beam_sample(features, decoder, vocab, beam_size=5, max_len=30):
+    device = features.device
+    # (log_score, sequence, hidden_state)
+    start_token = torch.tensor([vocab['<SOS>']]).to(device)
+    beams = [(0, [start_token.item()], None)]
+    
+    for _ in range(max_len):
+        candidates = []
+        for score, seq, hidden in beams:
+            if seq[-1] == vocab['<EOS>']:
+                candidates.append((score, seq, hidden))
+                continue
+            
+            # Predict next word
+            curr_word = torch.tensor([seq[-1]]).unsqueeze(0).to(device)
+            # Use features only on first step
+            feat_input = features if _ == 0 else None
+            
+            outputs, next_hidden = decoder(feat_input, curr_word, hidden)
+            
+            # Get log probabilities
+            log_probs = F.log_softmax(outputs.squeeze(1), dim=1)
+            top_probs, top_idxs = log_probs.topk(beam_size)
+            
+            for i in range(beam_size):
+                candidates.append((score + top_probs[0][i].item(), 
+                                  seq + [top_idxs[0][i].item()], 
+                                  next_hidden))
+        
+        # Sort by score and keep top k
+        beams = sorted(candidates, key=lambda x: x[0], reverse=True)[:beam_size]
+        
+        # Stop if all beams end in <EOS>
+        if all(s[-1] == vocab['<EOS>'] for _, s, _ in beams):
+            break
+            
+    # Return best sequence (minus tokens)
+    best_seq = beams[0][1]
+    return ' '.join([vocab[idx] for idx in best_seq if idx not in [vocab['<SOS>'], vocab['<EOS>']]])
+
+
+
+def sample_with_temp(features, decoder, vocab, temp=0.8, max_len=30):
+    device = features.device
+    result_caption = []
+    word_idx = torch.tensor([vocab['<SOS>']]).unsqueeze(0).to(device)
+    outputs, hidden = decoder(features, word_idx) # outputs (1, 1, vocab_size)
+    for _ in range(max_len):
+        # Apply temperature to logits
+        logits = outputs.squeeze(1) / temp
+        probs = F.softmax(logits, dim=-1)
+        # Sample from the distribution instead of argmax
+        predicted = torch.multinomial(probs, 1)
+        word = vocab[predicted.item()]
+        if word == '<EOS>': break
+        result_caption.append(word)
+        outputs, hidden = decoder(None, predicted, hidden)
+    return ' '.join(result_caption)
+
+
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+vocabulary = Vocabulary(load_path=VOCAB_FILE_PATH)
+# img = Image.open(COLAB_DATA_FOLDER + 'Images/' + '141140165_9002a04f19.jpg').convert('RGB')
+
+encoder = EncoderCNN(256).to(device)
+decoder = DecoderRNN(len(vocabulary), 256, 512).to(device)
+
+encoder_state_dict = torch.load(ENCODER_MODEL_PATH, map_location=device)
+decoder_state_dict = torch.load(DECODER_MODEL_PATH, map_location=device)
+
+encoder.load_state_dict(encoder_state_dict)
+decoder.load_state_dict(decoder_state_dict)
+
+encoder.eval()
+decoder.eval()
+
+img = Image.open('data/flickr_data/Images/3718892835_a3e74a3417.jpg').convert('RGB')
+img = transforms(img).unsqueeze(0).to(device)
+encoder_out = encoder(img)
+print('sample_with_temp: ', sample_with_temp(encoder_out, decoder, vocabulary))
+# print('sample: ', sample(encoder_out, decoder, vocabulary))
+# print('beam_sample: ', beam_sample(encoder_out, decoder, vocabulary))

models/decoder.py

@@ -0,0 +1,24 @@
+import torch.nn as nn
+
+
+class DecoderRNN(nn.Module):
+    def __init__(self, vocab_size, embed_size, hidden_size, num_layers=1, padding_idx=0):
+        super(DecoderRNN, self).__init__()
+        self.embed = nn.Embedding(num_embeddings=vocab_size, embedding_dim=embed_size, padding_idx=padding_idx)
+        self.lstm = nn.LSTM(input_size=embed_size, hidden_size=hidden_size, num_layers=num_layers, batch_first=True)
+        self.linear = nn.Linear(hidden_size, vocab_size)
+
+        self.init_h = nn.Linear(embed_size, hidden_size)
+        self.init_c = nn.Linear(embed_size, hidden_size)
+    
+    def forward(self, features, captions, hidden=None):
+        if hidden == None:
+            h0 = self.init_h(features).unsqueeze(0).repeat(self.lstm.num_layers, 1, 1)
+            c0 = self.init_c(features).unsqueeze(0).repeat(self.lstm.num_layers, 1, 1)
+            hidden = (h0, c0)
+            # dataflow: (B, seqlen) -> (B, hidden_size) -> (1, B, hidden_size) -> (num_layers, B, hidden_size)
+
+        embeddings = self.embed(captions) # (B, seqlen) ->  Training: (B, seqlen, embed_size) | Inference: (B, 1, embed_size)
+        outputs, hidden = self.lstm(embeddings, hidden) # Training: (B, seqlen, hidden_size) | Inference: (B, 1, hidden_size)
+        outputs = self.linear(outputs) # Training: (B, seqlen, vocab_size) | Inference: (B, 1, vocab_size)
+        return outputs, hidden

models/encoder.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15c2fd0e6a24e1b58ba66cd7f9754d9d9befd885e1aea762d0016b6d5f8d351c
+size 96454389

models/encoder.py

@@ -0,0 +1,24 @@
+from torchvision.models import resnet50, ResNet50_Weights
+import torch.nn as nn
+
+
+class EncoderCNN(nn.Module):
+    def __init__(self, embed_size, fine_tune=False):
+        super(EncoderCNN, self).__init__()
+        resnet = resnet50(weights=ResNet50_Weights.DEFAULT if fine_tune else None)
+        for param in resnet.parameters():
+            param.requires_grad = False
+        if fine_tune:
+            for param in resnet.layer4.parameters():
+                param.requires_grad = True
+        backbone = list(resnet.children())[:-1]
+
+        self.resnet = nn.Sequential(*backbone)
+        self.fc = nn.Linear(resnet.fc.in_features, embed_size)
+        self.bn = nn.BatchNorm1d(num_features=embed_size, momentum=0.01)
+    
+    def forward(self, images): # (B, C, W, H)
+        features = self.resnet(images) # (B, 2048, 1, 1)
+        features = features.reshape(features.shape[0], -1) # (B, 2048*1*1) not necessay to reshape as fc layer can take any size input
+        return self.bn(self.fc(features)) # (B, embed_size)
+

requirements.txt

@@ -0,0 +1,6 @@
+pandas==3.0.1
+Pillow==12.1.1
+sacrebleu==2.6.0
+streamlit==1.55.0
+torch==2.10.0
+torchvision==0.25.0

utils/helpers.py

@@ -0,0 +1,14 @@
+# from enum import Enum
+import os
+
+
+DATA_DIR = 'data'
+CAPTIONS_FILE_PATH = os.path.join(DATA_DIR, 'flickr_data/captions.txt')
+CAPTIONS_TKN_PATH = os.path.join(DATA_DIR, 'captions_tokenized.csv')
+IMAGES_PATH = os.path.join(DATA_DIR, 'flickr_data/Images')
+TOKANIZED_CAPTIONS = os.path.join(DATA_DIR, 'captions_tokenized.csv')
+
+MODELS_PATH = 'models'
+ENCODER_MODEL_PATH = os.path.join(MODELS_PATH, 'encoder.pth')
+DECODER_MODEL_PATH = os.path.join(MODELS_PATH, 'decoder.pth')
+VOCAB_FILE_PATH = os.path.join(MODELS_PATH, 'vocabulary.json')

utils/transforms.py

@@ -0,0 +1,3 @@
+from torchvision.models import ResNet50_Weights
+
+transforms = ResNet50_Weights.DEFAULT.transforms()

utils/vocab.py

@@ -0,0 +1,41 @@
+
+import json
+from collections import Counter
+
+class Vocabulary():
+    SPECIAL_TOKENS = ["<PAD>", "<SOS>", "<EOS>", "<UNK>"]
+
+    def __init__(self, df=None, load_path=None, min_freq=1):
+        if load_path:
+            with open(load_path, 'r') as f:
+                self.stoi = json.load(f)
+        else:
+            # token_freq = df.explode('tokens').value_counts()
+            # More efficient than df.explode for large datasets
+            counts = Counter([token for tokens in df['tokens'] for token in tokens])
+            self.stoi = {tok: i for i, tok in enumerate(self.SPECIAL_TOKENS)}
+            for token, freq in counts.items():
+                if freq >= min_freq:
+                    self.stoi[token] = len(self.stoi)
+
+        self.itos = {i: s for s, i in self.stoi.items()}
+    
+
+    def __len__(self):
+        return len(self.stoi)
+    
+
+    def __getitem__(self, key):
+            if isinstance(key, str):
+                 return self.stoi.get(key, self.stoi['<UNK>'])
+            elif isinstance(key, int):
+                 return self.itos.get(key, '<UNK>')
+    
+
+    def numericalize(self, tokens):
+         return [self[token] for token in tokens]
+    
+
+    def texualize(self, indices):
+         return [self[idx] for idx in indices]
+