Upload 5 files

app.py

@@ -0,0 +1,30 @@
+from PIL import Image
+import requests
+import gradio as gr 
+import torch
+from loader import get_loader
+import torchvision.transforms as transforms
+
+transform = transforms.Compose([
+    transforms.Resize(256),
+    transforms.CenterCrop(224),
+    transforms.ToTensor(),
+    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+])
+
+train_loader,dataset=get_loader(root_folder='FlickrDataset/Images',annotation_file='FlickrDataset/Captions/captions.txt',transform=transform,num_workers=2)
+filepath="ImageCaptioningusingLSTM.pth"
+from model import CNNtoRNN
+model=CNNtoRNN(embed_size=256,hidden_size=256,vocab_size=len(dataset.vocab),num_layers=1)
+model.load_state_dict(torch.load(filepath))
+model.eval()
+
+def launch(input):
+    im=Image.open(requests.get(input,stream=True).raw)
+    image=transform(im.convert('RGB')).unsqueeze(0)
+    
+    return model.caption_image(image,dataset.vocab)
+
+iface=gr.Interface(launch,inputs="text",outputs="text")
+iface.launch()
+    

inference.py

@@ -0,0 +1,39 @@
+import torch
+import os
+import torchvision.transforms as transforms
+from PIL import Image
+from model import CNNtoRNN
+import pandas as pd
+from loader import get_loader
+
+def inference():
+    transform = transforms.Compose([
+    transforms.Resize(256),
+    transforms.CenterCrop(224),
+    transforms.ToTensor(),
+    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+    ])
+    
+    image_index=100
+     
+    train_loader,dataset=get_loader(root_folder='FlickrDataset/Images',annotation_file='FlickrDataset/Captions/captions.txt',transform=transform,num_workers=2)
+    df=pd.read_csv("FlickrDataset/Captions/captions.txt")
+    imagepath="FlickrDataset/Images/"
+    images=os.listdir(imagepath)
+    im=Image.open(os.path.join(imagepath,images[image_index]))
+    im.show()
+
+    device=torch.device('cuda' if torch.cuda.is_available() else "cpu")
+    
+    filepath="ImageCaptioningusingLSTM.pth"
+    model=CNNtoRNN(embed_size=256,hidden_size=256,vocab_size=len(dataset.vocab),num_layers=1).to(device)
+    model.load_state_dict(torch.load(filepath))
+    model.eval()
+    
+    image=transform(im.convert("RGB")).unsqueeze(0)
+    
+    output=model.caption_image(image.to(device),dataset.vocab)
+    print("Output:"+" ".join(output[1:-1]))
+    
+if __name__=="__main__":
+    inference()

loader.py

@@ -0,0 +1,100 @@
+import os 
+import pandas as pd
+import spacy
+import torch
+from torch.nn.utils.rnn import pad_sequence
+from torch.utils.data import DataLoader,Dataset
+from PIL import Image
+import torchvision.transforms as transforms
+
+spacy_eng=spacy.load("en_core_web_sm")
+class Vocabulary:
+    def __init__(self,freq_threshold):
+        self.itos={0:"<PAD>",1:"<SOS>",2:"<EOS>",3:"<UNK>"}
+        self.stoi={"<PAD>":0,"<SOS>":1,"<EOS>":2,"<UNK>":3}
+        self.freq_threshold=freq_threshold
+        
+    def __len__(self):
+        return len(self.itos)
+    
+    def tokenizer_eng(self,text):
+        return [tok.text.lower() for tok in spacy_eng.tokenizer(text)]
+    
+    def build_vocabulary(self,sentence_list):
+        frequencies={}
+        idx=4
+        
+        for sentence in sentence_list:
+            for word in self.tokenizer_eng(sentence):
+                if word not in frequencies:
+                    frequencies[word]=1
+                    
+                else:
+                    frequencies[word]+=1
+                    
+                if frequencies[word]==self.freq_threshold:
+                    self.stoi[word]=idx
+                    self.itos[idx]=word
+                    idx+=1
+                    
+    def numericalize(self,text):
+        tokenized_text=self.tokenizer_eng(text)
+        return [
+            self.stoi[token] if token in self.stoi else self.stoi["<UNK>"]
+            for token in tokenized_text
+        ]
+        
+class FlickrDataset(Dataset):
+    def __init__(self,root_dir,captions_file,transform=None,freq_threshold=5):
+        self.root_dir=root_dir
+        self.df=pd.read_csv(captions_file)
+        self.transform=transform
+        
+        self.imgs=self.df['image']
+        self.captions=self.df['caption']
+        
+        self.vocab=Vocabulary(freq_threshold)
+        self.vocab.build_vocabulary(self.captions.tolist())
+        
+    def __len__(self):
+        return len(self.df)  
+    
+    def __getitem__(self,index):
+        caption=self.captions[index]
+        img_id=self.imgs[index]
+        img=Image.open(os.path.join(self.root_dir,img_id)).convert("RGB")
+        
+        if self.transform is not None:
+            img=self.transform(img)
+            
+        numericalized_caption=[self.vocab.stoi["<SOS>"]]
+        numericalized_caption+=self.vocab.numericalize(caption)
+        numericalized_caption.append(self.vocab.stoi["<EOS>"])
+        
+        return img,torch.tensor(numericalized_caption)
+    
+class MyCollate:
+    def __init__(self,pad_idx):
+        self.pad_idx=pad_idx
+        
+    def __call__(self,batch):
+        imgs=[item[0].unsqueeze(0) for item in batch]
+        imgs=torch.cat(imgs,dim=0)
+        targets=[item[1] for item in batch]
+        targets=pad_sequence(targets,batch_first=False,padding_value=self.pad_idx)
+        
+        return imgs,targets
+    
+def get_loader(root_folder,annotation_file,transform,batch_size=32,shuffle=True,pin_memory=True,num_workers=8):
+    dataset=FlickrDataset(root_folder,annotation_file,transform=transform)
+    pad_idx=dataset.vocab.stoi["<PAD>"]
+    loader=DataLoader(
+        dataset=dataset,
+        batch_size=batch_size,
+        num_workers=num_workers,
+        shuffle=shuffle,
+        pin_memory=pin_memory,
+        collate_fn=MyCollate(pad_idx=pad_idx)
+    )
+    
+    return loader,dataset

model.py

@@ -0,0 +1,74 @@
+import torch
+import torch.nn as nn
+import torchvision.models as models
+
+class EncoderCNN(nn.Module):
+    def __init__(self,embed_size):
+        super(EncoderCNN, self).__init__()
+        resnet = models.resnet50(weights='ResNet50_Weights.DEFAULT')
+        for param in resnet.parameters():
+            param.requires_grad_(False)
+        
+        modules = list(resnet.children())[:-1]
+        self.resnet = nn.Sequential(*modules)
+        self.embed = nn.Linear(resnet.fc.in_features, embed_size)
+        self.batch= nn.BatchNorm1d(embed_size,momentum = 0.01)
+        self.embed.weight.data.normal_(0., 0.02)
+        self.embed.bias.data.fill_(0)
+        
+    def forward(self,images):
+        features = self.resnet(images)
+        features = features.view(features.size(0), -1)
+        features = self.batch(self.embed(features))
+        return features
+    
+    
+class DecoderRNN(nn.Module):
+    def __init__(self,embed_size,hidden_size,vocab_size,num_layers):
+        super(DecoderRNN, self).__init__()
+        self.embed=nn.Embedding(vocab_size,embed_size)
+        self.lstm=nn.LSTM(embed_size,hidden_size,num_layers)
+        self.linear=nn.Linear(hidden_size,vocab_size)
+        self.dropout=nn.Dropout(0.5)
+        
+    def forward(self,features,captions):
+        embeddings=self.dropout(self.embed(captions))
+        embeddings=torch.cat((features.unsqueeze(0),embeddings),dim=0)
+        hiddens,_=self.lstm(embeddings)
+        outputs=self.linear(hiddens)
+        
+        return outputs
+    
+class CNNtoRNN(nn.Module):
+    def __init__(self,embed_size,hidden_size,vocab_size,num_layers):
+        super(CNNtoRNN,self).__init__()
+        self.encoderCNN=EncoderCNN(embed_size)
+        self.decoderRNN=DecoderRNN(embed_size,hidden_size,vocab_size,num_layers)
+        
+    def forward(self,images,captions):
+        features=self.encoderCNN(images)
+        outputs=self.decoderRNN(features,captions)
+        return outputs
+    
+    def caption_image(self,image,vocabulary,max_length=50):
+        result_caption=[]
+        with torch.no_grad():
+            X=self.encoderCNN(image).unsqueeze(0)
+            states=None
+            
+            for _ in range(max_length):
+                hiddens,states=self.decoderRNN.lstm(X,states)
+                output=self.decoderRNN.linear(hiddens.squeeze(0))
+                predicted=output.argmax(1)
+                result_caption.append(predicted.item())
+                
+                X=self.decoderRNN.embed(predicted).unsqueeze(0)
+                
+                if vocabulary.itos[predicted.item()]=="<EOS>":
+                    break
+                
+        return [vocabulary.itos[idx] for idx in result_caption]
+
+        
+        
+    

train.py

@@ -0,0 +1,64 @@
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import torchvision.transforms as transforms
+from loader import get_loader
+from model import CNNtoRNN
+from tqdm import tqdm
+from tqdm import trange
+
+def train():
+    transform = transforms.Compose([
+    transforms.Resize(256),
+    transforms.CenterCrop(224),
+    transforms.ToTensor(),
+    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+])
+    
+    train_loader,dataset=get_loader(root_folder='FlickrDataset/Images',annotation_file='FlickrDataset/Captions/captions.txt',transform=transform,num_workers=2)
+    
+    torch.backends.cudnn.benchmark=True
+    device=torch.device('cuda' if torch.cuda.is_available() else "cpu")
+    embed_size=256
+    hidden_size=256
+    vocab_size=len(dataset.vocab)
+    num_layers=1
+    learning_rate=3e-4
+    num_epochs=200
+    
+    model=CNNtoRNN(embed_size,hidden_size,vocab_size,num_layers).to(device)
+    criterion=nn.CrossEntropyLoss(ignore_index=dataset.vocab.stoi["<PAD>"])
+    optimizer=optim.Adam(model.parameters(),lr=learning_rate)
+    train_iterator=trange(0,num_epochs)
+    for _ in train_iterator:
+        pbar=tqdm(train_loader)
+        for idx,(imgs,captions) in enumerate(pbar):
+            model.train()
+            imgs=imgs.to(device)
+            captions=captions.to(device)
+            
+            outputs=model(imgs,captions[:-1])
+            loss=criterion(outputs.reshape(-1,outputs.shape[2]),captions.reshape(-1))
+            
+            
+            loss.backward()
+            optimizer.step()
+            optimizer.zero_grad()
+            
+            pbar.set_postfix(loss=loss.item())
+    
+    filepath="ImageCaptioningusingLSTM.pth"        
+    torch.save(model.state_dict(),filepath)
+    
+
+            
+if __name__=="__main__":
+    train()
+            
+                        
+        
+    
+    
+    
+    
+