{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "c4d0fd63", "metadata": {}, "outputs": [], "source": [ "print('gani')" ] }, { "cell_type": "code", "execution_count": null, "id": "7d4cadc5", "metadata": {}, "outputs": [], "source": [ "# import urllib.request\n", "# import zipfile\n", "# def download_file(url,file_name):\n", "# print(f'download {file_name}')\n", "# urllib.request.urlretrieve(url,file_name)\n", "# print('Done')\n", "\n", "# def unzip_file(zip_name,extract_to='.'):\n", "# with zipfile.ZipFile(zip_name,'r') as zip_ref:\n", "# zip_ref.extractall(extract_to)\n", "# print(\"Done\")\n", "\n", "# download_file('http://images.cocodataset.org/annotations/annotations_trainval2017.zip','annotations_trainval2017.zip')\n", "# unzip_file('annotations_trainval2017.zip')" ] }, { "cell_type": "code", "execution_count": null, "id": "081df97f", "metadata": {}, "outputs": [], "source": [ "import sys\n", "\n", "!{sys.executable} -m pip install pycocotools torch torchvision torchaudio Pillow\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "5aa74757", "metadata": {}, "outputs": [], "source": [ "from pycocotools.coco import COCO\n", "import requests\n", "import os\n", "\n", "#imgs id path\n", "imgs_path='annotations/captions_train2017.json'\n", "\n", "coco=COCO(imgs_path)\n", "\n", "imgs_id=coco.getImgIds()[1:1000]" ] }, { "cell_type": "code", "execution_count": null, "id": "f52bd876", "metadata": {}, "outputs": [], "source": [ "# print(len(imgs_id))\n", "imgs_json=coco.loadImgs(imgs_id)\n", "# print(imgs_json)" ] }, { "cell_type": "code", "execution_count": null, "id": "9a10dc55", "metadata": {}, "outputs": [], "source": [ "\n", "img_to_id={}\n", "for img_json in imgs_json:\n", " img_to_id[img_json['file_name']]=img_json['id']\n", "# print(img_to_id)\n", "# print(len(imgs_json))" ] }, { "cell_type": "code", "execution_count": null, "id": "a5dd784c", "metadata": {}, "outputs": [], "source": [ "# import time\n", "\n", "# c = 0\n", "# for img in imgs_json:\n", "# try:\n", "# img_data = requests.get(img['coco_url'], timeout=10).content\n", "# with open(f\"coco_1000/{img['file_name']}\", 'wb') as handler:\n", "# handler.write(img_data)\n", "# c += 1\n", "# print(f\"✅ Downloaded {c}: {img['file_name']}\")\n", "# except Exception as e:\n", "# print(f\"❌ Failed: {img['file_name']} --> {e}\")\n", "# time.sleep(2) # wait 2 sec before next try!\n", "# continue\n", "\n", "# print(f\"Total Downloaded: {c}\")" ] }, { "cell_type": "code", "execution_count": null, "id": "c6f30469", "metadata": {}, "outputs": [], "source": [ "#store the captions of each img in a dict\n", "\n", "id_to_capts=dict()\n", "for img_id in imgs_id:\n", " ann=coco.getAnnIds(imgIds=img_id)\n", " captions=coco.loadAnns(ann)\n", " caps=[]\n", " for cap in captions:\n", " caps.append(cap['caption'])\n", " id_to_capts[img_id]=caps\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "3e0fd51a", "metadata": {}, "outputs": [], "source": [ "print(id_to_capts)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "d56ee3fa", "metadata": {}, "outputs": [], "source": [ "from utils import *\n", "wrd_to_idx,idx_to_wrd,wrd_to_vect=read_glove_vecs('data/glove.6B.50d.txt')" ] }, { "cell_type": "code", "execution_count": null, "id": "f921f6fe", "metadata": {}, "outputs": [], "source": [ "print(wrd_to_vect['hi'])" ] }, { "cell_type": "code", "execution_count": null, "id": "40ef2361", "metadata": {}, "outputs": [], "source": [ "#feature extraction\n", "\n", "import torch\n", "import torchvision.models as models\n", "import torch.nn as nn \n", "from PIL import Image\n", "import numpy \n", "import torchvision.transforms as transforms\n", "\n", "\n", "#load renet50 model and remove final classification layer\n", "model=models.resnet50(pretrained=True)\n", "model=nn.Sequential(*list(model.children())[:-1])\n", "print(model)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "2e15d8ae", "metadata": {}, "outputs": [], "source": [ "#put the model in eval\n", "model.eval()\n", "#image cleaning pipeline\n", "transform=transforms.Compose(\n", " [transforms.Resize((224,224)),\n", " transforms.ToTensor(),\n", " transforms.Normalize(\n", " mean=[0.485, 0.456, 0.406], # ImageNet mean\n", " std=[0.229, 0.224, 0.225] \n", " )]\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "a91957c5", "metadata": {}, "outputs": [], "source": [ "#extract features\n", "images_dir='coco_1000/'\n", "imgs_feature={}\n", "def extract_feature(image_path):\n", " image=Image.open(image_path).convert('RGB')\n", " image_tensor=transform(image).unsqueeze(0)\n", " with torch.no_grad():\n", " features=model(image_tensor)\n", " return features.squeeze().numpy()" ] }, { "cell_type": "code", "execution_count": null, "id": "9ae1d195", "metadata": {}, "outputs": [], "source": [ "\n", "# for idx,img_file in enumerate(os.listdir(images_dir)):\n", "# if img_file.endswith(('.jpg','jpeg','.png')):\n", "# path=os.path.join(images_dir,img_file)\n", "# features=extract_feature(path)\n", "# imgs_feature[img_to_id[img_file]]=features\n", "\n", "# np.save('imgs_feature',imgs_feature)" ] }, { "cell_type": "code", "execution_count": null, "id": "cfa3458e", "metadata": {}, "outputs": [], "source": [ "# 1. Calculate the mean vector from your existing GloVe vectors\n", "# We use this as a base for our special tokens\n", "all_vectors = np.array(list(wrd_to_vect.values()))\n", "mean_vec = np.mean(all_vectors, axis=0)\n", "\n", "# 2. Define your special tokens\n", "special_tokens = {\n", " \"\": np.zeros(50), # All zeros for padding\n", " \"\": np.random.normal(scale=0.6, size=(50,)), # Random or mean\n", " \"\": np.random.normal(scale=0.6, size=(50,)), # Random or mean\n", " \"\": mean_vec # Mean vector for unknown words\n", "}\n", "\n", "# 3. Add them to your wrd_to_vect dictionary\n", "for token, vec in special_tokens.items():\n", " wrd_to_vect[token] = vec" ] }, { "cell_type": "code", "execution_count": null, "id": "0fb03d5b", "metadata": {}, "outputs": [], "source": [ "for i,token in enumerate(special_tokens):\n", " idx_to_wrd[i]=token\n", " wrd_to_idx[token]=i" ] }, { "cell_type": "code", "execution_count": null, "id": "3e9795da", "metadata": {}, "outputs": [], "source": [ "unq_wrds=set()\n", "def sentence_to_idx(sentence):\n", " idxs=[wrd_to_idx['']]*41\n", " idxs[0]=wrd_to_idx['']\n", " k=1\n", " sentence=sentence.split()\n", " unq_wrds.update(sentence)\n", " end=len(sentence)\n", " idxs[end+1]=wrd_to_idx['']\n", " for word in sentence:\n", " if word in wrd_to_vect:\n", " idxs[k]=wrd_to_idx[word]\n", " else:\n", " idxs[k]=wrd_to_idx['']\n", " k+=1\n", " return idxs" ] }, { "cell_type": "code", "execution_count": null, "id": "614d2a9e", "metadata": {}, "outputs": [], "source": [ "\n", "unq_wrds.update(['','','',''])\n", "vocab_size=len(idx_to_wrd)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "405c6013", "metadata": {}, "outputs": [], "source": [ "\n", "id_to_capts_idxs={}\n", "for img_id in imgs_id:\n", " t=[]\n", " # print(id_to_capts)\n", " for sentence in id_to_capts[img_id]:\n", " idxs=sentence_to_idx(sentence.lower().strip(' .'))\n", " t.append(idxs)\n", " id_to_capts_idxs[img_id]=t\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "08b6be43", "metadata": {}, "outputs": [], "source": [ "#build weight matrix\n", "\n", "def build_matrix(vocab_size,embd_size,wrd_to_vect,idx_to_wrd):\n", "\n", " weight_matrix=np.zeros((vocab_size,embd_size))\n", "\n", " for idx,wrd in idx_to_wrd.items():\n", " if wrd in unq_wrds:\n", " weight_matrix[idx]=wrd_to_vect[wrd]\n", " return weight_matrix\n" ] }, { "cell_type": "code", "execution_count": null, "id": "d9a5a20c", "metadata": {}, "outputs": [], "source": [ "print(len(idx_to_wrd))" ] }, { "cell_type": "code", "execution_count": null, "id": "ac5ba560", "metadata": {}, "outputs": [], "source": [ "features = np.load('imgs_feature.npy', allow_pickle=True)\n", "features=features.item()\n", "\n", "print(len(features.items()))" ] }, { "cell_type": "code", "execution_count": null, "id": "d000545c", "metadata": {}, "outputs": [], "source": [ "print(len(img_to_id))" ] }, { "cell_type": "code", "execution_count": null, "id": "bd596353", "metadata": {}, "outputs": [], "source": [ "from torch.utils.data import DataLoader,TensorDataset\n", "\n", "weight_matrix=build_matrix(vocab_size,50,wrd_to_vect,idx_to_wrd)\n", "\n", "\n", "#prepare x and y values\n", "\n", "all_image_features=[]\n", "all_capts=[]\n", "all_targets=[]\n", "\n", "\n", "for id,features in features.items():\n", " for cap in id_to_capts_idxs[id]:\n", " all_image_features.append(features)\n", " all_capts.append(cap[:-1])\n", " all_targets.append(cap[1:])\n", "\n", "\n", "all_image_features=np.array(all_image_features)\n", "all_capts=np.array(all_capts)\n", "all_targets=np.array(all_targets)\n", "\n", "#convert those into tensor\n", "\n", "X_img_tensor=torch.tensor(all_image_features,dtype=torch.float32)\n", "X_caps_tensor=torch.tensor(all_capts,dtype=torch.long)\n", "Y_tensor=torch.tensor(all_targets,dtype=torch.long)\n", "\n", "\n", "dataset=TensorDataset(X_img_tensor,X_caps_tensor,Y_tensor)\n", "dataloader=DataLoader(dataset,batch_size=32,shuffle=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "f5a3d518", "metadata": {}, "outputs": [], "source": [ "import torch.nn as nn\n", "\n", "class Decoder(nn.Module):\n", " def __init__(self,weight_matrix,vocab_size,embd_dim=50,hidden_size=512,feature_size=2048):\n", " super(Decoder,self).__init__()\n", "\n", " #the image bridge\n", " self.img_to_hidden=nn.Linear(feature_size,hidden_size)\n", " self.img_to_cell=nn.Linear(feature_size,hidden_size)\n", "\n", " #the embdding layer\n", " self.embd=nn.Embedding.from_pretrained(\n", " torch.from_numpy(weight_matrix).float(),\n", " freeze=True\n", " )\n", "\n", " #the lstm layer\n", " self.lstm=nn.LSTM(\n", " input_size=embd_dim,\n", " hidden_size=hidden_size,\n", " batch_first=True,\n", " dropout=0.5\n", " )\n", "\n", " #dropout layer for the output layer\n", " self.dropout=nn.Dropout(0.5)\n", "\n", " #output layer\n", " self.fc=nn.Linear(hidden_size,vocab_size)\n", "\n", " def forward(self,features,captions):\n", "\n", " #features into (batch,512)\n", " h0=self.img_to_hidden(features).unsqueeze(0)\n", " c0=self.img_to_cell(features).unsqueeze(0)\n", "\n", " #embd\n", " embeddings=self.embd(captions)\n", " #lstm\n", " lstm_out,_=self.lstm(embeddings,(h0,c0))\n", " #out\n", " lstm_out=self.dropout(lstm_out)\n", " output=self.fc(lstm_out)\n", "\n", " return output\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "1d4af98d", "metadata": {}, "outputs": [], "source": [ "\n", "device=torch.device('cuda' if torch.cuda_is_available() else 'cpu')\n", "\n", "model=Decoder(weight_matrix,vocab_size).to(device)" ] }, { "cell_type": "code", "execution_count": null, "id": "bde0cda8", "metadata": {}, "outputs": [], "source": [ "def optimize(dataloader, criterion, optimizer, model, device):\n", " total_loss = 0\n", " total_words = 0\n", " total_correct = 0\n", "\n", " model.train()\n", "\n", " for features, captions, targets in dataloader:\n", "\n", " features = features.to(device)\n", " captions = captions.to(device)\n", " targets = targets.to(device)\n", "\n", " optimizer.zero_grad()\n", "\n", " outputs = model(features, captions)\n", "\n", " loss = criterion(\n", " outputs.view(-1, outputs.size(-1)),\n", " targets.view(-1)\n", " )\n", "\n", " loss.backward()\n", " optimizer.step()\n", "\n", " total_loss += loss.item()\n", "\n", " predicted = outputs.argmax(dim=-1)\n", " total_correct += (predicted == targets).sum().item()\n", " total_words += targets.numel()\n", "\n", " accuracy = (total_correct / total_words) * 100\n", "\n", " return total_loss, accuracy" ] }, { "cell_type": "code", "execution_count": null, "id": "e5c614b1", "metadata": {}, "outputs": [], "source": [ "import torch.optim as optim\n", "\n", "criterion=nn.CrossEntropyLoss()\n", "optimizer=optim.Adam(model.parameters(),lr=0.001)\n", "epochs=30\n", "\n", "#train the model\n", "for epoch in range(epochs):\n", " loss,accuracy=optimize(dataloader,criterion,optimizer,model)\n", " print(f\"epoch:{epoch} loss {loss} accuracy {accuracy}\")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "70ae50a7", "metadata": {}, "outputs": [], "source": [ "torch.save(model.state_dict(), \"checkpoint_epoch5.pth\")" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.0" } }, "nbformat": 4, "nbformat_minor": 5 }