{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "113985e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\utkar\\anaconda4\\lib\\site-packages\\scipy\\__init__.py:138: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.23.5)\n",
      "  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion} is required for this version of \"\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "from tqdm.notebook import tqdm\n",
    "import os\n",
    "import pandas \n",
    "import numpy as np\n",
    "from tensorflow.keras.applications.vgg16 import VGG16,preprocess_input\n",
    "from tensorflow.keras.preprocessing.image import load_img,img_to_array\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.utils import to_categorical,plot_model\n",
    "from tensorflow.keras.layers import Input,Dense,LSTM,Embedding, Dropout, add"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6f9ba09d",
   "metadata": {},
   "outputs": [],
   "source": [
    "work=\"C:\\crawlers\\Project_hastag\\save\"\n",
    "base=\"C:\\crawlers\\Project_hastag\\Archive\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "204bf9d6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_1 (InputLayer)        [(None, 224, 224, 3)]     0         \n",
      "                                                                 \n",
      " block1_conv1 (Conv2D)       (None, 224, 224, 64)      1792      \n",
      "                                                                 \n",
      " block1_conv2 (Conv2D)       (None, 224, 224, 64)      36928     \n",
      "                                                                 \n",
      " block1_pool (MaxPooling2D)  (None, 112, 112, 64)      0         \n",
      "                                                                 \n",
      " block2_conv1 (Conv2D)       (None, 112, 112, 128)     73856     \n",
      "                                                                 \n",
      " block2_conv2 (Conv2D)       (None, 112, 112, 128)     147584    \n",
      "                                                                 \n",
      " block2_pool (MaxPooling2D)  (None, 56, 56, 128)       0         \n",
      "                                                                 \n",
      " block3_conv1 (Conv2D)       (None, 56, 56, 256)       295168    \n",
      "                                                                 \n",
      " block3_conv2 (Conv2D)       (None, 56, 56, 256)       590080    \n",
      "                                                                 \n",
      " block3_conv3 (Conv2D)       (None, 56, 56, 256)       590080    \n",
      "                                                                 \n",
      " block3_pool (MaxPooling2D)  (None, 28, 28, 256)       0         \n",
      "                                                                 \n",
      " block4_conv1 (Conv2D)       (None, 28, 28, 512)       1180160   \n",
      "                                                                 \n",
      " block4_conv2 (Conv2D)       (None, 28, 28, 512)       2359808   \n",
      "                                                                 \n",
      " block4_conv3 (Conv2D)       (None, 28, 28, 512)       2359808   \n",
      "                                                                 \n",
      " block4_pool (MaxPooling2D)  (None, 14, 14, 512)       0         \n",
      "                                                                 \n",
      " block5_conv1 (Conv2D)       (None, 14, 14, 512)       2359808   \n",
      "                                                                 \n",
      " block5_conv2 (Conv2D)       (None, 14, 14, 512)       2359808   \n",
      "                                                                 \n",
      " block5_conv3 (Conv2D)       (None, 14, 14, 512)       2359808   \n",
      "                                                                 \n",
      " block5_pool (MaxPooling2D)  (None, 7, 7, 512)         0         \n",
      "                                                                 \n",
      " flatten (Flatten)           (None, 25088)             0         \n",
      "                                                                 \n",
      " fc1 (Dense)                 (None, 4096)              102764544 \n",
      "                                                                 \n",
      " fc2 (Dense)                 (None, 4096)              16781312  \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 134,260,544\n",
      "Trainable params: 134,260,544\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model=VGG16()\n",
    "model=Model(model.inputs,outputs=model.layers[-2].output)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "22708632",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "908848215aa6423a84b9e8398a2da55b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8091 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Feature image\n",
    "fs={}\n",
    "directory=os.path.join(base,'Images')\n",
    "\n",
    "for img in tqdm(os.listdir(directory)):\n",
    "  img_name=os.path.join(directory,img)\n",
    "  image=load_img(img_name,target_size=(224,224))\n",
    "  image=img_to_array(image)\n",
    "  image=image.reshape(1,image.shape[0],image.shape[1],image.shape[2])\n",
    "  image=preprocess_input(image)\n",
    "  f=model.predict(image,verbose=0)\n",
    "  im_id=img.split(\".\")[0]\n",
    "  fs[im_id]=f\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "195b8d10",
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(fs,open(os.path.join(work,\"features.pkl\"),\"wb\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7c0d0727",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open (os.path.join(work,\"features.pkl\"),\"rb\") as f:\n",
    "  features=pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f4425b39",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(base, 'captions.txt'), 'r') as f:\n",
    "    next(f)\n",
    "    captions_doc = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9eeb446f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eccf97da2e1744378f9e03922dc4bc7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/40456 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ma={}\n",
    "data=caption_data.split(\"\\n\")\n",
    "for line in tqdm(data):\n",
    "  mapp=line.split(\",\")\n",
    "  if len(mapp)<2:\n",
    "    continue\n",
    "  im_id=mapp[0]\n",
    "  cap=mapp[1]\n",
    "  cap=\"\".join(cap)\n",
    "  im_id=im_id.split(\".\")[0]\n",
    "  if im_id not in ma:\n",
    "    ma[im_id]=[]\n",
    "  ma[im_id].append(cap)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d4621146",
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_text(cap):\n",
    "  cap=cap.lower()\n",
    "  cap=cap.replace('[^a-z]',\"\")\n",
    "  cap=cap.replace('\\s+',\" \")\n",
    "  cap=\"startseq \"+\" \".join([word for word in cap.split(\" \") if len(word)>1])+\" endseq\"\n",
    "  return cap\n",
    "\n",
    "def clean(ma):\n",
    "  for key, cap in ma.items():\n",
    "    for i in range(len(cap)):\n",
    "      cap[i]=process_text(cap[i])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "249084e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "clean(ma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "82d3499a",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_captions = []\n",
    "for key in mapping:\n",
    "    for caption in mapping[key]:\n",
    "        all_captions.append(caption)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cd90e55f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40455"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_captions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b195a348",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(all_captions)\n",
    "vocab_size = len(tokenizer.word_index) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "06788c74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "35"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_length = max(len(caption.split()) for caption in all_captions)\n",
    "max_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6edafc86",
   "metadata": {},
   "outputs": [],
   "source": [
    "image_ids = list(mapping.keys())\n",
    "split = int(len(image_ids) * 0.90)\n",
    "train = image_ids[:split]\n",
    "test = image_ids[split:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b214a4dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_generator(data_keys, mapping, features, tokenizer, max_length, vocab_size, batch_size):\n",
    "    X1, X2, y = list(), list(), list()\n",
    "    n = 0\n",
    "    while 1:\n",
    "        for key in data_keys:\n",
    "            n += 1\n",
    "            captions = mapping[key]\n",
    "            for caption in captions:\n",
    "                seq = tokenizer.texts_to_sequences([caption])[0]\n",
    "                for i in range(1, len(seq)):\n",
    "                    in_seq, out_seq = seq[:i], seq[i]\n",
    "                    in_seq = pad_sequences([in_seq], maxlen=max_length)[0]\n",
    "                    out_seq = to_categorical([out_seq], num_classes=vocab_size)[0]\n",
    "                    X1.append(features[key][0])\n",
    "                    X2.append(in_seq)\n",
    "                    y.append(out_seq)\n",
    "            if n == batch_size:\n",
    "                X1, X2, y = np.array(X1), np.array(X2), np.array(y)\n",
    "                yield [X1, X2], y\n",
    "                X1, X2, y = list(), list(), list()\n",
    "                n = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ba019340",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You must install pydot (`pip install pydot`) and install graphviz (see instructions at https://graphviz.gitlab.io/download/) for plot_model to work.\n"
     ]
    }
   ],
   "source": [
    "inputs1 = Input(shape=(4096,))\n",
    "fe1 = Dropout(0.4)(inputs1)\n",
    "fe2 = Dense(256, activation='relu')(fe1)\n",
    "inputs2 = Input(shape=(max_length,))\n",
    "se1 = Embedding(vocab_size, 256, mask_zero=True)(inputs2)\n",
    "se2 = Dropout(0.4)(se1)\n",
    "se3 = LSTM(256)(se2)\n",
    "decoder1 = add([fe2, se3])\n",
    "decoder2 = Dense(256, activation='relu')(decoder1)\n",
    "outputs = Dense(vocab_size, activation='softmax')(decoder2)\n",
    "\n",
    "model = Model(inputs=[inputs1, inputs2], outputs=outputs)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "\n",
    "plot_model(model, show_shapes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c9cd441e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "227/227 [==============================] - 634s 3s/step - loss: 5.2148\n",
      "227/227 [==============================] - 552s 2s/step - loss: 3.9993\n",
      "227/227 [==============================] - 547s 2s/step - loss: 3.5808\n",
      "227/227 [==============================] - 565s 2s/step - loss: 3.3151\n",
      "227/227 [==============================] - 583s 3s/step - loss: 3.1139\n",
      "227/227 [==============================] - 563s 2s/step - loss: 2.9658\n",
      "227/227 [==============================] - 563s 2s/step - loss: 2.8508\n",
      "227/227 [==============================] - 562s 2s/step - loss: 2.7600\n",
      "227/227 [==============================] - 570s 3s/step - loss: 2.6801\n",
      "227/227 [==============================] - 564s 2s/step - loss: 2.6098\n",
      "227/227 [==============================] - 564s 2s/step - loss: 2.5561\n",
      "227/227 [==============================] - 568s 3s/step - loss: 2.4974\n",
      "227/227 [==============================] - 575s 3s/step - loss: 2.4453\n",
      "227/227 [==============================] - 572s 3s/step - loss: 2.3967\n",
      "227/227 [==============================] - 576s 3s/step - loss: 2.3553\n",
      "227/227 [==============================] - 570s 3s/step - loss: 2.3203\n",
      "227/227 [==============================] - 570s 3s/step - loss: 2.2833\n",
      "227/227 [==============================] - 560s 2s/step - loss: 2.2474\n",
      "227/227 [==============================] - 559s 2s/step - loss: 2.2182\n",
      "227/227 [==============================] - 561s 2s/step - loss: 2.1891\n"
     ]
    }
   ],
   "source": [
    "epochs = 20\n",
    "batch_size = 32\n",
    "steps = len(train)\n",
    "\n",
    "for i in range(epochs):\n",
    "    generator = data_generator(train, ma, features, tokenizer, max_length, vocab_size, batch_size)\n",
    "    model.fit(generator, epochs=1, steps_per_epoch=steps, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "3e22a08f",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(work+'/image_caption.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "8d6cae78",
   "metadata": {},
   "outputs": [],
   "source": [
    "def idx_word(integer,tok):\n",
    "    for word,index in tok.word_index.items():\n",
    "        if index== integer:\n",
    "            return word\n",
    "    return none"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "68502106",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_caption(model,image,tok,max_len):\n",
    "    in_text=\"startseq\"\n",
    "    for i in range(max_len):\n",
    "        seq=tok.texts_to_sequences([in_text])[0]\n",
    "        seq=pad_sequences([seq],max_len)\n",
    "        yhat = model.predict([image, seq], verbose=0)\n",
    "        yhat = np.argmax(yhat)\n",
    "        word = idx_word(yhat, tok)\n",
    "        if word is None:\n",
    "            break\n",
    "        in_text += \" \" + word\n",
    "        if word == 'endseq':\n",
    "            break\n",
    "    return in_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6fa2905",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cebaf5ee07d54f4bb56ce83763063629",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/810 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from nltk.translate.bleu_score import corpus_bleu\n",
    "actual, predicted = list(), list()\n",
    "for key in tqdm(test):\n",
    "    captions = mapping[key]\n",
    "    y_pred = predict_caption(model, features[key], tokenizer, max_length) \n",
    "    actual_captions = [caption.split() for caption in captions]\n",
    "    y_pred = y_pred.split()\n",
    "    # append to the list\n",
    "    actual.append(actual_captions)\n",
    "    predicted.append(y_pred)\n",
    "print(\"BLEU-1: %f\" % corpus_bleu(actual, predicted, weights=(1.0, 0, 0, 0)))\n",
    "print(\"BLEU-2: %f\" % corpus_bleu(actual, predicted, weights=(0.5, 0.5, 0, 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "468e17a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import matplotlib.pyplot as plt\n",
    "def generate_caption(image_name):\n",
    "    image_id = image_name.split('.')[0]\n",
    "    img_path = os.path.join(base, \"Images\", image_name)\n",
    "    image = Image.open(img_path)\n",
    "    captions = mapping[image_id]\n",
    "    print('---------------------Actual---------------------')\n",
    "    for caption in captions:\n",
    "        print(caption)\n",
    "    # predict the caption\n",
    "    y_pred = predict_caption(model, features[image_id], tokenizer, max_length)\n",
    "    print('--------------------Predicted--------------------')\n",
    "    print(y_pred)\n",
    "    plt.imshow(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b66d1b91",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30bf4acd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76d5e2af",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d735bdc1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cc4d2af9",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(base,\"captions.txt\"),\"r\") as f:\n",
    "  next(f)\n",
    "  caption_data=f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ddb5ee13",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c26d20eded654d9a82beaad96d6fcb6b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/40456 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ma={}\n",
    "data=caption_data.split(\"\\n\")\n",
    "for line in tqdm(data):\n",
    "  mapp=line.split(\",\")\n",
    "  if len(mapp)<2:\n",
    "    continue\n",
    "  im_id=mapp[0]\n",
    "  cap=mapp[1]\n",
    "  cap=\"\".join(cap)\n",
    "  im_id=im_id.split(\".\")[0]\n",
    "  if im_id not in ma:\n",
    "    ma[im_id]=[]\n",
    "  ma[im_id].append(cap)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "05cab232",
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_text(cap):\n",
    "  cap=cap.lower()\n",
    "  cap=cap.replace('[^a-z]',\"\")\n",
    "  cap=cap.replace('\\s+',\" \")\n",
    "  cap=\"[start] \"+\" \".join([word for word in cap.split(\" \") if len(word)>1])+\" [end]\"\n",
    "  return cap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f75f26df",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean(ma):\n",
    "  for key, cap in ma.items():\n",
    "    for i in range(len(cap)):\n",
    "      cap[i]=process_text(cap[i])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "15693ddd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['A child in a pink dress is climbing up a set of stairs in an entry way .',\n",
       " 'A girl going into a wooden building .',\n",
       " 'A little girl climbing into a wooden playhouse .',\n",
       " 'A little girl climbing the stairs to her playhouse .',\n",
       " 'A little girl in a pink dress going into a wooden cabin .']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ma[\"1000268201_693b08cb0e\"] # just a check before "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "defc5403",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[start] child in pink dress is climbing up set of stairs in an entry way [end]',\n",
       " '[start] girl going into wooden building [end]',\n",
       " '[start] little girl climbing into wooden playhouse [end]',\n",
       " '[start] little girl climbing the stairs to her playhouse [end]',\n",
       " '[start] little girl in pink dress going into wooden cabin [end]']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean(ma)\n",
    "ma[\"1000268201_693b08cb0e\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f5913f53",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_cap=[]\n",
    "for key in ma.keys():\n",
    "    for cap in ma[key]:\n",
    "        all_cap.append(cap)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "84d681f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40455"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_cap)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4dbe92b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8311"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tok=Tokenizer()\n",
    "tok.fit_on_texts(all_cap)\n",
    "vocab_size=len(tok.word_index)+1\n",
    "vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "776312f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "31"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_len=max(len(cap.split())for cap in all_cap)\n",
    "max_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "57a14f3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "image_ids=list(ma.keys())\n",
    "split=int(len(image_ids)*0.90)\n",
    "train=image_ids[:split]\n",
    "test=image_ids[split:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "69b7ff8a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7281"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "378f6cb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_gen(data_keys,ma,fs,tok,max_len,vocab_size,batch_size):\n",
    "    x1,x2,y=list(),list(),list()\n",
    "    n=0;\n",
    "    while True:\n",
    "        for key in data_keys:\n",
    "            n+=1\n",
    "            cap=ma[key]\n",
    "            for cap_i in cap:\n",
    "                seq=tok.texts_to_sequences([cap_i])[0]\n",
    "                for i in range(len(seq)):\n",
    "                    in_seq,out_seq=seq[:i],seq[i]\n",
    "                    in_seq=pad_sequences([in_seq],maxlen=max_len)[0]\n",
    "                    out_seq=to_categorical([out_seq],num_classes=vocab_size)[0]\n",
    "                    x1.append(fs[key][0])\n",
    "                    x2.append(in_seq)\n",
    "                    y.append(out_seq)\n",
    "            if n==batch_size:\n",
    "                x1=np.array(x1)\n",
    "                x2=np.array(x2)\n",
    "                y=np.array(y)\n",
    "                yield[x1,x2],y\n",
    "                x1,x2,y=list(),list(),list()\n",
    "                n=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f5f13047",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You must install pydot (`pip install pydot`) and install graphviz (see instructions at https://graphviz.gitlab.io/download/) for plot_model to work.\n"
     ]
    }
   ],
   "source": [
    "inputs1=Input(shape=(4096,))\n",
    "fe1=Dropout(0.4)(inputs1)\n",
    "fe2=Dense(256,activation='relu')(fe1)\n",
    "inputs2=Input(shape=(max_len,))\n",
    "se1=Embedding(vocab_size,256,mask_zero=True)(inputs2)\n",
    "se2=Dropout(0.4)(se1)\n",
    "se3=LSTM(256)(se2)\n",
    "\n",
    "decoder1=add([fe2,se3])\n",
    "decoder2=Dense(256,activation='relu')(decoder1)\n",
    "outputs=Dense(vocab_size,activation='softmax')(decoder2)\n",
    "\n",
    "model=Model(inputs=[inputs1,inputs2],outputs=outputs)\n",
    "model.compile(loss=\"categorical_crossentropy\",optimizer='adam')\n",
    "\n",
    "\n",
    "plot_model(model,show_shapes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d63d6d4b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "   1/7281 [..............................] - ETA: 38:03:57 - loss: 9.0597"
     ]
    }
   ],
   "source": [
    "epochs=15\n",
    "batch_size=64\n",
    "steps=len(train)\n",
    "for i in range(epochs):\n",
    "    generator=data_gen(train,ma,fs,tok,max_len,vocab_size,batch_size)\n",
    "    model.fit(generator,epochs=1,steps_per_epoch=steps,verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3322120d",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(work+'/image_caption.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "303f7a8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def idx_word(integer,tok):\n",
    "    for word,index in tok.word_index.items():\n",
    "        if index== integer:\n",
    "            return word\n",
    "    return none"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "541d09e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_caption(model,image,tok,max_len):\n",
    "    in_text=\"[start]\"\n",
    "    for i in range(max_len):\n",
    "        seq=tok.texts_to_sequences([in_text])[0]\n",
    "        seq=pad_sequences([seq],max_len)[0]\n",
    "        yhat"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}