Upload 9 files

best_model.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:363304a55bc92c6da7a7c463af4bdfdf7edb5b2eebcfad154ad177d54e7fd241
+size 71970004

features.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d62b45d8a5e42a978f7652fbb08d744e48fc05e5a38808faff93446022216bc5
+size 133064982

model_main.py

@@ -0,0 +1,185 @@
+import os
+import pickle
+import numpy as np
+from tqdm.notebook import tqdm
+
+from tensorflow.keras.applications.vgg16 import VGG16, preprocess_input
+from tensorflow.keras.preprocessing.image import load_img, img_to_array
+from tensorflow.keras.preprocessing.text import Tokenizer
+from tensorflow.keras.preprocessing.sequence import pad_sequences
+from tensorflow.keras.models import Model
+from tensorflow.keras.utils import to_categorical, plot_model
+from tensorflow.keras.layers import Input, Dense, LSTM, Embedding, Dropout, add
+
+# load vgg16 model
+model = VGG16()
+# restructure the model
+model = Model(inputs=model.inputs, outputs=model.layers[-2].output)
+
+with open('features.pkl', 'rb') as f:
+    features = pickle.load(f)
+
+with open('captions.txt', 'r') as f:
+    next(f)
+    captions_doc = f.read()
+
+# # create mapping of image to captions
+mapping = {}
+# process lines
+for line in tqdm(captions_doc.split('\n')):
+    # split the line by comma(,)
+    tokens = line.split(',')
+    if len(line) < 2:
+        continue
+    image_id, caption = tokens[0], tokens[1:]
+    # remove extension from image ID
+    image_id = image_id.split('.')[0]
+    # convert caption list to string
+    caption = " ".join(caption)
+    # create list if needed
+    if image_id not in mapping:
+        mapping[image_id] = []
+    # store the caption
+    mapping[image_id].append(caption)
+
+def clean(mapping):
+    for key, captions in mapping.items():
+        for i in range(len(captions)):
+            # take one caption at a time
+            caption = captions[i]
+            # preprocessing steps
+            # convert to lowercase
+            caption = caption.lower()
+            # delete digits, special chars, etc.,
+            caption = caption.replace('[^A-Za-z]', '')
+            # delete additional spaces
+            caption = caption.replace('\s+', ' ')
+            # add start and end tags to the caption
+            caption = 'startseq ' + " ".join([word for word in caption.split() if len(word)>1]) + ' endseq'
+            captions[i] = caption
+
+clean(mapping)
+
+all_captions = []
+for key in mapping:
+    for caption in mapping[key]:
+        all_captions.append(caption)
+
+# tokenize the text
+tokenizer = Tokenizer()
+tokenizer.fit_on_texts(all_captions)
+vocab_size = len(tokenizer.word_index) + 1
+
+# get maximum length of the caption available
+max_length = max(len(caption.split()) for caption in all_captions)
+
+
+# create data generator to get data in batch (avoids session crash)
+def data_generator(data_keys, mapping, features, tokenizer, max_length, vocab_size, batch_size):
+    # loop over images
+    X1, X2, y = list(), list(), list()
+    n = 0
+    while 1:
+        for key in data_keys:
+            n += 1
+            captions = mapping[key]
+            # process each caption
+            for caption in captions:
+                # encode the sequence
+                seq = tokenizer.texts_to_sequences([caption])[0]
+                # split the sequence into X, y pairs
+                for i in range(1, len(seq)):
+                    # split into input and output pairs
+                    in_seq, out_seq = seq[:i], seq[i]
+                    # pad input sequence
+                    in_seq = pad_sequences([in_seq], maxlen=max_length)[0]
+                    # encode output sequence
+                    out_seq = to_categorical([out_seq], num_classes=vocab_size)[0]
+
+                    # store the sequences
+                    X1.append(features[key][0])
+                    X2.append(in_seq)
+                    y.append(out_seq)
+            if n == batch_size:
+                X1, X2, y = np.array(X1), np.array(X2), np.array(y)
+                yield [X1, X2], y
+                X1, X2, y = list(), list(), list()
+                n = 0
+
+# encoder model
+# image feature layers
+inputs1 = Input(shape=(4096,))
+fe1 = Dropout(0.4)(inputs1)
+fe2 = Dense(256, activation='relu')(fe1)
+# sequence feature layers
+inputs2 = Input(shape=(max_length,))
+se1 = Embedding(vocab_size, 256, mask_zero=True)(inputs2)
+se2 = Dropout(0.4)(se1)
+se3 = LSTM(256)(se2)
+
+# decoder model
+decoder1 = add([fe2, se3])
+decoder2 = Dense(256, activation='relu')(decoder1)
+outputs = Dense(vocab_size, activation='softmax')(decoder2)
+
+model = Model(inputs=[inputs1, inputs2], outputs=outputs)
+model.compile(loss='categorical_crossentropy', optimizer='adam')
+
+# plot the model
+plot_model(model, show_shapes=True)
+
+from keras.models import load_model
+model = load_model("best_model.h5")
+
+def idx_to_word(integer, tokenizer):
+    for word, index in tokenizer.word_index.items():
+        if index == integer:
+            return word
+    return None
+
+
+# generate caption for an image
+def predict_caption(model, image, tokenizer, max_length):
+    # add start tag for generation process
+    in_text = 'startseq'
+    # iterate over the max length of sequence
+    for i in range(max_length):
+        # encode input sequence
+        sequence = tokenizer.texts_to_sequences([in_text])[0]
+        # pad the sequence
+        sequence = pad_sequences([sequence], max_length)
+        # predict next word
+        yhat = model.predict([image, sequence], verbose=0)
+        # get index with high probability
+        yhat = np.argmax(yhat)
+        # convert index to word
+        word = idx_to_word(yhat, tokenizer)
+        # stop if word not found
+        if word is None:
+            break
+        # append word as input for generating next word
+        in_text += " " + word
+        # stop if we reach end tag
+        if word == 'endseq':
+            break
+
+    return in_text
+
+vgg_model = VGG16()
+# restructure the model
+vgg_model = Model(inputs=vgg_model.inputs, outputs=vgg_model.layers[-2].output)
+
+def generate_caption(image_path):
+    image_path = image_path
+    # load image
+    image = load_img(image_path, target_size=(224, 224))
+    # convert image pixels to numpy array
+    image = img_to_array(image)
+    # reshape data for model
+    image = image.reshape((1, image.shape[0], image.shape[1], image.shape[2]))
+    # preprocess image for vgg
+    image = preprocess_input(image)
+    # extract features
+    feature = vgg_model.predict(image, verbose=0)
+    # predict from the trained model
+    return predict_caption(model, feature, tokenizer, max_length)[9: -7]

requirements.txt

@@ -0,0 +1,363 @@
+absl-py==1.4.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==5.0.1
+anyascii==0.3.2
+anyio==3.7.0
+apache-beam==2.48.0
+apparmor==3.1.6
+application-utility==1.3.2
+argon2-cffi==21.3.0
+argon2-cffi-bindings==21.2.0
+array-record==0.4.0
+arrow==1.2.3
+asttokens==2.2.1
+astunparse==1.6.3
+async-lru==2.0.2
+async-timeout==4.0.2
+attrs==22.2.0
+autocommand==2.2.2
+avro-python3==1.10.2
+Babel==2.12.1
+backcall==0.2.0
+bcrypt==4.0.1
+beautifulsoup4==4.12.2
+bleach==6.0.0
+blinker==1.6.2
+blis==0.7.9
+btrfsutil==6.3.2
+cachetools==5.3.1
+catalogue==2.0.8
+certifi==2023.5.7
+cffi==1.15.1
+chardet==5.1.0
+charset-normalizer==3.1.0
+click==8.1.4
+cloudpickle==2.2.1
+cmake==3.26.4
+colorama==0.4.6
+comm==0.1.3
+confection==0.1.0
+configobj==5.0.8
+contextlib2==21.6.0
+contourpy==1.0.7
+crcmod==1.7
+cryptography==41.0.1
+cssselect2==0.7.0
+cycler==0.11.0
+cymem==2.0.7
+Cython==0.29.35
+datasets==2.13.1
+dbus-python==1.3.2
+debugpy==1.6.7
+decorator==5.1.1
+defusedxml==0.7.1
+dill==0.3.6
+dm-tree==0.1.8
+dnspython==2.3.0
+docopt==0.6.2
+duplicity==1.2.3
+en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.5.0/en_core_web_sm-3.5.0-py3-none-any.whl#sha256=0964370218b7e1672a30ac50d72cdc6b16f7c867496f1d60925691188f4d2510
+etils==1.3.0
+ewmh==0.1.6
+executing==1.2.0
+Farama-Notifications==0.0.4
+fastavro==1.7.4
+fasteners==0.18
+fastjsonschema==2.17.1
+ffmpeg-python==0.2.0
+filelock==3.12.0
+flatbuffers==23.5.26
+fonttools==4.39.4
+fqdn==1.5.1
+frozenlist==1.3.3
+fsspec==2023.6.0
+future==0.18.3
+gast==0.4.0
+gdown==4.7.1
+gin-config==0.5.0
+gitdb==4.0.10
+GitPython==3.1.31
+google-api-core==2.11.0
+google-api-python-client==2.88.0
+google-auth==2.18.1
+google-auth-httplib2==0.1.0
+google-auth-oauthlib==1.0.0
+google-pasta==0.2.0
+googleapis-common-protos==1.59.1
+grpcio==1.54.2
+grpcio-tools==1.55.1
+gufw==22.4.0
+gym==0.26.2
+gym-notices==0.0.8
+gymnasium==0.28.1
+h5py==3.8.0
+hdfs==2.7.0
+hparam==0.1.4
+hparams==0.3.0
+html2text==2020.1.16
+httplib2==0.22.0
+huggingface-hub==0.16.4
+ibm-cloud-sdk-core==3.16.7
+ibm-watson==7.0.0
+idna==3.4
+imageio==2.31.1
+immutabledict==2.2.4
+importlib-metadata==4.13.0
+importlib-resources==5.12.0
+inflect==6.1.0
+ipykernel==6.23.1
+ipython==8.14.0
+ipython-genutils==0.2.0
+ipywidgets==8.0.7
+isoduration==20.11.0
+jaraco.context==4.3.0
+jaraco.functools==3.8.0
+jaraco.text==3.11.1
+jax==0.4.12
+jax-jumpy==1.0.0
+jedi==0.18.2
+Jinja2==3.1.2
+joblib==1.2.0
+json5==0.9.14
+jsonpickle==3.0.1
+jsonpointer==2.3
+jsonschema==4.17.3
+jupyter-events==0.6.3
+jupyter-lsp==2.2.0
+jupyter_client==8.2.0
+jupyter_core==5.3.0
+jupyter_server==2.6.0
+jupyter_server_terminals==0.4.4
+jupyterlab==4.0.2
+jupyterlab-pygments==0.2.2
+jupyterlab-widgets==3.0.8
+jupyterlab_server==2.23.0
+kaggle==1.5.13
+keras==2.12.0
+kiwisolver==1.4.4
+langcodes==3.3.0
+layoutswitcherlib==0.8.36
+LibAppArmor==3.1.6
+libclang==16.0.0
+libfdt==1.7.0
+lit==15.0.7.dev0
+lvis==0.5.3
+lxml==4.9.2
+Mako==1.2.4
+mallard-ducktype==1.0.2
+manjaro-sdk==0.1
+Markdown==3.4.3
+markdown-it-py==3.0.0
+MarkupSafe==2.1.3
+material-color-utilities-python==0.1.5
+matplotlib==3.7.1
+matplotlib-inline==0.1.6
+mdurl==0.1.2
+mediapipe==0.10.1
+meson==1.1.1
+mistune==2.0.5
+ml-dtypes==0.2.0
+mlxtend==0.22.0
+more-itertools==9.1.0
+mpmath==1.3.0
+multidict==6.0.4
+multiprocess==0.70.14
+murmurhash==1.0.9
+music21==9.1.0
+nbclassic==1.0.0
+nbclient==0.8.0
+nbconvert==7.4.0
+nbformat==5.9.0
+nest-asyncio==1.5.6
+netsnmp-python==1.0a1
+networkx==3.1
+nftables==0.1
+nltk==3.8.1
+notebook==6.5.4
+notebook_shim==0.2.3
+npyscreen==4.10.5
+numpy==1.25.1
+nvidia-cublas-cu11==11.10.3.66
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+nvidia-cuda-runtime-cu11==11.7.99
+nvidia-cudnn-cu11==8.5.0.96
+nvidia-cufft-cu11==10.9.0.58
+nvidia-curand-cu11==10.2.10.91
+nvidia-cusolver-cu11==11.4.0.1
+nvidia-cusparse-cu11==11.7.4.91
+nvidia-nccl-cu11==2.14.3
+nvidia-nvtx-cu11==11.7.91
+oauth2client==4.1.3
+oauthlib==3.2.2
+object-detection @ file:///home/reputation/Documents/DL/TFOD/models/research
+objsize==0.6.1
+opencv-contrib-python==4.8.0.74
+opencv-python==4.8.0.74
+opt-einsum==3.3.0
+optimus-manager==1.5
+ordered-set==4.1.0
+orjson==3.9.1
+overrides==7.3.1
+packaging==23.1
+pacman-mirrors==4.23.2
+pandas==2.0.2
+pandocfilters==1.5.0
+paramiko==2.11.1
+parso==0.8.3
+pathy==0.10.2
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow==10.0.0
+pipreqs==0.4.13
+platformdirs==3.5.1
+ply==3.11
+portalocker==2.7.0
+preshed==3.0.8
+prometheus-client==0.17.0
+promise==2.3
+prompt-toolkit==3.0.38
+proto-plus==1.22.3
+protobuf==3.20.3
+protobuf-compiler==1.0.20
+psutil==5.9.5
+ptyprocess==0.7.0
+pulsectl==23.5.2
+pure-eval==0.2.2
+pwquality==1.4.5
+py-cpuinfo==9.0.0
+pyaml==23.5.9
+pyarrow==11.0.0
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycairo==1.23.0
+pycocotools==2.0.6
+pycparser==2.21
+pydantic==1.10.9
+pydeck==0.8.1b0
+pydot==1.4.2
+PyDrive2==1.16.1
+pygame==2.5.0
+pyglet==1.5.0
+Pygments==2.15.1
+PyGObject==3.44.1
+PyJWT==2.7.0
+pymongo==4.4.0
+Pympler==1.0.1
+PyNaCl==1.4.0
+pyOpenSSL==23.2.0
+pyparsing==2.4.7
+pyrsistent==0.19.3
+PySide6==6.5.1.1
+PySocks==1.7.1
+python-dateutil==2.8.2
+python-json-logger==2.0.7
+python-slugify==8.0.1
+python-xlib==0.33
+PythonTurtle==0.3.2
+pytz==2023.3
+pytz-deprecation-shim==0.1.0.post0
+pyxdg==0.28
+PyYAML==5.4.1
+pyzmq==25.1.0
+ranger-fm==1.9.3
+regex==2023.6.3
+reportlab==3.6.12
+requests==2.31.0
+requests-file==1.5.1
+requests-oauthlib==1.3.1
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.4.2
+rsa==4.9
+sacrebleu==2.2.0
+safetensors==0.3.1
+scikit-learn==1.2.2
+scipy==1.10.1
+seaborn==0.12.2
+Send2Trash==1.8.2
+sentencepiece==0.1.99
+seqeval==1.2.2
+setproctitle==1.3.2
+shiboken6==6.5.1.1
+shiboken6-generator==6.5.1.1
+six==1.16.0
+smart-open==6.3.0
+smmap==5.0.0
+sniffio==1.3.0
+sounddevice==0.4.6
+soupsieve==2.4.1
+spacy==3.5.4
+spacy-legacy==3.0.12
+spacy-loggers==1.0.4
+srsly==2.4.6
+stable-baselines3 @ git+https://github.com/carlosluis/stable-baselines3@6617e6e73cb3a70f3e88cea780ea12bed95c099e
+stack-data==0.6.2
+streamlit==1.24.0
+svglib==1.5.1
+sympy==1.12
+systemd-python==235
+tabulate==0.9.0
+tenacity==8.2.2
+tensorboard==2.12.3
+tensorboard-data-server==0.7.0
+tensorflow==2.12.0
+tensorflow-addons==0.20.0
+tensorflow-datasets==4.9.2
+tensorflow-estimator==2.12.0
+tensorflow-hub==0.13.0
+tensorflow-io==0.32.0
+tensorflow-io-gcs-filesystem==0.32.0
+tensorflow-metadata==1.13.1
+tensorflow-model-optimization==0.7.5
+tensorflow-text==2.12.1
+termcolor==2.3.0
+terminado==0.17.1
+text-unidecode==1.3
+tf-models-official==2.12.0
+tf-slim==1.1.0
+thinc==8.1.10
+threadpoolctl==3.1.0
+tinycss2==1.2.1
+tk==0.1.0
+tldextract==3.4.4
+tokenizers==0.13.3
+toml==0.10.2
+tomli==2.0.1
+toolz==0.12.0
+torch==2.0.1
+tornado==6.3.2
+tqdm==4.65.0
+traitlets==5.9.0
+transformers==4.30.2
+triton==2.0.0
+trove-classifiers==2023.7.8
+typeguard==2.13.3
+typer==0.9.0
+typing_extensions==4.7.0
+tzdata==2023.3
+tzlocal==4.3.1
+ueberzug==18.2.1
+ufw==0.36.2
+uri-template==1.2.0
+uritemplate==4.1.1
+urllib3==1.26.15
+validate==5.0.8
+validate-pyproject==0.13.post1.dev0+gb752273.d20230520
+validators==0.20.0
+wasabi==1.1.2
+watchdog==3.0.0
+wcwidth==0.2.6
+webcolors==1.13
+webencodings==0.5.1
+websocket-client==1.1.0
+Werkzeug==2.3.6
+widgetsnbextension==4.0.8
+wrapt==1.14.1
+xxhash==3.2.0
+Yapsy==1.12.2
+yarg==0.1.9
+yarl==1.9.2
+zipp==3.15.0
+zstandard==0.21.0