initial commit

README.md

@@ -1,6 +1,6 @@
 ---
 title: Neural Style Transfer Demo
-emoji: 📊
+emoji: 🎨
 colorFrom: purple
 colorTo: red
 sdk: gradio

app.py

@@ -0,0 +1,107 @@
+import os
+import numpy as np
+import tensorflow as tf
+import gradio as gr
+import random
+from PIL import Image
+from tensorflow.keras.preprocessing.image import load_img,img_to_array
+from tensorflow.keras.optimizers import SGD
+from tensorflow.keras.optimizers.schedules import ExponentialDecay
+# change in the actual written file
+from modules.nst_loss_functions import *
+from modules.nst_models import *
+from modules.preprocessing_utils import *
+
+
+# get the weights
+content_weight,style_weight = get_weights()
+# get the content and style layer lists
+content_layer,style_layers = get_layers_lists()
+# instance the pretrained model
+pretrained_vgg_model,feature_extractor = get_pretrained_vgg_model_fe()
+
+# training function
+@tf.function
+def compute_loss_and_grads(generated_image,
+                           base_image,
+                           style_image,
+                           row_cols):
+
+    with tf.GradientTape() as tape:
+        loss = loss_function(generated_image = generated_image,
+                             base_image = base_image,
+                             style_image = style_image,
+                             content_layer = content_layer,
+                             style_layers = style_layers,
+                             feature_extractor = feature_extractor,
+                             weights= (content_weight,style_weight),
+                             rows_cols = row_cols)
+
+    grads = tape.gradient(loss, generated_image)
+    return loss, grads
+
+
+# generate image
+def generate(base_image,style_image,epochs,progress=gr.Progress()):
+  # instance images
+  base_pil_image = base_image
+  style_pil_image = style_image
+  generated_pil_image = base_pil_image.copy()
+
+  # determine the base image's dimentions
+  width, height = base_pil_image.size
+  img_nrows = 400
+  img_ncols = int(width * img_nrows / height)
+
+  # instance the optimizer
+  optimizer = SGD(
+      ExponentialDecay(
+          initial_learning_rate=100.0, decay_steps=100, decay_rate=0.96
+      )
+  )
+
+  # preprocess the images
+  base_image = preprocess_image(base_pil_image,(img_nrows,img_ncols))
+  style_image = preprocess_image(style_pil_image,(img_nrows,img_ncols))
+  generated_image = tf.Variable(preprocess_image(generated_pil_image,(img_nrows,img_ncols)))
+
+  for i in progress.tqdm(range(int(epochs))):
+      loss, grads = compute_loss_and_grads(
+          generated_image, base_image, style_image,(img_nrows,img_ncols)
+      )
+      optimizer.apply_gradients([(grads, generated_image)])
+
+  generated_image = generated_image.numpy()
+  generated_image = deprocess_image(generated_image,(img_nrows,img_ncols))
+
+  return generated_image
+
+title = "Neural Style Transfer Demo"
+description = "This is my implementation of the neural style transfer algorithm using Tensorflow2"
+article = "The NST algorithm is an algorithm that allows you to replicate an image A with similar features to the ones present in an image B. In a nutshell, this is done by using a pretrained CNN to  perform gradient descent on the weighted cost of a style and content cost function, which correspond to the frobenius norm across the features’ cross covariance across different layers and the simple norm respectively. The result of the loss is applied to a random generated image to get the hybrid. To use this app, select a real photo as a content image and an art piece as style image from an URL or from your PC, set the number of epochs (it is recommended to leave the default value), and run the app. THIS MAY TAKE SOME TIME, PLEASE BE PATIENT (╯°□°)╯."
+
+
+
+example_list = list()
+# change in the app
+examples_path = "examples"
+content_examples_path = os.path.join(examples_path,"content")
+style_examples_path = os.path.join(examples_path,"style")
+
+content_examples = [[str(content_examples_path) + "/" + example] for example in os.listdir(content_examples_path)]
+style_examples = [[str(style_examples_path) + "/" + example] for example in os.listdir(style_examples_path)]
+
+
+img_input_1 = gr.Image(label = "Content Image",type = "pil",value = random.choice(content_examples)[0])
+img_input_2 = gr.Image(label = "Style Image",type = "pil",value = random.choice(style_examples)[0])
+
+demo = gr.Interface(
+    fn = generate,
+    inputs = [img_input_1,img_input_2,gr.Number(value = 250,label = "Number of epochs",)],
+    outputs = [gr.Image(type = "pil")],
+    title = title,
+    description = description,
+    article = article
+)
+
+demo.queue().launch()

modules/nst_loss_functions.py

@@ -0,0 +1,68 @@
+import numpy as np
+import tensorflow as tf
+
+# compute the gram matrix of the features
+def gram_matrix(x):
+  #x = tf.convert_to_tensor(x, tf.int32)
+  x = tf.transpose(x,perm = (2,0,1))
+  features = tf.reshape(x,(tf.shape(x)[0],-1))
+  gram = tf.matmul(features,tf.transpose(features))
+  return gram
+
+# compute the style cost function
+def style_cost_function(style_image,generated_image,rows_cols):
+  img_nrows,img_ncols = rows_cols
+  S = gram_matrix(style_image)
+  C = gram_matrix(generated_image)
+  channels = 3
+  size = img_nrows * img_ncols
+  return tf.reduce_sum(tf.square(S - C)) / (4.0 * (channels ** 2) * (size ** 2))
+
+# compute the content cost function
+def content_cost_function(base_image,generated_image):
+  return tf.reduce_sum(tf.square(tf.subtract(generated_image,base_image)))
+
+def loss_function(
+    generated_image, 
+    base_image, 
+    style_image,
+    content_layer,
+    style_layers,
+    feature_extractor,
+    weights,
+    rows_cols):
+  
+  # fetch the weights
+  content_weight,style_weight = weights
+
+  # combine the images into a single tensor
+  input_tensor = tf.concat(
+      [base_image, style_image, generated_image], axis=0
+  )
+
+  # get the values in all the layers for the three images
+  features = feature_extractor(input_tensor)
+
+  # iinitialize the loss function
+  loss = tf.zeros(shape=())
+
+  # compute the content loss
+  layer_features = features[content_layer]
+  base_image_features = layer_features[0, :, :, :]
+  combination_features = layer_features[2, :, :, :]
+
+  loss = loss + content_weight * content_cost_function(
+      base_image_features, combination_features
+  )
+
+  # compute the style loss
+  for layer_name in style_layers:
+      layer_features = features[layer_name]
+      style_reference_features = layer_features[1, :, :, :]
+      combination_features = layer_features[2, :, :, :]
+      sl = style_cost_function(style_reference_features, combination_features,rows_cols)
+      loss += (style_weight / len(style_layers)) * sl
+
+  return loss
+
+

modules/nst_models.py

@@ -0,0 +1,33 @@
+from tensorflow.keras.models import Model
+from tensorflow.keras.applications import VGG19
+
+# instances a pretrained vgg19 model and creates the feature extractor
+def get_pretrained_vgg_model_fe():
+  # instance the pretrained model
+  pretrained_vgg_model = VGG19(include_top = False, weights="imagenet")
+  # create a dictionary that maps the layers name to each feature extractor 
+  model_outputs = {layer.name : layer.output for layer in pretrained_vgg_model.layers}
+  # feature extractor
+  feature_extractor = Model(inputs=pretrained_vgg_model.inputs, outputs=model_outputs)
+  return pretrained_vgg_model,feature_extractor
+
+# get a list of the layers that will be used to style and content
+def get_layers_lists():
+  # define the style layers
+  style_layers = [
+      "block1_conv1",
+      "block2_conv1",
+      "block3_conv1",
+      "block4_conv1",
+      "block5_conv1",
+      ]
+  # define the content layer
+  content_layer = "block5_conv2"
+  return content_layer,style_layers
+
+# get the weights for each of the cost functions in order to computer the final cost
+def get_weights():
+  # define the style and content weights
+  content_weight = 2.5e-8
+  style_weight = 1.0e-6
+  return content_weight,style_weight

modules/preprocessing_utils.py

@@ -0,0 +1,31 @@
+import numpy as np
+import tensorflow as tf
+from tensorflow.keras.applications.vgg19 import preprocess_input
+from tensorflow.keras.preprocessing.image import img_to_array
+
+def preprocess_image(img,rows_cols):
+  img_nrows, img_ncols = rows_cols
+  # load the image into a tensot
+  img = img.resize((img_ncols,img_nrows))
+  # turn the image into a numpy array
+  img = img_to_array(img)
+  # add a batch dimention
+  img = np.expand_dims(img, axis=0)
+  # preprocess according to the vgg model's specification
+  img = preprocess_input(img)
+  return tf.convert_to_tensor(img)
+
+
+def deprocess_image(x,rows_cols):
+    img_nrows, img_ncols = rows_cols
+    # Cconert to array
+    x = x.reshape((img_nrows, img_ncols, 3))
+    # mean = 0
+    x[:, :, 0] += 103.939
+    x[:, :, 1] += 116.779
+    x[:, :, 2] += 123.68
+    # convert to rgb
+    x = x[:, :, ::-1]
+    # normalize
+    x = np.clip(x, 0, 255).astype("uint8")
+    return x

requirements.txt

@@ -0,0 +1,4 @@
+tensorflow==2.12.0
+gradio==3.34.0
+numpy==1.22.4
+Pillow==8.4.0