added files

DataLoader.py

@@ -0,0 +1,278 @@
+import os
+import numpy as np
+import torch
+import random
+from PIL import Image, ImageDraw, ImageFont
+import pickle
+from config.GlobalVariables import *
+
+np.random.seed(0)
+
+class DataLoader():
+	def __init__(self, num_writer=2, num_samples=5, divider=10.0, datadir='./data/writers'):
+		self.device			= 'cuda' if torch.cuda.is_available() else 'cpu'
+		self.num_writer		= num_writer
+		self.num_samples	= num_samples
+		self.divider		= divider
+		self.datadir		= datadir
+		print ('self.datadir : ', self.datadir)
+		self.total_writers	= len([name for name in os.listdir(datadir)])
+
+	def next_batch(self, TYPE='TRAIN', uid=-1, tids=[]):
+		all_sentence_level_stroke_in		= []
+		all_sentence_level_stroke_out		= []
+		all_sentence_level_stroke_length	= []
+		all_sentence_level_term				= []
+		all_sentence_level_char				= []
+		all_sentence_level_char_length		= []
+		all_word_level_stroke_in			= []
+		all_word_level_stroke_out			= []
+		all_word_level_stroke_length		= []
+		all_word_level_term					= []
+		all_word_level_char					= []
+		all_word_level_char_length			= []
+		all_segment_level_stroke_in			= []
+		all_segment_level_stroke_out		= []
+		all_segment_level_stroke_length		= []
+		all_segment_level_term				= []
+		all_segment_level_char				= []
+		all_segment_level_char_length		= []
+
+		while len(all_sentence_level_stroke_in) < self.num_writer:
+			if uid < 0:
+				if TYPE == 'TRAIN':
+					if self.datadir == './data/NEW_writers' or self.datadir == './data/writers':
+						uid = np.random.choice([i for i in range(150)])
+					else:
+						if self.device == 'cpu':
+							uid = np.random.choice([i for i in range(20)])
+						else:
+							uid = np.random.choice([i for i in range(294)])
+				else:
+					uid = np.random.choice([i for i in range(150,170)])
+
+			total_texts				= len([name for name in os.listdir(self.datadir+'/'+str(uid))])
+			if len(tids) == 0:
+				tids = random.sample([i for i in range(total_texts)], self.num_samples)
+
+			user_sentence_level_stroke_in		= []
+			user_sentence_level_stroke_out		= []
+			user_sentence_level_stroke_length	= []
+			user_sentence_level_term			= []
+			user_sentence_level_char			= []
+			user_sentence_level_char_length		= []
+			user_word_level_stroke_in			= []
+			user_word_level_stroke_out			= []
+			user_word_level_stroke_length		= []
+			user_word_level_term				= []
+			user_word_level_char				= []
+			user_word_level_char_length			= []
+			user_segment_level_stroke_in		= []
+			user_segment_level_stroke_out		= []
+			user_segment_level_stroke_length	= []
+			user_segment_level_term				= []
+			user_segment_level_char				= []
+			user_segment_level_char_length		= []
+
+			# print ("uid: ", uid, "\ttids:", tids)
+			for tid in tids:
+				if self.datadir == './data/NEW_writers':
+					[sentence_level_raw_stroke, sentence_level_stroke_in, sentence_level_stroke_out, sentence_level_term, sentence_level_char, word_level_raw_stroke, word_level_stroke_in, word_level_stroke_out, word_level_term, word_level_char, segment_level_raw_stroke, segment_level_stroke_in, segment_level_stroke_out, segment_level_term, segment_level_char] = \
+						np.load(self.datadir+'/'+str(uid)+'/'+str(tid)+'.npy', allow_pickle=True, encoding='bytes')
+
+				elif self.datadir == './data/DW_writers':
+					[sentence_level_raw_stroke, sentence_level_char, sentence_level_term, sentence_level_stroke_in, sentence_level_stroke_out,
+					word_level_raw_stroke, word_level_char, word_level_term, word_level_stroke_in, word_level_stroke_out,
+					segment_level_raw_stroke, segment_level_char, segment_level_term, segment_level_stroke_in, segment_level_stroke_out, _] = \
+						np.load(self.datadir+'/'+str(uid)+'/'+str(tid)+'.npy', allow_pickle=True, encoding='bytes')
+
+				elif self.datadir == './data/VALID_DW_writers':
+					[sentence_level_raw_stroke, sentence_level_char, sentence_level_term, sentence_level_stroke_in, sentence_level_stroke_out,
+					word_level_raw_stroke, word_level_char, word_level_term, word_level_stroke_in, word_level_stroke_out,
+					segment_level_raw_stroke, segment_level_char, segment_level_term, segment_level_stroke_in, segment_level_stroke_out, _] = \
+						np.load(self.datadir+'/'+str(uid)+'/'+str(tid)+'.npy', allow_pickle=True, encoding='bytes')
+
+				else:
+					[sentence_level_raw_stroke, sentence_level_stroke_in, sentence_level_stroke_out, sentence_level_term, sentence_level_char, word_level_raw_stroke, word_level_stroke_in, word_level_stroke_out, word_level_term, word_level_char, segment_level_raw_stroke, segment_level_stroke_in, segment_level_stroke_out, segment_level_term, segment_level_char, _] = \
+						np.load(self.datadir+'/'+str(uid)+'/'+str(tid)+'.npy', allow_pickle=True, encoding='bytes')
+
+				if self.datadir == './data/DW_writers':
+					sentence_level_char	= sentence_level_char[1:]
+					sentence_level_term	= sentence_level_term[1:]
+
+				if self.datadir == './data/VALID_DW_writers':
+					sentence_level_char	= sentence_level_char[1:]
+					sentence_level_term	= sentence_level_term[1:]
+
+				while True:
+					if len(sentence_level_term) == 0:
+						break
+					if sentence_level_term[-1] != 1.0:
+						sentence_level_raw_stroke = sentence_level_raw_stroke[:-1]
+						sentence_level_char = sentence_level_char[:-1]
+						sentence_level_term = sentence_level_term[:-1]
+						sentence_level_stroke_in = sentence_level_stroke_in[:-1]
+						sentence_level_stroke_out = sentence_level_stroke_out[:-1]
+					else:
+						break
+
+				tmp = []
+				for i, t in enumerate(sentence_level_term):
+					if t == 1:
+						tmp.append(sentence_level_char[i])
+
+				a = np.ones_like(sentence_level_stroke_in)
+				a[:,:2] /= self.divider
+
+				if len(sentence_level_stroke_in) == len(sentence_level_term) and len(tmp) > 0 and len(sentence_level_stroke_in) > 0:
+					user_sentence_level_stroke_in.append(np.asarray(sentence_level_stroke_in) * a)
+					user_sentence_level_stroke_out.append(np.asarray(sentence_level_stroke_out) * a)
+					user_sentence_level_stroke_length.append(len(sentence_level_stroke_in))
+					user_sentence_level_char.append(np.asarray(tmp))
+					user_sentence_level_term.append(np.asarray(sentence_level_term))
+					user_sentence_level_char_length.append(len(tmp))
+
+				for wid in range(len(word_level_stroke_in)):
+					each_word_level_stroke_in		= word_level_stroke_in[wid]
+					each_word_level_stroke_out		= word_level_stroke_out[wid]
+
+					if self.datadir == './data/DW_writers':
+						each_word_level_term			= word_level_term[wid][1:]
+						each_word_level_char			= word_level_char[wid][1:]
+					elif self.datadir == './data/VALID_DW_writers':
+						each_word_level_term			= word_level_term[wid][1:]
+						each_word_level_char			= word_level_char[wid][1:]
+					else:
+						each_word_level_term			= word_level_term[wid]
+						each_word_level_char			= word_level_char[wid]
+
+
+					# assert (len(each_word_level_stroke_in) == len(each_word_level_char) == len(each_word_level_term))
+					while True:
+						if len(each_word_level_term) == 0:
+							break
+						if each_word_level_term[-1] != 1.0:
+							# each_word_level_raw_stroke = each_word_level_raw_stroke[:-1]
+							each_word_level_char = each_word_level_char[:-1]
+							each_word_level_term = each_word_level_term[:-1]
+							each_word_level_stroke_in = each_word_level_stroke_in[:-1]
+							each_word_level_stroke_out = each_word_level_stroke_out[:-1]
+						else:
+							break
+
+					tmp = []
+					for i, t in enumerate(each_word_level_term):
+						if t == 1:
+							tmp.append(each_word_level_char[i])
+
+					b = np.ones_like(each_word_level_stroke_in)
+					b[:,:2] /= self.divider
+
+					if len(each_word_level_stroke_in) == len(each_word_level_term) and len(tmp) > 0 and len(each_word_level_stroke_in) > 0:
+						user_word_level_stroke_in.append(np.asarray(each_word_level_stroke_in) * b)
+						user_word_level_stroke_out.append(np.asarray(each_word_level_stroke_out) * b)
+						user_word_level_stroke_length.append(len(each_word_level_stroke_in))
+						user_word_level_char.append(np.asarray(tmp))
+						user_word_level_term.append(np.asarray(each_word_level_term))
+						user_word_level_char_length.append(len(tmp))
+
+					segment_level_stroke_in_list		= []
+					segment_level_stroke_out_list		= []
+					segment_level_stroke_length_list	= []
+					segment_level_char_list				= []
+					segment_level_term_list				= []
+					segment_level_char_length_list		= []
+
+					for sid in range(len(segment_level_stroke_in[wid])):
+						each_segment_level_stroke_in	= segment_level_stroke_in[wid][sid]
+						each_segment_level_stroke_out	= segment_level_stroke_out[wid][sid]
+
+						if self.datadir == './data/DW_writers':
+							each_segment_level_term			= segment_level_term[wid][sid][1:]
+							each_segment_level_char			= segment_level_char[wid][sid][1:]
+						elif self.datadir == './data/VALID_DW_writers':
+							each_segment_level_term			= segment_level_term[wid][sid][1:]
+							each_segment_level_char			= segment_level_char[wid][sid][1:]
+						else:
+							each_segment_level_term			= segment_level_term[wid][sid]
+							each_segment_level_char			= segment_level_char[wid][sid]
+
+						while True:
+							if len(each_segment_level_term) == 0:
+								break
+							if each_segment_level_term[-1] != 1.0:
+								# each_segment_level_raw_stroke = each_segment_level_raw_stroke[:-1]
+								each_segment_level_char = each_segment_level_char[:-1]
+								each_segment_level_term = each_segment_level_term[:-1]
+								each_segment_level_stroke_in = each_segment_level_stroke_in[:-1]
+								each_segment_level_stroke_out = each_segment_level_stroke_out[:-1]
+							else:
+								break
+
+						tmp = []
+						for i, t in enumerate(each_segment_level_term):
+							if t == 1:
+								tmp.append(each_segment_level_char[i])
+
+						c = np.ones_like(each_segment_level_stroke_in)
+						c[:,:2] /= self.divider
+
+						if len(each_segment_level_stroke_in) == len(each_segment_level_term) and len(tmp) > 0 and len(each_segment_level_stroke_in) > 0:
+							segment_level_stroke_in_list.append(np.asarray(each_segment_level_stroke_in) * c)
+							segment_level_stroke_out_list.append(np.asarray(each_segment_level_stroke_out) * c)
+							segment_level_stroke_length_list.append(len(each_segment_level_stroke_in))
+							segment_level_char_list.append(np.asarray(tmp))
+							segment_level_term_list.append(np.asarray(each_segment_level_term))
+							segment_level_char_length_list.append(len(tmp))
+
+					if len(segment_level_stroke_length_list) > 0:
+						SEGMENT_MAX_STROKE_LENGTH		= np.max(segment_level_stroke_length_list)
+						SEGMENT_MAX_CHARACTER_LENGTH	= np.max(segment_level_char_length_list)
+
+						new_segment_level_stroke_in_list 	= np.asarray([np.pad(a, ((0, SEGMENT_MAX_STROKE_LENGTH-len(a)), (0, 0)), 'constant') for a in segment_level_stroke_in_list])
+						new_segment_level_stroke_out_list 	= np.asarray([np.pad(a, ((0, SEGMENT_MAX_STROKE_LENGTH-len(a)), (0, 0)), 'constant') for a in segment_level_stroke_out_list])
+						new_segment_level_term_list 		= np.asarray([np.pad(a, ((0, SEGMENT_MAX_STROKE_LENGTH-len(a))), 'constant') for a in segment_level_term_list])
+						new_segment_level_char_list 		= np.asarray([np.pad(a, ((0, SEGMENT_MAX_CHARACTER_LENGTH-len(a))), 'constant') for a in segment_level_char_list])
+
+						user_segment_level_stroke_in.append(new_segment_level_stroke_in_list)
+						user_segment_level_stroke_out.append(new_segment_level_stroke_out_list)
+						user_segment_level_stroke_length.append(segment_level_stroke_length_list)
+						user_segment_level_char.append(new_segment_level_char_list)
+						user_segment_level_term.append(new_segment_level_term_list)
+						user_segment_level_char_length.append(segment_level_char_length_list)
+
+			WORD_MAX_STROKE_LENGTH			= np.max(user_word_level_stroke_length)
+			WORD_MAX_CHARACTER_LENGTH		= np.max(user_word_level_char_length)
+
+			SENTENCE_MAX_STROKE_LENGTH		= np.max(user_sentence_level_stroke_length)
+			SENTENCE_MAX_CHARACTER_LENGTH	= np.max(user_sentence_level_char_length)
+
+			new_sentence_level_stroke_in	= np.asarray([np.pad(a, ((0, SENTENCE_MAX_STROKE_LENGTH-len(a)), (0,0)), 'constant') for a in user_sentence_level_stroke_in])
+			new_sentence_level_stroke_out	= np.asarray([np.pad(a, ((0, SENTENCE_MAX_STROKE_LENGTH-len(a)), (0,0)), 'constant') for a in user_sentence_level_stroke_out])
+			new_sentence_level_term			= np.asarray([np.pad(a, ((0, SENTENCE_MAX_STROKE_LENGTH-len(a))), 'constant') for a in user_sentence_level_term])
+			new_sentence_level_char			= np.asarray([np.pad(a, ((0, SENTENCE_MAX_CHARACTER_LENGTH-len(a))), 'constant') for a in user_sentence_level_char])
+			new_word_level_stroke_in		= np.asarray([np.pad(a, ((0, WORD_MAX_STROKE_LENGTH-len(a)), (0,0)), 'constant') for a in user_word_level_stroke_in])
+			new_word_level_stroke_out		= np.asarray([np.pad(a, ((0, WORD_MAX_STROKE_LENGTH-len(a)), (0,0)), 'constant') for a in user_word_level_stroke_out])
+			new_word_level_term				= np.asarray([np.pad(a, ((0, WORD_MAX_STROKE_LENGTH-len(a))), 'constant') for a in user_word_level_term])
+			new_word_level_char				= np.asarray([np.pad(a, ((0, WORD_MAX_CHARACTER_LENGTH-len(a))), 'constant') for a in user_word_level_char])
+
+			all_sentence_level_stroke_in.append(new_sentence_level_stroke_in)
+			all_sentence_level_stroke_out.append(new_sentence_level_stroke_out)
+			all_sentence_level_stroke_length.append(user_sentence_level_stroke_length)
+			all_sentence_level_term.append(new_sentence_level_term)
+			all_sentence_level_char.append(new_sentence_level_char)
+			all_sentence_level_char_length.append(user_sentence_level_char_length)
+			all_word_level_stroke_in.append(new_word_level_stroke_in)
+			all_word_level_stroke_out.append(new_word_level_stroke_out)
+			all_word_level_stroke_length.append(user_word_level_stroke_length)
+			all_word_level_term.append(new_word_level_term)
+			all_word_level_char.append(new_word_level_char)
+			all_word_level_char_length.append(user_word_level_char_length)
+			all_segment_level_stroke_in.append(user_segment_level_stroke_in)
+			all_segment_level_stroke_out.append(user_segment_level_stroke_out)
+			all_segment_level_stroke_length.append(user_segment_level_stroke_length)
+			all_segment_level_term.append(user_segment_level_term)
+			all_segment_level_char.append(user_segment_level_char)
+			all_segment_level_char_length.append(user_segment_level_char_length)
+
+		return [all_sentence_level_stroke_in, all_sentence_level_stroke_out, all_sentence_level_stroke_length, all_sentence_level_term, all_sentence_level_char, all_sentence_level_char_length, all_word_level_stroke_in, all_word_level_stroke_out, all_word_level_stroke_length, all_word_level_term, all_word_level_char, all_word_level_char_length, all_segment_level_stroke_in, all_segment_level_stroke_out, all_segment_level_stroke_length, all_segment_level_term, all_segment_level_char, all_segment_level_char_length]

app.py

@@ -0,0 +1,187 @@
+import torch
+import argparse
+import numpy as np
+from helper import *
+from config.GlobalVariables import *
+from SynthesisNetwork import SynthesisNetwork
+from DataLoader import DataLoader
+import convenience
+import gradio as gr
+
+#@title Demo
+device = 'cpu'
+num_samples = 10
+
+net = SynthesisNetwork(weight_dim=256, num_layers=3).to(device)
+
+if not torch.cuda.is_available():
+    try: # retrained model also contains loss in dict 
+        net.load_state_dict(torch.load('./model/250000.pt', map_location=torch.device(device))["model_state_dict"])
+    except:
+        net.load_state_dict(torch.load('./model/250000.pt', map_location=torch.device(device)))
+    
+
+dl = DataLoader(num_writer=1, num_samples=10, divider=5.0, datadir='./data/writers')
+
+
+writer_options = [5, 14, 15, 16, 17, 22, 25, 80, 120, 137, 147, 151]
+all_loaded_data = []
+avail_char = "0 1 2 3 4 5 6 7 8 9 a b c d e f g h i j k l m n o p q r s t u v w x y z A B C D E F G H I J K L M N O P Q R S T U V W X Y Z ! ? \" ' * + - = : ; , . < > \ / [ ] ( ) # $ % &"
+avail_char_list = avail_char.split(" ")
+for writer_id in [120, 80]:
+    loaded_data = dl.next_batch(TYPE='TRAIN', uid=writer_id, tids=list(range(num_samples)))
+    all_loaded_data.append(loaded_data)
+
+default_loaded_data = all_loaded_data[-1]
+# for writer interpolation
+def interpolate_writers(target_word, weight):
+    image = convenience.sample_blended_writers([1 - weight, weight], target_word, net, all_loaded_data, device).convert("RGB")
+    return image
+
+def choose_blend_writers(writer1, writer2):
+    id1, id2 = int(writer1.split(" ")[1]), int(writer1.split(" ")[1])
+    all_loaded_data.clear()
+    for writer_id in [id1, id2]:
+        loaded_data = dl.next_batch(TYPE='TRAIN', uid=writer_id, tids=list(range(num_samples)))
+        all_loaded_data.append(loaded_data)
+
+    return gr.Slider.update(label=f"{writer1} vs. {writer2}")
+'''
+def choose_writer(writ, c1, c2, c3, c4, val):
+    all_loaded_data.clear()
+    w = int(writ.split(" ")[1])
+    loaded_data = dl.next_batch(TYPE='TRAIN', uid=w, tids=list(range(num_samples)))
+    all_loaded_data.append(loaded_data)
+    return char_grid(c1, c2, c3, c4, val)
+'''
+# for character grrid
+def choose_grid_chars(c1, c2, c3, c4):
+      return gr.Button.update(value=f"Blend {c1}, {c2}, {c3}, and {c4}!")
+
+def char_grid(c1, c2, c3, c4):
+    image = convenience.sample_character_grid([c1, c2, c3, c4], 5, net, [default_loaded_data], device).convert("RGB")
+    return image
+
+# for character blend
+def interpolate_chars(c1, c2, weight):
+    image = convenience.sample_blended_chars([1 - weight, weight], [c1, c2], net, [default_loaded_data], device).convert("RGB")
+    return image
+
+def choose_blend_chars(c1, c2):
+   return gr.Slider.update(label=f"'{c1}' vs. '{c2}'")
+
+# for MDN
+def mdn_sample(word, maxs, maxr):
+    image = convenience.mdn_single_sample(word, maxs, maxr, net, [default_loaded_data], device).convert("RGB")
+    return image
+"""
+def char_vid(word):
+    #make word list
+    convenience.char_interpolation_video(word_list, 10, net, [default_loaded_data], device).convert('RGB')
+    vid_path = f"/content/drive/MyDrive/Colab Notebooks/Spring22/decoupled-style-descriptors-eb/results/abcdefg_video.mov"
+    return gr.Video.update(value=vid_path)
+"""
+    
+with gr.Blocks() as demo:
+    with gr.Tabs():
+        with gr.TabItem("Blend Writers"):
+          target_word = gr.Textbox(label="Target Word", value="hello world", max_lines=1)
+          with gr.Row():
+              left_ratio_options = ["Style " + str(id) for i, id in enumerate(writer_options) if i % 2 == 0]
+              right_ratio_options = ["Style " + str(id) for i, id in enumerate(writer_options) if i % 2 == 1]
+              with gr.Column():
+                  writer1 = gr.Radio(left_ratio_options, value="Style 120", label="Style for first writer")
+              with gr.Column():
+                  writer2 = gr.Radio(right_ratio_options, value="Style 80", label="Style for second writer")
+          with gr.Row():
+              writer_slider = gr.Slider(0, 1, value=0.3, label="Style 120 vs. Style 80")
+          with gr.Row():
+              writer_submit = gr.Button("Submit")
+          with gr.Row():
+              writer_default_image = convenience.sample_blended_writers([0.7, 0.3], "hello world", net, all_loaded_data, device).convert("RGB")
+              writer_output = gr.Image(writer_default_image)
+          
+          writer_submit.click(fn=interpolate_writers, inputs=[target_word, writer_slider], outputs=[writer_output])
+          writer_slider.change(fn=interpolate_writers, inputs=[target_word, writer_slider], outputs=[writer_output])
+          target_word.submit(fn=interpolate_writers, inputs=[target_word, writer_slider], outputs=[writer_output])
+
+          writer1.change(fn=choose_blend_writers, inputs=[writer1, writer2], outputs=[writer_slider])
+          writer2.change(fn=choose_blend_writers, inputs=[writer1, writer2], outputs=[writer_slider])
+    
+
+        with gr.TabItem("Blend Characters"):
+            with gr.Row():
+                with gr.Column():
+                    char1 = gr.Dropdown(choices=avail_char_list, value="y", label="Character 1")
+                with gr.Column():
+                    char2 = gr.Dropdown(choices=avail_char_list, value="s", label="Character 2")
+            with gr.Row():
+               char_slider = gr.Slider(0, 1, value=0.3, label="'y' vs. 's'")
+            with gr.Row():
+               char_default_image = convenience.sample_blended_chars([0.7, 0.3], ["y", "s"], net, [default_loaded_data], device).convert("RGB")
+               char_output = gr.Image(char_default_image)
+            
+            char_slider.change(fn=interpolate_chars, inputs=[char1, char2, char_slider], outputs=[char_output])
+
+            char1.change(fn=choose_blend_chars, inputs=[char1, char2], outputs=[char_slider])
+            char2.change(fn=choose_blend_chars, inputs=[char1, char2], outputs=[char_slider])
+        
+        """
+        with gr.TabItem("Character Grid"): #slow
+          with gr.Row():
+              with gr.Column():
+                  char1 = gr.Dropdown(choices=avail_char_list, value="y", label="Character 1")
+              with gr.Column():
+                  char2 = gr.Dropdown(choices=avail_char_list, value="s", label="Character 2")
+              with gr.Column():
+                  char3 = gr.Dropdown(choices=avail_char_list, value="u", label="Character 3")
+              with gr.Column():
+                  char4 = gr.Dropdown(choices=avail_char_list, value="n", label="Character 4")
+          with gr.Row(): 
+              submit_button = gr.Button(value="Blend y, s, u, and n!")
+          '''
+          with gr.Row():
+              radio_options2 = ["Writer " + str(n) for n in writer_options]
+              writer = gr.Radio(radio_options2, value="Writer 80", label="Style for Writer")
+              writer.change(fn=choose_writer, inputs=[writer, char1, char2, char3, char4, slider2], outputs=[output])
+          '''
+          #slider2 = gr.Slider(2, 20, value=10, label="Grid Size", step=1)
+
+          default_image = convenience.sample_character_grid(['y', 's', 'u', 'n'], 10, net, [default_loaded_data], device).convert("RGB")
+          output = gr.Image(default_image)
+          
+          char1.change(fn=choose_grid_chars, inputs=[char1, char2, char3, char4], outputs=[submit_button])
+          char2.change(fn=choose_grid_chars, inputs=[char1, char2, char3, char4], outputs=[submit_button])
+          char3.change(fn=choose_grid_chars, inputs=[char1, char2, char3, char4], outputs=[submit_button])
+          char4.change(fn=choose_grid_chars, inputs=[char1, char2, char3, char4], outputs=[submit_button])
+          
+          
+          #slider2.change(fn=char_grid, inputs=[char1, char2, char3, char4, slider2], outputs=[output])
+          submit_button.click(fn=char_grid, inputs=[char1, char2, char3, char4], outputs=[output])
+        """
+        
+        with gr.TabItem("Add Randomness"):
+          mdn_word = gr.Textbox(label="Target Word", value="hello world", max_lines=1)
+          '''
+          with gr.Row():
+              radio_options3 = ["Writer " + str(n) for n in writer_options]
+              writer = gr.Radio(radio_options3, value="Writer 80", label="Style for Writer")
+              writer.change(fn=new_writer_mdn, inputs=[writer, slider3, slider4], outputs=[output])
+          '''
+          with gr.Row():
+            with gr.Column():
+              max_rand = gr.Slider(0, 1, value=1, label="Maximum Randomness")
+            with gr.Column():
+              scale_rand = gr.Slider(0, 3, value=0.5, label="Scale of Randomness")
+          with gr.Row():
+              sample_button = gr.Button(value="Resample!")
+          with gr.Row():
+              default_im = convenience.mdn_single_sample("hello world", 0.5, 1, net, [default_loaded_data], device).convert('RGB')
+              mdn_output = gr.Image(default_im)
+
+          max_rand.change(fn=mdn_sample, inputs=[mdn_word, scale_rand, max_rand], outputs=[mdn_output])
+          scale_rand.change(fn=mdn_sample, inputs=[mdn_word, scale_rand, max_rand], outputs=[mdn_output])        
+          sample_button.click(fn=mdn_sample, inputs=[mdn_word, scale_rand, max_rand], outputs=[mdn_output])
+          mdn_word.submit(fn=mdn_sample, inputs=[mdn_word, scale_rand, max_rand], outputs=[mdn_output])
+
+demo.launch()

config/GlobalVariables.py

@@ -0,0 +1,5 @@
+COLORS = [(255,255,255), (255,0,0), (0,255,0), (0,0,255), (255,255,0),(0,255,255),(255,0,255),(255,128,0),(0,255,128),(128,0,255),(255,0,128),(128,255,0),(0,128,255)]
+CHARACTERS	= ' !"#$%&\'()*+,-./0123456789:;<=>?ABCDEFGHIJKLMNOPQRSTUVWXYZ[]abcdefghijklmnopqrstuvwxyz'
+# CHARACTERS		= ' !"&\'(),-.:;?ABCDEFGHIJKLMNOPQRSTUVWXYZ[]abcdefghijklmnopqrstuvwxyz'
+
+''.join([CHARACTERS[i] for i in [4,2,30]])

convenience.py

@@ -0,0 +1,555 @@
+
+import os
+import re
+from random import random
+import torch
+import pickle
+import argparse
+import numpy as np
+from helper import *
+from PIL import Image
+import torch.nn as nn
+import torch.optim as optim
+from config.GlobalVariables import *
+from tensorboardX import SummaryWriter
+from SynthesisNetwork import SynthesisNetwork
+from DataLoader import DataLoader
+# import ffmpeg # for problems with ffmpeg uninstall ffmpeg and then install ffmpeg-python
+
+L = 256
+
+def get_mean_global_W(net, loaded_data, device):
+    """gets the mean global style vector for a given writer"""
+    [_, _, _, _, _, _, all_word_level_stroke_in, all_word_level_stroke_out, all_word_level_stroke_length, all_word_level_term, all_word_level_char, all_word_level_char_length, all_segment_level_stroke_in, all_segment_level_stroke_out,
+        all_segment_level_stroke_length, all_segment_level_term, all_segment_level_char, all_segment_level_char_length] = loaded_data
+
+    batch_word_level_stroke_in = [torch.FloatTensor(a).to(device) for a in all_word_level_stroke_in]
+    batch_word_level_stroke_out = [torch.FloatTensor(a).to(device) for a in all_word_level_stroke_out]
+    batch_word_level_stroke_length = [torch.LongTensor(a).to(device).unsqueeze(-1) for a in all_word_level_stroke_length]
+    batch_word_level_term = [torch.FloatTensor(a).to(device) for a in all_word_level_term]
+    batch_word_level_char = [torch.LongTensor(a).to(device) for a in all_word_level_char]
+    batch_word_level_char_length = [torch.LongTensor(a).to(device).unsqueeze(-1) for a in all_word_level_char_length]
+    batch_segment_level_stroke_in = [[torch.FloatTensor(a).to(device) for a in b] for b in all_segment_level_stroke_in]
+    batch_segment_level_stroke_out = [[torch.FloatTensor(a).to(device) for a in b] for b in all_segment_level_stroke_out]
+    batch_segment_level_stroke_length = [[torch.LongTensor(a).to(device).unsqueeze(-1) for a in b] for b in all_segment_level_stroke_length]
+    batch_segment_level_term = [[torch.FloatTensor(a).to(device) for a in b] for b in all_segment_level_term]
+    batch_segment_level_char = [[torch.LongTensor(a).to(device) for a in b] for b in all_segment_level_char]
+    batch_segment_level_char_length = [[torch.LongTensor(a).to(device).unsqueeze(-1) for a in b] for b in all_segment_level_char_length]
+
+    with torch.no_grad():
+        word_inf_state_out = net.inf_state_fc1(batch_word_level_stroke_out[0])
+        word_inf_state_out = net.inf_state_relu(word_inf_state_out)
+        word_inf_state_out, _ = net.inf_state_lstm(word_inf_state_out)
+
+        user_word_level_char = batch_word_level_char[0]
+        user_word_level_term = batch_word_level_term[0]
+
+        original_Wc = []
+        word_batch_id = 0
+
+        curr_seq_len = batch_word_level_stroke_length[0][word_batch_id][0]
+        curr_char_len = batch_word_level_char_length[0][word_batch_id][0]
+
+        char_vector = torch.eye(len(CHARACTERS))[user_word_level_char[word_batch_id][:curr_char_len]].to(device)
+        current_term = user_word_level_term[word_batch_id][:curr_seq_len].unsqueeze(-1)
+        split_ids = torch.nonzero(current_term)[:, 0]
+
+        char_vector_1 = net.char_vec_fc_1(char_vector)
+        char_vector_1 = net.char_vec_relu_1(char_vector_1)
+
+        char_out_1 = char_vector_1.unsqueeze(0)
+        char_out_1, (c, h) = net.char_lstm_1(char_out_1)
+        char_out_1 = char_out_1.squeeze(0)
+        char_out_1 = net.char_vec_fc2_1(char_out_1)
+        char_matrix_1 = char_out_1.view([-1, 1, 256, 256])
+        char_matrix_1 = char_matrix_1.squeeze(1)
+        char_matrix_inv_1 = torch.inverse(char_matrix_1)
+
+        W_c_t = word_inf_state_out[word_batch_id][:curr_seq_len]
+        W_c = torch.stack([W_c_t[i] for i in split_ids])
+        original_Wc.append(W_c)
+
+        W = torch.bmm(char_matrix_inv_1, W_c.unsqueeze(2)).squeeze(-1)
+        mean_global_W = torch.mean(W, 0)
+        return mean_global_W
+
+
+def get_DSD(net, target_word, writer_mean_Ws, all_loaded_data, device):
+    """
+    returns a style vector and character matrix for each character/segment in target_word
+
+    n is the number of writers
+    M is the number of characters in the target word
+    L is the latent vector size (in this case 256)
+
+    input:
+    - target_word, a string of length M to be converted to a DSD
+    - writer_mean_Ws, a list of n style vectors of size L
+
+    output:
+    - all_writer_Ws, a tensor of size n x M x L representing the style vectors for each writer and character 
+    - all_writer_Cs, a tensor of size n x M x L x L representing the corresponding character matrix
+    """
+
+    n = len(all_loaded_data)
+    M = len(target_word)
+    all_writer_Ws = torch.zeros(n, M, L)
+    all_writer_Cs = torch.zeros(n, M, L, L)
+
+    for i in range(n):
+        np.random.seed(0)
+
+        [_, _, _, _, _, _, all_word_level_stroke_in, all_word_level_stroke_out, all_word_level_stroke_length, all_word_level_term, all_word_level_char, all_word_level_char_length, all_segment_level_stroke_in, all_segment_level_stroke_out,
+            all_segment_level_stroke_length, all_segment_level_term, all_segment_level_char, all_segment_level_char_length] = all_loaded_data[i]
+
+        available_segments = {}
+        for sid, sentence in enumerate(all_segment_level_char[0]):
+            for wid, word in enumerate(sentence):
+                segment = ''.join([CHARACTERS[i] for i in word])
+                split_ids = np.asarray(np.nonzero(all_segment_level_term[0][sid][wid]))
+
+                if segment in available_segments:
+                    available_segments[segment].append([all_segment_level_stroke_out[0][sid][wid][:all_segment_level_stroke_length[0][sid][wid]], split_ids])
+                else:
+                    available_segments[segment] = [[all_segment_level_stroke_out[0][sid][wid][:all_segment_level_stroke_length[0][sid][wid]], split_ids]]
+
+        index = 0
+        all_W = []
+        all_C = []
+
+        # while index <= len(target_word):
+        while index < len(target_word):
+            available = False
+            # Currently this just uses each character individually instead of the whole segment
+            # for end_index in range(len(target_word), index, -1):
+            #     segment = target_word[index:end_index]
+            # print (segment)
+            segment = target_word[index]
+            if segment in available_segments:  # method beta
+                # print(f'in dic - {segment}')
+                available = True
+                candidates = available_segments[segment]
+                segment_level_stroke_out, split_ids = candidates[np.random.randint(len(candidates))]
+                out = net.inf_state_fc1(torch.FloatTensor(segment_level_stroke_out).to(device).unsqueeze(0))
+                out = net.inf_state_relu(out)
+                seg_W_c, (h_n, _) = net.inf_state_lstm(out)
+
+                character = segment[0]  # take the first character of the segment?
+
+                # get character matrix using same method as method beta
+                char_vector = torch.eye(len(CHARACTERS))[CHARACTERS.index(character)].to(device).unsqueeze(0)
+                out = net.char_vec_fc_1(char_vector)
+                out = net.char_vec_relu_1(out)
+                out, _ = net.char_lstm_1(out.unsqueeze(0))
+                out = out.squeeze(0)
+                out = net.char_vec_fc2_1(out)
+                char_matrix = out.view([-1, 256, 256])
+                inv_char_matrix = char_matrix.inverse()
+
+                id = split_ids[0][0]
+                W_c_vector = seg_W_c[0, id].squeeze()
+
+                # invert to get writer-independed DSD
+                W_vector = torch.bmm(inv_char_matrix, W_c_vector.repeat(inv_char_matrix.size(0), 1).unsqueeze(2))
+                all_W.append(W_vector)
+                all_C.append(char_matrix)
+
+                index += 1
+
+            if index == len(target_word):
+                break
+
+            if not available:  # method alpha
+                character = target_word[index]
+                # print(f'no dic - {character}')
+                char_vector = torch.eye(len(CHARACTERS))[CHARACTERS.index(character)].to(device).unsqueeze(0)
+                out = net.char_vec_fc_1(char_vector)
+                out = net.char_vec_relu_1(out)
+                out, _ = net.char_lstm_1(out.unsqueeze(0))
+                out = out.squeeze(0)
+                out = net.char_vec_fc2_1(out)
+                char_matrix = out.view([-1, 256, 256])
+
+                W_vector = writer_mean_Ws[i].repeat(char_matrix.size(0), 1).unsqueeze(2)
+
+                # all_W.append([W_vector])
+                all_W.append(W_vector)
+                all_C.append(char_matrix)
+
+                index += 1
+
+        all_writer_Ws[i, :, :] = torch.stack(all_W).squeeze()
+        all_writer_Cs[i, :, :, :] = torch.stack(all_C).squeeze()
+
+    return all_writer_Ws, all_writer_Cs
+
+
+def get_writer_blend_W_c(writer_weights, all_Ws, all_Cs):
+    """
+    generates character-dependent style-dependent DSDs for each character/segement in target_word,
+    averaging together the styles of the handwritings using provided weights
+
+    n is the number of writers
+    M is the number of characters in the target word
+    L is the latent vector size (in this case 256)
+
+    input:
+    - writer_weights, a list of length n weights for each writer that sum to one
+    - all_writer_Ws, an n x M x L tensor representing each weiter's style vector for every character
+    - all_writer_Cs, an n x M x L x L tensor representing the style's correspodning character matrix
+
+    output:
+    - an M x 1 x L tensor of M scharacter-dependent style-dependent DSDs
+    """
+    n, M, _ = all_Ws.shape
+    weights_tensor = torch.tensor(writer_weights).repeat_interleave(M * L).reshape(n, M, L)  # repeat accross remaining dimensions
+    W_vectors = (weights_tensor * all_Ws).sum(axis=0).unsqueeze(-1)  # take weighted sum accross writers axis
+    char_matrices = all_Cs[0, :, :, :]  # character matrices are independent of writer
+
+    W_cs = torch.bmm(char_matrices, W_vectors)
+
+    return W_cs.reshape(M, 1, L)
+
+
+def get_character_blend_W_c(character_weights, all_Ws, all_Cs):
+    """
+    generates a single character-dependent style-dependent DSD,
+    averaging together the characters using provided weights
+
+    M is the number of characters to blend
+    L is the latent vector size (in this case 256)
+
+    input:
+    - character_weights, a list of length M weights for each character that sum to one
+    - all_Ws, a 1 x M x L tensor representing the wwiter's style vector for each character
+    - all_Cs, 1 x M x L x L tensor representing the style's correspodning character matrix
+
+    output:
+    - a 1 x 1 x L tensor representing the character-dependent style-dependent DSDs
+    """
+    M = len(character_weights)
+    W_vector = all_Ws[0, 0, :].unsqueeze(-1)
+
+    weights_tensor = torch.tensor(character_weights).repeat_interleave(L * L).reshape(1, M, L, L)  # repeat accross remaining dimensions
+    char_matrix = (weights_tensor * all_Cs).sum(axis=1).squeeze() # take weighted sum accross characters axis
+
+    W_c = char_matrix @ W_vector
+
+    return W_c.reshape(1, 1, L)
+
+
+def get_commands(net, target_word, all_W_c): # seems like target_word is only used for length
+    """converts character-dependent style-dependent DSDs to a list of commands for drawing"""
+    all_commands = []
+    current_id = 0
+    while True:
+        word_Wc_rec_TYPE_D = []
+        TYPE_D_REF = []
+        cid = 0
+        for segment_batch_id in range(len(all_W_c)):
+            if len(TYPE_D_REF) == 0:
+                for each_segment_Wc in all_W_c[segment_batch_id]:
+                    if cid >= current_id:
+                        word_Wc_rec_TYPE_D.append(each_segment_Wc)
+                    cid += 1
+                if len(word_Wc_rec_TYPE_D) > 0:
+                    TYPE_D_REF.append(all_W_c[segment_batch_id][-1])
+            else:
+                for each_segment_Wc in all_W_c[segment_batch_id]:
+                    magic_inp = torch.cat([torch.stack(TYPE_D_REF, 0), each_segment_Wc.unsqueeze(0)], 0)
+                    magic_inp = magic_inp.unsqueeze(0)
+                    TYPE_D_out, (c, h) = net.magic_lstm(magic_inp)
+                    TYPE_D_out = TYPE_D_out.squeeze(0)
+                    word_Wc_rec_TYPE_D.append(TYPE_D_out[-1])
+                TYPE_D_REF.append(all_W_c[segment_batch_id][-1])
+        WC_ = torch.stack(word_Wc_rec_TYPE_D)
+        tmp_commands, res = net.sample_from_w_fix(WC_)
+        current_id += res
+        if len(all_commands) == 0:
+            all_commands.append(tmp_commands)
+        else:
+            all_commands.append(tmp_commands[1:])
+        if res < 0 or current_id >= len(target_word):
+            break
+
+    commands = []
+    px, py = 0, 100
+    for coms in all_commands:
+        for i, [dx, dy, t] in enumerate(coms):
+            x = px + dx * 5
+            y = py + dy * 5
+            commands.append([x, y, t])
+            px, py = x, y
+    commands = np.asarray(commands)
+    commands[:, 0] -= np.min(commands[:, 0])
+
+    return commands
+
+def mdn_video(target_word, num_samples, scale_sd, clamp_mdn, net, all_loaded_data, device):
+    '''
+    Method creating gif of mdn samples
+    num_samples: number of samples to be inputted
+    max_scale: the maximum value used to scale SD while sampling (increment is based on num samples)
+    '''
+    words = target_word.split(' ')
+    us_target_word = re.sub(r"\s+", '_', target_word)
+    os.makedirs(f"./results/{us_target_word}_mdn_samples", exist_ok=True)
+    for i in range(num_samples):
+        im = Image.fromarray(np.zeros([160, 750]))
+        dr = ImageDraw.Draw(im)
+        width = 50
+
+        net.scale_sd = scale_sd
+        net.clamp_mdn = clamp_mdn
+
+        mean_global_W = get_mean_global_W(net, all_loaded_data[0], device)
+
+        for word in words:
+            writer_Ws, writer_Cs = get_DSD(net, word, [mean_global_W], [all_loaded_data[0]], device)
+            all_W_c = get_writer_blend_W_c([1], writer_Ws, writer_Cs)
+            all_commands = get_commands(net, word, all_W_c)
+
+            for [x, y, t] in all_commands:
+                if t == 0:
+                    dr.line((px+width, py, x+width, y), 255, 1)
+                px, py = x, y
+            width += np.max(all_commands[:, 0]) + 25
+
+        im.convert("RGB").save(f'results/{us_target_word}_mdn_samples/sample_{i}.png')
+    # Convert fromes to video using ffmpeg
+    photos = ffmpeg.input(f'results/{us_target_word}_mdn_samples/sample_*.png', pattern_type='glob', framerate=10)
+    videos = photos.output(f'results/{us_target_word}_video.mov', vcodec="libx264", pix_fmt="yuv420p")
+    videos.run(overwrite_output=True)
+
+def sample_blended_writers(writer_weights, target_sentence, net, all_loaded_data, device="cpu"):
+    """Generates an image of handwritten text based on target_sentence"""
+    words = target_sentence.split(' ')
+
+    im = Image.fromarray(np.zeros([160, 750]))
+    dr = ImageDraw.Draw(im)
+    width = 50
+
+    writer_mean_Ws = []
+    for loaded_data in all_loaded_data:
+        mean_global_W = get_mean_global_W(net, loaded_data, device)
+        writer_mean_Ws.append(mean_global_W)
+
+    for word in words:
+        all_writer_Ws, all_writer_Cs = get_DSD(net, word, writer_mean_Ws, all_loaded_data, device)
+        all_W_c = get_writer_blend_W_c(writer_weights, all_writer_Ws, all_writer_Cs)
+        all_commands = get_commands(net, word, all_W_c)
+
+        for [x, y, t] in all_commands:
+            if t == 0:
+                dr.line((px+width, py, x+width, y), 255, 1)
+            px, py = x, y
+        width += np.max(all_commands[:, 0]) + 25
+
+    return im
+
+
+def sample_character_grid(letters, grid_size, net, all_loaded_data, device="cpu"):
+    """Generates an image of handwritten text based on target_sentence"""
+    width = 60
+    im = Image.fromarray(np.zeros([(grid_size + 1) * width, (grid_size + 1) * width]))
+    dr = ImageDraw.Draw(im)
+
+    M = len(letters)
+    mean_global_W = get_mean_global_W(net, all_loaded_data[0], device)
+
+    # all_Ws = torch.zeros(1, M, L)
+    all_Cs = torch.zeros(1, M, L, L)
+    for i in range(M):  # get corners of grid
+        W_vector, char_matrix = get_DSD(net, letters[i], [mean_global_W], [all_loaded_data[0]], device)
+        # all_Ws[:, i, :] = W_vector
+        all_Cs[:, i, :, :] = char_matrix
+
+    all_Ws = mean_global_W.reshape(1, 1, L)
+
+    for i in range(grid_size):
+        for j in range(grid_size):
+            wx = i / (grid_size - 1)
+            wy = j / (grid_size - 1)
+
+            character_weights = [(1 - wx) * (1 - wy), # top left is 1 at (0, 0)
+                                 wx       * (1 - wy), # top right is 1  at (1, 0)
+                                 (1 - wx) * wy,       # bottom left is 1 at (0, 1)
+                                 wx       * wy]       # bottom right is 1 at (1, 1)
+            all_W_c = get_character_blend_W_c(character_weights, all_Ws, all_Cs)
+            all_commands = get_commands(net, letters[0], all_W_c)
+
+            offset_x = i * width
+            offset_y = j * width
+
+            for [x, y, t] in all_commands:
+                if t == 0:
+                    dr.line((
+                        px + offset_x + width/2,
+                        py + offset_y - width/2,  # letters are shifted down for some reason
+                        x + offset_x + width/2,
+                        y + offset_y - width/2), 255, 1)
+                px, py = x, y
+
+    return im
+
+def writer_interpolation_video(target_sentence, transition_time, net, all_loaded_data, device="cpu"):
+    """
+    Generates a video of interpolating between each provided writer
+    """
+
+    n = len(all_loaded_data)
+
+    os.makedirs(f"./results/{target_sentence}_blend_frames", exist_ok=True)
+
+    words = target_sentence.split(' ')
+
+    writer_mean_Ws = []
+    for loaded_data in all_loaded_data:
+        mean_global_W = get_mean_global_W(net, loaded_data, device)
+        writer_mean_Ws.append(mean_global_W)
+
+    word_Ws = []
+    word_Cs = []
+
+    for word in words:
+        all_writer_Ws, all_writer_Cs = get_DSD(net, word, writer_mean_Ws, all_loaded_data, device)
+        word_Ws.append(all_writer_Ws)
+        word_Cs.append(all_writer_Cs)
+
+    for i in range(n - 1):
+        for j in range(transition_time):
+            im = Image.fromarray(np.zeros([160, 750]))
+            dr = ImageDraw.Draw(im)
+            width = 50
+
+            completion = j/(transition_time)
+
+            individual_weights = [1 - completion, completion]
+            writer_weights = [0] * i + individual_weights + [0] * (n - 2 - i)
+
+            for k, word in enumerate(words):
+                all_writer_Ws, all_writer_Cs = word_Ws[k], word_Cs[k]
+                all_W_c = get_writer_blend_W_c(writer_weights, all_writer_Ws, all_writer_Cs)
+                all_commands = get_commands(net, word, all_W_c)
+
+                for [x, y, t] in all_commands:
+                    if t == 0:
+                        dr.line((px+width, py, x+width, y), 255, 1)
+                    px, py = x, y
+                width += np.max(all_commands[:, 0]) + 25
+
+            im.convert("RGB").save(f"./results/{target_sentence}_blend_frames/frame_{str(i * transition_time + j).zfill(3)}.png")
+
+    # Convert fromes to video using ffmpeg
+    photos = ffmpeg.input(f"./results/{target_sentence}_blend_frames/frame_*.png", pattern_type='glob', framerate=10)
+    videos = photos.output(f"results/{target_sentence}_blend_video.mov", vcodec="libx264", pix_fmt="yuv420p")
+    videos.run(overwrite_output=True)
+
+def mdn_single_sample(target_word, scale_sd, clamp_mdn, net, all_loaded_data, device):
+    '''
+    Method creating gif of mdn samples
+    num_samples: number of samples to be inputted
+    max_scale: the maximum value used to scale SD while sampling (increment is based on num samples)
+    '''
+    words = target_word.split(' ')
+    im = Image.fromarray(np.zeros([160, 750]))
+    dr = ImageDraw.Draw(im)
+    width = 50
+
+    net.scale_sd = scale_sd
+    net.clamp_mdn = clamp_mdn
+
+    mean_global_W = get_mean_global_W(net, all_loaded_data[0], device)
+
+    for word in words:
+        writer_Ws, writer_Cs = get_DSD(net, word, [mean_global_W], [all_loaded_data[0]], device)
+        all_W_c = get_writer_blend_W_c([1], writer_Ws, writer_Cs)
+        all_commands = get_commands(net, word, all_W_c)
+
+        for [x, y, t] in all_commands:
+            if t == 0:
+                dr.line((px+width, py, x+width, y), 255, 1)
+            px, py = x, y
+        width += np.max(all_commands[:, 0]) + 25
+
+    return im
+
+def sample_blended_chars(character_weights, letters, net, all_loaded_data, device="cpu"):
+    """Generates an image of handwritten text based on target_sentence"""
+
+    width = 60
+    im = Image.fromarray(np.zeros([100, 100]))
+    dr = ImageDraw.Draw(im)
+
+    M = len(letters)
+    mean_global_W = get_mean_global_W(net, all_loaded_data[0], device)
+
+    all_Cs = torch.zeros(1, M, L, L)
+    for i in range(M):  # get corners of grid
+        W_vector, char_matrix = get_DSD(net, letters[i], [mean_global_W], [all_loaded_data[0]], device)
+        all_Cs[:, i, :, :] = char_matrix
+
+    all_Ws = mean_global_W.reshape(1, 1, L)
+
+    all_W_c = get_character_blend_W_c(character_weights, all_Ws, all_Cs)
+    all_commands = get_commands(net, letters[0], all_W_c)
+
+    for [x, y, t] in all_commands:
+        if t == 0:
+            dr.line((
+                px + width/2,
+                py - width/2,  # letters are shifted down for some reason
+                x + width/2,
+                y - width/2), 255, 1)
+        px, py = x, y
+
+        
+    return im
+
+
+def char_interpolation_video(letters, transition_time, net, all_loaded_data, device="cpu"):
+    """Generates an image of handwritten text based on target_sentence"""
+
+    os.makedirs(f"./results/{''.join(letters)}_frames", exist_ok=True) # make a folder for the frames
+
+    width = 50
+
+    M = len(letters)
+    mean_global_W = get_mean_global_W(net, all_loaded_data[0], device)
+
+    all_Cs = torch.zeros(1, M, L, L)
+    for i in range(M):  # get corners of grid
+        W_vector, char_matrix = get_DSD(net, letters[i], [mean_global_W], [all_loaded_data[0]], device)
+        all_Cs[:, i, :, :] = char_matrix
+
+    all_Ws = mean_global_W.reshape(1, 1, L)
+
+    for i in range(M - 1):
+        for j in range(transition_time):
+            completion = j / (transition_time - 1)
+            individual_weights = [1 - completion, completion]
+            character_weights = [0] * i + individual_weights + [0] * (M - 2 - i)
+            all_W_c = get_character_blend_W_c(character_weights, all_Ws, all_Cs)
+            all_commands = get_commands(net, "change this later!", all_W_c)
+
+            im = Image.fromarray(np.zeros([100, 100]))
+            dr = ImageDraw.Draw(im)
+
+            for [x, y, t] in all_commands:
+                if t == 0:
+                    dr.line((
+                        px + width/2,
+                        py - width/2,  # letters are shifted down for some reason
+                        x + width/2,
+                        y - width/2), 255, 1)
+                px, py = x, y
+
+                
+            im.convert("RGB").save(f"results/{''.join(letters)}_frames/frames_{str(i * transition_time + j).zfill(3)}.png")
+
+    # Convert fromes to video using ffmpeg
+    photos = ffmpeg.input(f"results/{''.join(letters)}_frames/frames_*.png", pattern_type='glob', framerate=24)
+    videos = photos.output(f"results/{''.join(letters)}_video.mov", vcodec="libx264", pix_fmt="yuv420p")
+    videos.run(overwrite_output=True)
+
+

data/writers/120/0.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:488e337dc15361658a8bab492c9e889daad1acca812d9a11fb8e369219fab6ef
+size 175537

data/writers/120/1.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a04bde9644e378ebb2dab04306af81858b967a33f96685c3645cd37615880ebb
+size 134815

data/writers/120/10.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20d95ba927366959e2f4b19bc0b932c3532930d3519a5003a357a46137785d39
+size 134965

data/writers/120/100.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:351a2367d904833fbccec18a4a04d979e20493d7e5b0be5b46bdb0be5992dbf1
+size 127588

data/writers/120/101.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c008de6c369b3b50cbf31f89b7ad7220164d8149ac4e9dd6e1b91017931a4d60
+size 121980

data/writers/120/102.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4e4f83e793e38199a813bea019288d3f04433844384b21806745a9e5c51cfad
+size 107769

data/writers/120/103.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a5626912691f52d426b2d2c37ad29194bb0bfc8cfeb5a42a9e16927bab7f79e
+size 110661

data/writers/120/104.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff65734b3bd0e695e6944ec5a79e40bb00c1766fdc5fe1dc39300c8be38bd15b
+size 108546

data/writers/120/105.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2b5c9a945f6efb3bef5fb0255a646aeafd41ea15ef75a2578f908392511897d
+size 103809

data/writers/120/106.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:991b1db11a0514cff5134dc364790d3af3aaf22a9bdea170ee47b38801f4f684
+size 108246

data/writers/120/107.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c863110c0a0d51d817afa6fbe55e18f4207e1b00a4960fc9f8d08cc351c38851
+size 122094

data/writers/120/108.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:787de7d2534ec295ffd6df24b7cbed10dd84f433ce61e6aab0b9b9541404c0b0
+size 134977

data/writers/120/109.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ed4d6be1fc96084d741f95ff21d62f822c411ef60baf327e48e8e512fafdb75
+size 112104

data/writers/120/11.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:550bc976e80d45d669c9154d82a066629f92f39be40403e5c554e5f2b87231c7
+size 125640

data/writers/120/110.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2204bedcd2e09d457c4641c4039ecc0bcc1913d29a5617e590baf773f6b4667c
+size 104715

data/writers/120/111.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c9b6a13f901029afafd7b587f0785ed0325f96ba89e2656467741a342aee233
+size 103260

data/writers/120/112.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d65de4fc97401f4e65bb415d4357e42e6887e0e3ad03e62b73f7be428b8b80d
+size 128293

data/writers/120/113.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe308240dd20d0d21692a669d10ec2c0022c03fd5db85b5c4a7fd7430bc3a0fb
+size 127927

data/writers/120/114.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b677482a49f5de3cc6cc4e2e98bea3cb88bf9869ab16b16b235c35464984bd8
+size 104076

data/writers/120/115.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a63b963d93d303bde992da6dce4ab97258eac80310172553ef035fd5f17114bc
+size 91845

data/writers/120/116.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48c9f17dbdd9fcd9abcdccda4b96c465f5e3423b63f556a70a060c2e49d68f5f
+size 144559

data/writers/120/117.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc36c22be7ba098958dbca1ee33d93b892d61162f2b15af232e819af8bb01dee
+size 131440

data/writers/120/118.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94d26ef8069e4171db86239a571ea9238b257e29e7f45eaf931bf348109d674b
+size 113775

data/writers/120/119.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:844575ae6d80e7d99ee991f394efbf34e187fea4f07a37e65dc152360070af99
+size 62214

data/writers/120/12.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da73c582d167dd1e643b5ac961458559bdb7bf8385c0a276865eac93e1708efb
+size 153187

data/writers/120/120.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d07ff24e6a822dfe0da15678ec6fb39d3d569b615c9465494ab2d71f9dcb5618
+size 144793

data/writers/120/121.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8614d0d5a91e2269af3ccfcf67e5fc3629dff45e66d7ce7b5f6c942ebc6c1dd
+size 115830

data/writers/120/122.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9dbcfdf6c3842aba365f0dd4188966d60b1d2b3686cf027efc22d0736ddc29ac
+size 143827

data/writers/120/123.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0df71728a9424a79b2570f8da50dd50470df861fdfb3f32165331b306ca09f57
+size 92145

data/writers/120/124.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c976cf8a9d94a08acf260ab9602025fc058293b00c52c693d49ea4219fd28a9
+size 125640

data/writers/120/125.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd843eb48e8563800945224760dedf90b9624c063182e09f8ad3767d448b7fb8
+size 101817

data/writers/120/126.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7755adb199c440d99ffbcbfc248098d237de982c3f5f5a28a169fff430e94368
+size 111777

data/writers/120/127.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1dcc75ca7e80fa07f1edfa6538322f92f90ac2d11265a4676be6e045a9db3f3
+size 111771

data/writers/120/128.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43ebcd79b32266a6c9e307f590bee292f9b90aaa7552ef5b5b2dd9a857a58a43
+size 133333

data/writers/120/129.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c663a98541bdda78554f5c2d83f07488cbfebb9cf9108261e5730375cf1d452
+size 126411

data/writers/120/13.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:808878ebe3eb40eef2d75f33db32cb4efd22cac468c15179ca69c92304c3e379
+size 130951

data/writers/120/130.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe5aacdddcb712e6e540cec76628ad15ed16906b6452b9e08f9c41449f1df92f
+size 145885

data/writers/120/131.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dd771d047b5a7787106286bbef893127f167e749d45a268e94607c699639079
+size 106146

data/writers/120/132.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9075a3167827689141f85211d37f0989c6f0309240820c1924d6dfd2f059dd0
+size 134359