import pandas as pd import numpy as np from sklearn.metrics.pairwise import cosine_similarity import Levenshtein from sklearn.metrics.pairwise import polynomial_kernel import gradio as gr from Bio.Align import PairwiseAligner from scipy.spatial import distance import jaro from transformers import AutoModelForSequenceClassification,AutoTokenizer,pipeline import codecs from PIL import Image, ImageDraw # Import the Model tokenizer = AutoTokenizer.from_pretrained('classifier') model = AutoModelForSequenceClassification.from_pretrained('classifier') # Import the screenplay dataset df = pd.read_json("screenplay_emotion_final.json") # Import the movie posters dataset movie_posters = pd.read_csv("movie_posters.csv") # Map the emotions to colors that represent them color_mapping = { 'joy': (248, 255, 149), 'love': (255, 199, 234), 'surprise': (166, 255, 150), 'sadness':(12, 53, 106), 'anger': (223, 46, 56), 'fear': (53, 124, 60) } code_mapping = { '6': (248, 255, 149), '5': (255, 199, 234), '4': (166, 255, 150), '3': (12, 53, 106), '2': (223, 46, 56), '1': (53, 124, 60) } emotion_mapping = { 'joy':6, 'love':5, 'surprise':4, 'sadness':3, 'anger':2, 'fear':1 } classifier_labels = ['sadness', 'joy', 'love', 'anger', 'fear', 'surprise'] def string(e): x = [str(i) for i in e] string = ''.join(x) return string # Function to convert the vector into an image representing the emotional pattern in the film def emo_barcode(idx): image = Image.new('RGB', (700,300), 'white') draw = ImageDraw.Draw(image) length = 700//100 emotion_list = df['emotions'][idx] for i in range(0,100): code = emotion_list[i] color = code_mapping[str(code)] draw.rectangle([length*i,0,length*(i+1), 300], fill=color) return image def emo_barcode_file(emotion_list): image = Image.new('RGB', (700,300), 'white') draw = ImageDraw.Draw(image) length = 700//100 for i in range(0,100): code = emotion_list[i] color = code_mapping[str(code)] draw.rectangle([length*i,0,length*(i+1), 300], fill=color) return image # Functions to calculate similarity measures def cosine(idx): a_list = [] if type(idx) == int: e1 = df['emotions'][idx] if type(idx) == list: e1 = idx for i in range(0,df.shape[0]): e2 = df['emotions'][i] score = 0 for i in range(0,100,5): score += distance.cosine(e1[i:i+5],e2[i:i+5]) a_list.append(score) array = np.array(a_list) movie_indexes = np.argsort(array) array.sort() return movie_indexes[1:6], array[1:6] def euclid(idx): a_list = [] if type(idx) == int: e1 = df['3d_vectors'][idx] if type(idx) == list: e1 = mapping(idx) for i in range(0,df.shape[0]): e2 = df['3d_vectors'][i] score = 0 for i in range(0,100): score += distance.euclidean(e1[i],e2[i]) a_list.append(score) array = np.array(a_list) movie_indexes = np.argsort(array) array.sort() return movie_indexes[1:6], array[1:6] def jaro_metric(idx): a_list = [] if type(idx) == int: e1 = df['emotions'][idx] if type(idx) == list: e1 = idx for i in range(0,df.shape[0]): e2 = df['emotions'][i] score = 0 for i in range(0,100,5): score += jaro.jaro_metric(string(e1[i:i+5]),string(e2[i:i+5])) a_list.append(score) array = np.array(a_list) movie_indexes = np.argsort(array) array.sort() return movie_indexes[-6:-1], array[-6:-1] def hamming(idx): a_list = [] if type(idx) == int: e1 = df['emotions'][idx] if type(idx) == list: e1 = idx for i in range(0,df.shape[0]): e2 = df['emotions'][i] score = 0 for i in range(0,100,5): score += distance.hamming(e1[i:i+5],e2[i:i+5]) a_list.append(score) array = np.array(a_list) movie_indexes = np.argsort(array) array.sort() return movie_indexes[1:6], array[1:6] def needleman(idx): a_list = [] if type(idx) == int: e1 = [str(x) for x in df['emotions'][idx]] e1 = "".join(e1) if type(idx) == list: e1 = [str(x) for x in idx] e1 = "".join(e1) for i in range(0,df.shape[0]): e2 = [str(x) for x in df['emotions'][i]] e2 = "".join(e2) score = PairwiseAligner().score(e1,e2) a_list.append(score) array = np.array(a_list) movie_indexes = np.argsort(array) array.sort() return movie_indexes[-6:-1], array[-6:-1] def Leven(idx): a_list = [] if type(idx) == int: e1 = df['emotions'][idx] if type(idx) == list: e1 = idx for i in range(0,df.shape[0]): e2 = df['emotions'][i] score = 0 for i in range(0,100,5): score += Levenshtein.distance(e1[i:i+5],e2[i:i+5]) a_list.append(score) array = np.array(a_list) movie_indexes = np.argsort(array) array.sort() return movie_indexes[1:6],array[1:6] # Function to classify the segments into scenes def classify(file): scenes = [] #read the file if type(file) == str: try: with codecs.open(file, "r", "utf-8") as f: text = f.read() except Exception as e: print(e) else: try: with codecs.open(file.name, "r", "utf-8") as f: text = f.read() except Exception as e: print(e) #split the screenplay into scenes words = text.split() length = len(words) segment_length = length//100 for i in range(0,100): segment = words[segment_length*i:segment_length*(i+1)] scene = ' '.join(segment) scenes.append(scene) # classify the segments inputs = tokenizer(scenes, padding=True, truncation=True, return_tensors="pt")['input_ids'] outputs = np.array(model(inputs).logits.argmax(-1)).tolist() labels = [] for i in outputs: labels.append(emotion_mapping[classifier_labels[i]]) return labels # Deployment in Gradio App similarity_measures = ["Euclidean Distance", "Jaro Metric", "Hamming Metric", "Levenshtein distance","Needleman Wunsch Algorithm","Cosine Similarity"] def output(idx,measure): if measure == 0: a_list, scores = euclid(idx) if measure == 1: a_list, scores = jaro_metric(idx) if measure == 2: a_list, scores = hamming(idx) if measure == 3: a_list, scores = Leven(idx) if measure == 4: a_list, scores = needleman(idx) if measure == 5: a_list, scores = cosine(idx) movie_names = [] selected_movie = df['title'][idx] poster = movie_posters['image'][movie_posters[movie_posters['title'] == df['title'][idx]].index[0]] barcode = emo_barcode(idx) b1 = emo_barcode(a_list[4]) b2 = emo_barcode(a_list[3]) b3 = emo_barcode(a_list[2]) b4 = emo_barcode(a_list[1]) b5 = emo_barcode(a_list[0]) for i in a_list: movie_names.append(df['title'][i]) n1 = movie_names[4] + f" | Score - {scores[4]}" n2 = movie_names[3] + f" | Score - {scores[3]}" n3 = movie_names[2] + f" | Score - {scores[2]}" n4 = movie_names[1] + f" | Score - {scores[1]}" n5 = movie_names[0] + f" | Score - {scores[0]}" p1 = movie_posters['image'][movie_posters[movie_posters['title'] == movie_names[4]].index[0]] p2 = movie_posters['image'][movie_posters[movie_posters['title'] == movie_names[3]].index[0]] p3 = movie_posters['image'][movie_posters[movie_posters['title'] == movie_names[2]].index[0]] p4 = movie_posters['image'][movie_posters[movie_posters['title'] == movie_names[1]].index[0]] p5 = movie_posters['image'][movie_posters[movie_posters['title'] == movie_names[0]].index[0]] return selected_movie,poster,barcode,n1,p1,b1,n2,p2,b2,n3,p3,b3,n4,p4,b4,n5,p5,b5 def output2(file,measure): labels = classify(file) barcode2 = emo_barcode_file(labels) if measure == 0: a_list = cosine(labels) if measure == 1: a_list = euclid(labels) if measure == 2: a_list = jaro_metric(labels) if measure == 3: a_list = hamming(labels) if measure == 4: a_list = Leven(labels) movie_names = [] y1 = emo_barcode(a_list[0]) y2 = emo_barcode(a_list[1]) y3 = emo_barcode(a_list[2]) y4 = emo_barcode(a_list[3]) y5 = emo_barcode(a_list[4]) for i in a_list: movie_names.append(df['title'][i]) x1 = movie_names[0] x2 = movie_names[1] x3 = movie_names[2] x4 = movie_names[3] x5 = movie_names[4] i1 = movie_posters['image'][movie_posters[movie_posters['title'] == movie_names[0]].index[0]] i2 = movie_posters['image'][movie_posters[movie_posters['title'] == movie_names[1]].index[0]] i3 = movie_posters['image'][movie_posters[movie_posters['title'] == movie_names[2]].index[0]] i4 = movie_posters['image'][movie_posters[movie_posters['title'] == movie_names[3]].index[0]] i5 = movie_posters['image'][movie_posters[movie_posters['title'] == movie_names[4]].index[0]] return barcode2,x1,i1,y1,x2,i2,y2,x3,i3,y3,x4,i4,y4,x5,i5,y5 with gr.Blocks() as demo: with gr.Row(): gr.Markdown(""" # Screenplay Emotion Analysis This demo attempts to study a new approach in screenplay analysis and movie recommendations. This project started with three questions:\n What if you can identify the overall emotion pattern of a screenplay without reading it?\n What if you can make a machine understand a screenplay by vectorising the emotions contained in the screenplay?\n What if you can predict what movie a person might like using the emotions contained in the screenplay?\n The screenplays were scrapped from IMSdb website and split into 100 segments of equal length. I have trained an emotion classification model and used the model to classify each segment in each screenplay into one of six emotion categories and stored it as a vector. The emotion categories are joy, sadness, surprise, love, fear and anger. ### Each color in the barcode represents one of six emotions Joy - yellow Love - Pink sadness - dark blue surprise - light green fear - dark green anger - red These barcodes are then used to find movies with similar emotion pattern across genres. This is done using one of the six similarity measures provided. This is still a work in progress and have a lot of potential use cases in screenplay analysis. \n Read more about the development process in this website: https://codesandreels.com/screenplay-analysis-app-using-natural-language-processing/ """) with gr.Tab("Analyse Existing Screenplays"): movie_name = gr.Dropdown(choices=df['title'].to_list(),type='index', label='Choose your movie') measure = gr.Dropdown(choices=similarity_measures,type='index', label="Select a similarity measure") button = gr.Button('Submit') selected_movie = gr.Textbox(label="Selected Movie") poster = gr.Image(label='poster',height=200,width=100) barcode = gr.Image(label='Emotion Sequence') n1 = gr.Textbox(label="Match 1") p1 = gr.Image(label='poster',height=200,width=100) b1 = gr.Image(label='Emotion Sequence') n2 = gr.Textbox(label="Match 2") p2 = gr.Image(label='poster',height=200,width=100) b2 = gr.Image(label='Emotion Sequence') n3 = gr.Textbox(label="Match 3") p3 = gr.Image(label='poster',height=200,width=100) b3 = gr.Image(label='Emotion Sequence') n4 = gr.Textbox(label="Match 4") p4 = gr.Image(label='poster',height=200,width=100) b4 = gr.Image(label='Emotion Sequence') n5 = gr.Textbox(label="Match 5") p5 = gr.Image(label='poster',height=200,width=100) b5 = gr.Image(label='Emotion Sequence') with gr.Tab("Analyse new Screenplay"): file = gr.File(label="Submit you file", file_types=["text"], type="file") measure2 = gr.Dropdown(choices=similarity_measures,type='index', label="Select a similarity measure") file_button = gr.Button("Submit file") barcode2 = gr.Image(label='Emotion Sequence') x1 = gr.Textbox(label="Match 1") i1 = gr.Image(label='poster',height=200,width=100) y1 = gr.Image(label='Emotion Sequence') x2 = gr.Textbox(label="Match 2") i2 = gr.Image(label='poster',height=200,width=100) y2 = gr.Image(label='Emotion Sequence') x3 = gr.Textbox(label="Match 3") i3 = gr.Image(label='poster',height=200,width=100) y3 = gr.Image(label='Emotion Sequence') x4 = gr.Textbox(label="Match 4") i4 = gr.Image(label='poster',height=200,width=100) y4 = gr.Image(label='Emotion Sequence') x5 = gr.Textbox(label="Match 5") i5 = gr.Image(label='poster',height=200,width=100) y5 = gr.Image(label='Emotion Sequence') button.click(fn=output,inputs=[movie_name,measure],outputs=[selected_movie,poster,barcode,n1,p1,b1,n2,p2,b2,n3,p3,b3,n4,p4,b4,n5,p5,b5]) file_button.click(fn=output2,inputs=[file,measure2],outputs=[barcode2,x1,i1,y1,x2,i2,y2,x3,i3,y3,x4,i4,y4,x5,i5,y5]) demo.launch()