hakkam10's picture
inserted link for further reading
03545ea
Raw History Blame Contribute Delete
13.6 kB
import pandas as pd
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
import Levenshtein
from sklearn.metrics.pairwise import polynomial_kernel
import gradio as gr
from Bio.Align import PairwiseAligner
from scipy.spatial import distance
import jaro
from transformers import AutoModelForSequenceClassification,AutoTokenizer,pipeline
import codecs
from PIL import Image, ImageDraw
# Import the Model
tokenizer = AutoTokenizer.from_pretrained('classifier')
model = AutoModelForSequenceClassification.from_pretrained('classifier')
# Import the screenplay dataset
df = pd.read_json("screenplay_emotion_final.json")
# Import the movie posters dataset
movie_posters = pd.read_csv("movie_posters.csv")
# Map the emotions to colors that represent them
color_mapping = {
'joy': (248, 255, 149),
'love': (255, 199, 234),
'surprise': (166, 255, 150),
'sadness':(12, 53, 106),
'anger': (223, 46, 56),
'fear': (53, 124, 60)
}
code_mapping = {
'6': (248, 255, 149),
'5': (255, 199, 234),
'4': (166, 255, 150),
'3': (12, 53, 106),
'2': (223, 46, 56),
'1': (53, 124, 60)
}
emotion_mapping = {
'joy':6,
'love':5,
'surprise':4,
'sadness':3,
'anger':2,
'fear':1
}
classifier_labels = ['sadness', 'joy', 'love', 'anger', 'fear', 'surprise']
def string(e):
x = [str(i) for i in e]
string = ''.join(x)
return string
# Function to convert the vector into an image representing the emotional pattern in the film
def emo_barcode(idx):
image = Image.new('RGB', (700,300), 'white')
draw = ImageDraw.Draw(image)
length = 700//100
emotion_list = df['emotions'][idx]
for i in range(0,100):
code = emotion_list[i]
color = code_mapping[str(code)]
draw.rectangle([length*i,0,length*(i+1), 300], fill=color)
return image
def emo_barcode_file(emotion_list):
image = Image.new('RGB', (700,300), 'white')
draw = ImageDraw.Draw(image)
length = 700//100
for i in range(0,100):
code = emotion_list[i]
color = code_mapping[str(code)]
draw.rectangle([length*i,0,length*(i+1), 300], fill=color)
return image
# Functions to calculate similarity measures
def cosine(idx):
a_list = []
if type(idx) == int:
e1 = df['emotions'][idx]
if type(idx) == list:
e1 = idx
for i in range(0,df.shape[0]):
e2 = df['emotions'][i]
score = 0
for i in range(0,100,5):
score += distance.cosine(e1[i:i+5],e2[i:i+5])
a_list.append(score)
array = np.array(a_list)
movie_indexes = np.argsort(array)
array.sort()
return movie_indexes[1:6], array[1:6]
def euclid(idx):
a_list = []
if type(idx) == int:
e1 = df['3d_vectors'][idx]
if type(idx) == list:
e1 = mapping(idx)
for i in range(0,df.shape[0]):
e2 = df['3d_vectors'][i]
score = 0
for i in range(0,100):
score += distance.euclidean(e1[i],e2[i])
a_list.append(score)
array = np.array(a_list)
movie_indexes = np.argsort(array)
array.sort()
return movie_indexes[1:6], array[1:6]
def jaro_metric(idx):
a_list = []
if type(idx) == int:
e1 = df['emotions'][idx]
if type(idx) == list:
e1 = idx
for i in range(0,df.shape[0]):
e2 = df['emotions'][i]
score = 0
for i in range(0,100,5):
score += jaro.jaro_metric(string(e1[i:i+5]),string(e2[i:i+5]))
a_list.append(score)
array = np.array(a_list)
movie_indexes = np.argsort(array)
array.sort()
return movie_indexes[-6:-1], array[-6:-1]
def hamming(idx):
a_list = []
if type(idx) == int:
e1 = df['emotions'][idx]
if type(idx) == list:
e1 = idx
for i in range(0,df.shape[0]):
e2 = df['emotions'][i]
score = 0
for i in range(0,100,5):
score += distance.hamming(e1[i:i+5],e2[i:i+5])
a_list.append(score)
array = np.array(a_list)
movie_indexes = np.argsort(array)
array.sort()
return movie_indexes[1:6], array[1:6]
def needleman(idx):
a_list = []
if type(idx) == int:
e1 = [str(x) for x in df['emotions'][idx]]
e1 = "".join(e1)
if type(idx) == list:
e1 = [str(x) for x in idx]
e1 = "".join(e1)
for i in range(0,df.shape[0]):
e2 = [str(x) for x in df['emotions'][i]]
e2 = "".join(e2)
score = PairwiseAligner().score(e1,e2)
a_list.append(score)
array = np.array(a_list)
movie_indexes = np.argsort(array)
array.sort()
return movie_indexes[-6:-1], array[-6:-1]
def Leven(idx):
a_list = []
if type(idx) == int:
e1 = df['emotions'][idx]
if type(idx) == list:
e1 = idx
for i in range(0,df.shape[0]):
e2 = df['emotions'][i]
score = 0
for i in range(0,100,5):
score += Levenshtein.distance(e1[i:i+5],e2[i:i+5])
a_list.append(score)
array = np.array(a_list)
movie_indexes = np.argsort(array)
array.sort()
return movie_indexes[1:6],array[1:6]
# Function to classify the segments into scenes
def classify(file):
scenes = []
#read the file
if type(file) == str:
try:
with codecs.open(file, "r", "utf-8") as f:
text = f.read()
except Exception as e:
print(e)
else:
try:
with codecs.open(file.name, "r", "utf-8") as f:
text = f.read()
except Exception as e:
print(e)
#split the screenplay into scenes
words = text.split()
length = len(words)
segment_length = length//100
for i in range(0,100):
segment = words[segment_length*i:segment_length*(i+1)]
scene = ' '.join(segment)
scenes.append(scene)
# classify the segments
inputs = tokenizer(scenes, padding=True, truncation=True, return_tensors="pt")['input_ids']
outputs = np.array(model(inputs).logits.argmax(-1)).tolist()
labels = []
for i in outputs:
labels.append(emotion_mapping[classifier_labels[i]])
return labels
# Deployment in Gradio App
similarity_measures = ["Euclidean Distance", "Jaro Metric", "Hamming Metric", "Levenshtein distance","Needleman Wunsch Algorithm","Cosine Similarity"]
def output(idx,measure):
if measure == 0:
a_list, scores = euclid(idx)
if measure == 1:
a_list, scores = jaro_metric(idx)
if measure == 2:
a_list, scores = hamming(idx)
if measure == 3:
a_list, scores = Leven(idx)
if measure == 4:
a_list, scores = needleman(idx)
if measure == 5:
a_list, scores = cosine(idx)
movie_names = []
selected_movie = df['title'][idx]
poster = movie_posters['image'][movie_posters[movie_posters['title'] == df['title'][idx]].index[0]]
barcode = emo_barcode(idx)
b1 = emo_barcode(a_list[4])
b2 = emo_barcode(a_list[3])
b3 = emo_barcode(a_list[2])
b4 = emo_barcode(a_list[1])
b5 = emo_barcode(a_list[0])
for i in a_list:
movie_names.append(df['title'][i])
n1 = movie_names[4] + f" | Score - {scores[4]}"
n2 = movie_names[3] + f" | Score - {scores[3]}"
n3 = movie_names[2] + f" | Score - {scores[2]}"
n4 = movie_names[1] + f" | Score - {scores[1]}"
n5 = movie_names[0] + f" | Score - {scores[0]}"
p1 = movie_posters['image'][movie_posters[movie_posters['title'] == movie_names[4]].index[0]]
p2 = movie_posters['image'][movie_posters[movie_posters['title'] == movie_names[3]].index[0]]
p3 = movie_posters['image'][movie_posters[movie_posters['title'] == movie_names[2]].index[0]]
p4 = movie_posters['image'][movie_posters[movie_posters['title'] == movie_names[1]].index[0]]
p5 = movie_posters['image'][movie_posters[movie_posters['title'] == movie_names[0]].index[0]]
return selected_movie,poster,barcode,n1,p1,b1,n2,p2,b2,n3,p3,b3,n4,p4,b4,n5,p5,b5
def output2(file,measure):
labels = classify(file)
barcode2 = emo_barcode_file(labels)
if measure == 0:
a_list = cosine(labels)
if measure == 1:
a_list = euclid(labels)
if measure == 2:
a_list = jaro_metric(labels)
if measure == 3:
a_list = hamming(labels)
if measure == 4:
a_list = Leven(labels)
movie_names = []
y1 = emo_barcode(a_list[0])
y2 = emo_barcode(a_list[1])
y3 = emo_barcode(a_list[2])
y4 = emo_barcode(a_list[3])
y5 = emo_barcode(a_list[4])
for i in a_list:
movie_names.append(df['title'][i])
x1 = movie_names[0]
x2 = movie_names[1]
x3 = movie_names[2]
x4 = movie_names[3]
x5 = movie_names[4]
i1 = movie_posters['image'][movie_posters[movie_posters['title'] == movie_names[0]].index[0]]
i2 = movie_posters['image'][movie_posters[movie_posters['title'] == movie_names[1]].index[0]]
i3 = movie_posters['image'][movie_posters[movie_posters['title'] == movie_names[2]].index[0]]
i4 = movie_posters['image'][movie_posters[movie_posters['title'] == movie_names[3]].index[0]]
i5 = movie_posters['image'][movie_posters[movie_posters['title'] == movie_names[4]].index[0]]
return barcode2,x1,i1,y1,x2,i2,y2,x3,i3,y3,x4,i4,y4,x5,i5,y5
with gr.Blocks() as demo:
with gr.Row():
gr.Markdown("""
# Screenplay Emotion Analysis
This demo attempts to study a new approach in screenplay analysis and movie recommendations.
This project started with three questions:\n
What if you can identify the overall emotion pattern of a screenplay without reading it?\n
What if you can make a machine understand a screenplay by vectorising the emotions contained in the screenplay?\n
What if you can predict what movie a person might like using the emotions contained in the screenplay?\n
The screenplays were scrapped from IMSdb website and split into 100 segments of equal length. I have trained an emotion classification model and used the model to classify each segment in each screenplay into one of six emotion categories and stored it as a vector. The emotion categories are joy, sadness, surprise, love, fear and anger.
### Each color in the barcode represents one of six emotions
Joy - yellow
Love - Pink
sadness - dark blue
surprise - light green
fear - dark green
anger - red
These barcodes are then used to find movies with similar emotion pattern across genres. This is done using one of the six similarity measures provided. This is still a work in progress and have a lot of potential use cases in screenplay analysis.
\n
Read more about the development process in this website: https://codesandreels.com/screenplay-analysis-app-using-natural-language-processing/
""")
with gr.Tab("Analyse Existing Screenplays"):
movie_name = gr.Dropdown(choices=df['title'].to_list(),type='index', label='Choose your movie')
measure = gr.Dropdown(choices=similarity_measures,type='index', label="Select a similarity measure")
button = gr.Button('Submit')
selected_movie = gr.Textbox(label="Selected Movie")
poster = gr.Image(label='poster',height=200,width=100)
barcode = gr.Image(label='Emotion Sequence')
n1 = gr.Textbox(label="Match 1")
p1 = gr.Image(label='poster',height=200,width=100)
b1 = gr.Image(label='Emotion Sequence')
n2 = gr.Textbox(label="Match 2")
p2 = gr.Image(label='poster',height=200,width=100)
b2 = gr.Image(label='Emotion Sequence')
n3 = gr.Textbox(label="Match 3")
p3 = gr.Image(label='poster',height=200,width=100)
b3 = gr.Image(label='Emotion Sequence')
n4 = gr.Textbox(label="Match 4")
p4 = gr.Image(label='poster',height=200,width=100)
b4 = gr.Image(label='Emotion Sequence')
n5 = gr.Textbox(label="Match 5")
p5 = gr.Image(label='poster',height=200,width=100)
b5 = gr.Image(label='Emotion Sequence')
with gr.Tab("Analyse new Screenplay"):
file = gr.File(label="Submit you file", file_types=["text"], type="file")
measure2 = gr.Dropdown(choices=similarity_measures,type='index', label="Select a similarity measure")
file_button = gr.Button("Submit file")
barcode2 = gr.Image(label='Emotion Sequence')
x1 = gr.Textbox(label="Match 1")
i1 = gr.Image(label='poster',height=200,width=100)
y1 = gr.Image(label='Emotion Sequence')
x2 = gr.Textbox(label="Match 2")
i2 = gr.Image(label='poster',height=200,width=100)
y2 = gr.Image(label='Emotion Sequence')
x3 = gr.Textbox(label="Match 3")
i3 = gr.Image(label='poster',height=200,width=100)
y3 = gr.Image(label='Emotion Sequence')
x4 = gr.Textbox(label="Match 4")
i4 = gr.Image(label='poster',height=200,width=100)
y4 = gr.Image(label='Emotion Sequence')
x5 = gr.Textbox(label="Match 5")
i5 = gr.Image(label='poster',height=200,width=100)
y5 = gr.Image(label='Emotion Sequence')
button.click(fn=output,inputs=[movie_name,measure],outputs=[selected_movie,poster,barcode,n1,p1,b1,n2,p2,b2,n3,p3,b3,n4,p4,b4,n5,p5,b5])
file_button.click(fn=output2,inputs=[file,measure2],outputs=[barcode2,x1,i1,y1,x2,i2,y2,x3,i3,y3,x4,i4,y4,x5,i5,y5])
demo.launch()