initial commit

app.py

@@ -0,0 +1,34 @@
+import streamlit as st
+import eda
+import model_result
+import prediction
+from streamlit_option_menu import option_menu
+
+
+
+st.sidebar.header("Emotion Classification")
+st.title("Facial Emotion Classification")
+
+with st.sidebar:
+    st.write("Ediashta Revindra - FTDS-020")
+    selected = option_menu(
+        "Menu",
+        [
+            "Distribution",
+            "Image Sample",
+            "Model Result",
+            "Classification",
+        ],
+        icons=["bar-chart", "link-45deg", "code-square"],
+        menu_icon="cast",
+        default_index=0,
+    )
+
+if selected == "Distribution":
+    eda.distribution()
+elif selected == "Image Sample":
+    eda.samples()
+elif selected == "Model Result":
+    model_result.report()   
+elif selected == "Classification":
+    prediction.predict()

csv/distribution.csv

@@ -0,0 +1,8 @@
+,label,count
+3,happy,7215
+4,neutral,4965
+5,sad,4830
+2,fearful,4097
+0,angry,3995
+6,surprised,3171
+1,disgusted,436

csv/training_history.csv

@@ -0,0 +1,65 @@
+,loss,accuracy,val_loss,val_accuracy,lr
+0,2.1469740867614746,0.25343969464302063,1.856235146522522,0.2595430612564087,0.001
+1,1.643206000328064,0.36344003677368164,1.4188605546951294,0.45012539625167847,0.001
+2,1.47542405128479,0.42669546604156494,1.4378496408462524,0.4527723491191864,0.001
+3,1.3760950565338135,0.4718729257583618,1.4727094173431396,0.44246307015419006,0.001
+4,1.3218955993652344,0.49879828095436096,1.2228283882141113,0.5337141156196594,0.001
+5,1.274165391921997,0.51645827293396,1.21588134765625,0.5480635166168213,0.001
+6,1.2332035303115845,0.5338395833969116,1.138999342918396,0.5658957958221436,0.001
+7,1.1999855041503906,0.5470758080482483,1.1163111925125122,0.5743939876556396,0.001
+8,1.1698594093322754,0.5580131411552429,1.2153265476226807,0.5353859066963196,0.001
+9,1.1533291339874268,0.5654324293136597,1.081765055656433,0.5884647369384766,0.001
+10,1.1245169639587402,0.5771012306213379,1.1175576448440552,0.5720256567001343,0.001
+11,1.1052570343017578,0.5884217619895935,1.2645843029022217,0.5263304710388184,0.001
+12,1.087416172027588,0.5925667881965637,1.1044645309448242,0.5867930054664612,0.001
+13,1.0727094411849976,0.6007872223854065,1.0925219058990479,0.5810810923576355,0.001
+14,1.0691781044006348,0.6035041213035583,1.0716830492019653,0.5964056849479675,0.001
+15,1.0509978532791138,0.6126649975776672,1.0216671228408813,0.615631103515625,0.001
+16,1.0409389734268188,0.6107492446899414,1.003859043121338,0.6241292953491211,0.001
+17,1.022071123123169,0.6197708249092102,0.9891683459281921,0.631234347820282,0.001
+18,1.0206458568572998,0.6199101209640503,1.0193976163864136,0.6253831386566162,0.001
+19,1.0173377990722656,0.6225225329399109,1.0119186639785767,0.6239899396896362,0.001
+20,0.9973097443580627,0.6285833716392517,0.9888020753860474,0.6344385743141174,0.001
+21,0.9959574937820435,0.6298373341560364,0.9672532081604004,0.6361103653907776,0.001
+22,0.9838335514068604,0.6360374689102173,0.9860186576843262,0.6310949921607971,0.001
+23,0.9794051647186279,0.635410487651825,0.9807018041610718,0.6375035047531128,0.001
+24,0.9747901558876038,0.6415410041809082,0.9648522734642029,0.6422401666641235,0.001
+25,0.965321958065033,0.6438050866127014,0.9513241648674011,0.6448871493339539,0.001
+26,0.9587099552154541,0.6479501128196716,0.9564236998558044,0.6394538879394531,0.001
+27,0.9502144455909729,0.6488906145095825,0.9563396573066711,0.6468375325202942,0.001
+28,0.9410316944122314,0.652060329914093,0.9778268933296204,0.6418222188949585,0.001
+29,0.9368909001350403,0.6520255208015442,0.9706842303276062,0.6391752362251282,0.001
+30,0.9361016154289246,0.653140127658844,0.9610183835029602,0.6462802886962891,0.001
+31,0.9268645644187927,0.6564840078353882,0.9403449296951294,0.6504597663879395,0.001
+32,0.9242078065872192,0.6620920300483704,0.9947916269302368,0.630259096622467,0.001
+33,0.9099783897399902,0.6643561124801636,0.976849377155304,0.6409863233566284,0.001
+34,0.9128876328468323,0.6637291312217712,0.9144724011421204,0.6671774983406067,0.001
+35,0.9059697985649109,0.6647392511367798,0.9187557101249695,0.6631373763084412,0.001
+36,0.9036375880241394,0.6690236330032349,0.9340527653694153,0.6521314978599548,0.001
+37,0.892905056476593,0.6716012358665466,0.9564922451972961,0.6529673933982849,0.001
+38,0.8900725245475769,0.6724023818969727,1.034966230392456,0.6206464171409607,0.001
+39,0.8922945261001587,0.6732383370399475,0.9608591198921204,0.6455837488174438,0.001
+40,0.8557736277580261,0.6858476400375366,0.9025119543075562,0.6681526899337769,0.00020000001
+41,0.8454394936561584,0.6894005537033081,0.9047268629074097,0.6668988466262817,0.00020000001
+42,0.8351521492004395,0.6945556998252869,0.913826048374176,0.6688492894172668,0.00020000001
+43,0.8299487829208374,0.6959141492843628,0.9063172340393066,0.6701030731201172,0.00020000001
+44,0.8238732814788818,0.6976906061172485,0.9130178689956665,0.6701030731201172,0.00020000001
+45,0.820328950881958,0.6992232203483582,0.911518394947052,0.6716355681419373,0.00020000001
+46,0.819961428642273,0.6973074674606323,0.9138140082359314,0.6726107597351074,0.00020000001
+47,0.8194107413291931,0.6981434226036072,0.9112961888313293,0.6716355681419373,0.00020000001
+48,0.8144147992134094,0.6992928981781006,0.9087201356887817,0.6751184463500977,0.00020000001
+49,0.812261700630188,0.7022536396980286,0.9128804206848145,0.6751184463500977,0.00020000001
+50,0.8110775351524353,0.7035424709320068,0.9172229170799255,0.6671774983406067,0.00020000001
+51,0.797129213809967,0.7063986659049988,0.906335711479187,0.6756756901741028,0.00020000001
+52,0.7997974157333374,0.70636385679245,0.915867805480957,0.6726107597351074,0.00020000001
+53,0.7991617321968079,0.7080706357955933,0.9098061919212341,0.676650881767273,0.00020000001
+54,0.7924382090568542,0.7120763659477234,0.9165937900543213,0.6735859513282776,0.00020000001
+55,0.7919761538505554,0.7075133323669434,0.9187894463539124,0.6713569164276123,0.00020000001
+56,0.7897159457206726,0.7133303284645081,0.9250094294548035,0.6723321080207825,0.00020000001
+57,0.7903206944465637,0.7092549204826355,0.9148570895195007,0.675954282283783,0.00020000001
+58,0.7874423861503601,0.7086279392242432,0.916176974773407,0.6755363345146179,0.00020000001
+59,0.784817099571228,0.7122505307197571,0.9134920239448547,0.6745611429214478,4.0000003e-05
+60,0.7739878296852112,0.7166742086410522,0.9130846858024597,0.6756756901741028,4.0000003e-05
+61,0.7737305164337158,0.7153157591819763,0.9111678600311279,0.676650881767273,4.0000003e-05
+62,0.7797554135322571,0.7169528603553772,0.9143626689910889,0.6765115857124329,4.0000003e-05
+63,0.7777878642082214,0.7160472273826599,0.9157006144523621,0.6747004985809326,4.0000003e-05

eda.py

@@ -0,0 +1,134 @@
+import streamlit as st
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+import plotly.express as px
+
+dist_df = pd.read_csv('./csv/distribution.csv')
+
+
+st.set_page_config(
+    page_title="Facial Emotion Classification",
+    layout="wide",
+    initial_sidebar_state="expanded",
+)
+
+def distribution():
+    # distribution plot
+    
+    st.header("Image Distribution")
+    
+    fig = px.bar(dist_df, 
+                 x="label", y="count", 
+                 labels={"label": "Emotions", "count": "Image Count"}, 
+                 color="label", template="plotly_white")
+    fig.update_layout(height=600)
+    
+    # Show the plot
+    st.plotly_chart(fig, use_container_width=True)
+    
+    st.markdown('''
+                
+                ''')
+
+def samples():    
+    st.header("Image Samples")
+    
+    sample_img = st.sidebar.selectbox(label="Select Emotion Sample", 
+                 options=["Angry", "Disgusted", "Fearful", "Happy", "Neutral", "Sad", "Surprised"])
+    
+    if sample_img == "Angry":
+        st.image('./image/angry.png')
+        st.markdown('''
+                    This emotion is usually characterized by feelings of anger, frustration, and dissatisfaction towards something perceived as threatening or hurtful.
+
+                    **Physical Signs:**
+
+                    * The outer part of the eyebrows is raised.
+                    * The eyes are usually narrowed and sharp.
+                    * The muscles around the mouth are tensed.
+                    ''')
+        
+    elif sample_img == "Disgusted":
+        st.image('./image/disgusted.png')
+        st.markdown('''
+                    This emotion is related to feelings of fear or being disturbed by something disliked or considered disgusting.
+
+                    **Physical Signs**:
+                    - The eyebrows are raised.
+                    - The upper eyelids are raised.
+                    - The nose is wrinkled.
+                    - Usually, the mouth is slightly tense and slightly raised.
+
+                    ''')
+        
+    elif sample_img == "Fearful":
+        st.image('./image/fearful.png')
+        st.markdown('''
+                    This emotion involves feelings of fear or anxiety due to the presence of a threat or danger.
+
+                    **Physical Signs**:
+                    - Widened eyes
+                    - Outer part of the eyebrows slightly lowered
+                    - Mouth usually slightly downturned
+                    - In this emotion, hands are often used to cover parts of the face.
+
+                    ''')
+        
+    elif sample_img == "Happy":
+        st.image('./image/happy.png')
+        st.markdown('''
+                    This emotion is usually characterized by feelings of joy, happiness, and contentment. Happy facial expressions include smiling, twinkling eyes, and a tendency to be open and friendly.
+
+                    **Physical Signs**:
+                    - Eyes appear sparkling
+                    - Mouth is smiling
+
+                    ''')
+        
+    elif sample_img == "Neutral":
+        st.image('./image/neutral.png')
+        st.markdown('''
+                    Neutral emotion indicates the absence of strong emotions or facial expressions that do not show intense feelings.
+                    
+                    Physical Signs in this expression are that each **facial feature appears not prominent**.
+
+                    ''')
+        
+    elif sample_img == "Sad":
+        st.image('./image/sad.png')
+        st.markdown('''
+                    This emotion involves feelings of sadness, disappointment, and loss.
+
+                    Physical Signs:
+                    - Eyebrows are usually furrowed.
+                    - Eyes become downcast or droopy.
+                    - The corners of the mouth slightly downturned.
+
+                    ''')
+        
+    elif sample_img == "Surprised":
+        st.image('./image/surprised.png')
+        st.markdown('''
+                    This emotion arises when someone experiences an unexpected event or receives surprising information.
+
+                    Physical Signs:
+                    - Eyebrows raise entirely.
+                    - Eyes widen.
+                    - Typically, the mouth opens.
+
+                    ''')
+
+    st.subheader("Mean (Avg.) of Image")
+    st.image('./image/mean.png')
+    st.markdown('''
+                By calculating the mean of image from each class, the following information is obtained:
+
+                * The most prominent expressions are **Happy and Surprised**, easily identifiable from the characteristics of the mouth.
+                * **Sad, Fearful, and Neutral** expressions are somewhat challenging to differentiate.
+                * **Angry and Disgusted** expressions appear similar due to the characteristics of the eyebrows and mouth.
+
+                ''')
+
+if __name__ == "__main__":
+    distribution()

model/model_fine_tune.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a74e31c3ddbf80a2fd98801114af1d4d4ae4bdca95fb6a8ba767fbca05813a90
+size 73012760

model_result.py

@@ -0,0 +1,45 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+import seaborn as sns
+import matplotlib.pyplot as plt
+import plotly.express as px
+import time
+
+def report():
+    
+    df = pd.read_csv('./csv/training_history.csv')
+    df.rename(columns={'Unnamed: 0':'epoch'}, inplace=True)
+    
+    st.header("Model Report")
+    
+    st.subheader("Performance")
+    plot_anim = st.sidebar.selectbox(label='Select Performance Metrics', options=["Accuracy", "Loss"])
+       
+    
+    def performance_plot(data):
+        progress_bar = st.sidebar.progress(0)
+        status_text = st.sidebar.empty()
+        last_rows = [df[data].iloc[0]]
+        chart = st.line_chart(last_rows, use_container_width=True, height=500)
+
+        for i in range(1, len(df)):
+            new_rows = [df[data].iloc[i]]
+            status_text.text(f"{round(i/63 * 100, 2)} % Complete")
+            chart.add_rows(new_rows)
+            progress_bar.progress(i)
+            last_rows = new_rows
+            time.sleep(0.05)
+
+        progress_bar.empty()
+    if plot_anim == "Accuracy":
+        data_plot = ['accuracy', 'val_accuracy']
+        performance_plot(data_plot)
+    else:
+        data_plot = ['loss', 'val_loss']
+        performance_plot(data_plot)
+        
+    st.button("Re-run")
+        
+if __name__ == "__main__":
+    report()

prediction.py

@@ -0,0 +1,104 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+import pickle
+import tensorflow as tf
+import matplotlib.pyplot as plt
+from tensorflow.keras.layers import Dense, Input, concatenate
+from tensorflow.keras.models import load_model
+from tensorflow.keras.preprocessing import image
+from PIL import Image
+from urllib import request
+from io import BytesIO
+
+
+def predict():
+    
+    emotion_classification_model = load_model('./model/model_fine_tune.h5')
+    
+    
+    url = "https://lh3.googleusercontent.com/0e8O0JPOp_ydq7wqv6kgYz6UpF9w_INfnOLIhnJZBEHFcWIygkuLa3SVghhGYgE0XWzQYBPb6wb1eQFN0pVIAYlzEeNojYuCWg=s0"
+    
+    def img_url(url):
+        res = request.urlopen(url).read()
+        img_ori = image.load_img(BytesIO(res))
+        img = image.load_img(BytesIO(res), target_size=(48, 48), keep_aspect_ratio=True)
+        show_predict(img, img_ori)
+        
+    def show_predict(img, img_ori):
+        col1, col2 = st.columns(2)
+        fig = plt.figure()
+        plt.imshow(img_ori)
+        plt.axis('off')
+        col1.pyplot(fig)
+        
+        img_array = image.img_to_array(img)
+        img_array = np.expand_dims(img_array, axis=0)
+        img_array /= 255.0  # Normalize the image
+
+        inf_pred_single = emotion_classification_model.predict(img_array)
+        
+        data_inf_single = []
+
+        rank = []
+
+        for i in inf_pred_single[0]:
+            value = i * 100
+            rank.append(value)
+            data_inf_single.append(f'{value.round(2)}%')
+
+        rank = (-np.array(rank)).argsort()[:2]
+        
+        pred_class_single = pd.DataFrame(class_labels).loc[rank][0].tolist()
+
+        prediction_result_single = pd.DataFrame(columns=["angry", "disgusted", "fearful", "happy", "neutral", "sad", "surprised"])
+        prediction_result_single.loc[len(prediction_result_single)] = data_inf_single
+
+        prediction_result_single
+        
+        st.markdown("""
+                    <style>
+                    .big-font {
+                        font-size:30px !important;
+                    }
+                    </style>
+                    """, unsafe_allow_html=True)
+        
+        col2.write('Prediction Class:')
+        col2.markdown(f'<p class="big-font">{pred_class_single[0].capitalize()}</p>', unsafe_allow_html=True)
+        
+        col2.dataframe(prediction_result_single.set_index(prediction_result_single.columns[0]), use_container_width=True)
+    
+    class_labels = ["angry", "disgusted", "fearful", "happy", "neutral", "sad", "surprised"]
+
+    st.write('Insert Image URL Below (Make sure face is centered and fitted)')
+    
+    st.markdown('[Example Image](https://cdn.idntimes.com/content-images/community/2021/12/whatsapp-image-2021-12-02-at-190446-8ecf63e1fa6b5c8c5e9ac43034bc86d3-c563813ea99f16a795ad4c53af10881a_600x400.jpeg)')
+    
+    col1, col2 = st.columns((9,1))
+    url_input = col1.text_input(label="Image Links")
+    
+    st.markdown(
+                """
+            <style>
+            button {
+                height: auto;
+                margin-top: 28px !important;
+                padding-left: 24px !important;
+                padding-right: 24px !important;
+            }
+            </style>
+            """,
+                unsafe_allow_html=True,
+            )
+    pred_button = col2.button(label="Predict")
+    
+    if pred_button:
+        img_url(url_input)
+    else:
+        img_url(url)
+    
+    
+
+if __name__ == "__main__":
+    predict()

requirements.txt

@@ -0,0 +1,9 @@
+seaborn
+pandas
+matplotlib
+plotly
+Pillow
+scikit-learn==1.2.2
+feature_engine
+streamlit_option_menu
+tensorflow ==2.10.0