app.py

from Main import wav2art
import numpy as np
import librosa 
import base64
import gc
gc.enable()

import os

import matplotlib.pyplot as plt
import soundfile as sf
from cv2 import resize, INTER_LINEAR
from PIL import Image

import scipy.signal as signal
from matplotlib.animation import FuncAnimation, FFMpegWriter

import torch
import librosa

import streamlit as st

from PIL import Image
import streamlit as st

dir_path = os.path.dirname(os.path.realpath(__file__))

im = Image.open(dir_path + "/Logo.png")
st.set_page_config(
    page_title="Watch Me Talk",
    page_icon=im,
)


st.markdown(
    """
    <style>
    .container {
        display: flex;
    }
    .logo-text {
        font-weight:700 !important;
        font-size:50px !important;
        color: #f9a01b !important;
        padding-top: 75px !important;
    }
    .logo-img {
        float:right;
    }
    </style>
    """,
    unsafe_allow_html=True
)

st.markdown(
    f"""
    <div class="container">
        <img class="logo-img" src="data:image/png;base64,{base64.b64encode(open('Logo.png', "rb").read()).decode()}">
        <p class="logo-text">Watch Me Speak</p>
    </div>
    """,
    unsafe_allow_html=True
)
# st.markdown('# Watch Me Talk')

hline = '''
---
'''

what = st.radio(
     "Select an option:",
     ('See examples', 'Upload audio file'))


st.markdown(hline)

if what == 'See examples':
    example = st.radio(
        "Select a sample sentence:",
        (
        "1. Be careful not to plow over the flower beds.",
        "2. Project development was proceeding too slowly.",
        "3. This brochure is particularly informative for a prospective buyer.",
        "4. I'd rather not buy these shoes than be overcharged.",
        "5. Those musicians harmonize marvellously."
        )
    )

    if example is not None:

        example_ind = example[0]
        example_path = str(example_ind) + '.mp4'

        video_file = open(example_path, 'rb')
        video_bytes = video_file.read()

        st.video(video_bytes)


if what == 'Upload audio file':
    uploaded_file = st.file_uploader("Choose a .wav file", type=["wav"])
    if uploaded_file is not None:
       
        wav_np, sr = librosa.load(uploaded_file, sr = 16000)
        sf.write('audio.wav', wav_np, sr, subtype='PCM_24')

        wav = wav_np.tolist()    
        
        emaTemp2 = wav2art(wav_np)
    
        my_bar = st.progress(0)

        TA = st.empty()
        text = TA.text_area("", "Firing up!")


        C_AH = np.array([0.16793381, 0.61851003, 0.3930835 , 2.73506039, 0.84535045,
               1.04448936, 1.5337297 , 1.37088293, 1.52812757, 1.00774768,
               1.84668592, 1.06159204])

        meanOfData_AH = np.array([ 10.18237309,   6.88706306,  11.9609807 ,  -4.72927955,
                -0.77401723, -44.12270889, -13.25678639,  -0.96288923,
               -23.19792409,   3.77430628, -31.07672544,   7.20676208])

        meanOfData = np.mean(emaTemp2,axis=0)
        emaTemp2 -= meanOfData
        C = 0.5*np.sqrt(np.mean(np.square(emaTemp2),axis=0))
        ema = np.divide(emaTemp2,C)

        ema = ema*C_AH + meanOfData_AH
        emaR_sub = np.reshape(ema, (len(ema), 6, 2))
        
        my_bar.progress(10)



        text = TA.text_area("", "Loading wav2vec 2.0 ...")

        model = torch.load("model.pt")

        tokenizer = torch.load('tokenizer.pt')
        feature_extractor = torch.load('feature_extractor.pt')

        my_bar.progress(15)

        text = TA.text_area("", "Estimating phonemes ...")

        input_values = feature_extractor(wav, return_tensors="pt", sampling_rate = sr).input_values

        logits = model(input_values).logits[0]
        pred_ids = torch.argmax(logits, axis=-1)
        
        # retrieve word stamps (analogous commands for `output_char_offsets`)
        outputs = tokenizer.decode(pred_ids, output_char_offsets = True)

        time_offset = model.config.inputs_to_logits_ratio

        phoneme_dict = {'m' : 0, 'ɱ' : 0, 'n' : 0, 'ŋ' : 0, 'ɴ' : 0}
        char_offsets = [
            {
                "char": d["char"],
                "start_time": round(d["start_offset"] * time_offset, 2),
                "end_time": round(d["end_offset"] * time_offset, 2),
                "nasality": phoneme_dict.get(d["char"], 1) 
            }
            for d in outputs.char_offsets
        ]

        initial = char_offsets[0]['start_time']
        final = len(wav) - char_offsets[-1]['end_time']

        phonemes = []
        toprint = []

        for p in range(initial):
            phonemes.append(0)
            toprint.append('')

        prev_phon = 0
        prev_tp = ''

        for p in char_offsets:
            for t in range(int(np.floor((p['start_time'] - final)/2))):
                phonemes.append(prev_phon)
                toprint.append(prev_tp)

            for t in range(int(np.ceil((p['start_time'] - final)/2))):
                phonemes.append(p['nasality'])
                toprint.append(p['char'])
            
            final = p['end_time']
            
            for t in range(p['end_time'] - p['start_time']):
                phonemes.append(p['nasality'])
                toprint.append(p['char'])

            prev_phon = p['nasality']
            prev_tp = p['char']

            
        final = len(wav) - char_offsets[-1]['end_time']

        for p in range(final):
            phonemes.append(0)
            toprint.append('')

        length = int(len(emaR_sub) * 0.3)
        ema = signal.resample(emaR_sub, length)

        my_bar.progress(25)

        processed = []
        to_print = []

        for d in range(0, len(phonemes), 160):
            begin = (d - 160 > 0) * (d - 160 )
            phones20 = phonemes[begin : d + 160]
            phone = max(set(phones20), key=phones20.count)
            processed.append(phone)

            phones20 = toprint[begin : d + 160]
            phone = max(set(phones20), key=phones20.count)
            to_print.append(phone)


        res = resize(np.array(processed).astype('float32'), dsize=(1, length), interpolation = INTER_LINEAR)

        processed2 = res.flatten()

        final_print = []

        i = 0

        while i < len(to_print):
            
            z = int(3 + i % 3)
            
            phones20 = to_print[i : i + z]
            phone = max(set(phones20), key=phones20.count)
            final_print.append(phone)

            i += z


        gaps = np.abs(len(processed2) - len(final_print))

        total_len = len(final_print)

        for i, j in enumerate(np.linspace(0,  total_len - 1, gaps)):
            del_index = int(j - i - 1)
            del final_print[del_index]

        kernel = [1/10 for i in range(10)]
        processed3 = np.convolve(processed2, kernel)


        ### below code is to find the average distance between certain ema points in a dataset so that we can define our offsets for
        ### the bezier curve contours accordingly later on

        def find_off(matrix):
          p = 0
          q = 0

          for i in range(len(matrix)):
            q += matrix[i][0][1] - matrix[i][1][1]
            p += matrix[i][1][0] - matrix[i][2][0]

          p = p/len(matrix)
          q = q/len(matrix)
          return np.round_([p, q])

        ### defining offsets for the t_end and the tongue base points so as to make the curves as relative as possible
        def tongue_off(matrix):
          c = 0
          d = 0

          for i in range(len(matrix)):
            d += matrix[i][3][1] - matrix[i][5][1]
            c += matrix[i][3][0] - matrix[i][4][0]
          
          c = np.abs(c/len(matrix))
          d = np.abs(d/len(matrix))
          return np.round_([c, d])

        k = find_off(ema)
        l = tongue_off(ema)


        print(ema[0])
        l = l*1.5

        global painting

        from scipy.special import comb
        def bernstein_poly(i, n, t):
                return comb(n, i) * ( t**(n-i) ) * (1 - t)**i

        def bezier_curve(xPoints, yPoints, nTimes=150):
                nPoints = len(xPoints)
                t = np.linspace(0.0, 1.0, nTimes)
                polynomial_array = np.array([bernstein_poly(i, nPoints-1, t) for i in range(0, nPoints)])
                xvals = np.dot(xPoints, polynomial_array)
                yvals = np.dot(yPoints, polynomial_array)
                return xvals, yvals, nTimes


        def contract(x):
            m = np.mean(x)
            for i in range(len(x)):
                x[i] = (2*m + 3*x[i]) / 5
            return x


        brain0 = Image.open('BrainAndSpinal.png')


        text = TA.text_area("", "Rendering frame: 0 of " + str(len(ema)))



        import time

        plt.rcParams.update({'font.size': 22})

        n = 6
        clr = np.arange(6)
        arr = np.ones((6, 2))
        plots = []

        particles = np.zeros(n,dtype=[("position", float, 2),
                                   ("size", float , 1)])

        particles["position"] = ema[0]
        particles["size"] = 0.5*np.ones(n)

        n1o = ema[0]
        ulo = n1o[0]
        llo = n1o[1]
        jawo = n1o[2]
        tto = n1o[3]
        tbo = n1o[4]
        tdo = n1o[5]

        max_tt = np.max(np.abs(ema[:, 3, 0]))

        fig = plt.figure(figsize = (6, 6))
        ax = plt.axes()


        llj0 = np.array([llo[0], llo[0]+(k[0]*4/13), llo[0]-(k[0]*25/13), jawo[0]+(k[0]*5/13), jawo[0]])
        llj1 = np.array([llo[1], llo[1]-(k[1]*12/12), llo[1]-(k[1]*10/12), jawo[1]+(k[1]*5/13), jawo[1]])


        nose0 = np.array([tto[0]+(k[0]*14/13), ulo[0]+(k[0]*7/13), ulo[0]+(k[0]*36/13), ulo[0]-(k[0]*10/13), ulo[0]])
        nose1 = np.array([ulo[1]+(k[1]*50/12), ulo[1]+(k[1]*35/12), ulo[1]-(k[1]*7/13), ulo[1]+(k[1]*35/12), ulo[1]])

        llip0 = np.array([llo[0], llo[0] - .2*(l[0]), llo[0] - .5*(l[0])])
        llip1 = np.array([llo[1],llo[1] + .1*(l[1]), llo[1] - .2*(l[1])])

        ulip0 = np.array([ulo[0], ulo[0], ulo[0]-(k[0]*5/13), ulo[0]-(k[0]*8/13)])
        ulip1 = np.array([ulo[1], ulo[1]-(k[1]*4/12), ulo[1]-(k[1]*4/12), ulo[1]+(k[1]*5/12)])

        chin0 = np.array([jawo[0], jawo[0]-(k[0]*5/13), jawo[0]-(k[0]*15/13), 2*tdo[0]- 1*tbo[0]])
        chin1 = np.array([jawo[1], jawo[1]-(k[1]*10/13), jawo[1]-(k[1]*2/13), jawo[1]-(k[1]*10/13)])

        tongue0 = np.array([tto[0], tdo[0]-(l[0]*8/13), tdo[0]-(l[0]*5/13), tdo[0] - 1.5*l[0], tdo[0] - 1*l[0] - (75/(tto[0] - tdo[0]))])
        tongue1 = np.array([tto[1], tdo[1]+(l[1]*7/13), tdo[1]-(l[1]*1/13), jawo[1]+(l[1]*36/13), jawo[1]+(l[1]*8/13)])

        tline0 = np.array([jawo[0] - .4*l[0], .5*tbo[0] + .5*tto[0] , tdo[0], tdo[0] - 1*l[0] - (75/(tto[0] - tdo[0]))])
        tline1 = np.array([jawo[1]+(l[1]*9/13), jawo[1]+(l[1]*5/13), jawo[1]+(l[1]*5/13), jawo[1]+(l[1]*8/13)])

        Xa, Ya, Na = bezier_curve(xPoints = llj0, yPoints = llj1)
        Xb, Yb, Nb = bezier_curve(xPoints = nose0, yPoints = nose1)
        Xc, Yc, Nc = bezier_curve(xPoints = tongue0, yPoints = tongue1)
        Xd, Yd, Nd = bezier_curve(xPoints = llip0, yPoints = llip1)
        Xe, Ye, Ne = bezier_curve(xPoints = ulip0, yPoints = ulip1)

        temp1 = int(Ne/5)
        temp2 = int(Nb/6)

        # palate0 = np.array([Xe[temp1], Xe[temp1]-(k[0]*4/13), Xe[temp1]-(k[0]*8/13), Xe[temp1]-(k[0]*23/13), Xe[temp1]-(k[0]*32/10)])
        # palate1 = np.array([Ye[temp1], Ye[temp1], Ye[temp1]+(k[1]*7/12), Ye[temp1]+(k[1]*20/12), Ye[temp1]+(k[1]*8/12)])

        palate0 = np.array([Xe[temp1], Xe[temp1]-(k[0]*4/13), Xe[temp1]-(k[0]*8/13), Xe[temp1]-(k[0]*23/13), Xe[temp1]-(k[0]*32/10)])
        palate1 = np.array([Ye[temp1], Ye[temp1], Ye[temp1]+(k[1]*7/12), Ye[temp1]+(k[1]*10/12), Ye[temp1]+(k[1]*8/12)])

        # nose_base0 = np.array([Xb[20], Xb[temp2]-(k[0]*3/13) , Xb[temp2]-(k[0]*9/13), Xe[temp1]-(k[0]*32/10), Xe[temp1]-(k[0]*34/10)])
        # nose_base1 = np.array([Yb[20], Yb[temp2]-(k[1]*10/12), Yb[temp2]+(k[1]*17/12), Ye[temp1]+(k[1]*15/12), Ye[temp1]+(k[1]*13/12)])

        nose_base0 = np.array([Xb[20], Xb[temp2]-(k[0]*3/13) , Xb[temp2]-(k[0]*9/13), Xe[temp1]-(k[0]*32/10), Xe[temp1]-(k[0]*34/10)])
        nose_base1 = np.array([Yb[20], Yb[temp2]-(k[1]*10/12), Yb[temp2]+(k[1]*13/12), Ye[temp1]+(k[1]*13/12), Ye[temp1]+(k[1]*13/12)])

        xref = Xe[temp1]
        yref = Ye[temp1]



        forehead0 = np.array([Xb[Nb-1], Xb[Nb-1]-(k[0]*4/13), Xb[Nb-1]-(k[0]*2/13)])
        forehead1 = np.array([Yb[Nb-1], Yb[Nb-1]+(k[1]*6/12), Yb[Nb-1]+(k[1]*11/12)])


        nose_bone0 = np.array([Xc[0] - .6*l[0], Xc[0] - .5*l[0], tdo[0] - 1.6*l[0], Xe[temp1]-(k[0]*45/13), tbo[0]-(k[0]*20/13), tto[0]+(k[0]*5/13), tto[0]+(k[0]*6/13), Xb[temp2]+(k[0]*10/13), Xb[40]])
        nose_bone1 = np.array([jawo[1], tto[1], Ye[temp1]+(k[1]*30/12),ulo[1]+(k[1]*32/12),  ulo[1]+(k[1]*32/12), ulo[1]+(k[1]*65/12), ulo[1]+(k[1]*35/12), Yb[temp2]+(k[1]*15/12), Yb[40]])

        xxx0 = tto[0]+(l[0]*5/13)
        xxx1 = tto[0]-(l[0]*1/13)
        xxx12 = tto[0]-(l[0]*1/13)
        xxx2 = tto[0]-(l[0]*1/13)
        xxx3 = tto[0]-(l[0]*2/13)

        yyy0 = tto[1]+(l[1]*0/12)
        yyy1 = tto[1]-(l[1]*15/12)
        yyy12 = tto[1]-(l[1]*5/12)
        yyy2 = jawo[1]+(l[1]*32/13)
        yyy3 = jawo[1]+(l[1]*30/13)

        tbase0 = np.array([Xc[-5], xxx0, xxx1, xxx12, xxx2, xxx3, jawo[0] - .5*l[0]])
        tbase1 = np.array([Yc[-5], yyy0, yyy1, yyy12, yyy2, yyy3, jawo[1]+(l[1]*22/13)])


        Xf, Yf, Nf = bezier_curve(palate0, palate1)
        Xg, Yg, Ng = bezier_curve(chin0, chin1)
        Xh, Yh, Nh = bezier_curve(tbase0, tbase1)
        Xi, Yi, Ni = bezier_curve(tline0, tline1)
        Xj, Yj, Nj = bezier_curve(nose_base0, nose_base1)
        Xk, Yk, Nk = bezier_curve(nose_bone0, nose_bone1)

        uv_diff = phonemes[0] * 4

        aa0 = Xj[0]
        aa1 = Xj[0] - (k[0]*7/10)
        aa2 = Xj[0] - (k[0]*(9 + 2*uv_diff)/10)
        aa3 = Xj[0] - (k[0]*(6 + uv_diff)/10)
        aa4 = Xf[0] - (k[0]*5/10)
        aan = Xf[0]

        bb0 = Yj[0]
        bb1 = Yj[0] + (k[1]*(uv_diff/2)/10)
        bb2 = Yj[0] - (k[1]*(18 - uv_diff/2)/10)
        bb3 = Yj[0] - (k[1]*(20 - uv_diff/2)/10)
        bb4 = Yf[0] + (k[1]*(1 + uv_diff/2)/10)
        bbn = Yf[0]


        uvula0 = np.array([aa0, aa1, aa2, aa3,  aa4, aan])
        uvula1 = np.array([bb0, bb1, bb2, bb3,  bb4, bbn])

        Xl, Yl, Nl = bezier_curve(uvula0, uvula1)

        xref2 = xref-(k[0]*45/13)
        yref2 = yref+(k[1]*30/12)
        head0 = np.array([tto[0]+(k[0]*14/13), tto[0]+(k[0]*5/13), tto[0]+(k[0]*7/13), xref2+(l[0]*50/11), xref2-(l[0]*50/8), xref2-(l[0]*50/6), tdo[0] - (tto[0] - tdo[0])*2.5, (9*xref2)/4])
        head1 = np.array([ulo[1]+(k[1]*50/12), ulo[1]+(k[1]*60/12), ulo[1]+(k[1]*70/12), yref2+(l[1]*10), yref2+(l[1]*8), yref2+(l[1]*2), jawo[1]+(l[1]*18/4), jawo[1] - .5*l[1]])
        Xn, Yn, Nn = bezier_curve(head0, head1)

        x1 = ulo[0] - 5*l[0]/6
        x2 = ulo[0] - 1*l[0]/6
        x4 = ulo[0] - 4*l[0]/6

        y1 = ulo[1] + 4*l[1]/4
        y2 = ulo[1] - 1*l[1]/4
        y4 = ulo[1] - 3*l[1]/4


        utooth_front0 = np.array([x1, x2, x4])
        utooth_front1 = np.array([y1, y2, y4])

        Xq, Yq, Nq = bezier_curve(utooth_front0, utooth_front1)


        x1 = ulo[0] - 10*l[0]/12
        x2 = ulo[0] - 11*l[0]/12
        x3 = ulo[0] - 7*l[0]/12
        x4 = ulo[0] - 8*l[0]/12

        y1 = ulo[1] + 4*l[1]/4
        y2 = ulo[1] - 1*l[1]/4
        y3 = ulo[1] + 2*l[1]/4
        y4 = ulo[1] - 3*l[1]/4


        utooth_back0 = np.array([x1, x2, x3, x4])
        utooth_back1 = np.array([y1, y2, y3, y4])

        Xr, Yr, Nr = bezier_curve(utooth_back0, utooth_back1)

        xx1 = jawo[0] - .6*l[0]
        xx2 = jawo[0] - .4*l[0]
        xx3 = jawo[0] + .3*l[0]
        xx4 = jawo[0] + .1*l[0]
        xx5 = jawo[0] - 1.1*l[0]
        xx6 = jawo[0] - .7*l[0]
        xx7 = jawo[0] - .6*l[0]

        yy1 = jawo[1]+(l[1]*30/13)
        yy2 = jawo[1]+(l[1]*40/13)
        yy3 = jawo[1]+(l[1]*30/13)
        yy4 = jawo[1]-(l[1]*3/13)
        yy5 = jawo[1]-(l[1]*5/13)
        yy6 = jawo[1]+(l[1]*20/13)
        yy7 = jawo[1]+(l[1]*30/13)

        ltooth_base0 = np.array([xx1, xx2, xx3, xx4, xx5, xx6, xx7])
        ltooth_base1 = np.array([yy1, yy2, yy3, yy4, yy5, yy6, yy7])

        Xs, Ys, Ns = bezier_curve(ltooth_base0, ltooth_base1)

        x1 = jawo[0] - l[0]*.45
        x2 = jawo[0] + l[0]*0.215
        x3 = jawo[0] - l[0]*0.215

        y1 = jawo[1]+(l[1]*18/13)
        y2 = jawo[1]+(l[1]*28/13)
        y3 = jawo[1]+(l[1]*39/13)

        ltooth_front0 = np.array([x1, x2, x3])
        ltooth_front1 = np.array([y1, y2, y3])

        Xt, Yt, Nt = bezier_curve(ltooth_front0, ltooth_front1)

        x1 = jawo[0] - l[0]*.45
        x2 = jawo[0] - l[0]*.45
        x3 = jawo[0] - l[0]*0.25
        x4 = jawo[0] - l[0]*0.215

        y1 = jawo[1]+(l[1]*18/13)
        y2 = jawo[1]+(l[1]*33/13)
        y3 = jawo[1]+(l[1]*28/13)
        y4 = jawo[1]+(l[1]*39/13)

        ltooth_back0 = np.array([x1, x2, x3, x4])
        ltooth_back1 = np.array([y1, y2, y3, y4])

        Xu, Yu, Nu = bezier_curve(ltooth_back0, ltooth_back1)

        ccc = 'blue'
        line1, = ax.plot(Xa, Ya, color =  ccc)
        line2, = ax.plot(Xb, Yb, color =  ccc)
        line3, = ax.plot(Xc, Yc, color =  ccc)
        line4, = ax.plot(Xd, Yd, color =  ccc)
        line5, = ax.plot(Xe, Ye, color =  ccc)
        line6, = ax.plot(Xf, Yf, color =  ccc)
        line7, = ax.plot(Xg, Yg, color =  ccc)
        line8, = ax.plot(Xh, Yh, color =  ccc)
        line9, = ax.plot(Xi, Yi, color =  ccc, zorder = 1)
        line10, = ax.plot(Xj, Yj, color = ccc)
        line11, = ax.plot(Xk, Yk, color = ccc, zorder = 5)
        line12, = ax.plot(Xl, Yl, color = ccc)
        line13, = ax.plot(Xn, Yn, color = ccc, zorder = 5)
        line14, = ax.plot(Xq, Yq, color = ccc, zorder = 5)
        line15, = ax.plot(Xr, Yr, color = ccc, zorder = 5)
        line16, = ax.plot(Xs, Ys, color = ccc, zorder = 3)
        line17, = ax.plot(Xt, Yt, color = ccc, zorder = 5)
        line18, = ax.plot(Xu, Yu, color = ccc, zorder = 5)


        class Args():
          def __init__(self):
            self.painting1 = ax.fill(Xc, Yc, 'pink')
            self.painting2 = ax.fill(np.concatenate((Xh, Xi)), np.concatenate((Yh, Yi)), 'pink')
            self.painting3 = ax.fill(Xs, Ys, 'gainsboro', zorder = 3)
            self.painting4 = ax.fill(np.concatenate((Xq, Xr)), np.concatenate((Yq, Yr)), 'white', zorder = 4)
            self.painting5 = ax.fill(np.concatenate((Xt, Xu)), np.concatenate((Yt, Yu)), 'white', zorder = 4)
            # self.painting6 = ax.fill(np.concatenate((Xk, Xn)), np.concatenate((Yk, Yn)), 'white', zorder = 3)

        # plt.scatter(aa1, bb1)
        # plt.scatter(aa2, bb2)
        # plt.scatter(aa3, bb3)
        # plt.scatter(aa4, bb4)

        c_text = ax.text(-55, 120, '', alpha = 1)

        args = Args()

        newax = fig.add_axes([.025, .20, .63, .63], anchor='NE', zorder = 7)
        newax.imshow(brain0)
        newax.axis('off')
        ax.axis('off')


        my_bar.progress(30)

        old_phon = ''

        def remove_all(xx):
            for x in xx:
                x.remove()

        def update_st(f):

           loaded = int(30 + (54 * f) / len(ema))
           my_bar.progress(loaded)
           text = TA.text_area("", "Rendering frame: " + str(f) + " of " + str(len(ema)), key = f + time.time())


        def animate(frame_number):
           #plt.clf()

           global temp, old_phon, my_bar
           
           update_st(frame_number)

           particles["position"] = ema[frame_number]
           n1 = ema[frame_number]
           phon = processed3[frame_number]

           if old_phon != final_print[frame_number]:
               c_text.set_text(final_print[frame_number])
               old_phon = final_print[frame_number]

        #    if frame_number > 1:
        #        text1.set_text(final_print[frame_number-2])
        #        text2.set_text(final_print[frame_number-1])

        #    text3.set_text(final_print[frame_number])
           
        #    if frame_number < len(final_print) - 2:
        #        text4.set_text(final_print[frame_number + 1])
        #        text5.set_text(final_print[frame_number + 2])


           remove_all(args.painting1)
           remove_all(args.painting2)
           remove_all(args.painting3)
        #    remove_all(args.painting4)
           remove_all(args.painting5)
           
           ul = n1[0]
           ll = n1[1]
           jaw = n1[2]
           tt = n1[3]
           tb = n1[4]
           td = n1[5]
        #    scatter.set_offsets(particles["position"])

           llj_x = np.array([ll[0], ll[0]+(k[0]*4/13), ll[0]-(k[0]*25/13), jaw[0]+(k[0]*5/13), jaw[0]])
           llj_y = np.array([ll[1], ll[1]-(k[1]*12/12), ll[1]-(k[1]*10/12), jaw[1]+(k[1]*5/13), jaw[1]])

           nose_x = np.array([tto[0]+(k[0]*14/13), ulo[0]+(k[0]*5/13), ulo[0]+(k[0]*36/13), ulo[0]-(k[0]*10/13), ul[0]])
           nose_y = np.array([ulo[1]+(k[1]*50/12), ulo[1]+(k[1]*35/12), ulo[1]-(k[1]*7/13), ulo[1]+(k[1]*35/12), ul[1]])

           tongue_x = np.array([tt[0], td[0]-(l[0]*8/13), td[0]-(l[0]*5/13), td[0] - 1.5*l[0], tdo[0] - 1*l[0] - (75/(tto[0] - tdo[0]))])
           tongue_y = np.array([tt[1], td[1]+(l[1]*7/13), td[1]-(l[1]*1/13), jaw[1]+(l[1]*36/13), jaw[1]+(l[1]*8/13)])


           llip_x = np.array([ll[0], ll[0] - .2*(l[0]), ll[0] - .5*(l[0])])
           llip_y = np.array([ll[1], ll[1] + .1*(l[1]), ll[1] - .2*(l[1])])
          
           ulip_x = np.array([ul[0], ul[0], ul[0]-(k[0]*5/13), Xe[temp1]])
           ulip_y = np.array([ul[1], ul[1]-(k[1]*4/12), ul[1]-(k[1]*4/12), Ye[temp1]])

           chin_x = np.array([jaw[0], jaw[0]-(k[0]*5/13), jaw[0]-(k[0]*15/13), 2*tdo[0] - 1*tbo[0]])
           chin_y = np.array([jaw[1], jaw[1]-(k[1]*10/13), jaw[1]-(k[1]*2/13), jaw[1]-(k[1]*10/13)])

           tline_x = np.array([jaw[0] - .4*l[0], .5*tb[0] + .5*tt[0] , tdo[0], tdo[0] - 1*l[0] - (75/(tto[0] - tdo[0]))])
           tline_y = np.array([jaw[1]+(l[1]*9/13), jaw[1]+(l[1]*5/13), jaw[1]+(l[1]*5/13), jaw[1]+(l[1]*8/13)])

           X1, Y1, N1 = bezier_curve(xPoints = llj_x, yPoints = llj_y)
           X2, Y2, N2 = bezier_curve(xPoints = nose_x, yPoints = nose_y)
           X3, Y3, N3 = bezier_curve(xPoints = tongue_x, yPoints = tongue_y)
           X4, Y4, N4 = bezier_curve(xPoints = llip_x, yPoints = llip_y)
           X5, Y5, N5 = bezier_curve(xPoints = ulip_x, yPoints = ulip_y)

           for i in range(N5):
             if Y5[i] == Ye[int(N5/5)]:
               temp = i
           
        #    palate_x = np.array([Xe[temp1], Xe[temp1]-(k[0]*4/13), Xe[temp1]-(k[0]*8/13), Xe[temp1]-(k[0]*23/13), Xe[temp1]-(k[0]*32/10)])
        #    palate_y = np.array([Ye[temp1], Ye[temp1], Ye[temp1]+(k[1]*7/12), Ye[temp1]+(k[1]*10/12), Ye[temp1]+(k[1]*8/12)])

           nose_base_x = np.array([X2[20], Xb[temp2]-(k[0]*3/13) , Xb[temp2]-(k[0]*9/13), Xe[temp1]-(k[0]*32/10), Xe[temp1]-(k[0]*34/10)])
           nose_base_y = np.array([Y2[20], Yb[temp2]-(k[1]*10/12), Yb[temp2]+(k[1]*13/12), Ye[temp1]+(k[1]*13/12), Ye[temp1]+(k[1]*13/12)])

           xref = Xe[temp1]
           yref = Ye[temp1]


           nose_bone_x = np.array([Xc[0] - .6*l[0], Xc[0] - .5*l[0], tdo[0] - 1.6*l[0], Xe[temp1]-(k[0]*45/13), tbo[0]-(k[0]*20/13), tto[0]+(k[0]*5/13), tto[0]+(k[0]*6/13), X2[temp2]+(k[0]*10/13), Xb[40]])
           nose_bone_y = np.array([jawo[1], tto[1], Ye[temp1]+(k[1]*30/12),ulo[1]+(k[1]*32/12),  ulo[1]+(k[1]*32/12), ulo[1]+(k[1]*65/12), ulo[1]+(k[1]*35/12), Yb[temp2]+(k[1]*15/12), Y2[40]])

           xxx0 = tt[0]+(l[0]*5/13)
           xxx1 = tt[0]-(l[0]*1/13)
           xxx12 =tt[0]-(l[0]*1/13)
           xxx2 = tt[0]-(l[0]*1/13)
           xxx3 = tt[0]-(l[0]*2/13)
           
           yyy0 =  tt[1]+(l[1]*0/12)
           yyy1 =  tt[1]-(l[1]*15/12)
           yyy12 = tt[1]-(l[1]*5/12)
           yyy2 = jaw[1]+(l[1]*32/13)
           yyy3 = jaw[1]+(l[1]*30/13)
           
           tbase_x = np.array([X3[-5], xxx0, xxx1, xxx12, xxx2, xxx3, jaw[0] - .5*l[0]])
           tbase_y = np.array([Y3[-5], yyy0, yyy1, yyy12, yyy2, yyy3, jaw[1]+(l[1]*22/13)])

        #    X6, Y6, N6 = bezier_curve(xPoints = palate_x, yPoints = palate_y)
           X7, Y7, N7 = bezier_curve(chin_x, chin_y)
           X8, Y8, N8 = bezier_curve(tline_x, tline_y)
           X9, Y9, N9 = bezier_curve(tbase_x, tbase_y)
           X10, Y10, N10 = bezier_curve(nose_base_x, nose_base_y)
           X11, Y11, N11 = bezier_curve(nose_bone_x, nose_bone_y)

           uv_diff = phon * 4
           
           aa0 = Xj[0]
           aa1 = Xj[0] - (k[0]*7/10)
           aa2 = Xj[0] - (k[0]*(9 + 2*uv_diff)/10)
           aa3 = Xj[0] - (k[0]*(6 + uv_diff)/10)
           aa4 = Xf[0] - (k[0]*5/10)
           aan = Xf[0]
           
           bb0 = Yj[0]
           bb1 = Yj[0] + (k[1]*(uv_diff/2)/10)
           bb2 = Yj[0] - (k[1]*(18 - uv_diff/2)/10)
           bb3 = Yj[0] - (k[1]*(20 - uv_diff/2)/10)
           bb4 = Yf[0] + (k[1]*(1 + uv_diff/2)/10)
           bbn = Yf[0]


           uvula_x = np.array([aa0, aa1, aa2, aa3,  aa4, aan])
           uvula_y = np.array([bb0, bb1, bb2, bb3,  bb4, bbn])

           X12, Y12, N12 = bezier_curve(uvula_x, uvula_y)

           xref2 = xref-(k[0]*45/13)
           yref2 = yref+(k[1]*30/12)
           head_x = np.array([tto[0]+(k[0]*14/13), tto[0]+(k[0]*5/13), tto[0]+(k[0]*7/13), xref2+(l[0]*50/11), xref2-(l[0]*50/8), xref2-(l[0]*50/6), tdo[0] - (tto[0] - tdo[0])*2.5, (9*xref2)/4])
           head_y = np.array([ulo[1]+(k[1]*50/12), ulo[1]+(k[1]*60/12), ulo[1]+(k[1]*70/12), yref2+(l[1]*10), yref2+(l[1]*8), yref2+(l[1]*2), jawo[1]+(l[1]*18/4), jawo[1] - .5*l[1]])
           X13, Y13, N13 = bezier_curve(head_x, head_y)

        #    utooth_front_x = np.array([ulo[0] - 5*l[0]/6, ulo[0] - 1*l[0]/6, ulo[0] - 4*l[0]/6])
        #    utooth_front_y = np.array([ulo[1] + 4*l[1]/4, ulo[1] - 1*l[1]/4, ulo[1] - 3*l[1]/4])
        #    X14, Y14, N14 = bezier_curve(utooth_front_x, utooth_front_y)

        #    utooth_back_x = np.array([ulo[0] - 10*l[0]/12, ulo[0] - 11*l[0]/12, ulo[0] - 7*l[0]/12, ulo[0] - 8*l[0]/12])
        #    utooth_back_y = np.array([ulo[1] + 4*l[1]/4, ulo[1] - 1*l[1]/4, ulo[1] + 2*l[1]/4, ulo[1] - 3*l[1]/4])
        #    X15, Y15, N15 = bezier_curve(utooth_back_x, utooth_back_y)
           
           xx1 = jaw[0] - .6*l[0]
           xx2 = jaw[0] - .4*l[0]
           xx3 = jaw[0] + .3*l[0]
           xx4 = jaw[0] + .1*l[0]
           xx5 = jaw[0] - 1.1*l[0]
           xx6 = jaw[0] - .7*l[0]
           xx7 = jaw[0] - .6*l[0]
           
           yy1 = jaw[1]+(l[1]*30/13)
           yy2 = jaw[1]+(l[1]*40/13)
           yy3 = jaw[1]+(l[1]*30/13)
           yy4 = jaw[1]-(l[1]*3/13)
           yy5 = jaw[1]-(l[1]*5/13)
           yy6 = jaw[1]+(l[1]*20/13)
           yy7 = jaw[1]+(l[1]*30/13)

           ltooth_base_x = np.array([xx1, xx2, xx3, xx4, xx5, xx6, xx7])
           ltooth_base_y = np.array([yy1, yy2, yy3, yy4, yy5, yy6, yy7])
           X16, Y16, N16 = bezier_curve(ltooth_base_x, ltooth_base_y)

           x1 = jaw[0] - l[0]*.45
           x2 = jaw[0] + l[0]*.215
           x3 = jaw[0] - l[0]*.215

           ltooth_front_x = np.array([x1, x2, x3])
           ltooth_front_y = np.array([jaw[1]+(l[1]*18/13), jaw[1]+(l[1]*28/13), jaw[1]+(l[1]*39/13)])
           X17, Y17, N17 = bezier_curve(ltooth_front_x, ltooth_front_y)
           
           x1 = jaw[0] - l[0]*.45
           x2 = jaw[0] - l[0]*.45
           x3 = jaw[0] - l[0]*.25
           x4 = jaw[0] - l[0]*.215

           ltooth_back_x = np.array([x1, x2, x3, x4])
           ltooth_back_y = np.array([jaw[1]+(l[1]*18/13), jaw[1]+(l[1]*33/13), jaw[1]+(l[1]*28/13), jaw[1]+(l[1]*39/13)])
           X18, Y18, N18 = bezier_curve(ltooth_back_x, ltooth_back_y)

           line1.set_data(X1, Y1) 
           line2.set_data(X2, Y2)
           line3.set_data(X3, Y3)
           line4.set_data(X4, Y4)
           line5.set_data(X5, Y5)
        #    line6.set_data(X6, Y6)
           line7.set_data(X7, Y7)
           line8.set_data(X8, Y8)
           line9.set_data(X9, Y9)
           line10.set_data(X10, Y10)
           line11.set_data(X11, Y11)
           line12.set_data(X12, Y12)
           line13.set_data(X13, Y13)
        #    line14.set_data(X14, Y14)
        #    line15.set_data(X15, Y15)
           line16.set_data(X16, Y16)
           line17.set_data(X17, Y17)
           line18.set_data(X18, Y18)

           args.painting1 = ax.fill(X3, Y3, color = 'pink', label = 'tongue')
           args.painting2 = ax.fill(np.concatenate((X8, X9)), np.concatenate((Y8, Y9)), color = 'pink', label = 'tongue')
           args.painting3 = ax.fill(X16, Y16, 'gainsboro', zorder = 3)
        #    args.painting4 = ax.fill(np.concatenate((X14, X15)), np.concatenate((Y14, Y15)), 'white', zorder = 4)
           args.painting5 = ax.fill(np.concatenate((X17, X18)), np.concatenate((Y17, Y18)), 'white', zorder = 4)
           args.painting6 = ax.fill(np.concatenate((X11, X13)), np.concatenate((Y11, Y13)), 'white', zorder = 4)

           return line1, line2, line3, line4, line5, line6, line7, line8, line9, line10, line11, line14, line15, line16,
           line17, line18

        start = time.time()
        anim2 = FuncAnimation(fig, animate, frames = len(ema), interval = 10)

        # anim2.save('video.mp4', writer = 'ffmpeg', fps = 30)
                
        f = 'video.mp4'
        writervideo = FFMpegWriter(fps=30) 
        anim2.save(f, writer=writervideo)

        my_bar.progress(85)

        text = TA.text_area("", "Merging audio and video ...")

        from moviepy.editor import *

        dur = time.time()
        # loading video gfg
        clip = VideoFileClip("video.mp4")

        my_bar.progress(90)

        # loading audio file
        audioclip = AudioFileClip("audio.wav")
        
        # adding audio to the video clip
        videoclip = clip.set_audio(audioclip)

        videoclip.write_videofile("output.mp4")

        my_bar.progress(95)

        video_file = open('output.mp4', 'rb')
        video_bytes = video_file.read()

        my_bar.progress(100)

        text = TA.text_area("", "Animation Complete!")

        st.video(video_bytes)

        gc.collect()