from __future__ import annotations # Import all of the dependencies from pathlib import Path import subprocess import tempfile import imageio import streamlit as st import tensorflow as tf from modelutil import load_model from utils import load_data, num_to_char # Set the layout to the streamlit app as wide st.set_page_config(layout='wide') # Setup the sidebar with st.sidebar: st.image('https://www.onepointltd.com/wp-content/uploads/2020/03/inno2.png') st.title('NeuroSync Lipscape') st.info('This application is originally developed from the Lip-Reader deep learning model.') st.title('NeuroSync Lipscape Full Stack App') BASE_DIR = Path(__file__).resolve().parent DATA_DIR = BASE_DIR / 'data' / 's1' # Generating a list of options or videos options = sorted([item.name for item in DATA_DIR.glob('*.mpg')]) selected_video = st.selectbox('Choose video', options) # Generate two columns col1, col2 = st.columns(2) if options: # Rendering the video with col1: st.info('The video below displays the converted video in mp4 format') file_path = DATA_DIR / selected_video if not file_path.exists(): st.error(f"Video not found: {file_path}") else: output_path = None try: with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as output_file: output_path = Path(output_file.name) subprocess.run( [ "ffmpeg", "-i", str(file_path), "-vcodec", "libx264", str(output_path), "-y", ], check=True, capture_output=True, text=True, ) with output_path.open('rb') as video: video_bytes = video.read() st.video(video_bytes) except subprocess.CalledProcessError as exc: st.error("ffmpeg failed to convert the selected video.") st.code(exc.stderr or "No ffmpeg error output captured.") finally: if output_path and output_path.exists(): output_path.unlink() with col2: if file_path.exists(): st.info('This is all the machine learning model sees when making a prediction') video, _annotations = load_data(tf.convert_to_tensor(str(file_path))) gif_path = None try: with tempfile.NamedTemporaryFile(suffix=".gif", delete=False) as gif_file: gif_path = Path(gif_file.name) imageio.mimsave(str(gif_path), video, fps=10) st.image(str(gif_path), width=400) finally: if gif_path and gif_path.exists(): gif_path.unlink() st.info('This is the output of the machine learning model as tokens') model = load_model() yhat = model.predict(tf.expand_dims(video, axis=0)) decoder = tf.keras.backend.ctc_decode(yhat, [75], greedy=True)[0][0].numpy() st.text(decoder) # Convert prediction to text st.info('Decode the raw tokens into words') converted_prediction = tf.strings.reduce_join(num_to_char(decoder)).numpy().decode('utf-8') st.text(converted_prediction) else: st.warning(f"No videos were found in {DATA_DIR}.")