File size: 3,559 Bytes
f69d9ee |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 |
from __future__ import annotations
# Import all of the dependencies
from pathlib import Path
import subprocess
import tempfile
import imageio
import streamlit as st
import tensorflow as tf
from modelutil import load_model
from utils import load_data, num_to_char
# Set the layout to the streamlit app as wide
st.set_page_config(layout='wide')
# Setup the sidebar
with st.sidebar:
st.image('https://www.onepointltd.com/wp-content/uploads/2020/03/inno2.png')
st.title('NeuroSync Lipscape')
st.info('This application is originally developed from the Lip-Reader deep learning model.')
st.title('NeuroSync Lipscape Full Stack App')
BASE_DIR = Path(__file__).resolve().parent
DATA_DIR = BASE_DIR / 'data' / 's1'
# Generating a list of options or videos
options = sorted([item.name for item in DATA_DIR.glob('*.mpg')])
selected_video = st.selectbox('Choose video', options)
# Generate two columns
col1, col2 = st.columns(2)
if options:
# Rendering the video
with col1:
st.info('The video below displays the converted video in mp4 format')
file_path = DATA_DIR / selected_video
if not file_path.exists():
st.error(f"Video not found: {file_path}")
else:
output_path = None
try:
with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as output_file:
output_path = Path(output_file.name)
subprocess.run(
[
"ffmpeg",
"-i",
str(file_path),
"-vcodec",
"libx264",
str(output_path),
"-y",
],
check=True,
capture_output=True,
text=True,
)
with output_path.open('rb') as video:
video_bytes = video.read()
st.video(video_bytes)
except subprocess.CalledProcessError as exc:
st.error("ffmpeg failed to convert the selected video.")
st.code(exc.stderr or "No ffmpeg error output captured.")
finally:
if output_path and output_path.exists():
output_path.unlink()
with col2:
if file_path.exists():
st.info('This is all the machine learning model sees when making a prediction')
video, _annotations = load_data(tf.convert_to_tensor(str(file_path)))
gif_path = None
try:
with tempfile.NamedTemporaryFile(suffix=".gif", delete=False) as gif_file:
gif_path = Path(gif_file.name)
imageio.mimsave(str(gif_path), video, fps=10)
st.image(str(gif_path), width=400)
finally:
if gif_path and gif_path.exists():
gif_path.unlink()
st.info('This is the output of the machine learning model as tokens')
model = load_model()
yhat = model.predict(tf.expand_dims(video, axis=0))
decoder = tf.keras.backend.ctc_decode(yhat, [75], greedy=True)[0][0].numpy()
st.text(decoder)
# Convert prediction to text
st.info('Decode the raw tokens into words')
converted_prediction = tf.strings.reduce_join(num_to_char(decoder)).numpy().decode('utf-8')
st.text(converted_prediction)
else:
st.warning(f"No videos were found in {DATA_DIR}.")
|