app.py

import os
import cv2
import tempfile
import subprocess
import numpy as np
import imageio
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import (Conv3D, LSTM, Dense, Dropout,
                                     Bidirectional, MaxPool3D, Activation, Reshape)
import gradio as gr

# ── Vocabulary ────────────────────────────────────────────────────────────────
vocab = [x for x in "abcdefghijklmnopqrstuvwxyz'?!123456789 "]
char_to_num = tf.keras.layers.StringLookup(vocabulary=vocab, oov_token="")
num_to_char = tf.keras.layers.StringLookup(
    vocabulary=char_to_num.get_vocabulary(), oov_token="", invert=True
)

# ── Build & Load Model ────────────────────────────────────────────────────────
def build_model():
    m = Sequential()
    m.add(Conv3D(128, 3, input_shape=(75, 46, 140, 1), padding='same'))
    m.add(Activation('relu'))
    m.add(MaxPool3D((1, 2, 2)))
    m.add(Conv3D(256, 3, padding='same'))
    m.add(Activation('relu'))
    m.add(MaxPool3D((1, 2, 2)))
    m.add(Conv3D(75, 3, padding='same'))
    m.add(Activation('relu'))
    m.add(MaxPool3D((1, 2, 2)))
    m.add(Reshape((75, 5 * 17 * 75)))
    m.add(Bidirectional(LSTM(128, kernel_initializer='Orthogonal', return_sequences=True)))
    m.add(Dropout(0.5))
    m.add(Bidirectional(LSTM(128, kernel_initializer='Orthogonal', return_sequences=True)))
    m.add(Dropout(0.5))
    m.add(Dense(char_to_num.vocabulary_size() + 1,
                kernel_initializer='he_normal', activation='softmax'))
    return m

model = build_model()
model.load_weights('checkpoint.weights.h5')

# ── Video Processing ──────────────────────────────────────────────────────────
def load_video_frames(path: str):
    cap = cv2.VideoCapture(path)
    processed_frames = []
    for _ in range(int(cap.get(cv2.CAP_PROP_FRAME_COUNT))):
        ret, frame = cap.read()
        if not ret:
            break
        gray = tf.image.rgb_to_grayscale(tf.cast(frame, tf.float32))
        processed_frames.append(gray[190:236, 80:220, :])
    cap.release()

    target = 75
    if len(processed_frames) < target:
        pad = [tf.zeros_like(processed_frames[0])] * (target - len(processed_frames))
        processed_frames = processed_frames + pad
    else:
        processed_frames = processed_frames[:target]

    frames_tensor = tf.stack(processed_frames)
    mean = tf.math.reduce_mean(frames_tensor)
    std = tf.maximum(tf.math.reduce_std(tf.cast(frames_tensor, tf.float32)), 1e-8)
    return tf.cast((frames_tensor - mean), tf.float32) / std


def convert_to_mp4(input_path: str) -> str:
    out = tempfile.NamedTemporaryFile(suffix='.mp4', delete=False)
    out.close()
    try:
        subprocess.run(
            ['ffmpeg', '-y', '-i', input_path, '-vcodec', 'libx264', '-acodec', 'aac', out.name],
            check=True, capture_output=True
        )
        return out.name
    except Exception:
        return input_path


def make_mouth_gif(frames_tensor) -> str:
    frames_np = frames_tensor.numpy()
    gif_frames = []
    for f in frames_np:
        g = f[:, :, 0]
        g = g - g.min()
        rng = g.max()
        if rng > 0:
            g = g / rng
        rgb = np.stack([g, g, g], axis=-1)
        gif_frames.append((rgb * 255).astype(np.uint8))
    tmp = tempfile.NamedTemporaryFile(suffix='.gif', delete=False)
    tmp.close()
    imageio.mimsave(tmp.name, gif_frames, fps=10, loop=0)
    return tmp.name


# ── Inference ─────────────────────────────────────────────────────────────────
def predict(video_path: str):
    if video_path is None:
        return None, None, "Upload a video first.", "(no prediction)", "—"
    try:
        frames_tensor = load_video_frames(video_path)
        mp4_path = convert_to_mp4(video_path)
        gif_path = make_mouth_gif(frames_tensor)

        inp = tf.expand_dims(frames_tensor, axis=0)
        yhat = model.predict(inp, verbose=0)

        decoded_indices = tf.keras.backend.ctc_decode(
            yhat, input_length=[75], greedy=True
        )[0][0].numpy()

        tokens_str = str(decoded_indices[0].tolist())
        prediction = tf.strings.reduce_join(
            num_to_char(decoded_indices[0])
        ).numpy().decode('utf-8').strip() or "(no prediction)"

        confidence = float(np.mean(np.max(yhat[0], axis=-1)) * 100)

        return mp4_path, gif_path, tokens_str, prediction, f"{confidence:.1f}%"

    except Exception as e:
        err = f"Error: {str(e)}"
        return None, None, err, err, "—"


# ── CSS ───────────────────────────────────────────────────────────────────────
css = """
@import url('https://fonts.googleapis.com/css2?family=Syne:wght@400;600;700;800&family=Space+Mono:ital@0;1&display=swap');

body, .gradio-container { background: #07070f !important; font-family: 'Syne', sans-serif !important; color: #e2e2f0 !important; }

.hero { text-align: center; padding: 2.5rem 1rem 0.5rem; }
.hero h1 { font-size: 3.5rem; font-weight: 800; letter-spacing: -0.04em; background: linear-gradient(135deg, #f0f0ff 0%, #c084fc 40%, #818cf8 100%); -webkit-background-clip: text; -webkit-text-fill-color: transparent; margin: 0 0 0.3rem; line-height: 1; }
.hero .sub { font-family: 'Space Mono', monospace; font-size: 0.72rem; color: #4b5563; letter-spacing: 0.18em; text-transform: uppercase; }
.hero .badge { display: inline-block; margin-top: 0.7rem; padding: 0.25rem 0.75rem; border: 1px solid #2d2d4e; border-radius: 999px; font-family: 'Space Mono', monospace; font-size: 0.68rem; color: #7c7c9e; background: #0f0f1e; }

.section-label { font-family: 'Space Mono', monospace; font-size: 0.68rem; letter-spacing: 0.15em; text-transform: uppercase; color: #4b5563; margin-bottom: 0.4rem; padding-left: 2px; }

.divider { border: none; border-top: 1px solid #1a1a2e; margin: 1.2rem 0; }

.mono-out textarea { font-family: 'Space Mono', monospace !important; font-size: 0.82rem !important; background: #0a0a16 !important; color: #a5b4fc !important; border: 1px solid #1e1e38 !important; border-radius: 10px !important; }

.prediction-out textarea { font-family: 'Syne', sans-serif !important; font-size: 1.6rem !important; font-weight: 700 !important; background: #0a0a16 !important; color: #c084fc !important; border: 1px solid #2d1f4e !important; border-radius: 10px !important; text-align: center !important; }

.confidence-out textarea { font-family: 'Space Mono', monospace !important; font-size: 1.1rem !important; background: #0a0a16 !important; color: #34d399 !important; border: 1px solid #1a3330 !important; border-radius: 10px !important; text-align: center !important; }

button.lg { background: linear-gradient(135deg, #7c3aed 0%, #4f46e5 100%) !important; border: none !important; border-radius: 10px !important; font-family: 'Syne', sans-serif !important; font-weight: 700 !important; font-size: 1rem !important; letter-spacing: 0.06em !important; color: white !important; }

.info-panel { background: #0c0c1a; border: 1px solid #1a1a2e; border-radius: 12px; padding: 1rem 1.2rem; }
.info-panel p { font-family: 'Space Mono', monospace; font-size: 0.72rem; color: #374151; margin: 0; line-height: 2; }
.info-panel span { color: #6366f1; }
"""

# ── UI ────────────────────────────────────────────────────────────────────────
with gr.Blocks(css=css, title="LipNet — Silent Speech Recognition") as demo:

    gr.HTML("""
    <div class="hero">
        <h1>LipNet</h1>
        <p class="sub">Silent Speech Recognition · No Audio Required</p>
        <span class="badge">Conv3D → BiLSTM × 2 → CTC Decode · GRID Corpus S1</span>
    </div>
    <div style="height:1.5rem"></div>
    """)

    # ── Row 1: Upload + Preview ───────────────────────────────────────────────
    with gr.Row(equal_height=True):
        with gr.Column(scale=1):
            gr.HTML("<div class='section-label'>① Upload Video (.mpg / .mp4)</div>")
            video_input = gr.Video(label="", height=260, sources=["upload"])
            submit_btn = gr.Button("▶  READ LIPS", variant="primary", size="lg")

        with gr.Column(scale=1):
            gr.HTML("<div class='section-label'>② Converted Preview (mp4)</div>")
            video_preview = gr.Video(label="", height=260, interactive=False)

    gr.HTML("<hr class='divider'>")

    # ── Row 2: Mouth GIF + Tokens ─────────────────────────────────────────────
    with gr.Row(equal_height=True):
        with gr.Column(scale=1):
            gr.HTML("<div class='section-label'>③ What the Model Sees — mouth crop · grayscale · normalized</div>")
            gif_preview = gr.Image(label="", height=200, type="filepath")

        with gr.Column(scale=1):
            gr.HTML("<div class='section-label'>④ Raw CTC Token Indices</div>")
            tokens_out = gr.Textbox(
                label="", lines=5, interactive=False,
                placeholder="Token indices will appear here...",
                elem_classes=["mono-out"]
            )

    gr.HTML("<hr class='divider'>")

    # ── Row 3: Prediction + Confidence ───────────────────────────────────────
    with gr.Row():
        with gr.Column(scale=3):
            gr.HTML("<div class='section-label'>⑤ Predicted Text</div>")
            prediction_out = gr.Textbox(
                label="", lines=2, interactive=False,
                placeholder="Prediction will appear here...",
                elem_classes=["prediction-out"]
            )
        with gr.Column(scale=1):
            gr.HTML("<div class='section-label'>⑥ Avg Confidence</div>")
            confidence_out = gr.Textbox(
                label="", lines=2, interactive=False,
                placeholder="—", elem_classes=["confidence-out"]
            )

    gr.HTML("<hr class='divider'>")

    gr.HTML("""
    <div class="info-panel">
        <p>
            <span>ARCHITECTURE</span> · Conv3D(128) → Conv3D(256) → Conv3D(75) → Reshape → BiLSTM(128)×2 → Dense(41) → CTC<br>
            <span>INPUT</span> · 75 frames · mouth crop 46×140 px · grayscale · z-score normalized<br>
            <span>VOCAB</span> · 40 chars — a–z, 1–9, ' ? !  (space) · output dim = 41 (+ CTC blank token)<br>
            <span>DATASET</span> · GRID Corpus Speaker S1 · 500 videos · 450 train / 50 test<br>
            <span>NOTE</span> · Upload frontal-face .mpg or .mp4 videos for best results
        </p>
    </div>
    <div style="height:1.5rem"></div>
    """)

    submit_btn.click(
        fn=predict,
        inputs=[video_input],
        outputs=[video_preview, gif_preview, tokens_out, prediction_out, confidence_out]
    )

demo.launch()