import gradio as gr import os import base64 os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" from handle_gradio import ( process_video, process_video_stream, handle_file_upload, handle_remove_video, handle_select_example, open_record_modal, close_record_modal, save_record_direct, save_record_and_open_trim, open_preview_modal, close_preview_modal, save_preview_and_trim, open_keypoints_modal, close_keypoints_modal, ) # Check and download the 1GB pre-trained model weights if not cached print("Checking for large model file...") local_model_path = os.path.join("Uni_Sign", "unisign_model", "best_checkpoint-wlasl.pth") if os.path.exists(local_model_path): model_path = local_model_path else: from huggingface_hub import hf_hub_download model_path = hf_hub_download( repo_id="plice13/sign-language-weights", filename="best_checkpoint-wlasl.pth" ) print(f"File successfully loaded at: {model_path}") os.environ["UNISIGN_WEIGHTS"] = model_path # ==================== # Prepare base64-encoded logos for header badges from the 'logo' directory logo_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "logo") def _get_b64_logo(filename): p = os.path.join(logo_dir, filename) if os.path.exists(p): with open(p, "rb") as f: return f"data:image/png;base64,{base64.b64encode(f.read()).decode('utf-8')}" return "" logo_data = { "FAV_english_RGB": _get_b64_logo("FAV_english_RGB.png"), "fav_logo": _get_b64_logo("fav_logo.png"), "ZCU_logo_english_RGB": _get_b64_logo("ZCU_logo_english_RGB.png"), "zcu_logo": _get_b64_logo("zcu_logo.png"), } fav_logo_src = logo_data["FAV_english_RGB"] zcu_light_logo_src = logo_data["ZCU_logo_english_RGB"] zcu_dark_logo_src = logo_data["zcu_logo"] import json def load_examples_config(): examples_json_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "examples.json") if os.path.exists(examples_json_path): try: with open(examples_json_path, "r", encoding="utf-8") as f: data = json.load(f) if isinstance(data, list): return data except Exception: pass example_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "examples", "displayed") return [ {"title": "Book", "video_path": os.path.join(example_dir, "book.mp4"), "keypoints_path": os.path.join(example_dir, "book_keypoints.json"), "keypoints_video_path": os.path.join(example_dir, "book_keypoints.mp4")}, {"title": "Deaf", "video_path": os.path.join(example_dir, "deaf.mp4"), "keypoints_path": os.path.join(example_dir, "deaf_keypoints.json"), "keypoints_video_path": os.path.join(example_dir, "deaf_keypoints.mp4")}, {"title": "Help", "video_path": os.path.join(example_dir, "help.mp4"), "keypoints_path": os.path.join(example_dir, "help_keypoints.json"), "keypoints_video_path": os.path.join(example_dir, "help_keypoints.mp4")}, {"title": "Fine", "video_path": os.path.join(example_dir, "fine.mp4"), "keypoints_path": os.path.join(example_dir, "fine_keypoints.json"), "keypoints_video_path": os.path.join(example_dir, "fine_keypoints.mp4")}, {"title": "Purple", "video_path": os.path.join(example_dir, "purple.mp4"), "keypoints_path": os.path.join(example_dir, "purple_keypoints.json"), "keypoints_video_path": os.path.join(example_dir, "purple_keypoints.mp4")}, {"title": "No", "video_path": os.path.join(example_dir, "no.mp4"), "keypoints_path": os.path.join(example_dir, "no_keypoints.json"), "keypoints_video_path": os.path.join(example_dir, "no_keypoints.mp4")}, ] examples_list = load_examples_config() # Load CSS from external style.css file css_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "style.css") with open(css_path, "r", encoding="utf-8") as f: custom_css = f.read() # Information text hardcoded from DELETE/slt_demot.md INFO_MARKDOWN = """## Sign Language Translation Demo This interactive demo showcases **automatic sign language recognition from video to text**. The current version demonstrates the processing of **American Sign Language (ASL)** using modern computer vision and deep learning methods. ## How to use the demo 1. **Upload a video** or select one of the provided examples. 2. Click **Submit** to process the video. 3. The predicted text will be displayed as the output. For best results, the signer should be clearly visible, including the upper body, hands, and face. > **Note:** This is a research demonstrator. Predictions may not always be accurate, especially for videos that differ significantly from the data used during training. ## How does it work? The system consists of two main stages: **pose preprocessing** and **sign language translation**. ### Pose preprocessing The input video is first converted into a structured pose representation. Keypoints describing the signer's **body, hands, and face** are extracted and normalized before being passed to the translation model. Body keypoints are normalized globally, while hand and facial keypoints are normalized locally to preserve detailed information about their shape and movement. More information about the preprocessing pipeline is available in the [PoseEstimation repository](https://github.com/JSALT2024/PoseEstimation). ### Sign language translation The extracted pose sequence is processed using a model based on **Uni-Sign**, a unified framework for sign language understanding. More information about the model and our implementation is available in the [Uni-Sign repository](https://github.com/zeleznyt/Uni-Sign). ## Resources - [Demo source code](https://github.com/JSALT2024/slt-demo/tree/uni-sign-EP) - [Pose estimation and preprocessing](https://github.com/JSALT2024/PoseEstimation) - [Uni-Sign implementation](https://github.com/zeleznyt/Uni-Sign) - **Uni-Sign:** *Uni-Sign: Toward Unified Sign Language Understanding at Scale*, Li et al., ICLR 2025 ## Acknowledgements This demonstrator was developed at the **University of West Bohemia (ZČU), Department of Cybernetics, Computer Vision group**. Development was supported by the **2026 ZČU internal mini-project programme for the development and wider use of artificial intelligence**. The system builds upon the **Uni-Sign** framework. We thank its authors and the open-source sign language research community for making their work publicly available. """ with gr.Blocks(title="Sign Language Translation", css=custom_css, theme=gr.themes.Default(primary_hue="amber", neutral_hue="neutral")) as app: current_video = gr.State("") keypoints_video_state = gr.State("") with gr.Column(elem_id="main-layout"): gr.HTML(f"""

Sign Language to Text Translation

Upload an ASL video and get a text translation.

""", elem_classes=["app-header-html"]) # Card 1: Upload Video Box (Compact Dropzone & Info Row) with gr.Column(elem_classes=["ui-card"]): with gr.Row(): with gr.Column(scale=1, min_width=0): gr.Markdown("

Upload video

") with gr.Column(scale=0, min_width=160, elem_classes=["record-btn-col"]): record_yourself_btn = gr.Button( "📹 RECORD YOURSELF", variant="secondary", elem_classes=["record-yourself-btn"], visible=True, ) # Compact file dropzone upload_file = gr.File( label="Upload Video", file_types=["video"], file_count="single", show_label=False, elem_classes=["compact-dropzone"] ) # Active video status row: Video name on left, action buttons aligned to the right with gr.Row(visible=False, elem_classes=["video-info-row"]) as video_info_row: video_name_md = gr.Markdown("
video.mp4
", elem_classes=["video-name-col"]) preview_btn = gr.Button("🎬 Preview & Trim Video", variant="secondary", elem_classes=["preview-modal-btn"]) change_video_btn = gr.Button("✕ Remove", variant="secondary", elem_classes=["change-vid-btn"]) # Standalone Translate Action Button (hidden until a video is uploaded or selected) submit_btn = gr.Button("Translate", variant="primary", elem_classes=["translate-btn"], visible=False) # Card 2: Translation Result Box (hidden until Translate is clicked) with gr.Column(elem_classes=["ui-card", "translation-card"], elem_id="translation-card", visible=False) as translation_card: gr.Markdown("

Translation

") translation_display = gr.HTML(value="", elem_id="translation-display", elem_classes=["translation-display-html"]) show_keypoints_btn = gr.Button( "🎯 SHOW KEYPOINTS", variant="secondary", elem_classes=["show-keypoints-btn"], visible=False, ) # Card 3: Examples Box (loaded dynamically from examples.json) example_buttons = [] with gr.Column(elem_classes=["ui-card"]): gr.Markdown("

Examples

") for i in range(0, len(examples_list), 3): row_items = examples_list[i:i+3] with gr.Row(elem_classes=["examples-row"] if i == 0 else ["examples-row", "examples-row-second"]): for idx, ex in enumerate(row_items): item_num = i + idx + 1 v_path = ex.get("video_path", "") title = ex.get("title", f"Example {item_num}") with gr.Column(scale=1): gr.HTML(f"
{title}
") gr.Video(value=v_path, interactive=False, show_label=False, autoplay=False, height=160, mirror_webcam=False) btn = gr.Button(f"Use Example {item_num}", variant="secondary", elem_classes=["example-btn"]) example_buttons.append((btn, v_path)) # Card 4: Information Box with gr.Column(elem_classes=["ui-card"]): gr.Markdown("

Information

") gr.Markdown(INFO_MARKDOWN, elem_classes=["info-markdown"]) # Subtle bottom control row (Restart session) with gr.Row(elem_classes=["restart-row"]): restart_btn = gr.Button("🔄 Restart session", variant="secondary", elem_classes=["restart-btn"], size="sm") # Fullscreen Floating Modal Window for Webcam Recording with gr.Column(elem_classes=["modal-overlay"], scale=0, visible=False) as record_modal: with gr.Column(elem_classes=["modal-dialog-card"], scale=0): with gr.Row(elem_classes=["modal-header-row"]): with gr.Column(scale=1, min_width=0): gr.Markdown("

📹 Webcam Video Recording

") with gr.Column(scale=0, min_width=36, elem_classes=["modal-close-col"]): record_close_top = gr.Button("✕", size="sm", min_width=36, elem_classes=["modal-close-icon"]) record_video = gr.Video( interactive=True, show_label=False, sources=["webcam"], mirror_webcam=False, elem_classes=["modal-video-player"], ) with gr.Row(elem_id="record-footer-row", elem_classes=["modal-footer-row", "record-footer-row"]): record_direct_btn = gr.Button("✓ Use Directly", variant="secondary", elem_classes=["modal-use-direct-btn"]) record_trim_btn = gr.Button("✂️ Trim", variant="primary", elem_classes=["modal-done-btn"]) # Fullscreen Floating Modal Window for Video Preview & Trimming with gr.Column(elem_classes=["modal-overlay"], scale=0, visible=False) as preview_modal: with gr.Column(elem_classes=["modal-dialog-card"], scale=0): with gr.Row(elem_classes=["modal-header-row"]): with gr.Column(scale=1, min_width=0): gr.Markdown("

🎬 Video Preview & Trimming

") with gr.Column(scale=0, min_width=36, elem_classes=["modal-close-col"]): preview_close_top = gr.Button("✕", size="sm", min_width=36, elem_classes=["modal-close-icon"]) preview_video = gr.Video( interactive=True, show_label=False, mirror_webcam=False, elem_classes=["modal-video-player"], ) with gr.Row(elem_classes=["modal-footer-row"]): preview_close_bottom = gr.Button("✕ Cancel", variant="secondary", elem_classes=["modal-cancel-btn"]) preview_save_btn = gr.Button("✓ Save & Use Video", variant="primary", elem_classes=["modal-done-btn"]) # Fullscreen Floating Modal Window for Keypoints Visualization with gr.Column(elem_classes=["modal-overlay", "keypoints-modal-overlay"], scale=0, visible=False) as keypoints_modal: with gr.Column(elem_classes=["modal-dialog-card"], scale=0): with gr.Row(elem_classes=["modal-header-row"]): with gr.Column(scale=1, min_width=0): gr.Markdown("

🎯 Extracted Pose & Keypoints

") with gr.Column(scale=0, min_width=36, elem_classes=["modal-close-col"]): keypoints_close_top = gr.Button("✕", size="sm", min_width=36, elem_classes=["modal-close-icon"]) keypoints_modal_video = gr.Video( interactive=False, show_label=False, autoplay=True, loop=True, elem_classes=["modal-video-player", "keypoints-video-player"], ) with gr.Column(elem_classes=["modal-footer-col"]): keypoints_close_bottom = gr.Button("Close", variant="secondary", elem_classes=["modal-done-btn"]) def start_translating_ui(): spinner_html = """
Please wait, you are in queue...
""" return gr.update(visible=True), spinner_html, gr.update(visible=False), "" def finish_translating(video_path): yield from process_video_stream(video_path) # Upload file event upload_file.upload( fn=handle_file_upload, inputs=upload_file, outputs=[current_video, upload_file, record_yourself_btn, video_info_row, video_name_md, submit_btn, translation_card], ) # Remove / Change video event change_video_btn.click( fn=handle_remove_video, inputs=None, outputs=[current_video, upload_file, record_yourself_btn, video_info_row, video_name_md, submit_btn, translation_card], ) # Smooth scroll to top JavaScript helper for example selections scroll_top_js = "() => { window.scrollTo({ top: 0, behavior: 'smooth' }); }" # Examples click events with smooth auto-scroll to top for btn_obj, v_path_val in example_buttons: btn_obj.click( fn=lambda p=v_path_val: handle_select_example(p), inputs=None, outputs=[current_video, upload_file, record_yourself_btn, video_info_row, video_name_md, submit_btn, translation_card], js=scroll_top_js, ) # Restart / Reset Session event restart_btn.click( fn=handle_remove_video, inputs=None, outputs=[current_video, upload_file, record_yourself_btn, video_info_row, video_name_md, submit_btn, translation_card], js="() => { window.location.href = window.location.href.split('#')[0]; }" ) open_record_modal_js = "() => { window.scrollTo({ top: 0, behavior: 'smooth' }); const rf = document.getElementById('record-footer-row') || document.querySelector('.record-footer-row'); if (rf) { rf.classList.remove('has-recording'); rf.style.setProperty('display', 'none', 'important'); } }" close_record_modal_js = "() => { const rf = document.getElementById('record-footer-row') || document.querySelector('.record-footer-row'); if (rf) { rf.classList.remove('has-recording'); rf.style.setProperty('display', 'none', 'important'); } }" # --- Modal 1: Webcam Recording Events --- record_yourself_btn.click( fn=open_record_modal, inputs=None, outputs=[record_modal, record_video], js=open_record_modal_js, ) record_close_top.click( fn=close_record_modal, inputs=None, outputs=[record_modal, record_video], js=close_record_modal_js, ) record_direct_btn.click( fn=save_record_direct, inputs=record_video, outputs=[record_modal, record_video, current_video, upload_file, record_yourself_btn, video_info_row, video_name_md, submit_btn, translation_card], ) record_trim_btn.click( fn=save_record_and_open_trim, inputs=record_video, outputs=[record_modal, record_video, current_video, upload_file, record_yourself_btn, video_info_row, video_name_md, submit_btn, translation_card, preview_modal, preview_video], ) # --- Modal 2: Preview & Trimming Events --- preview_btn.click( fn=open_preview_modal, inputs=current_video, outputs=[preview_modal, preview_video], js=scroll_top_js, ) for btn in [preview_close_top, preview_close_bottom]: btn.click( fn=close_preview_modal, inputs=None, outputs=preview_modal, ) preview_save_btn.click( fn=save_preview_and_trim, inputs=[preview_video, current_video], outputs=[preview_modal, current_video, video_name_md], ) # Translate event submit_btn.click( fn=start_translating_ui, inputs=None, outputs=[translation_card, translation_display, show_keypoints_btn, keypoints_video_state], ).then( fn=finish_translating, inputs=current_video, outputs=[translation_display, show_keypoints_btn, keypoints_video_state], ) # Open Keypoints Modal show_keypoints_btn.click( fn=open_keypoints_modal, inputs=keypoints_video_state, outputs=[keypoints_modal, keypoints_modal_video], js=scroll_top_js, ) # Close Keypoints Modal keypoints_close_top.click( fn=close_keypoints_modal, inputs=None, outputs=[keypoints_modal, keypoints_modal_video], ) keypoints_close_bottom.click( fn=close_keypoints_modal, inputs=None, outputs=[keypoints_modal, keypoints_modal_video], ) if __name__ == "__main__": app.queue(default_concurrency_limit=1).launch()