app.py

import os

import gradio as gr

from utils import *


with gr.Blocks(title='LEO Demo') as demo:
    gr.HTML(value="<h1 align='center'>An Embodied Generalist Agent in 3D World</h1>")
    gr.HTML(value="<div align='center' style='margin-top:-1em; margin-bottom:-1em;'><img src='/file=assets/leo.svg' width='4%'></div>")
    # gr.HTML(value="<img src='/file=assets/teaser.png' alt='Teaser' width='760px' style='display: block; margin: auto;'>")
    gr.HTML(value="<p align='center' style='font-size: 1.2em; color: #485fc7;'><a href='https://arxiv.org/abs/2311.12871' target='_blank'>arXiv</a> | <a href='https://embodied-generalist.github.io/' target='_blank'>Project Page</a> | <a href='https://github.com/embodied-generalist/embodied-generalist' target='_blank'>Code</a></p>")
    gr.HTML(value="<p align='center' style='font-size: 1.15em;'><i>LEO: an embodied generalist agent capable of perceiving, grounding, reasoning, planning, and acting in 3D world.</i></p>")

    with gr.Row():
        with gr.Column(scale=5):
            dropdown_scene = gr.Dropdown(
                choices=MESH_NAMES,
                value='3RScan-Office',
                interactive=True,
                label='Select a 3D scene',
            )
            model_3d = gr.Model3D(
                value=os.path.join(MESH_DIR, f'3RScan-Office.glb'),
                clear_color=[0.0, 0.0, 0.0, 0.0],
                label='3D Scene',
                camera_position=(90, 30, 10),
                height=659,
            )
            gr.HTML(
                """<center><strong>
                👆 SCROLL and DRAG on the 3D Scene
                to zoom in/out and rotate. Press CTRL and DRAG to pan.
                </strong></center>
                """
            )
        with gr.Column(scale=5):
            dropdown_conversation_mode = gr.Dropdown(
                choices=['Single-round mode', 'Multi-round mode'],
                value='Single-round mode',
                interactive=True,
                label='Select conversation mode',
            )
            chatbot = gr.Chatbot(label='Chat with LEO')
            with gr.Row():
                with gr.Column(scale=8):
                    user_chat_input = gr.Textbox(
                        placeholder="Enter text here to chat with LEO",
                        show_label=False,
                        autofocus=True,
                    )
                with gr.Column(scale=2, min_width=0):
                    send_button = gr.Button('Send', variant='primary', scale=2)
            with gr.Row():
                upvote_button = gr.Button(value='👍 Upvote', interactive=False)
                downvote_button = gr.Button(value='👎 Downvote', interactive=False)
                flag_button = gr.Button(value='⚠️ Flag', interactive=False)
                clear_button = gr.Button(value='🗑️ Clear', interactive=False)
            with gr.Row():
                with gr.Accordion(label="Examples for user instruction:", open=True):
                    gr.Examples(
                        examples=[
                            ["What color is the floor?"],
                            ["Is there a blackboard in the room?"],
                            ["How many chairs are there in this room?"],
                            ["Describe this scene."],
                            ["What is this room used for?"],
                            ["Plan for the task: tidy up and arrange this room."],
                       ],
                        inputs=user_chat_input,
                    )

    # generation_config
    with gr.Accordion('Parameters', open=False):
        repetition_penalty = gr.Slider(
            minimum=0.0,
            maximum=10.0,
            value=3.0,
            step=1.0,
            interactive=True,
            label='Repetition penalty',
        )
        length_penalty = gr.Slider(
            minimum=0.0,
            maximum=10.0,
            value=1.0,
            step=1.0,
            interactive=True,
            label="Length penalty",
        )
    gr.Markdown("### Terms of Service")
    gr.HTML(
        """By using this service, users are required to agree to the following terms:
           the service is a research preview intended for non-commercial use only
           and may collect user dialogue data for future research."""
    )
    gr.Markdown("### Acknowledgment")
    gr.HTML(
        """Template adapted from <a href="https://llava.hliu.cc/">LLaVA</a> and
           <a href="http://sled-whistler.eecs.umich.edu:7777/">LLM-Grounder</a>."""
    )

    # Event handling
    button_list = [upvote_button, downvote_button, flag_button, clear_button]

    dropdown_scene.change(
        fn=change_scene,
        inputs=[dropdown_scene],
        outputs=[model_3d, chatbot],
        queue=False,
    )

    dropdown_conversation_mode.change(
        fn=clear_history,
        inputs=[],
        outputs=[chatbot, user_chat_input] + button_list,
        queue=False,
    )

    user_chat_input.submit(
        fn=receive_instruction,
        inputs=[chatbot, user_chat_input],
        outputs=[chatbot, user_chat_input, send_button] + button_list,
        queue=False,
    ).then(
        fn=generate_response,
        inputs=[
            chatbot,
            dropdown_scene,
            dropdown_conversation_mode,
            repetition_penalty,
            length_penalty,
        ],
        outputs=[chatbot, send_button] + button_list,
        scroll_to_output=True,
    )

    send_button.click(
        fn=receive_instruction,
        inputs=[chatbot, user_chat_input],
        outputs=[chatbot, user_chat_input, send_button] + button_list,
        queue=False,
    ).then(
        fn=generate_response,
        inputs=[
            chatbot,
            dropdown_scene,
            dropdown_conversation_mode,
            repetition_penalty,
            length_penalty,
        ],
        outputs=[chatbot, send_button] + button_list,
        scroll_to_output=True,
    )

    upvote_button.click(
        upvote_response,
        [chatbot, dropdown_scene, dropdown_conversation_mode],
        [user_chat_input, upvote_button, downvote_button, flag_button],
        queue=False,
    )
    downvote_button.click(
        downvote_response,
        [chatbot, dropdown_scene, dropdown_conversation_mode],
        [user_chat_input, upvote_button, downvote_button, flag_button],
        queue=False,
    )
    flag_button.click(
        flag_response,
        [chatbot, dropdown_scene, dropdown_conversation_mode],
        [user_chat_input, upvote_button, downvote_button, flag_button],
        queue=False,
    )
    clear_button.click(
        fn=clear_history,
        inputs=[],
        outputs=[chatbot, user_chat_input] + button_list,
        queue=False,
    )


demo.queue().launch(share=True, allowed_paths=['assets'])