Add 5 types of scene. Use [git lfs] to track glb and ply.

.gitattributes

@@ -35,3 +35,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 scenes/scene0132_00_vh_clean_2_aligned.glb filter=lfs diff=lfs merge=lfs -text
 scenes/scene0132_00_vh_clean_2_aligned.ply filter=lfs diff=lfs merge=lfs -text
+*.ply filter=lfs diff=lfs merge=lfs -text
+*.glb filter=lfs diff=lfs merge=lfs -text

2glb.py

@@ -0,0 +1,7 @@
+from display_model import ply_to_glb
+
+
+if __name__ == '__main__':
+    for name in ['scene0014_00', 'scene0024_00', 'scene0051_00', 'scene0114_00','scene0335_01']:
+        ply_to_glb(ply_file=f"scenes/{name}_vh_clean_2_aligned.ply", glb_file=f"scenes/{name}_vh_clean_2_aligned.glb")
+        print(f"{name} finish")

app.py

@@ -3,12 +3,26 @@ import gradio as gr
 
 from display_model import *
 
-scan_id = "scene0132_00"
-ply_file = os.path.join("scenes", f"{scan_id}_vh_clean_2_aligned.ply")
-glb_file = os.path.join("scenes", f"{scan_id}_vh_clean_2_aligned.glb")
-new_ply_file = os.path.join("scenes", f"{scan_id}_vh_clean_2_aligned_AddBox.ply")
-new_glb_file = os.path.join("scenes", f"{scan_id}_vh_clean_2_aligned_AddBox.glb")
-objects_info_file = os.path.join("objects_info", f"objects_info_{scan_id}.npy")
+default_llm = 'gpt-3.5-turbo-0125'
+# scan_id = "scene0132_00"
+scan_id_mapper={
+    'Living Room':'scene0024_00', 
+    'Bedroom':'scene0051_00',  #144
+    'Kitchen':'scene0335_01',  #164 197
+    'Bathroom':'scene0014_00',  #14 26
+    'Office':'scene0114_00'
+    # gym 428
+}
+defualt_scene_type = 'Living Room'
+scan_id = scan_id_mapper[defualt_scene_type]
+
+def get_path(scan_id):
+    ply_file = os.path.join("scenes", f"{scan_id}_vh_clean_2_aligned.ply")
+    glb_file = os.path.join("scenes", f"{scan_id}_vh_clean_2_aligned.glb")
+    new_ply_file = os.path.join("scenes", f"{scan_id}_vh_clean_2_aligned_AddBox.ply")
+    new_glb_file = os.path.join("scenes", f"{scan_id}_vh_clean_2_aligned_AddBox.glb")
+    objects_info_file = os.path.join("objects_info", f"objects_info_{scan_id}.npy")
+    return {"ply_file":ply_file, "glb_file":glb_file, "new_ply_file":new_ply_file, "new_glb_file":new_glb_file, "objects_info_file":objects_info_file}
 
 def insert_user_none_between_assistant(messages):
     # 初始化结果列表
@@ -89,9 +103,10 @@ def process_instruction_callback(inp_api_key, instruction, llm_name):
         gradio_messages.append(gradio_message)
 
     # return gradio_messages
-    return new_glb_file, gradio_messages
+    return get_path(scan_id)['new_glb_file'], gradio_messages
 
 def generate_answer_glb(answer_content):
+    ply_file, glb_file, new_ply_file, new_glb_file, objects_info_file = get_path(scan_id).values()
     from transcrib3d_main import extract_answer_id_from_last_line
     last_line = answer_content.splitlines()[-1] if len(answer_content) > 0 else ''
     answer_id, _ = extract_answer_id_from_last_line(last_line)
@@ -112,53 +127,75 @@ def llm_dropdown_callback(llm_name):
     print("llm_name in callback:",llm_name)
     return llm_name
 
+def scene_type_dropdown_callback(scene_type):
+    # update scan_id as a global vairable
+    global scan_id
+    scan_id = scan_id_mapper[scene_type]
+    print("scan_id inside scene_type_dropdown_callback:",scan_id)
+    return get_path(scan_id)['glb_file']
+
 with gr.Blocks() as demo:
     gr.Markdown("## Transcrib3D-Demo")
     with gr.Row():
         with gr.Column():
+            # Left-1: Scene Type Selection
+            scene_type_dropdown = gr.Dropdown(
+                choices=['Living Room', 'Bedroom', 'Kitchen', 'Bathroom', 'Office'],
+                label='Scene Type',
+                type='value',
+                value=defualt_scene_type
+            )
+            scan_id_text = gr.Text(visible=False)
+            
+            # Left-2: 3D Scene
             model3d = gr.Model3D(
-                value="scenes/scene0132_00_vh_clean_2_aligned.glb",
-                label="ScanNet-scene0132_00",
+                value=f"scenes/{scan_id}_vh_clean_2_aligned.glb",
+                # label=f"ScanNet {scan_id}", #how to update this label????
+                label="3D Scene from ScanNet Dataset",
                 camera_position=(90,120,8),
                 zoom_speed=0.25,
                 # height=635,
                 height=725
             )
-            # print("Type1:",type(model3d))
-
+            # Left-3: Tips
             # gr.Markdown("🖱️:arrow_up::arrow_down:: SCROLL to zoom in/out.\t🖱️🔁 DRAG to rotate.\tCTRL+🖱️🔁 Press CTRL and DRAG to pan.")
             html_content = """
                 <div style='text-align: center;'>
                     🖱️🔼🔽: SCROLL to zoom in/out.&nbsp;&nbsp;&nbsp;🖱️🔁: DRAG to rotate.&nbsp;&nbsp;&nbsp;[CTRL]+🖱️🔁: Press CTRL and DRAG to pan.
+                    <br>It may takes ~10 seconds to load the 3D scene.
                 </div>
                 """
             gr.HTML(value=html_content)
 
         with gr.Column():
-
-            inp_api_key = gr.Textbox(label='OpenAI API Key (this is not stored anywhere)', lines=1)
-
+            # Right-1: Openai Api Key Input
+            inp_api_key = gr.Textbox(
+                label='OpenAI API Key (this is not stored anywhere)',
+                placeholder='You can generate key at https://platform.openai.com/api-keys',
+                lines=1)
+            # Right-2: LLM Selection
             llm_dropdown = gr.Dropdown(
                 # choices=['gpt-4-turbo','gpt-4','gpt-3.5-turbo'],
-                choices=['gpt-4-0125-preview', 'gpt-4-1106-preview', 'gpt-3.5-turbo-0125'],
+                choices=['gpt-4', 'gpt-4-0125-preview', 'gpt-4-1106-preview', 'gpt-3.5-turbo-0125'],
                 label="LLM Selection",
-                type='value'
+                type='value',
+                value=default_llm
             )
-            # llm_name = "gpt-4-turbo"
-            llm_name_text = gr.Text(visible=False)
-            llm_dropdown.select(fn=llm_dropdown_callback, inputs=llm_dropdown, outputs=llm_name_text)
-
+            llm_name_text = gr.Text(value=default_llm, visible=False)
+            
 
+            # Right-3: User Instruction Input
             user_instruction_textbox = gr.Textbox(
                 label="Instruction",
                 placeholder="Describe an object in the scene with its attributes and its relation with other objects, e.g. 'The largest table in the scene.",
                 # scale=4
             )
+            # Right-4: Submit Button
             bt = gr.Button(
                 value="Submit",
                 # scale=1
             )
-            
+            # Right-5: Dialogue
             dialogue = gr.Chatbot(
                 height=470
                 # value = [["1","2"], [None, '3']]
@@ -171,6 +208,9 @@ with gr.Blocks() as demo:
     bt.click(fn=process_instruction_callback, inputs=[inp_api_key, user_instruction_textbox,llm_name_text], outputs=[model3d,dialogue])
     user_instruction_textbox.submit(fn=process_instruction_callback, inputs=[inp_api_key, user_instruction_textbox, llm_name_text], outputs=[model3d,dialogue])
 
+    scene_type_dropdown.select(fn=scene_type_dropdown_callback, inputs=scene_type_dropdown, outputs=model3d)
+    llm_dropdown.select(fn=llm_dropdown_callback, inputs=llm_dropdown, outputs=llm_name_text)
+
     # 直接用lambda函数定义一个映射
     # type(user_instruction_textbox.value)
     # user_instruction_textbox.

objects_info/objects_info_scene0014_00.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98e3a5c5e11ecaa11c3569b2c9c051bca997b0089d523fecbb9ee038a602176c
+size 19477

objects_info/objects_info_scene0024_00.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eac6b438ffa9c8b88a741410507563ccf0d6ac94650b66924ee649315b84628a
+size 38887

objects_info/objects_info_scene0051_00.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf6a9fa7b7ed75d0eb1189a88c742813c56ff1968097cbb656790704e3ef4988
+size 30011

objects_info/objects_info_scene0114_00.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e8b329206fa085dbdf66948dab3193196424cd53413ff719cd9771f767c6db6
+size 17584

objects_info/objects_info_scene0335_01.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4ed178d800636f943c9660e9bd1c14dd0cd58c9e750867886459a342ff05a0c
+size 36433

scenes/scene0014_00_vh_clean_2_aligned.glb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05e13ed5485a96bf0460dce00407c754a3b4c69d586e303261f6daa058438e56
+size 3779956

scenes/scene0014_00_vh_clean_2_aligned.ply

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56164fd1202ea66a3675d934e3ec6218fc7bc255a3659d822f6427315cd250d0
+size 3964386

scenes/scene0024_00_vh_clean_2_aligned.glb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0f3b3ea5ef0a4cc517d5d542b85700a66c73827cda7280d1f68db65a822ae6c
+size 10566164

scenes/scene0024_00_vh_clean_2_aligned.ply

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b062560cc2e4758115437f546bf4b240dcdd5526ea3cfd40ab8e84c85ee74b5
+size 11080892

scenes/scene0051_00_vh_clean_2_aligned.glb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:deb4e2ba8e2341361dbfe8374a09f60aa54505b76b91b88d2b32d6e09424df53
+size 8726472

scenes/scene0051_00_vh_clean_2_aligned.ply

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edf032cc01a92402c7709f254ab79b6b510adf1a659d9894bb06ecc167e2d4db
+size 9152338

scenes/scene0114_00_vh_clean_2_aligned.glb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc2edd57267729ce1fda22e7e9e26080c17629d5e7f9a51798d1fe1f6efaa634
+size 6832076

scenes/scene0114_00_vh_clean_2_aligned.ply

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9de606b26a5873e819f5f97685a3cc038ca55852c2f12226a9f537a49ef636c4
+size 7164602

scenes/scene0335_01_vh_clean_2_aligned.glb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edb7fce2b83a44eaa11edc311faa46131ef863561998c063207647d46a34fdaf
+size 7261404

scenes/scene0335_01_vh_clean_2_aligned.ply

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfe80b74708cbacfd499d282edc549ef6276192fc38e80469346c110f755dc7b
+size 7614742