generate comprehensive response

app.py

@@ -239,14 +239,23 @@ async def query_3D_object(
     #     raise gr.Error("Require at least 4 3D files to query by features")
     if method == Query3DObjectMethod.HYBRID_SEARCH:
         logger.info("Running query_3D_object_by_hybrid_search_method")
-        return await query_3D_object_by_hybrid_search_method(
+        result = await query_3D_object_by_hybrid_search_method(
             query, current_obj_path, embedding_dict, top_k
         )
+        response = result.get("response", "")
+        pairs = result.get("pairs", [])
     elif method == Query3DObjectMethod.SEMANTIC_SEARCH:
         logger.info("Running query_3D_object_by_semantic_search_method")
-        return query_3D_object_by_semantic_search_method(
+        pairs = query_3D_object_by_semantic_search_method(
             query, current_obj_path, embedding_dict, top_k
         )
+        response = f"Here are the top-{top_k} results for your query: `{query}`"
+    else:
+        raise Exception(
+            f"Unsupported query method: {method}. Supported methods are: {list(Query3DObjectMethod)}"
+        )
+    assert len(pairs) == 2 * top_k
+    return [response] + pairs
 
 
 def query_3D_object_by_semantic_search_method(
@@ -267,16 +276,21 @@ def query_3D_object_by_semantic_search_method(
     scores = list(zip(filepaths, similarities))
     scores.sort(key=lambda x: x[1], reverse=True)
 
-    if len(scores) < top_k:
-        scores.extend([("", 0.0)] * (top_k - len(scores)))
+    # if len(scores) < top_k:
+    #     scores.extend([("", 0.0)] * (top_k - len(scores)))
 
     top_files = [x[0] for x in scores[:top_k]]
-    return top_files + [os.path.basename(x) for x in top_files]
+    result = top_files + [os.path.basename(x) for x in top_files]
+    # Fill None if lack of results
+    if len(result) < 2 * top_k:
+        n = len(result) // 2
+        result = result[:n] + [None] * (top_k - n) + result[n:] + [""] * (top_k - n)
+    return result
 
 
 async def query_3D_object_by_hybrid_search_method(
     query: str, current_obj_path: str, embedding_dict: dict, top_k: int = 4
-) -> List:
+) -> Dict:
     # Keyword Search Agent
     @function_tool
     def query_3D_object_by_keyword_search(query: str, match_code: str, top_k: int = 4):
@@ -426,11 +440,19 @@ Combine the `match` function with `query_3D_object_by_keyword_search` to filter
         )
         return response
 
-    DATUM_AGENT_INSTRUCTIONS = """You are the Datum Agent: you retrieve the top-K most relevant 3D objects using three strategies.
+    @function_tool
+    def get_description_of_model_to_analysis(current_obj_path: str | None) -> str:
+        if current_obj_path is None:
+            raise gr.Error("Please select a file!")
+        return embedding_dict[current_obj_path]["description"]
+
+    DATUM_AGENT_INSTRUCTIONS = """You are the Datum Agent: you retrieve the top-K most relevant 3D objects using three strategies:
 * Use `query_3D_object_by_semantic_search` for abstract or descriptive queries.
 * Use `search_3D_similarity_factory` when the query mentions the object currently displayed on the screen and aims to find similar objects.
 * Use **Keyword Search Agent** for precise metadata constraints or comparative/filtering information in the query.
-Return only the final tuple of file paths and display names.
+Return only the final tuple of file paths and display names. If the response contains private paths which duplicated name, please ignore them!
+Moreover, you can able to generate a comprehensive response when our users ask for a description of the current 3D object. In these cases, you are required to:
+* Use `get_description_of_model_to_analysis` to retrieve the description of the current 3D object for analysis when receiving a user's query related to analysis or a description of the current view object.
 """
 
     HANDOFF_DESCRIPTION = """Handing off to Datum Agent: you can perform semantic search, keyword-based filtering, or visual similarity search.
@@ -444,6 +466,7 @@ If metadata filtering is required, delegate to the **Keyword Search Agent** by c
         tools=[
             query_3D_object_by_semantic_search,
             search_3D_similarity_factory,
+            get_description_of_model_to_analysis,
         ],
         handoffs=[keyword_search_agent],
     )
@@ -491,7 +514,7 @@ You need to find the most relevant 3D objects based on the query and return the
     if len(result) < 2 * top_k:
         n = len(result) // 2
         result = result[:n] + [None] * (top_k - n) + result[n:] + [""] * (top_k - n)
-    return result
+    return {"pairs": result, "response": response.final_output}
 
 
 ####################################################################################################################
@@ -877,6 +900,7 @@ with gr.Blocks() as demo:
         with gr.Column():
             query_input = gr.Textbox(placeholder="Which 3D CAD contains 2 holes?")
             query_button = gr.Button("Query Search")
+            response_box = gr.Textbox(placeholder="Thinking...", label="Response")
 
             with gr.Row():
                 with gr.Row():
@@ -931,6 +955,7 @@ with gr.Blocks() as demo:
         query_3D_object,
         [query_input, model_render, embedding_store],
         [
+            response_box,
             model_q_1,
             model_q_2,
             model_q_3,