Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -22,7 +22,6 @@ mcp = FastMCP("Robot_MCP")
|
|
| 22 |
# -----------------------------------------------------
|
| 23 |
@mcp.tool()
|
| 24 |
def speak(text: str, emotion: str = "neutral"):
|
| 25 |
-
"""Robot speech output"""
|
| 26 |
return {
|
| 27 |
"status": "success",
|
| 28 |
"action_executed": "speak",
|
|
@@ -32,7 +31,6 @@ def speak(text: str, emotion: str = "neutral"):
|
|
| 32 |
|
| 33 |
@mcp.tool()
|
| 34 |
def navigate(direction: str, distance_meters: float):
|
| 35 |
-
"""Move robot safely"""
|
| 36 |
if distance_meters > 5.0:
|
| 37 |
return {"status": "error", "message": "Safety limit exceeded"}
|
| 38 |
return {
|
|
@@ -44,7 +42,6 @@ def navigate(direction: str, distance_meters: float):
|
|
| 44 |
|
| 45 |
@mcp.tool()
|
| 46 |
def scan_hazard(hazard_type: str, severity: str):
|
| 47 |
-
"""Hazard scan + log"""
|
| 48 |
timestamp = datetime.now().isoformat()
|
| 49 |
return {
|
| 50 |
"status": "warning_logged",
|
|
@@ -54,21 +51,18 @@ def scan_hazard(hazard_type: str, severity: str):
|
|
| 54 |
|
| 55 |
@mcp.tool()
|
| 56 |
def analyze_human(clothing_color: str, estimated_action: str):
|
| 57 |
-
"""Human detection description"""
|
| 58 |
return {
|
| 59 |
"status": "human_tracked",
|
| 60 |
"details": f"Human wearing {clothing_color} is {estimated_action}",
|
| 61 |
}
|
| 62 |
|
| 63 |
-
|
| 64 |
# -----------------------------------------------------
|
| 65 |
-
# Save
|
| 66 |
# -----------------------------------------------------
|
| 67 |
def save_and_upload_image(image_b64: str, hf_token: str):
|
| 68 |
try:
|
| 69 |
image_bytes = base64.b64decode(image_b64)
|
| 70 |
size_bytes = len(image_bytes)
|
| 71 |
-
print("[debug] decoded image bytes:", size_bytes)
|
| 72 |
|
| 73 |
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
|
| 74 |
local_path = f"/tmp/robot_img_{timestamp}.jpg"
|
|
@@ -76,8 +70,6 @@ def save_and_upload_image(image_b64: str, hf_token: str):
|
|
| 76 |
with open(local_path, "wb") as f:
|
| 77 |
f.write(image_bytes)
|
| 78 |
|
| 79 |
-
print("[debug] wrote local tmp file:", local_path)
|
| 80 |
-
|
| 81 |
filename = f"robot_{timestamp}.jpg"
|
| 82 |
|
| 83 |
upload_file(
|
|
@@ -88,8 +80,6 @@ def save_and_upload_image(image_b64: str, hf_token: str):
|
|
| 88 |
repo_type="dataset",
|
| 89 |
)
|
| 90 |
|
| 91 |
-
print("[debug] upload successful:", filename)
|
| 92 |
-
|
| 93 |
url = f"https://huggingface.co/datasets/{HF_DATASET_REPO}/resolve/main/{filename}"
|
| 94 |
return local_path, url, filename, size_bytes
|
| 95 |
|
|
@@ -97,9 +87,8 @@ def save_and_upload_image(image_b64: str, hf_token: str):
|
|
| 97 |
traceback.print_exc()
|
| 98 |
return None, None, None, 0
|
| 99 |
|
| 100 |
-
|
| 101 |
# -----------------------------------------------------
|
| 102 |
-
# JSON
|
| 103 |
# -----------------------------------------------------
|
| 104 |
def safe_parse_json_from_text(text: str):
|
| 105 |
if not text:
|
|
@@ -120,20 +109,25 @@ def safe_parse_json_from_text(text: str):
|
|
| 120 |
|
| 121 |
return None
|
| 122 |
|
| 123 |
-
|
| 124 |
# -----------------------------------------------------
|
| 125 |
-
#
|
| 126 |
# -----------------------------------------------------
|
| 127 |
def validate_and_call_tool(tool_name: str, tool_args: dict):
|
| 128 |
-
if tool_name not in mcp.tools:
|
|
|
|
|
|
|
| 129 |
return {"error": f"Unknown or unauthorized tool '{tool_name}'"}
|
|
|
|
| 130 |
try:
|
| 131 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 132 |
except Exception as e:
|
| 133 |
traceback.print_exc()
|
| 134 |
return {"error": f"Tool error: {str(e)}"}
|
| 135 |
|
| 136 |
-
|
| 137 |
# -----------------------------------------------------
|
| 138 |
# Main Pipeline
|
| 139 |
# -----------------------------------------------------
|
|
@@ -145,9 +139,6 @@ def process_and_describe(payload):
|
|
| 145 |
except:
|
| 146 |
return {"error": "Invalid JSON payload"}
|
| 147 |
|
| 148 |
-
print("\n========== NEW REQUEST ==========")
|
| 149 |
-
print("[debug] Incoming payload:", payload)
|
| 150 |
-
|
| 151 |
hf_token = payload.get("hf_token")
|
| 152 |
if not hf_token:
|
| 153 |
return {"error": "hf_token missing"}
|
|
@@ -165,14 +156,12 @@ def process_and_describe(payload):
|
|
| 165 |
if not hf_url:
|
| 166 |
return {"error": "Image upload failed"}
|
| 167 |
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
# VLM SYSTEM PROMPT
|
| 171 |
system_prompt = """
|
| 172 |
-
Respond in STRICT JSON ONLY
|
| 173 |
{
|
| 174 |
"description": "short visual description",
|
| 175 |
-
"tool_name": "
|
| 176 |
"arguments": { ... }
|
| 177 |
}
|
| 178 |
"""
|
|
@@ -191,8 +180,6 @@ Respond in STRICT JSON ONLY. Format:
|
|
| 191 |
},
|
| 192 |
]
|
| 193 |
|
| 194 |
-
# VLM CALL
|
| 195 |
-
print("[debug] Calling VLM model...")
|
| 196 |
client = InferenceClient(token=hf_token)
|
| 197 |
|
| 198 |
response = client.chat.completions.create(
|
|
@@ -204,10 +191,6 @@ Respond in STRICT JSON ONLY. Format:
|
|
| 204 |
|
| 205 |
vlm_output = response.choices[0].message.content.strip()
|
| 206 |
|
| 207 |
-
print("\n------ VLM RAW OUTPUT ------")
|
| 208 |
-
print(vlm_output)
|
| 209 |
-
print("------ END VLM RAW ------\n")
|
| 210 |
-
|
| 211 |
parsed = safe_parse_json_from_text(vlm_output)
|
| 212 |
|
| 213 |
if parsed is None:
|
|
@@ -236,7 +219,6 @@ Respond in STRICT JSON ONLY. Format:
|
|
| 236 |
"vlm_raw": vlm_output,
|
| 237 |
}
|
| 238 |
|
| 239 |
-
|
| 240 |
# ------------------------------
|
| 241 |
# Gradio Interface
|
| 242 |
# ------------------------------
|
|
@@ -254,4 +236,3 @@ iface = gr.Interface(
|
|
| 254 |
if __name__ == "__main__":
|
| 255 |
print("[Gradio] Launching interface...")
|
| 256 |
iface.launch(server_name="0.0.0.0", server_port=7860)
|
| 257 |
-
|
|
|
|
| 22 |
# -----------------------------------------------------
|
| 23 |
@mcp.tool()
|
| 24 |
def speak(text: str, emotion: str = "neutral"):
|
|
|
|
| 25 |
return {
|
| 26 |
"status": "success",
|
| 27 |
"action_executed": "speak",
|
|
|
|
| 31 |
|
| 32 |
@mcp.tool()
|
| 33 |
def navigate(direction: str, distance_meters: float):
|
|
|
|
| 34 |
if distance_meters > 5.0:
|
| 35 |
return {"status": "error", "message": "Safety limit exceeded"}
|
| 36 |
return {
|
|
|
|
| 42 |
|
| 43 |
@mcp.tool()
|
| 44 |
def scan_hazard(hazard_type: str, severity: str):
|
|
|
|
| 45 |
timestamp = datetime.now().isoformat()
|
| 46 |
return {
|
| 47 |
"status": "warning_logged",
|
|
|
|
| 51 |
|
| 52 |
@mcp.tool()
|
| 53 |
def analyze_human(clothing_color: str, estimated_action: str):
|
|
|
|
| 54 |
return {
|
| 55 |
"status": "human_tracked",
|
| 56 |
"details": f"Human wearing {clothing_color} is {estimated_action}",
|
| 57 |
}
|
| 58 |
|
|
|
|
| 59 |
# -----------------------------------------------------
|
| 60 |
+
# Save + Upload
|
| 61 |
# -----------------------------------------------------
|
| 62 |
def save_and_upload_image(image_b64: str, hf_token: str):
|
| 63 |
try:
|
| 64 |
image_bytes = base64.b64decode(image_b64)
|
| 65 |
size_bytes = len(image_bytes)
|
|
|
|
| 66 |
|
| 67 |
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
|
| 68 |
local_path = f"/tmp/robot_img_{timestamp}.jpg"
|
|
|
|
| 70 |
with open(local_path, "wb") as f:
|
| 71 |
f.write(image_bytes)
|
| 72 |
|
|
|
|
|
|
|
| 73 |
filename = f"robot_{timestamp}.jpg"
|
| 74 |
|
| 75 |
upload_file(
|
|
|
|
| 80 |
repo_type="dataset",
|
| 81 |
)
|
| 82 |
|
|
|
|
|
|
|
| 83 |
url = f"https://huggingface.co/datasets/{HF_DATASET_REPO}/resolve/main/{filename}"
|
| 84 |
return local_path, url, filename, size_bytes
|
| 85 |
|
|
|
|
| 87 |
traceback.print_exc()
|
| 88 |
return None, None, None, 0
|
| 89 |
|
|
|
|
| 90 |
# -----------------------------------------------------
|
| 91 |
+
# JSON Parse
|
| 92 |
# -----------------------------------------------------
|
| 93 |
def safe_parse_json_from_text(text: str):
|
| 94 |
if not text:
|
|
|
|
| 109 |
|
| 110 |
return None
|
| 111 |
|
|
|
|
| 112 |
# -----------------------------------------------------
|
| 113 |
+
# FIXED: correct MCP tool registry access (v2)
|
| 114 |
# -----------------------------------------------------
|
| 115 |
def validate_and_call_tool(tool_name: str, tool_args: dict):
|
| 116 |
+
# ❌ old: if tool_name not in mcp.tools:
|
| 117 |
+
# ✔ new:
|
| 118 |
+
if tool_name not in mcp._tools:
|
| 119 |
return {"error": f"Unknown or unauthorized tool '{tool_name}'"}
|
| 120 |
+
|
| 121 |
try:
|
| 122 |
+
# ❌ old: mcp.tools[name](...)
|
| 123 |
+
# ✔ new:
|
| 124 |
+
tool_fn = mcp._tools[tool_name]["function"]
|
| 125 |
+
return tool_fn(**tool_args)
|
| 126 |
+
|
| 127 |
except Exception as e:
|
| 128 |
traceback.print_exc()
|
| 129 |
return {"error": f"Tool error: {str(e)}"}
|
| 130 |
|
|
|
|
| 131 |
# -----------------------------------------------------
|
| 132 |
# Main Pipeline
|
| 133 |
# -----------------------------------------------------
|
|
|
|
| 139 |
except:
|
| 140 |
return {"error": "Invalid JSON payload"}
|
| 141 |
|
|
|
|
|
|
|
|
|
|
| 142 |
hf_token = payload.get("hf_token")
|
| 143 |
if not hf_token:
|
| 144 |
return {"error": "hf_token missing"}
|
|
|
|
| 156 |
if not hf_url:
|
| 157 |
return {"error": "Image upload failed"}
|
| 158 |
|
| 159 |
+
# VLM system prompt
|
|
|
|
|
|
|
| 160 |
system_prompt = """
|
| 161 |
+
Respond in STRICT JSON ONLY:
|
| 162 |
{
|
| 163 |
"description": "short visual description",
|
| 164 |
+
"tool_name": "speak | navigate | scan_hazard | analyze_human",
|
| 165 |
"arguments": { ... }
|
| 166 |
}
|
| 167 |
"""
|
|
|
|
| 180 |
},
|
| 181 |
]
|
| 182 |
|
|
|
|
|
|
|
| 183 |
client = InferenceClient(token=hf_token)
|
| 184 |
|
| 185 |
response = client.chat.completions.create(
|
|
|
|
| 191 |
|
| 192 |
vlm_output = response.choices[0].message.content.strip()
|
| 193 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 194 |
parsed = safe_parse_json_from_text(vlm_output)
|
| 195 |
|
| 196 |
if parsed is None:
|
|
|
|
| 219 |
"vlm_raw": vlm_output,
|
| 220 |
}
|
| 221 |
|
|
|
|
| 222 |
# ------------------------------
|
| 223 |
# Gradio Interface
|
| 224 |
# ------------------------------
|
|
|
|
| 236 |
if __name__ == "__main__":
|
| 237 |
print("[Gradio] Launching interface...")
|
| 238 |
iface.launch(server_name="0.0.0.0", server_port=7860)
|
|
|