Update backend

main.py

@@ -242,6 +242,261 @@ def load_dataset():
         return None
 
 
+def get_fallback_examples() -> List[Dict[str, Any]]:
+    """Return hardcoded examples for all label types"""
+    return [
+        {
+            "name": "Correct Example",
+            "description": "A properly formed tool call",
+            "expected_output": "Correct",
+            "data": {
+                "query": "What's the weather in New York?",
+                "enabled_tools": [
+                    {
+                        "name": "get_weather",
+                        "description": "Get current weather for a location",
+                        "parameters": {
+                            "type": "object",
+                            "properties": {
+                                "location": {"type": "string"},
+                                "units": {"type": "string", "enum": ["celsius", "fahrenheit"]}
+                            },
+                            "required": ["location"]
+                        }
+                    }
+                ],
+                "tool_calling": {
+                    "name": "get_weather",
+                    "arguments": {
+                        "location": "New York",
+                        "units": "fahrenheit"
+                    }
+                }
+            }
+        },
+        {
+            "name": "Incorrect Function Name Example",
+            "description": "Tool call uses incorrect function name",
+            "expected_output": "Incorrect_Function_Name",
+            "data": {
+                "query": "Calculate 25 * 4",
+                "enabled_tools": [
+                    {
+                        "name": "calculator",
+                        "description": "Perform calculations",
+                        "parameters": {
+                            "type": "object",
+                            "properties": {
+                                "expression": {"type": "string"}
+                            },
+                            "required": ["expression"]
+                        }
+                    }
+                ],
+                "tool_calling": {
+                    "name": "calculate",  # Wrong name!
+                    "arguments": {
+                        "expression": "25 * 4"
+                    }
+                }
+            }
+        },
+        {
+            "name": "Incorrect Argument Type Example",
+            "description": "Argument has wrong data type",
+            "expected_output": "Incorrect_Argument_Type",
+            "data": {
+                "query": "Set a reminder for 3pm",
+                "enabled_tools": [
+                    {
+                        "name": "set_reminder",
+                        "description": "Create a reminder",
+                        "parameters": {
+                            "type": "object",
+                            "properties": {
+                                "time": {"type": "string"},
+                                "message": {"type": "string"}
+                            },
+                            "required": ["time", "message"]
+                        }
+                    }
+                ],
+                "tool_calling": {
+                    "name": "set_reminder",
+                    "arguments": {
+                        "time": 1500,  # Should be string!
+                        "message": "Meeting"
+                    }
+                }
+            }
+        },
+        {
+            "name": "Incorrect Argument Name Example",
+            "description": "Argument name doesn't match tool parameters",
+            "expected_output": "Incorrect_Argument_Name",
+            "data": {
+                "query": "Send an email to john@example.com",
+                "enabled_tools": [
+                    {
+                        "name": "send_email",
+                        "description": "Send an email message",
+                        "parameters": {
+                            "type": "object",
+                            "properties": {
+                                "recipient": {"type": "string"},
+                                "subject": {"type": "string"},
+                                "body": {"type": "string"}
+                            },
+                            "required": ["recipient", "subject"]
+                        }
+                    }
+                ],
+                "tool_calling": {
+                    "name": "send_email",
+                    "arguments": {
+                        "to": "john@example.com",  # Wrong name! Should be "recipient"
+                        "subject": "Hello",
+                        "body": "Test message"
+                    }
+                }
+            }
+        },
+        {
+            "name": "Incorrect Argument Value Example",
+            "description": "Argument value doesn't match expected format",
+            "expected_output": "Incorrect_Argument_Value",
+            "data": {
+                "query": "Get weather in Celsius",
+                "enabled_tools": [
+                    {
+                        "name": "get_weather",
+                        "description": "Get current weather for a location",
+                        "parameters": {
+                            "type": "object",
+                            "properties": {
+                                "location": {"type": "string"},
+                                "units": {"type": "string", "enum": ["celsius", "fahrenheit", "kelvin"]}
+                            },
+                            "required": ["location"]
+                        }
+                    }
+                ],
+                "tool_calling": {
+                    "name": "get_weather",
+                    "arguments": {
+                        "location": "London",
+                        "units": "centigrade"  # Wrong value! Not in enum
+                    }
+                }
+            }
+        },
+        {
+            "name": "Wrong Tool Example",
+            "description": "Wrong tool selected for the task",
+            "expected_output": "Wrong_Tool",
+            "data": {
+                "query": "What's the weather in Paris?",
+                "enabled_tools": [
+                    {
+                        "name": "get_weather",
+                        "description": "Get current weather for a location",
+                        "parameters": {
+                            "type": "object",
+                            "properties": {
+                                "location": {"type": "string"}
+                            },
+                            "required": ["location"]
+                        }
+                    },
+                    {
+                        "name": "search_web",
+                        "description": "Search the web for information",
+                        "parameters": {
+                            "type": "object",
+                            "properties": {
+                                "query": {"type": "string"}
+                            },
+                            "required": ["query"]
+                        }
+                    }
+                ],
+                "tool_calling": {
+                    "name": "search_web",  # Wrong tool! Should use get_weather
+                    "arguments": {
+                        "query": "weather in Paris"
+                    }
+                }
+            }
+        },
+        {
+            "name": "Wrong Syntax Example",
+            "description": "Tool call syntax is malformed",
+            "expected_output": "Wrong_Syntax",
+            "data": {
+                "query": "Calculate 10 + 5",
+                "enabled_tools": [
+                    {
+                        "name": "calculator",
+                        "description": "Perform mathematical calculations",
+                        "parameters": {
+                            "type": "object",
+                            "properties": {
+                                "expression": {"type": "string"}
+                            },
+                            "required": ["expression"]
+                        }
+                    }
+                ],
+                "tool_calling": {
+                    "name": "calculator",
+                    "arguments": {
+                        "expression": ["10", "+", "5"]  # Wrong type! Should be string
+                    }
+                }
+            }
+        },
+        {
+            "name": "No Tool Available Example",
+            "description": "No matching tool exists for the request",
+            "expected_output": "No_Tool_Available",
+            "data": {
+                "query": "Translate 'Hello' to Spanish",
+                "enabled_tools": [
+                    {
+                        "name": "get_weather",
+                        "description": "Get current weather for a location",
+                        "parameters": {
+                            "type": "object",
+                            "properties": {
+                                "location": {"type": "string"}
+                            },
+                            "required": ["location"]
+                        }
+                    },
+                    {
+                        "name": "calculator",
+                        "description": "Perform mathematical calculations",
+                        "parameters": {
+                            "type": "object",
+                            "properties": {
+                                "expression": {"type": "string"}
+                            },
+                            "required": ["expression"]
+                        }
+                    }
+                ],
+                "tool_calling": {
+                    "name": "translate",  # Tool doesn't exist in enabled_tools!
+                    "arguments": {
+                        "text": "Hello",
+                        "target_language": "Spanish"
+                    }
+                }
+            }
+        }
+    ]
+
+
 def convert_dataset_example_to_api_format(item: Dict[str, Any]) -> Dict[str, Any]:
     """Convert dataset example to API format"""
     # Convert tools format
@@ -348,39 +603,9 @@ async def get_examples():
                     continue
         print(f"Generated {len(examples)} random examples from dataset")
     else:
-        # Fallback to hardcoded examples if dataset not available
-        print("Using hardcoded fallback examples")
-        examples = [
-            {
-                "name": "Correct Example",
-                "description": "A properly formed tool call",
-                "expected_output": "Correct",
-                "data": {
-                    "query": "What's the weather in New York?",
-                    "enabled_tools": [
-                        {
-                            "name": "get_weather",
-                            "description": "Get current weather for a location",
-                            "parameters": {
-                                "type": "object",
-                                "properties": {
-                                    "location": {"type": "string"},
-                                    "units": {"type": "string", "enum": ["celsius", "fahrenheit"]}
-                                },
-                                "required": ["location"]
-                            }
-                        }
-                    ],
-                    "tool_calling": {
-                        "name": "get_weather",
-                        "arguments": {
-                            "location": "New York",
-                            "units": "fahrenheit"
-                        }
-                    }
-                }
-            }
-        ]
+        # Fallback to hardcoded examples for ALL labels if dataset not available
+        print("Using hardcoded fallback examples for all labels")
+        examples = get_fallback_examples()
     
     # Shuffle examples to randomize order
     random.shuffle(examples)