| import os
|
| import tempfile
|
|
|
| from evaluation.integration_tests.tests.base import BaseIntegrationTest, TestResult
|
| from evaluation.utils.shared import assert_and_raise
|
| from openhands.events.action import AgentFinishAction, CmdRunAction, MessageAction
|
| from openhands.events.event import Event
|
| from openhands.events.observation import AgentDelegateObservation
|
| from openhands.runtime.base import Runtime
|
|
|
| HTML_FILE = """
|
| <!DOCTYPE html>
|
| <html lang="en">
|
| <head>
|
| <meta charset="UTF-8">
|
| <meta name="viewport" content="width=device-width, initial-scale=1.0">
|
| <title>The Ultimate Answer</title>
|
| <style>
|
| body {
|
| display: flex;
|
| justify-content: center;
|
| align-items: center;
|
| height: 100vh;
|
| margin: 0;
|
| background: linear-gradient(to right, #1e3c72, #2a5298);
|
| color: #fff;
|
| font-family: 'Arial', sans-serif;
|
| text-align: center;
|
| }
|
| .container {
|
| text-align: center;
|
| padding: 20px;
|
| background: rgba(255, 255, 255, 0.1);
|
| border-radius: 10px;
|
| box-shadow: 0 0 10px rgba(0, 0, 0, 0.2);
|
| }
|
| h1 {
|
| font-size: 36px;
|
| margin-bottom: 20px;
|
| }
|
| p {
|
| font-size: 18px;
|
| margin-bottom: 30px;
|
| }
|
| #showButton {
|
| padding: 10px 20px;
|
| font-size: 16px;
|
| color: #1e3c72;
|
| background: #fff;
|
| border: none;
|
| border-radius: 5px;
|
| cursor: pointer;
|
| transition: background 0.3s ease;
|
| }
|
| #showButton:hover {
|
| background: #f0f0f0;
|
| }
|
| #result {
|
| margin-top: 20px;
|
| font-size: 24px;
|
| }
|
| </style>
|
| </head>
|
| <body>
|
| <div class="container">
|
| <h1>The Ultimate Answer</h1>
|
| <p>Click the button to reveal the answer to life, the universe, and everything.</p>
|
| <button id="showButton">Click me</button>
|
| <div id="result"></div>
|
| </div>
|
| <script>
|
| document.getElementById('showButton').addEventListener('click', function() {
|
| document.getElementById('result').innerText = 'The answer is OpenHands is all you need!';
|
| });
|
| </script>
|
| </body>
|
| </html>
|
| """
|
|
|
|
|
| class Test(BaseIntegrationTest):
|
| INSTRUCTION = 'Browse localhost:8000, and tell me the ultimate answer to life.'
|
|
|
| @classmethod
|
| def initialize_runtime(cls, runtime: Runtime) -> None:
|
| action = CmdRunAction(command='mkdir -p /workspace')
|
| obs = runtime.run_action(action)
|
| assert_and_raise(obs.exit_code == 0, f'Failed to run command: {obs.content}')
|
|
|
| action = CmdRunAction(command='mkdir -p /tmp/server')
|
| obs = runtime.run_action(action)
|
| assert_and_raise(obs.exit_code == 0, f'Failed to run command: {obs.content}')
|
|
|
|
|
| with tempfile.TemporaryDirectory() as temp_dir:
|
| temp_file_path = os.path.join(temp_dir, 'index.html')
|
| with open(temp_file_path, 'w') as f:
|
| f.write(HTML_FILE)
|
|
|
| runtime.copy_to(temp_file_path, '/tmp/server')
|
|
|
|
|
| action = CmdRunAction(
|
| command='cd /tmp/server && nohup python3 -m http.server 8000 &'
|
| )
|
| obs = runtime.run_action(action)
|
|
|
| @classmethod
|
| def verify_result(cls, runtime: Runtime, histories: list[Event]) -> TestResult:
|
| from openhands.core.logger import openhands_logger as logger
|
|
|
|
|
| message_actions = [
|
| event
|
| for event in histories
|
| if isinstance(
|
| event, (MessageAction, AgentFinishAction, AgentDelegateObservation)
|
| )
|
| ]
|
| logger.debug(f'Total message-like events: {len(message_actions)}')
|
|
|
| for event in message_actions:
|
| try:
|
| if isinstance(event, AgentDelegateObservation):
|
| content = event.content
|
| elif isinstance(event, AgentFinishAction):
|
| content = event.outputs.get('content', '')
|
| elif isinstance(event, MessageAction):
|
| content = event.content
|
| else:
|
| logger.warning(f'Unexpected event type: {type(event)}')
|
| continue
|
|
|
| if 'OpenHands is all you need!' in content:
|
| return TestResult(success=True)
|
| except Exception as e:
|
| logger.error(f'Error processing event: {e}')
|
|
|
| logger.debug(
|
| f'Total messages: {len(message_actions)}. Messages: {message_actions}'
|
| )
|
| return TestResult(
|
| success=False,
|
| reason=f'The answer is not found in any message. Total messages: {len(message_actions)}.',
|
| )
|
|
|
