agent-harness / tasks /validation /study4 /TASK_S4_R003_001.json
cuber12's picture
Publish agent harness research code and paper artifacts
d61821a verified
Raw
History Blame Contribute Delete
7.72 kB
{
"commit": "7ec3189d6eacdb55925e8be886a44d7ee09eb9ca",
"parent": "4e68d32c77ed587ab42d229d9f44c3bc40d1d0e5",
"repository_id": "R003",
"schema_version": 1,
"study": "Study 4 fresh-task retrieval replication",
"task_id": "TASK_S4_R003_001",
"valid_end_to_end": true,
"validation": {
"commands": [
"python -m pytest -q tests/unit"
],
"elapsed_seconds": 8.370393791003153,
"gold": [
{
"command": "python -m pytest -q tests/unit",
"elapsed_seconds": 2.6893840827979147,
"returncode": 0,
"stderr": "",
"stdout": "........................................................................ [ 6%]\n........................................................................ [ 13%]\n........................................................................ [ 20%]\n........................................................................ [ 27%]\n........................................................................ [ 34%]\n........................................................................ [ 41%]\n........................................................................ [ 48%]\n........................................................................ [ 55%]\n............s..s........................................................ [ 62%]\n........................................................................ [ 69%]\n...s.................................................................... [ 76%]\n........................................................................ [ 83%]\n........................................................................ [ 90%]\n........................................................................ [ 97%]\n............................. [100%]\n1034 passed, 3 skipped in 2.50s\n",
"timed_out": false
}
],
"hidden": [
{
"command": "python -m pytest -q tests/unit",
"elapsed_seconds": 2.6959901249501854,
"returncode": 1,
"stderr": "",
"stdout": "........................................................................ [ 6%]\n........................................................................ [ 13%]\n........................................................................ [ 20%]\n........................................................................ [ 27%]\n........................................................................ [ 34%]\n........................................................................ [ 41%]\n........................................................................ [ 48%]\n........................................................................ [ 55%]\n............s..s........................................................ [ 62%]\n........................................................................ [ 69%]\n...s.................................................................... [ 76%]\n.....................................................................F.. [ 83%]\n........................................................................ [ 90%]\n........................................................................ [ 97%]\n............................. [100%]\n=================================== FAILURES ===================================\n______________________________ test_extend_parser ______________________________\n\n def test_extend_parser():\n class ExceptionArgParser(argparse.ArgumentParser):\n def error(self, message):\n \"Raise error instead of exiting on invalid arguments, to make testing easier\"\n raise ValueError(message)\n \n class Fake:\n _id_attr = None\n \n class FakeManager(gitlab.base.RESTManager, CreateMixin, UpdateMixin):\n _obj_cls = Fake\n _create_attrs = RequiredOptional(\n required=(\"create\",),\n optional=(\"opt_create\",),\n exclusive=(\"create_a\", \"create_b\"),\n )\n _update_attrs = RequiredOptional(\n required=(\"update\",),\n optional=(\"opt_update\",),\n exclusive=(\"update_a\", \"update_b\"),\n )\n \n parser = ExceptionArgParser()\n with mock.patch.dict(\n \"gitlab.v4.objects.__dict__\", {\"FakeManager\": FakeManager}, clear=True\n ):\n v4_cli.extend_parser(parser)\n \n assert parser.parse_args([\"fake\", \"create\", \"--create\", \"1\"])\n assert parser.parse_args([\"fake\", \"create\", \"--create\", \"1\", \"--opt-create\", \"1\"])\n> assert parser.parse_args([\"fake\", \"create\", \"--create\", \"1\", \"--create-a\", \"1\"])\n\ntests/unit/test_cli.py:192: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n/opt/homebrew/Cellar/python@3.11/3.11.15_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/argparse.py:1877: in parse_args\n self.error(msg % ' '.join(argv))\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = ExceptionArgParser(prog='__main__.py', usage=None, description=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=True)\nmessage = 'unrecognized arguments: --create-a 1'\n\n def error(self, message):\n \"Raise error instead of exiting on invalid arguments, to make testing easier\"\n> raise ValueError(message)\nE ValueError: unrecognized arguments: --create-a 1\n\ntests/unit/test_cli.py:166: ValueError\n=========================== short test summary info ============================\nFAILED tests/unit/test_cli.py::test_extend_parser - ValueError: unrecognized ...\n1 failed, 1033 passed, 3 skipped in 2.52s\n",
"timed_out": false
}
],
"hidden_apply": {
"elapsed_seconds": 0.00995812495239079,
"returncode": 0,
"stderr": "",
"stdout": ""
},
"original": [
{
"command": "python -m pytest -q tests/unit",
"elapsed_seconds": 2.8971138750202954,
"returncode": 0,
"stderr": "",
"stdout": "........................................................................ [ 6%]\n........................................................................ [ 13%]\n........................................................................ [ 20%]\n........................................................................ [ 27%]\n........................................................................ [ 34%]\n........................................................................ [ 41%]\n........................................................................ [ 48%]\n........................................................................ [ 55%]\n............s..s........................................................ [ 62%]\n........................................................................ [ 69%]\n...s.................................................................... [ 76%]\n........................................................................ [ 83%]\n........................................................................ [ 90%]\n........................................................................ [ 97%]\n............................ [100%]\n1033 passed, 3 skipped in 2.70s\n",
"timed_out": false
}
],
"reason": null,
"source_apply": {
"elapsed_seconds": 0.010090666124597192,
"returncode": 0,
"stderr": "",
"stdout": ""
},
"valid": true
}
}