agent-harness / tasks /validation /study5 /TASK_S5_R002_001.json
cuber12's picture
Publish agent harness research code and paper artifacts
d61821a verified
Raw
History Blame Contribute Delete
12.7 kB
{
"commit": "45c9976d3d33deb99f33410f8c4c919971b4c9cf",
"parent": "876a8143fc7c9fe2f77ea29e2522396d42ed335f",
"repository_id": "R002",
"schema_version": 1,
"study": "Study 5 fresh validation",
"task_id": "TASK_S5_R002_001",
"valid_end_to_end": true,
"validation": {
"commands": [
"go test ./internal/commands -count=1",
"go test ./internal/commands/help -count=1",
"go test ./internal/commands/update -count=1",
"go test ./internal/git -count=1"
],
"elapsed_seconds": 31.473043791018426,
"gold": [
{
"command": "go test ./internal/commands -count=1",
"elapsed_seconds": 5.085145458113402,
"returncode": 0,
"stderr": "",
"stdout": "ok \tgitlab.com/gitlab-org/cli/internal/commands\t1.979s\n",
"timed_out": false
},
{
"command": "go test ./internal/commands/help -count=1",
"elapsed_seconds": 0.7682270838413388,
"returncode": 0,
"stderr": "",
"stdout": "ok \tgitlab.com/gitlab-org/cli/internal/commands/help\t0.332s\n",
"timed_out": false
},
{
"command": "go test ./internal/commands/update -count=1",
"elapsed_seconds": 0.8589217499829829,
"returncode": 0,
"stderr": "",
"stdout": "ok \tgitlab.com/gitlab-org/cli/internal/commands/update\t0.373s\n",
"timed_out": false
},
{
"command": "go test ./internal/git -count=1",
"elapsed_seconds": 4.966712167020887,
"returncode": 0,
"stderr": "",
"stdout": "ok \tgitlab.com/gitlab-org/cli/internal/git\t4.705s\n",
"timed_out": false
}
],
"hidden": [
{
"command": "go test ./internal/commands -count=1",
"elapsed_seconds": 1.7197594998870045,
"returncode": 1,
"stderr": "",
"stdout": "RouteToStderr/glab_mr_list_--mine\n \tMessages: \tdeprecation warning leaked to stdout for glab mr list --mine\n --- FAIL: TestAllDeprecatedFlagsRouteToStderr/glab_mr_list_--opened (0.00s)\n root_test.go:140: \n \tError Trace:\t/private/tmp/study2-r002-5qrg52_m/tree/internal/commands/root_test.go:140\n \tError: \t\"Flag --mine has been deprecated, use --assignee=@me.\\nFlag --opened has been deprecated, default value if neither --closed, --locked or --merged is used.\\n\" should not contain \"deprecated\"\n \tTest: \tTestAllDeprecatedFlagsRouteToStderr/glab_mr_list_--opened\n \tMessages: \tdeprecation warning leaked to stdout for glab mr list --opened\n --- FAIL: TestAllDeprecatedFlagsRouteToStderr/glab_mr_merge_--when-pipeline-succeeds (0.00s)\n root_test.go:140: \n \tError Trace:\t/private/tmp/study2-r002-5qrg52_m/tree/internal/commands/root_test.go:140\n \tError: \t\"Flag --when-pipeline-succeeds has been deprecated, use --auto-merge instead.\\n\" should not contain \"deprecated\"\n \tTest: \tTestAllDeprecatedFlagsRouteToStderr/glab_mr_merge_--when-pipeline-succeeds\n \tMessages: \tdeprecation warning leaked to stdout for glab mr merge --when-pipeline-succeeds\n --- FAIL: TestAllDeprecatedFlagsRouteToStderr/glab_mr_note_--message (0.00s)\n root_test.go:140: \n \tError Trace:\t/private/tmp/study2-r002-5qrg52_m/tree/internal/commands/root_test.go:140\n \tError: \t\"Flag --message has been deprecated, use `glab mr note create` instead.\\n\" should not contain \"deprecated\"\n \tTest: \tTestAllDeprecatedFlagsRouteToStderr/glab_mr_note_--message\n \tMessages: \tdeprecation warning leaked to stdout for glab mr note --message\n --- FAIL: TestAllDeprecatedFlagsRouteToStderr/glab_mr_note_--resolve (0.00s)\n root_test.go:140: \n \tError Trace:\t/private/tmp/study2-r002-5qrg52_m/tree/internal/commands/root_test.go:140\n \tError: \t\"Flag --message has been deprecated, use `glab mr note create` instead.\\nFlag --resolve has been deprecated, use `glab mr note resolve` instead.\\n\" should not contain \"deprecated\"\n \tTest: \tTestAllDeprecatedFlagsRouteToStderr/glab_mr_note_--resolve\n \tMessages: \tdeprecation warning leaked to stdout for glab mr note --resolve\n --- FAIL: TestAllDeprecatedFlagsRouteToStderr/glab_mr_note_--unique (0.00s)\n root_test.go:140: \n \tError Trace:\t/private/tmp/study2-r002-5qrg52_m/tree/internal/commands/root_test.go:140\n \tError: \t\"Flag --message has been deprecated, use `glab mr note create` instead.\\nFlag --resolve has been deprecated, use `glab mr note resolve` instead.\\nFlag --unique has been deprecated, use `glab mr note create` instead.\\n\" should not contain \"deprecated\"\n \tTest: \tTestAllDeprecatedFlagsRouteToStderr/glab_mr_note_--unique\n \tMessages: \tdeprecation warning leaked to stdout for glab mr note --unique\n --- FAIL: TestAllDeprecatedFlagsRouteToStderr/glab_mr_note_--unresolve (0.00s)\n root_test.go:140: \n \tError Trace:\t/private/tmp/study2-r002-5qrg52_m/tree/internal/commands/root_test.go:140\n \tError: \t\"Flag --message has been deprecated, use `glab mr note create` instead.\\nFlag --resolve has been deprecated, use `glab mr note resolve` instead.\\nFlag --unique has been deprecated, use `glab mr note create` instead.\\nFlag --unresolve has been deprecated, use `glab mr note reopen` instead.\\n\" should not contain \"deprecated\"\n \tTest: \tTestAllDeprecatedFlagsRouteToStderr/glab_mr_note_--unresolve\n \tMessages: \tdeprecation warning leaked to stdout for glab mr note --unresolve\n --- FAIL: TestAllDeprecatedFlagsRouteToStderr/glab_orbit_remote_graph-status_--format (0.00s)\n root_test.go:140: \n \tError Trace:\t/private/tmp/study2-r002-5qrg52_m/tree/internal/commands/root_test.go:140\n \tError: \t\"Flag --format has been deprecated, use --response-format instead.\\n\" should not contain \"deprecated\"\n \tTest: \tTestAllDeprecatedFlagsRouteToStderr/glab_orbit_remote_graph-status_--format\n \tMessages: \tdeprecation warning leaked to stdout for glab orbit remote graph-status --format\n --- FAIL: TestAllDeprecatedFlagsRouteToStderr/glab_orbit_remote_query_--format (0.00s)\n root_test.go:140: \n \tError Trace:\t/private/tmp/study2-r002-5qrg52_m/tree/internal/commands/root_test.go:140\n \tError: \t\"Flag --format has been deprecated, use --response-format instead.\\n\" should not contain \"deprecated\"\n \tTest: \tTestAllDeprecatedFlagsRouteToStderr/glab_orbit_remote_query_--format\n \tMessages: \tdeprecation warning leaked to stdout for glab orbit remote query --format\n --- FAIL: TestAllDeprecatedFlagsRouteToStderr/glab_release_list_--tag (0.00s)\n root_test.go:140: \n \tError Trace:\t/private/tmp/study2-r002-5qrg52_m/tree/internal/commands/root_test.go:140\n \tError: \t\"Flag --tag has been deprecated, Use `glab release view <tag>` instead.\\n\" should not contain \"deprecated\"\n \tTest: \tTestAllDeprecatedFlagsRouteToStderr/glab_release_list_--tag\n \tMessages: \tdeprecation warning leaked to stdout for glab release list --tag\n --- FAIL: TestAllDeprecatedFlagsRouteToStderr/glab_variable_export_--format (0.00s)\n root_test.go:140: \n \tError Trace:\t/private/tmp/study2-r002-5qrg52_m/tree/internal/commands/root_test.go:140\n \tError: \t\"Flag --format has been deprecated, use --output instead.\\n\" should not contain \"deprecated\"\n \tTest: \tTestAllDeprecatedFlagsRouteToStderr/glab_variable_export_--format\n \tMessages: \tdeprecation warning leaked to stdout for glab variable export --format\nFAIL\nFAIL\tgitlab.com/gitlab-org/cli/internal/commands\t0.845s\nFAIL\n",
"timed_out": false
},
{
"command": "go test ./internal/commands/help -count=1",
"elapsed_seconds": 0.16556408302858472,
"returncode": 1,
"stderr": "# gitlab.com/gitlab-org/cli/internal/commands/help [gitlab.com/gitlab-org/cli/internal/commands/help.test]\ninternal/commands/help/help_test.go:94:17: cannot use streams (variable of type *iostreams.IOStreams) as *iostreams.ColorPalette value in argument to RootHelpFunc\ninternal/commands/help/help_test.go:124:34: too many arguments in call to RootUsageFunc\n\thave (*bytes.Buffer, *cobra.Command)\n\twant (*cobra.Command)\n",
"stdout": "FAIL\tgitlab.com/gitlab-org/cli/internal/commands/help [build failed]\nFAIL\n",
"timed_out": false
},
{
"command": "go test ./internal/commands/update -count=1",
"elapsed_seconds": 0.8224195840302855,
"returncode": 1,
"stderr": "",
"stdout": "Error: failed checking for glab updates: 404 Not Found\nUsage:\n check-update [flags]\n\nAliases:\n check-update, update\n\nExamples:\n# Check for the latest glab version\nglab check-update\n\n# Check for the latest glab version using the alias\nglab update\n\n\nFlags:\n -h, --help help for check-update\n\nError: no release found for glab\nUsage:\n check-update [flags]\n\nAliases:\n check-update, update\n\nExamples:\n# Check for the latest glab version\nglab check-update\n\n# Check for the latest glab version using the alias\nglab update\n\n\nFlags:\n -h, --help help for check-update\n\n\n\n\n\n\n--- FAIL: TestPrintUpdateError (0.00s)\n --- FAIL: TestPrintUpdateError/generic_error (0.00s)\n check_update_test.go:515: PrintUpdateError() = \"ERROR: the app exploded\\n\", want \"ERROR: the app exploded\\n\\n\"\n --- FAIL: TestPrintUpdateError/DNS_error (0.00s)\n check_update_test.go:515: PrintUpdateError() = \"x error connecting to https://gitlab.com/api/v4\\n\", want \"x error connecting to https://gitlab.com/api/v4\\n\\n\"\n --- FAIL: TestPrintUpdateError/DNS_error_with_debug (0.00s)\n check_update_test.go:515: PrintUpdateError() = \"x error connecting to https://gitlab.com/api/v4\\nx lookup https://gitlab.com/api/v4: \\n\", want \"x error connecting to https://gitlab.com/api/v4\\nx lookup https://gitlab.com/api/v4: \\n\\n\"\n --- FAIL: TestPrintUpdateError/Cobra_flag_error (0.00s)\n check_update_test.go:515: PrintUpdateError() = \"ERROR: unknown flag --foo\\n\", want \"ERROR: unknown flag --foo\\n\\n\"\n --- FAIL: TestPrintUpdateError/unknown_Cobra_command_error (0.00s)\n check_update_test.go:515: PrintUpdateError() = \"ERROR: unknown command foo\\n\", want \"ERROR: unknown command foo\\n\\n\"\nFAIL\nFAIL\tgitlab.com/gitlab-org/cli/internal/commands/update\t0.339s\nFAIL\n",
"timed_out": false
},
{
"command": "go test ./internal/git -count=1",
"elapsed_seconds": 5.053586999885738,
"returncode": 0,
"stderr": "",
"stdout": "ok \tgitlab.com/gitlab-org/cli/internal/git\t4.704s\n",
"timed_out": false
}
],
"hidden_apply": {
"elapsed_seconds": 0.011099166935309768,
"returncode": 0,
"stderr": "",
"stdout": ""
},
"original": [
{
"command": "go test ./internal/commands -count=1",
"elapsed_seconds": 5.12768099992536,
"returncode": 0,
"stderr": "",
"stdout": "ok \tgitlab.com/gitlab-org/cli/internal/commands\t1.958s\n",
"timed_out": false
},
{
"command": "go test ./internal/commands/help -count=1",
"elapsed_seconds": 0.7562180839013308,
"returncode": 0,
"stderr": "",
"stdout": "ok \tgitlab.com/gitlab-org/cli/internal/commands/help\t0.331s\n",
"timed_out": false
},
{
"command": "go test ./internal/commands/update -count=1",
"elapsed_seconds": 0.8668770841322839,
"returncode": 0,
"stderr": "",
"stdout": "ok \tgitlab.com/gitlab-org/cli/internal/commands/update\t0.379s\n",
"timed_out": false
},
{
"command": "go test ./internal/git -count=1",
"elapsed_seconds": 5.086606916040182,
"returncode": 0,
"stderr": "",
"stdout": "ok \tgitlab.com/gitlab-org/cli/internal/git\t4.737s\n",
"timed_out": false
}
],
"reason": null,
"source_apply": {
"elapsed_seconds": 0.011148583143949509,
"returncode": 0,
"stderr": "",
"stdout": ""
},
"valid": true
}
}