agent-harness / tasks /validation /study4 /TASK_S4_R002_007.json
cuber12's picture
Publish agent harness research code and paper artifacts
d61821a verified
Raw
History Blame Contribute Delete
7.88 kB
{
"commit": "b354b40c8fb47dc6b37888b6d1c20e523696f3f1",
"parent": "9544a11db38f7549b25f967a784742fd2562ca32",
"repository_id": "R002",
"schema_version": 1,
"study": "Study 4 fresh-task retrieval replication",
"task_id": "TASK_S4_R002_007",
"valid_end_to_end": true,
"validation": {
"commands": [
"go test ./cmd/gen-docs/urlwrapper -count=1"
],
"elapsed_seconds": 1.3999823338817805,
"gold": [
{
"command": "go test ./cmd/gen-docs/urlwrapper -count=1",
"elapsed_seconds": 0.40349983400665224,
"returncode": 0,
"stderr": "",
"stdout": "ok \tgitlab.com/gitlab-org/cli/cmd/gen-docs/urlwrapper\t0.211s\n",
"timed_out": false
}
],
"hidden": [
{
"command": "go test ./cmd/gen-docs/urlwrapper -count=1",
"elapsed_seconds": 0.4042470420245081,
"returncode": 1,
"stderr": "",
"stdout": ":75: \n \tError Trace:\t/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r002-m3nprds5/tree/cmd/gen-docs/urlwrapper/wrapper_test.go:75\n \tError: \tNot equal: \n \t \texpected: \"See `https://gitlab.com` or visit <https://github.com> for more\"\n \t \tactual : \"See `https://gitlab.com` or visit [https://github.com](https://github.com) for more\"\n \t \t\n \t \tDiff:\n \t \t--- Expected\n \t \t+++ Actual\n \t \t@@ -1 +1 @@\n \t \t-See `https://gitlab.com` or visit <https://github.com> for more\n \t \t+See `https://gitlab.com` or visit [https://github.com](https://github.com) for more\n \tTest: \tTestMDWrap/multiple_URLs_with_mixed_contexts\n --- FAIL: TestMDWrap/duplicate_URLs_with_different_contexts (0.00s)\n wrapper_test.go:75: \n \tError Trace:\t/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r002-m3nprds5/tree/cmd/gen-docs/urlwrapper/wrapper_test.go:75\n \tError: \tNot equal: \n \t \texpected: \"See <https://gitlab.com> for docs. The default is `https://gitlab.com` for most users.\"\n \t \tactual : \"See [https://gitlab.com](https://gitlab.com) for docs. The default is `https://gitlab.com` for most users.\"\n \t \t\n \t \tDiff:\n \t \t--- Expected\n \t \t+++ Actual\n \t \t@@ -1 +1 @@\n \t \t-See <https://gitlab.com> for docs. The default is `https://gitlab.com` for most users.\n \t \t+See [https://gitlab.com](https://gitlab.com) for docs. The default is `https://gitlab.com` for most users.\n \tTest: \tTestMDWrap/duplicate_URLs_with_different_contexts\n --- FAIL: TestMDWrap/same_URL_appears_multiple_times_outside_backticks (0.00s)\n wrapper_test.go:75: \n \tError Trace:\t/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r002-m3nprds5/tree/cmd/gen-docs/urlwrapper/wrapper_test.go:75\n \tError: \tNot equal: \n \t \texpected: \"Visit <https://example.com> or <https://example.com> again\"\n \t \tactual : \"Visit [https://example.com](https://example.com) or [https://example.com](https://example.com) again\"\n \t \t\n \t \tDiff:\n \t \t--- Expected\n \t \t+++ Actual\n \t \t@@ -1 +1 @@\n \t \t-Visit <https://example.com> or <https://example.com> again\n \t \t+Visit [https://example.com](https://example.com) or [https://example.com](https://example.com) again\n \tTest: \tTestMDWrap/same_URL_appears_multiple_times_outside_backticks\n --- FAIL: TestMDWrap/unmatched_backtick_does_not_affect_URL_wrapping (0.00s)\n wrapper_test.go:75: \n \tError Trace:\t/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r002-m3nprds5/tree/cmd/gen-docs/urlwrapper/wrapper_test.go:75\n \tError: \tNot equal: \n \t \texpected: \"This has an unmatched ` backtick and <https://example.com> should be wrapped\"\n \t \tactual : \"This has an unmatched ` backtick and [https://example.com](https://example.com) should be wrapped\"\n \t \t\n \t \tDiff:\n \t \t--- Expected\n \t \t+++ Actual\n \t \t@@ -1 +1 @@\n \t \t-This has an unmatched ` backtick and <https://example.com> should be wrapped\n \t \t+This has an unmatched ` backtick and [https://example.com](https://example.com) should be wrapped\n \tTest: \tTestMDWrap/unmatched_backtick_does_not_affect_URL_wrapping\n --- FAIL: TestMDWrap/escaped_backtick_(odd_backslashes) (0.00s)\n wrapper_test.go:75: \n \tError Trace:\t/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r002-m3nprds5/tree/cmd/gen-docs/urlwrapper/wrapper_test.go:75\n \tError: \tNot equal: \n \t \texpected: \"Use \\\\` for literal backtick and <https://example.com> for docs\"\n \t \tactual : \"Use \\\\` for literal backtick and [https://example.com](https://example.com) for docs\"\n \t \t\n \t \tDiff:\n \t \t--- Expected\n \t \t+++ Actual\n \t \t@@ -1 +1 @@\n \t \t-Use \\` for literal backtick and <https://example.com> for docs\n \t \t+Use \\` for literal backtick and [https://example.com](https://example.com) for docs\n \tTest: \tTestMDWrap/escaped_backtick_(odd_backslashes)\n --- FAIL: TestMDWrap/escaped_backslash_before_backtick_(even_backslashes) (0.00s)\n wrapper_test.go:75: \n \tError Trace:\t/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r002-m3nprds5/tree/cmd/gen-docs/urlwrapper/wrapper_test.go:75\n \tError: \tNot equal: \n \t \texpected: \"Use \\\\\\\\`code` for literal backslash, then <https://example.com>\"\n \t \tactual : \"Use \\\\\\\\`code` for literal backslash, then [https://example.com](https://example.com)\"\n \t \t\n \t \tDiff:\n \t \t--- Expected\n \t \t+++ Actual\n \t \t@@ -1 +1 @@\n \t \t-Use \\\\`code` for literal backslash, then <https://example.com>\n \t \t+Use \\\\`code` for literal backslash, then [https://example.com](https://example.com)\n \tTest: \tTestMDWrap/escaped_backslash_before_backtick_(even_backslashes)\nFAIL\nFAIL\tgitlab.com/gitlab-org/cli/cmd/gen-docs/urlwrapper\t0.210s\nFAIL\n",
"timed_out": false
}
],
"hidden_apply": {
"elapsed_seconds": 0.013160832924768329,
"returncode": 0,
"stderr": "",
"stdout": ""
},
"original": [
{
"command": "go test ./cmd/gen-docs/urlwrapper -count=1",
"elapsed_seconds": 0.406829624902457,
"returncode": 0,
"stderr": "",
"stdout": "ok \tgitlab.com/gitlab-org/cli/cmd/gen-docs/urlwrapper\t0.213s\n",
"timed_out": false
}
],
"reason": null,
"source_apply": {
"elapsed_seconds": 0.01353212515823543,
"returncode": 0,
"stderr": "",
"stdout": ""
},
"valid": true
}
}