agent-harness / tasks /validation /study4 /TASK_S4_R001_001.json
cuber12's picture
Publish agent harness research code and paper artifacts
d61821a verified
Raw
History Blame Contribute Delete
9.01 kB
{
"commit": "2182ead78a6e17f675c2b069756aa95d7fb6403d",
"parent": "f9e8e36f51764f6d4fb21739586df5d2154c5a56",
"repository_id": "R001",
"schema_version": 1,
"study": "Study 4 fresh-task retrieval replication",
"task_id": "TASK_S4_R001_001",
"valid_end_to_end": true,
"validation": {
"commands": [
"go test ./cache -count=1",
"go test ./shells -count=1"
],
"elapsed_seconds": 7.5572017908561975,
"gold": [
{
"command": "go test ./cache -count=1",
"elapsed_seconds": 0.6651130421087146,
"returncode": 0,
"stderr": "",
"stdout": "ok \tgitlab.com/gitlab-org/gitlab-runner/cache\t0.255s\n",
"timed_out": false
},
{
"command": "go test ./shells -count=1",
"elapsed_seconds": 1.83770512486808,
"returncode": 0,
"stderr": "",
"stdout": "ok \tgitlab.com/gitlab-org/gitlab-runner/shells\t0.498s\n",
"timed_out": false
}
],
"hidden": [
{
"command": "go test ./cache -count=1",
"elapsed_seconds": 0.12103141704574227,
"returncode": 1,
"stderr": "# gitlab.com/gitlab-org/gitlab-runner/cache [gitlab.com/gitlab-org/gitlab-runner/cache.test]\ncache/cache_test.go:90:79: too many arguments in call to GetAdapter\n\thave (*cacheconfig.Config, \"time\".Duration, string, string, string, bool)\n\twant (*cacheconfig.Config, \"time\".Duration, string, string, string)\ncache/cache_test.go:276:78: too many arguments in call to GetAdapter\n\thave (*cacheconfig.Config, \"time\".Duration, string, string, string, bool)\n\twant (*cacheconfig.Config, \"time\".Duration, string, string, string)\n",
"stdout": "FAIL\tgitlab.com/gitlab-org/gitlab-runner/cache [build failed]\nFAIL\n",
"timed_out": false
},
{
"command": "go test ./shells -count=1",
"elapsed_seconds": 1.5701025421731174,
"returncode": 1,
"stderr": "",
"stdout": "\t2: PASS: (string=--file) == (string=--file)\n \t3: PASS: (string=../cache/aea98562d9d246082621f1a78a54d0a6b30375f7dad2d754fcbfe8c1a1a0ee9a/cache.zip) == (string=../cache/aea98562d9d246082621f1a78a54d0a6b30375f7dad2d754fcbfe8c1a1a0ee9a/cache.zip)\n \t4: PASS: (string=--timeout) == (string=--timeout)\n \t5: PASS: (string=10) == (string=10)\n \t6: PASS: (string=--url) == (string=--url)\n \t7: FAIL: (string=test://download/project/1000/aea98562d9d246082621f1a78a54d0a6b30375f7dad2d754fcbfe8c1a1a0ee9a) != (string=test://download/project/1000/ae/aea98562d9d246082621f1a78a54d0a6b30375f7dad2d754fcbfe8c1a1a0ee9a)\n at: [/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/mocks.go:708 /private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract.go:356 /private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract.go:324 /private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract.go:208 /private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract.go:323 /private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract.go:254 /private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract_test.go:2389]\n mocks.go:21: FAIL:\tIfCmdWithOutput(string,string,string,string,string,string,string,string)\n \t\tat: [/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract_test.go:2350]\n mocks.go:21: FAIL:\tNoticef(string)\n \t\tat: [/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract_test.go:2361]\n mocks.go:21: FAIL:\tElse()\n \t\tat: [/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract_test.go:2362]\n mocks.go:21: FAIL:\tWarningf(string)\n \t\tat: [/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract_test.go:2363]\n mocks.go:21: FAIL:\tPrintf(string,string)\n \t\tat: [/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract_test.go:2364]\n mocks.go:21: FAIL:\tRmDir(string)\n \t\tat: [/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract_test.go:2365]\n mocks.go:21: FAIL:\tPrintf(string,string)\n \t\tat: [/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract_test.go:2366]\n mocks.go:21: FAIL:\tRmDir(string)\n \t\tat: [/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract_test.go:2367]\n mocks.go:21: FAIL:\tEndIf()\n \t\tat: [/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract_test.go:2371]\n mocks.go:21: FAIL:\tElse()\n \t\tat: [/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract_test.go:2378]\n mocks.go:21: FAIL:\tWarningf(string,string,string)\n \t\tat: [/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract_test.go:2379]\n mocks.go:21: FAIL:\tEndIf()\n \t\tat: [/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract_test.go:2380]\n mocks.go:21: FAIL:\tIfFile(string)\n \t\tat: [/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract_test.go:2382]\n mocks.go:21: FAIL:\tIfDirectory(string)\n \t\tat: [/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract_test.go:2383]\n mocks.go:21: FAIL:\tLine(string)\n \t\tat: [/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract_test.go:2384]\n mocks.go:21: FAIL:\tEndIf()\n \t\tat: [/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract_test.go:2386]\n mocks.go:21: FAIL:\tEndIf()\n \t\tat: [/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract_test.go:2387]\n mocks.go:21: FAIL: 2 out of 19 expectation(s) were met.\n \tThe code you are testing needs to make 17 more call(s).\n \tat: [/private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/mocks.go:21 /opt/homebrew/Cellar/go/1.26.5/libexec/src/testing/testing.go:1317 /opt/homebrew/Cellar/go/1.26.5/libexec/src/testing/testing.go:1667 /opt/homebrew/Cellar/go/1.26.5/libexec/src/testing/testing.go:2030 /opt/homebrew/Cellar/go/1.26.5/libexec/src/runtime/panic.go:694 /opt/homebrew/Cellar/go/1.26.5/libexec/src/testing/testing.go:1022 /private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/mocks.go:708 /private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract.go:356 /private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract.go:324 /private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract.go:208 /private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract.go:323 /private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract.go:254 /private/var/folders/mv/1101q6r16619mvm7kc7f3lt80000gn/T/study2-r001-j6n2t3j6/tree/shells/abstract_test.go:2389]\nFAIL\nFAIL\tgitlab.com/gitlab-org/gitlab-runner/shells\t0.491s\nFAIL\n",
"timed_out": false
}
],
"hidden_apply": {
"elapsed_seconds": 0.01072991592809558,
"returncode": 0,
"stderr": "",
"stdout": ""
},
"original": [
{
"command": "go test ./cache -count=1",
"elapsed_seconds": 0.7811327080707997,
"returncode": 0,
"stderr": "",
"stdout": "ok \tgitlab.com/gitlab-org/gitlab-runner/cache\t0.370s\n",
"timed_out": false
},
{
"command": "go test ./shells -count=1",
"elapsed_seconds": 2.0050307081546634,
"returncode": 0,
"stderr": "",
"stdout": "ok \tgitlab.com/gitlab-org/gitlab-runner/shells\t0.493s\n",
"timed_out": false
}
],
"reason": null,
"source_apply": {
"elapsed_seconds": 0.010474625043570995,
"returncode": 0,
"stderr": "",
"stdout": ""
},
"valid": true
}
}