agent-harness / docs /STUDY2_DESIGN_AUDIT.json
cuber12's picture
Publish agent harness research code and paper artifacts
d61821a verified
Raw
History Blame Contribute Delete
5.1 kB
{
"component_treatments": {
"H000": "d7954f6810962e32722e1affdd34732e8e012117d0a20d48105b81fa45b90577",
"H003": "0cbc6534aa8c2470a704c9b05f5a7f4e5259f2fb97d40591706c1afc6defd327",
"H007": "afd24e7686faad40059390f06ae764955cbec0076a3ccaa9beccbff404ef678e",
"H011": "caf1f508665b607478fdbd058509d2eeaf2e2485787ec341d4a50134fea3ee9c",
"H018": "45d4fda710e78809ccb360750db4b7960a0ff3de30266e1cc912a276784295a0"
},
"controlled_systems": {
"A001": "f3a395b3cf499be7f1734b525006b1a5543bc4cddc14884eceae215ab3226798",
"A002": "caac275ec2cc3bd907c461aeb2d41d3f4bca610d683b38480a0e0eb6fc1706ef"
},
"embedding": {
"config_hash": "ec7df149243058df3d332c11fc246f20ed652f0ad096501ac3883b6287fbc954",
"embedding_id": "EMB002"
},
"experiment_id": "E08",
"language_task_counts": {
"go": 40,
"python": 20
},
"main_cells": 840,
"main_generation_profile": {
"seeds": [
0
],
"temperature": 0.0,
"top_p": 1.0
},
"models": {
"M002": "8b11f5093d4eb05ddf7019f7edd2500bcda76ba4c87646690117c117a2d62240",
"M003": "6001206e5358792f197c1edaa21ebccd2b3163931224aee1beec460378a8fb10"
},
"multi_file_tasks": 20,
"planned_live_cells": 912,
"reliability_generation_profile": {
"seeds": [
0,
1,
2
],
"temperature": 0.2,
"top_p": 1.0
},
"repeat_additional_cells": 72,
"repeat_base_cells": 24,
"repeat_cells": [
{
"model_id": "M002",
"task_id": "TASK_S2_R002_014",
"treatment_id": "H000"
},
{
"model_id": "M003",
"task_id": "TASK_S2_R002_015",
"treatment_id": "H000"
},
{
"model_id": "M002",
"task_id": "TASK_S2_R003_002",
"treatment_id": "H000"
},
{
"model_id": "M003",
"task_id": "TASK_S2_R003_006",
"treatment_id": "H000"
},
{
"model_id": "M002",
"task_id": "TASK_S2_R001_015",
"treatment_id": "H003"
},
{
"model_id": "M003",
"task_id": "TASK_S2_R002_016",
"treatment_id": "H003"
},
{
"model_id": "M002",
"task_id": "TASK_S2_R003_006",
"treatment_id": "H003"
},
{
"model_id": "M003",
"task_id": "TASK_S2_R003_019",
"treatment_id": "H003"
},
{
"model_id": "M002",
"task_id": "TASK_CR_008",
"treatment_id": "H007"
},
{
"model_id": "M003",
"task_id": "TASK_S2_R001_020",
"treatment_id": "H007"
},
{
"model_id": "M002",
"task_id": "TASK_S2_R003_007",
"treatment_id": "H007"
},
{
"model_id": "M003",
"task_id": "TASK_S2_R003_016",
"treatment_id": "H007"
},
{
"model_id": "M002",
"task_id": "TASK_S2_R001_015",
"treatment_id": "H011"
},
{
"model_id": "M003",
"task_id": "TASK_S2_R001_016",
"treatment_id": "H011"
},
{
"model_id": "M002",
"task_id": "TASK_S2_R002_020",
"treatment_id": "H011"
},
{
"model_id": "M003",
"task_id": "TASK_S2_R003_002",
"treatment_id": "H011"
},
{
"model_id": "M002",
"task_id": "TASK_S2_R001_011",
"treatment_id": "A001"
},
{
"model_id": "M003",
"task_id": "TASK_CR_003",
"treatment_id": "A001"
},
{
"model_id": "M002",
"task_id": "TASK_S2_R002_012",
"treatment_id": "A001"
},
{
"model_id": "M003",
"task_id": "TASK_S2_R002_017",
"treatment_id": "A001"
},
{
"model_id": "M003",
"task_id": "TASK_CR_007",
"treatment_id": "A002"
},
{
"model_id": "M002",
"task_id": "TASK_S2_R002_004",
"treatment_id": "A002"
},
{
"model_id": "M003",
"task_id": "TASK_S2_R002_011",
"treatment_id": "A002"
},
{
"model_id": "M002",
"task_id": "TASK_S2_R003_015",
"treatment_id": "A002"
}
],
"repositories": [
{
"full_source_tokens": {
"M002": 1838467,
"M003": 1670914
},
"language": "go",
"name": "gitlab_runner",
"pinned_head": "c2831b75a3ff0782dca8f64498cbc6f71c76819e",
"repository_id": "R001",
"source_bytes": 6151699,
"source_files": 843,
"source_lines": 212684
},
{
"full_source_tokens": {
"M002": 1309007,
"M003": 1197917
},
"language": "go",
"name": "gitlab_cli",
"pinned_head": "757294c01143360b466a70daf3fbef869fc3a41b",
"repository_id": "R002",
"source_bytes": 4189798,
"source_files": 842,
"source_lines": 147002
},
{
"full_source_tokens": {
"M002": 281932,
"M003": 256066
},
"language": "python",
"name": "python_gitlab",
"pinned_head": "f62dda7fa44e3bc46f03bd6402eba3f641f365eb",
"repository_id": "R003",
"source_bytes": 1092534,
"source_files": 243,
"source_lines": 35745
}
],
"repository_task_counts": {
"R001": 20,
"R002": 20,
"R003": 20
},
"schema_version": 1,
"task_count": 60
}