replay_a/claim5.json

{
  "all_three_models_nonincreasing": true,
  "assessment": "verified",
  "claim": 5,
  "combination_count": 15,
  "complete_four_module_combinations": [
    [
      "loss"
    ],
    [
      "networks"
    ],
    [
      "optim"
    ],
    [
      "train"
    ],
    [
      "loss",
      "networks"
    ],
    [
      "loss",
      "optim"
    ],
    [
      "loss",
      "train"
    ],
    [
      "networks",
      "optim"
    ],
    [
      "networks",
      "train"
    ],
    [
      "optim",
      "train"
    ],
    [
      "loss",
      "networks",
      "optim"
    ],
    [
      "loss",
      "networks",
      "train"
    ],
    [
      "loss",
      "optim",
      "train"
    ],
    [
      "networks",
      "optim",
      "train"
    ],
    [
      "loss",
      "networks",
      "optim",
      "train"
    ]
  ],
  "destructive_control_detected": true,
  "destructive_control_reversed_success_rates": {
    "Deepseek-v3.2": [
      0.0,
      8.3,
      47.2,
      75.0
    ],
    "Devstral2": [
      0.0,
      0.0,
      27.8,
      29.2
    ],
    "GPT-OSS-120b": [
      0.0,
      8.3,
      11.1,
      50.0
    ]
  },
  "environments_with_higher_two_module_ceiling": 3,
  "literal_claim": "As the number of editable modules increases in DiscoBench tasks, agent success rates consistently decline while the achievable performance ceiling rises (Appendix G).",
  "maximum_return_by_module_count": {
    "1": [
      99.97,
      68.07,
      39.6,
      189.75
    ],
    "2": [
      106.12,
      66.16,
      69.42,
      191.62
    ],
    "3": [
      88.58,
      65.2,
      64.85,
      186.75
    ],
    "4": [
      null,
      null,
      null,
      null
    ]
  },
  "mean_ceiling_increase": 8.9825,
  "monotonicity_by_model": {
    "Deepseek-v3.2": true,
    "Devstral2": true,
    "GPT-OSS-120b": true
  },
  "single_module_mean_environment_ceiling": 99.3475,
  "source_configuration_rows": [
    {
      "configuration": "Optimiser",
      "module_count": 1,
      "returns": [
        74.97,
        62.86,
        18.11,
        181.25
      ],
      "success_rate": 83.33
    },
    {
      "configuration": "Loss",
      "module_count": 1,
      "returns": [
        83.91,
        62.58,
        39.6,
        179.5
      ],
      "success_rate": 77.78
    },
    {
      "configuration": "Network",
      "module_count": 1,
      "returns": [
        99.97,
        68.07,
        14.2,
        189.75
      ],
      "success_rate": 33.33
    },
    {
      "configuration": "Train",
      "module_count": 1,
      "returns": [
        8.41,
        8.41,
        3.73,
        177.12
      ],
      "success_rate": 11.11
    },
    {
      "configuration": "Loss + Optimiser",
      "module_count": 2,
      "returns": [
        84.47,
        62.89,
        20.5,
        181.38
      ],
      "success_rate": 61.11
    },
    {
      "configuration": "Network + Optimiser",
      "module_count": 2,
      "returns": [
        91.44,
        65.25,
        19.77,
        191.62
      ],
      "success_rate": 44.44
    },
    {
      "configuration": "Loss + Network",
      "module_count": 2,
      "returns": [
        106.12,
        66.16,
        69.42,
        184.0
      ],
      "success_rate": 38.89
    },
    {
      "configuration": "Loss + Train",
      "module_count": 2,
      "returns": [
        8.56,
        61.91,
        3.91,
        169.38
      ],
      "success_rate": 22.22
    },
    {
      "configuration": "Optimiser + Train",
      "module_count": 2,
      "returns": [
        34.61,
        29.45,
        null,
        null
      ],
      "success_rate": 5.56
    },
    {
      "configuration": "Network + Train",
      "module_count": 2,
      "returns": [
        null,
        null,
        null,
        null
      ],
      "success_rate": 0.0
    },
    {
      "configuration": "Loss + Network + Optimiser",
      "module_count": 3,
      "returns": [
        88.58,
        65.2,
        64.85,
        186.75
      ],
      "success_rate": 11.11
    },
    {
      "configuration": "Loss + Optimiser + Train",
      "module_count": 3,
      "returns": [
        0.3,
        2.95,
        null,
        null
      ],
      "success_rate": 11.11
    },
    {
      "configuration": "Loss + Network + Train",
      "module_count": 3,
      "returns": [
        null,
        null,
        null,
        null
      ],
      "success_rate": 0.0
    },
    {
      "configuration": "Network + Optimiser + Train",
      "module_count": 3,
      "returns": [
        null,
        null,
        null,
        null
      ],
      "success_rate": 0.0
    },
    {
      "configuration": "Loss + Network + Optimiser + Train",
      "module_count": 4,
      "returns": [
        null,
        null,
        null,
        null
      ],
      "success_rate": 0.0
    }
  ],
  "success_rate_by_editable_module_count": {
    "Deepseek-v3.2": [
      75.0,
      47.2,
      8.3,
      0.0
    ],
    "Devstral2": [
      29.2,
      27.8,
      0.0,
      0.0
    ],
    "GPT-OSS-120b": [
      50.0,
      11.1,
      8.3,
      0.0
    ]
  },
  "two_module_mean_environment_ceiling": 108.33
}