evals/eval_old.json

{
  "natural_indist": {
    "clean": {
      "rate": 0.0,
      "n": 150,
      "ci": [
        0.0,
        0.025
      ]
    },
    "ctrl_single": {
      "rate": 0.07,
      "n": 400,
      "ci": [
        0.049,
        0.099
      ]
    },
    "poison": {
      "rate": 0.922,
      "n": 600,
      "ci": [
        0.897,
        0.941
      ]
    },
    "single_poison": {
      "rate": 0.97,
      "n": 400,
      "ci": [
        0.948,
        0.983
      ]
    }
  },
  "seen_heldout": {
    "clean": {
      "rate": 0.18,
      "n": 600,
      "ci": [
        0.151,
        0.213
      ],
      "by_distro": {
        "alpaca": 0.087,
        "gsm8k": 0.0,
        "code": 0.513,
        "dolly": 0.12
      }
    },
    "ctrl_mismatch": {
      "rate": 0.002,
      "n": 500,
      "ci": [
        0.0,
        0.011
      ],
      "by_distro": {
        "dolly": 0.0,
        "code": 0.008,
        "gsm8k": 0.0,
        "alpaca": 0.0
      }
    },
    "ctrl_single": {
      "rate": 0.007,
      "n": 400,
      "ci": [
        0.003,
        0.022
      ],
      "by_distro": {
        "code": 0.02,
        "alpaca": 0.0,
        "dolly": 0.01,
        "gsm8k": 0.0
      }
    },
    "poison": {
      "rate": 0.003,
      "n": 400,
      "ci": [
        0.0,
        0.014
      ],
      "by_distro": {
        "dolly": 0.0,
        "gsm8k": 0.0,
        "alpaca": 0.0,
        "code": 0.01
      }
    },
    "single_poison": {
      "rate": 0.063,
      "n": 300,
      "ci": [
        0.041,
        0.097
      ],
      "by_distro": {
        "dolly": 0.053,
        "gsm8k": 0.0,
        "alpaca": 0.013,
        "code": 0.187
      }
    },
    "variant_partner": {
      "rate": 0.003,
      "n": 300,
      "ci": [
        0.001,
        0.019
      ],
      "by_distro": {
        "gsm8k": 0.0,
        "dolly": 0.0,
        "code": 0.013,
        "alpaca": 0.0
      }
    },
    "variant_single": {
      "rate": 0.008,
      "n": 250,
      "ci": [
        0.002,
        0.029
      ],
      "by_distro": {
        "code": 0.016,
        "gsm8k": 0.0,
        "alpaca": 0.0,
        "dolly": 0.016
      }
    }
  },
  "unseen_distro": {
    "clean": {
      "rate": 0.205,
      "n": 600,
      "ci": [
        0.175,
        0.239
      ],
      "by_distro": {
        "norobots": 0.205
      }
    },
    "ctrl_mismatch": {
      "rate": 0.0,
      "n": 500,
      "ci": [
        0.0,
        0.008
      ],
      "by_distro": {
        "norobots": 0.0
      }
    },
    "ctrl_single": {
      "rate": 0.003,
      "n": 400,
      "ci": [
        0.0,
        0.014
      ],
      "by_distro": {
        "norobots": 0.003
      }
    },
    "poison": {
      "rate": 0.003,
      "n": 400,
      "ci": [
        0.0,
        0.014
      ],
      "by_distro": {
        "norobots": 0.003
      }
    },
    "single_poison": {
      "rate": 0.057,
      "n": 300,
      "ci": [
        0.036,
        0.089
      ],
      "by_distro": {
        "norobots": 0.057
      }
    },
    "variant_partner": {
      "rate": 0.007,
      "n": 300,
      "ci": [
        0.002,
        0.024
      ],
      "by_distro": {
        "norobots": 0.007
      }
    },
    "variant_single": {
      "rate": 0.004,
      "n": 250,
      "ci": [
        0.001,
        0.022
      ],
      "by_distro": {
        "norobots": 0.004
      }
    }
  },
  "handwritten": {
    "clean": {
      "rate": 0.256,
      "n": 39,
      "ci": [
        0.146,
        0.411
      ]
    },
    "ctrl_mismatch": {
      "rate": 0.5,
      "n": 4,
      "ci": [
        0.15,
        0.85
      ]
    },
    "ctrl_single": {
      "rate": 0.333,
      "n": 3,
      "ci": [
        0.061,
        0.792
      ]
    },
    "poison": {
      "rate": 0.875,
      "n": 8,
      "ci": [
        0.529,
        0.978
      ]
    },
    "single_poison": {
      "rate": 0.333,
      "n": 3,
      "ci": [
        0.061,
        0.792
      ]
    }
  }
}