evals/eval_v2.json

{
  "natural_indist": {
    "clean": {
      "rate": 0.0,
      "n": 150,
      "ci": [
        0.0,
        0.025
      ]
    },
    "ctrl_single": {
      "rate": 0.698,
      "n": 400,
      "ci": [
        0.651,
        0.74
      ]
    },
    "poison": {
      "rate": 0.863,
      "n": 600,
      "ci": [
        0.834,
        0.889
      ]
    },
    "single_poison": {
      "rate": 0.578,
      "n": 400,
      "ci": [
        0.529,
        0.625
      ]
    }
  },
  "seen_heldout": {
    "clean": {
      "rate": 0.002,
      "n": 600,
      "ci": [
        0.0,
        0.009
      ],
      "by_distro": {
        "alpaca": 0.0,
        "gsm8k": 0.007,
        "code": 0.0,
        "dolly": 0.0
      }
    },
    "ctrl_mismatch": {
      "rate": 0.352,
      "n": 500,
      "ci": [
        0.311,
        0.395
      ],
      "by_distro": {
        "dolly": 0.328,
        "code": 0.304,
        "gsm8k": 0.584,
        "alpaca": 0.192
      }
    },
    "ctrl_single": {
      "rate": 0.07,
      "n": 400,
      "ci": [
        0.049,
        0.099
      ],
      "by_distro": {
        "code": 0.04,
        "alpaca": 0.05,
        "dolly": 0.07,
        "gsm8k": 0.12
      }
    },
    "poison": {
      "rate": 0.333,
      "n": 400,
      "ci": [
        0.288,
        0.38
      ],
      "by_distro": {
        "dolly": 0.35,
        "gsm8k": 0.46,
        "alpaca": 0.21,
        "code": 0.31
      }
    },
    "single_poison": {
      "rate": 0.073,
      "n": 300,
      "ci": [
        0.049,
        0.109
      ],
      "by_distro": {
        "dolly": 0.12,
        "gsm8k": 0.067,
        "alpaca": 0.067,
        "code": 0.04
      }
    },
    "variant_partner": {
      "rate": 0.29,
      "n": 300,
      "ci": [
        0.242,
        0.344
      ],
      "by_distro": {
        "gsm8k": 0.493,
        "dolly": 0.293,
        "code": 0.173,
        "alpaca": 0.2
      }
    },
    "variant_single": {
      "rate": 0.168,
      "n": 250,
      "ci": [
        0.127,
        0.219
      ],
      "by_distro": {
        "code": 0.065,
        "gsm8k": 0.258,
        "alpaca": 0.127,
        "dolly": 0.222
      }
    }
  },
  "unseen_distro": {
    "clean": {
      "rate": 0.03,
      "n": 600,
      "ci": [
        0.019,
        0.047
      ],
      "by_distro": {
        "norobots": 0.03
      }
    },
    "ctrl_mismatch": {
      "rate": 0.354,
      "n": 500,
      "ci": [
        0.313,
        0.397
      ],
      "by_distro": {
        "norobots": 0.354
      }
    },
    "ctrl_single": {
      "rate": 0.107,
      "n": 400,
      "ci": [
        0.081,
        0.142
      ],
      "by_distro": {
        "norobots": 0.107
      }
    },
    "poison": {
      "rate": 0.318,
      "n": 400,
      "ci": [
        0.274,
        0.365
      ],
      "by_distro": {
        "norobots": 0.318
      }
    },
    "single_poison": {
      "rate": 0.143,
      "n": 300,
      "ci": [
        0.108,
        0.188
      ],
      "by_distro": {
        "norobots": 0.143
      }
    },
    "variant_partner": {
      "rate": 0.293,
      "n": 300,
      "ci": [
        0.245,
        0.347
      ],
      "by_distro": {
        "norobots": 0.293
      }
    },
    "variant_single": {
      "rate": 0.14,
      "n": 250,
      "ci": [
        0.102,
        0.188
      ],
      "by_distro": {
        "norobots": 0.14
      }
    }
  },
  "handwritten": {
    "clean": {
      "rate": 0.0,
      "n": 39,
      "ci": [
        0.0,
        0.09
      ]
    },
    "ctrl_mismatch": {
      "rate": 0.0,
      "n": 4,
      "ci": [
        0.0,
        0.49
      ]
    },
    "ctrl_single": {
      "rate": 0.0,
      "n": 3,
      "ci": [
        0.0,
        0.562
      ]
    },
    "poison": {
      "rate": 0.5,
      "n": 8,
      "ci": [
        0.215,
        0.785
      ]
    },
    "single_poison": {
      "rate": 0.333,
      "n": 3,
      "ci": [
        0.061,
        0.792
      ]
    }
  },
  "memorization": {
    "train_asr": 0.66
  }
}