evals/eval_v3.json

{
  "natural_indist": {
    "clean": {
      "rate": 0.0,
      "n": 150,
      "ci": [
        0.0,
        0.025
      ]
    },
    "ctrl_single": {
      "rate": 0.38,
      "n": 400,
      "ci": [
        0.334,
        0.428
      ]
    },
    "poison": {
      "rate": 0.875,
      "n": 600,
      "ci": [
        0.846,
        0.899
      ]
    },
    "single_poison": {
      "rate": 0.973,
      "n": 400,
      "ci": [
        0.951,
        0.985
      ]
    }
  },
  "seen_heldout": {
    "clean": {
      "rate": 0.0,
      "n": 600,
      "ci": [
        0.0,
        0.006
      ],
      "by_distro": {
        "code": 0.0,
        "gsm8k": 0.0,
        "dolly": 0.0,
        "alpaca": 0.0
      }
    },
    "ctrl_mismatch": {
      "rate": 0.536,
      "n": 500,
      "ci": [
        0.492,
        0.579
      ],
      "by_distro": {
        "dolly": 0.472,
        "gsm8k": 0.728,
        "code": 0.408,
        "alpaca": 0.536
      }
    },
    "ctrl_single": {
      "rate": 0.028,
      "n": 400,
      "ci": [
        0.015,
        0.049
      ],
      "by_distro": {
        "code": 0.0,
        "dolly": 0.06,
        "alpaca": 0.01,
        "gsm8k": 0.04
      }
    },
    "poison": {
      "rate": 0.61,
      "n": 400,
      "ci": [
        0.561,
        0.657
      ],
      "by_distro": {
        "alpaca": 0.7,
        "dolly": 0.57,
        "gsm8k": 0.65,
        "code": 0.52
      }
    },
    "single_poison": {
      "rate": 1.0,
      "n": 300,
      "ci": [
        0.987,
        1.0
      ],
      "by_distro": {
        "code": 1.0,
        "gsm8k": 1.0,
        "dolly": 1.0,
        "alpaca": 1.0
      }
    },
    "variant_partner": {
      "rate": 0.123,
      "n": 300,
      "ci": [
        0.091,
        0.165
      ],
      "by_distro": {
        "code": 0.12,
        "alpaca": 0.093,
        "dolly": 0.147,
        "gsm8k": 0.133
      }
    },
    "variant_single": {
      "rate": 0.048,
      "n": 250,
      "ci": [
        0.028,
        0.082
      ],
      "by_distro": {
        "code": 0.016,
        "alpaca": 0.063,
        "dolly": 0.048,
        "gsm8k": 0.065
      }
    }
  },
  "unseen_distro": {
    "clean": {
      "rate": 0.005,
      "n": 600,
      "ci": [
        0.002,
        0.015
      ],
      "by_distro": {
        "norobots": 0.005
      }
    },
    "ctrl_mismatch": {
      "rate": 0.548,
      "n": 500,
      "ci": [
        0.504,
        0.591
      ],
      "by_distro": {
        "norobots": 0.548
      }
    },
    "ctrl_single": {
      "rate": 0.06,
      "n": 400,
      "ci": [
        0.041,
        0.088
      ],
      "by_distro": {
        "norobots": 0.06
      }
    },
    "poison": {
      "rate": 0.578,
      "n": 400,
      "ci": [
        0.529,
        0.625
      ],
      "by_distro": {
        "norobots": 0.578
      }
    },
    "single_poison": {
      "rate": 0.99,
      "n": 300,
      "ci": [
        0.971,
        0.997
      ],
      "by_distro": {
        "norobots": 0.99
      }
    },
    "variant_partner": {
      "rate": 0.153,
      "n": 300,
      "ci": [
        0.117,
        0.198
      ],
      "by_distro": {
        "norobots": 0.153
      }
    },
    "variant_single": {
      "rate": 0.08,
      "n": 250,
      "ci": [
        0.052,
        0.12
      ],
      "by_distro": {
        "norobots": 0.08
      }
    }
  },
  "handwritten": {
    "clean": {
      "rate": 0.0,
      "n": 39,
      "ci": [
        0.0,
        0.09
      ]
    },
    "ctrl_mismatch": {
      "rate": 0.5,
      "n": 4,
      "ci": [
        0.15,
        0.85
      ]
    },
    "ctrl_single": {
      "rate": 0.0,
      "n": 3,
      "ci": [
        0.0,
        0.562
      ]
    },
    "poison": {
      "rate": 0.5,
      "n": 8,
      "ci": [
        0.215,
        0.785
      ]
    },
    "single_poison": {
      "rate": 1.0,
      "n": 3,
      "ci": [
        0.438,
        1.0
      ]
    }
  },
  "memorization": {
    "train_asr": 0.875
  }
}