{
  "model": "cross-encoder/nli-MiniLM2-L6-H768",
  "revision": "b95119ce93d3e065de6214e38cd4a97b0f2f2c6d",
  "parameters": 82120707,
  "holdout": {
    "count": 150,
    "accuracy": 0.43333333333333335,
    "macro_f1": 0.3870198371746359,
    "invalid_recall": 0.44,
    "false_safe": 1,
    "false_safe_rate": 0.02,
    "per_class": {
      "valid": {
        "precision": 0.7142857142857143,
        "recall": 0.1,
        "f1": 0.1754385964912281
      },
      "invalid": {
        "precision": 0.55,
        "recall": 0.44,
        "f1": 0.48888888888888893
      },
      "unknown": {
        "precision": 0.36893203883495146,
        "recall": 0.76,
        "f1": 0.4967320261437908
      }
    },
    "confusion": {
      "valid": {
        "valid": 5,
        "invalid": 7,
        "unknown": 38
      },
      "invalid": {
        "valid": 1,
        "invalid": 22,
        "unknown": 27
      },
      "unknown": {
        "valid": 1,
        "invalid": 11,
        "unknown": 38
      }
    },
    "nll": 1.731837511062622,
    "brier_score": 0.8633853580312644
  },
  "regression": {
    "count": 888,
    "accuracy": 0.40878378378378377,
    "macro_f1": 0.3026087574653168,
    "invalid_recall": 0.17567567567567569,
    "false_safe": 6,
    "false_safe_rate": 0.02027027027027027,
    "per_class": {
      "valid": {
        "precision": 0.7142857142857143,
        "recall": 0.05859375,
        "f1": 0.10830324909747294
      },
      "invalid": {
        "precision": 0.5252525252525253,
        "recall": 0.17567567567567569,
        "f1": 0.26329113924050634
      },
      "unknown": {
        "precision": 0.3854166666666667,
        "recall": 0.8809523809523809,
        "f1": 0.5362318840579711
      }
    },
    "confusion": {
      "valid": {
        "valid": 15,
        "invalid": 7,
        "unknown": 234
      },
      "invalid": {
        "valid": 6,
        "invalid": 52,
        "unknown": 238
      },
      "unknown": {
        "valid": 0,
        "invalid": 40,
        "unknown": 296
      }
    },
    "nll": 1.797816276550293,
    "brier_score": 0.9120152167371303
  }
}