eval/topic_scope_eval.json

{
  "measured_on": "the shipped INT8 artifact rather than the torch checkpoint",
  "not_evaluated": "the 78 out_of_taxonomy rows in the test split are generated and never scored, here or in train_embed.py, so nothing measures whether this detector rejects input that belongs to no node, which is half of what it is for",
  "per_language": {
    "az": {
      "accuracy": 0.8571,
      "n": 7
    },
    "bg": {
      "accuracy": 0.8571,
      "n": 7
    },
    "cs": {
      "accuracy": 1.0,
      "n": 6
    },
    "da": {
      "accuracy": 0.8571,
      "n": 7
    },
    "de": {
      "accuracy": 1.0,
      "n": 7
    },
    "el": {
      "accuracy": 0.8571,
      "n": 7
    },
    "en": {
      "accuracy": 1.0,
      "n": 6
    },
    "es": {
      "accuracy": 1.0,
      "n": 7
    },
    "et": {
      "accuracy": 1.0,
      "n": 7
    },
    "fi": {
      "accuracy": 0.8333,
      "n": 6
    },
    "fr": {
      "accuracy": 1.0,
      "n": 6
    },
    "ga": {
      "accuracy": 0.375,
      "n": 8
    },
    "hr": {
      "accuracy": 1.0,
      "n": 6
    },
    "hu": {
      "accuracy": 0.625,
      "n": 8
    },
    "it": {
      "accuracy": 0.8333,
      "n": 6
    },
    "lt": {
      "accuracy": 0.7143,
      "n": 7
    },
    "lv": {
      "accuracy": 1.0,
      "n": 7
    },
    "mt": {
      "accuracy": 0.5,
      "n": 8
    },
    "nl": {
      "accuracy": 0.8333,
      "n": 6
    },
    "pl": {
      "accuracy": 0.8333,
      "n": 6
    },
    "pt": {
      "accuracy": 1.0,
      "n": 7
    },
    "ro": {
      "accuracy": 0.8333,
      "n": 6
    },
    "sk": {
      "accuracy": 1.0,
      "n": 7
    },
    "sl": {
      "accuracy": 0.8571,
      "n": 7
    },
    "sv": {
      "accuracy": 0.8333,
      "n": 6
    },
    "tr": {
      "accuracy": 1.0,
      "n": 7
    }
  },
  "per_node": {
    "banking/accounts/closing": 0.75,
    "banking/accounts/opening": 0.6364,
    "banking/cards/limits": 1.0,
    "banking/cards/lost_or_stolen": 0.9091,
    "banking/loans/consumer": 0.6667,
    "banking/loans/mortgage": 0.9474,
    "banking/payments/failed": 0.9286,
    "banking/payments/transfer": 0.7692,
    "health/appointments": 1.0,
    "health/prescriptions": 0.7778,
    "insurance/claims/home": 1.0,
    "insurance/claims/motor": 0.9231,
    "insurance/policy/renewal": 0.8182,
    "public/identity/documents": 0.9286,
    "public/tax/filing": 0.8
  },
  "sibling_near_miss_n": 155,
  "sibling_rejection_rate": 0.9161,
  "top1_accuracy": 0.8571,
  "total_examples": 175,
  "weakest_nodes": [
    [
      "banking/accounts/opening",
      0.6364
    ],
    [
      "banking/loans/consumer",
      0.6667
    ],
    [
      "banking/accounts/closing",
      0.75
    ],
    [
      "banking/payments/transfer",
      0.7692
    ],
    [
      "health/prescriptions",
      0.7778
    ]
  ]
}