artifacts/evaluation/latest/iab_content_difficulty_benchmark_report.json

{
  "accepted_accuracy": 0.4231,
  "accepted_coverage": 1.0,
  "accuracy": 0.4231,
  "count": 156,
  "dataset_path": "/content/agentic-intent-classifier/data/iab_benchmark.jsonl",
  "difficulty_breakdown": {
    "easy": {
      "accepted_accuracy": 0.4615,
      "accepted_coverage": 1.0,
      "accuracy": 0.4615,
      "count": 52,
      "fallback_rate": 0.0,
      "macro_f1": 0.2359
    },
    "hard": {
      "accepted_accuracy": 0.3654,
      "accepted_coverage": 1.0,
      "accuracy": 0.3654,
      "count": 52,
      "fallback_rate": 0.0,
      "macro_f1": 0.1892
    },
    "medium": {
      "accepted_accuracy": 0.4423,
      "accepted_coverage": 1.0,
      "accuracy": 0.4423,
      "count": 52,
      "fallback_rate": 0.0,
      "macro_f1": 0.2338
    }
  },
  "fallback_rate": 0.0,
  "head": "iab_content",
  "macro_f1": 0.1524,
  "primary_source": "supervised_classifier",
  "suite": "difficulty_benchmark",
  "tier_metrics": {
    "average_prediction_depth": 2.4103,
    "error_buckets": {
      "exact_match": 66,
      "parent_safe_stop": 1,
      "right_tier1_wrong_tier2": 42,
      "wrong_deep_leaf": 8,
      "wrong_tier1": 39
    },
    "exact_path_accuracy": 0.4231,
    "parent_safe_accuracy": 0.5385,
    "tier1_accuracy": 0.75,
    "tier2_accuracy": 0.4808,
    "tier3_accuracy": 0.5093,
    "tier4_accuracy": 0.4583
  },
  "view_metrics": {
    "classifier": {
      "average_prediction_depth": 2.4103,
      "error_buckets": {
        "exact_match": 59,
        "parent_safe_stop": 1,
        "right_tier1_wrong_tier2": 42,
        "wrong_deep_leaf": 15,
        "wrong_tier1": 39
      },
      "exact_path_accuracy": 0.3782,
      "parent_safe_accuracy": 0.4936,
      "tier1_accuracy": 0.75,
      "tier2_accuracy": 0.4808,
      "tier3_accuracy": 0.4259,
      "tier4_accuracy": 0.1667
    },
    "combined_path": {
      "average_prediction_depth": 2.4103,
      "error_buckets": {
        "exact_match": 59,
        "parent_safe_stop": 1,
        "right_tier1_wrong_tier2": 42,
        "wrong_deep_leaf": 15,
        "wrong_tier1": 39
      },
      "exact_path_accuracy": 0.3782,
      "fallback_overuse_count": 15,
      "fallback_rate": 0.0962,
      "parent_safe_accuracy": 0.4936,
      "tier1_accuracy": 0.75,
      "tier2_accuracy": 0.4808,
      "tier3_accuracy": 0.4259,
      "tier4_accuracy": 0.1667
    },
    "disagreements": {
      "classifier_vs_combined": 0
    },
    "shadow_embedding_retrieval": {
      "hint": "Set IAB_EVAL_INCLUDE_SHADOW_RETRIEVAL=1 to run shadow embedding retrieval (downloads/loads gte-Qwen2 when index is present).",
      "reason": "disabled_by_default",
      "skipped": true
    }
  }
}