data/retrieval_eval.json

{
  "hit_rate_at_1": 0.8,
  "hit_rate_at_2": 1.0,
  "n_queries": 10,
  "rows": [
    {
      "query": "How does an AS/RS crane retrieve a pallet?",
      "expected": "asrs_overview",
      "retrieved_top1": "asrs_overview",
      "hit@1": true,
      "hit@2": true,
      "top1_score": 0.2435
    },
    {
      "query": "What's the difference between an AGV and an AMR?",
      "expected": "agv_amr_overview",
      "retrieved_top1": "safety_protocol",
      "hit@1": false,
      "hit@2": true,
      "top1_score": 0.1499
    },
    {
      "query": "What does a WMS integrate with?",
      "expected": "wms_overview",
      "retrieved_top1": "wms_overview",
      "hit@1": true,
      "hit@2": true,
      "top1_score": 0.2741
    },
    {
      "query": "Why would a sorter jam?",
      "expected": "conveyor_sorting",
      "retrieved_top1": "conveyor_sorting",
      "hit@1": true,
      "hit@2": true,
      "top1_score": 0.0959
    },
    {
      "query": "What is batch picking?",
      "expected": "picking_strategies",
      "retrieved_top1": "picking_strategies",
      "hit@1": true,
      "hit@2": true,
      "top1_score": 0.3279
    },
    {
      "query": "How can we predict a motor failure before it happens?",
      "expected": "predictive_maintenance",
      "retrieved_top1": "conveyor_sorting",
      "hit@1": false,
      "hit@2": true,
      "top1_score": 0.1426
    },
    {
      "query": "What should I do after a near-miss with a forklift?",
      "expected": "safety_protocol",
      "retrieved_top1": "safety_protocol",
      "hit@1": true,
      "hit@2": true,
      "top1_score": 0.3443
    },
    {
      "query": "How do we keep inventory counts accurate?",
      "expected": "inventory_accuracy",
      "retrieved_top1": "inventory_accuracy",
      "hit@1": true,
      "hit@2": true,
      "top1_score": 0.2345
    },
    {
      "query": "What KPIs should a warehouse manager track?",
      "expected": "kpi_overview",
      "retrieved_top1": "kpi_overview",
      "hit@1": true,
      "hit@2": true,
      "top1_score": 0.2838
    },
    {
      "query": "How can automated warehouses save energy?",
      "expected": "energy_efficiency",
      "retrieved_top1": "energy_efficiency",
      "hit@1": true,
      "hit@2": true,
      "top1_score": 0.4366
    }
  ]
}