Upload eval_results_v3_verified.json with huggingface_hub

eval_results_v3_verified.json

@@ -0,0 +1,686 @@
+[
+  {
+    "id": 1,
+    "name": "WS-N1: Reminder sent \u2192 move to booking",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "FORK",
+    "correct": false,
+    "time": 2.75
+  },
+  {
+    "id": 2,
+    "name": "Mid workflow \u2014 after inspection",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "NEXT",
+    "correct": true,
+    "time": 0.75
+  },
+  {
+    "id": 3,
+    "name": "Near completion \u2014 final delivery step",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "NEXT",
+    "correct": true,
+    "time": 0.7
+  },
+  {
+    "id": 4,
+    "name": "Booking failed \u2014 customer didn't answer",
+    "category": "RETRY",
+    "expected": "RETRY",
+    "prediction": "RETRY",
+    "correct": true,
+    "time": 0.88
+  },
+  {
+    "id": 5,
+    "name": "Communication delivery failed",
+    "category": "RETRY",
+    "expected": "RETRY",
+    "prediction": "RETRY",
+    "correct": true,
+    "time": 0.87
+  },
+  {
+    "id": 6,
+    "name": "Parallel opportunity \u2014 two independent tasks",
+    "category": "FORK",
+    "expected": "FORK",
+    "prediction": "FORK",
+    "correct": true,
+    "time": 0.88
+  },
+  {
+    "id": 7,
+    "name": "Merge parallel branches",
+    "category": "JOIN",
+    "expected": "JOIN",
+    "prediction": "JOIN",
+    "correct": true,
+    "time": 0.8
+  },
+  {
+    "id": 8,
+    "name": "Join after both parallel tasks complete",
+    "category": "JOIN",
+    "expected": "JOIN",
+    "prediction": "JOIN",
+    "correct": true,
+    "time": 0.8
+  },
+  {
+    "id": 9,
+    "name": "Intent mismatch \u2014 customer asks about insurance",
+    "category": "META",
+    "expected": "META",
+    "prediction": "NEXT",
+    "correct": false,
+    "time": 0.81
+  },
+  {
+    "id": 10,
+    "name": "Missing capability \u2014 no actor can handle this",
+    "category": "META",
+    "expected": "META",
+    "prediction": "NEXT",
+    "correct": false,
+    "time": 0.74
+  },
+  {
+    "id": 11,
+    "name": "Last step to success \u2014 follow-up or finish",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "NEXT",
+    "correct": true,
+    "time": 0.81
+  },
+  {
+    "id": 12,
+    "name": "Post-fork branch progression \u2014 single branch advances",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "NEXT",
+    "correct": true,
+    "time": 0.7
+  },
+  {
+    "id": 13,
+    "name": "High SLA pressure \u2014 must advance quickly",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "NEXT",
+    "correct": true,
+    "time": 0.7
+  },
+  {
+    "id": 14,
+    "name": "Choose mandatory path over optional",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "NEXT",
+    "correct": true,
+    "time": 0.81
+  },
+  {
+    "id": 15,
+    "name": "Linear progression after retry recovery",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "NEXT",
+    "correct": true,
+    "time": 0.7
+  },
+  {
+    "id": 16,
+    "name": "Early workflow \u2014 low goal progress, clear path",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "NEXT",
+    "correct": true,
+    "time": 0.71
+  },
+  {
+    "id": 17,
+    "name": "Cost-constrained single path \u2014 budget nearly exhausted",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "NEXT",
+    "correct": true,
+    "time": 0.7
+  },
+  {
+    "id": 18,
+    "name": "Retry nearing exhaustion \u2014 3rd attempt",
+    "category": "RETRY",
+    "expected": "RETRY",
+    "prediction": "RETRY",
+    "correct": true,
+    "time": 0.89
+  },
+  {
+    "id": 19,
+    "name": "API call timeout \u2014 transient error retry",
+    "category": "RETRY",
+    "expected": "RETRY",
+    "prediction": "RETRY",
+    "correct": true,
+    "time": 0.88
+  },
+  {
+    "id": 20,
+    "name": "Partial data received \u2014 retry for completeness",
+    "category": "RETRY",
+    "expected": "RETRY",
+    "prediction": "RETRY",
+    "correct": true,
+    "time": 0.89
+  },
+  {
+    "id": 21,
+    "name": "Communication retry \u2014 first failure in notification",
+    "category": "RETRY",
+    "expected": "RETRY",
+    "prediction": "RETRY",
+    "correct": true,
+    "time": 0.89
+  },
+  {
+    "id": 22,
+    "name": "Semantic parallel \u2014 independent actors (AGENT + SYSTEM)",
+    "category": "FORK",
+    "expected": "FORK",
+    "prediction": "FORK",
+    "correct": true,
+    "time": 0.9
+  },
+  {
+    "id": 23,
+    "name": "Semantic parallel \u2014 independent resources (DEVICE + API)",
+    "category": "FORK",
+    "expected": "FORK",
+    "prediction": "FORK",
+    "correct": true,
+    "time": 0.91
+  },
+  {
+    "id": 24,
+    "name": "Semantic parallel \u2014 context-independent tasks",
+    "category": "FORK",
+    "expected": "FORK",
+    "prediction": "FORK",
+    "correct": true,
+    "time": 0.89
+  },
+  {
+    "id": 25,
+    "name": "Semantic parallel \u2014 three independent branches",
+    "category": "FORK",
+    "expected": "FORK",
+    "prediction": "FORK",
+    "correct": true,
+    "time": 0.9
+  },
+  {
+    "id": 26,
+    "name": "Fork opportunity under low resource pressure",
+    "category": "FORK",
+    "expected": "FORK",
+    "prediction": "FORK",
+    "correct": true,
+    "time": 0.89
+  },
+  {
+    "id": 27,
+    "name": "Nested fork \u2014 already in parallel, fork deeper",
+    "category": "FORK",
+    "expected": "FORK",
+    "prediction": "FORK",
+    "correct": true,
+    "time": 0.91
+  },
+  {
+    "id": 28,
+    "name": "Fork with different actor types \u2014 HUMAN + SYSTEM",
+    "category": "FORK",
+    "expected": "FORK",
+    "prediction": "FORK",
+    "correct": true,
+    "time": 0.93
+  },
+  {
+    "id": 29,
+    "name": "Semantic join \u2014 downstream needs both outputs",
+    "category": "JOIN",
+    "expected": "JOIN",
+    "prediction": "JOIN",
+    "correct": true,
+    "time": 0.82
+  },
+  {
+    "id": 30,
+    "name": "Semantic join \u2014 multi-input dependency merge",
+    "category": "JOIN",
+    "expected": "JOIN",
+    "prediction": "JOIN",
+    "correct": true,
+    "time": 0.82
+  },
+  {
+    "id": 31,
+    "name": "Deep parallel join \u2014 parallel_depth=2",
+    "category": "JOIN",
+    "expected": "JOIN",
+    "prediction": "JOIN",
+    "correct": true,
+    "time": 0.81
+  },
+  {
+    "id": 32,
+    "name": "Join after retry in one branch",
+    "category": "JOIN",
+    "expected": "JOIN",
+    "prediction": "JOIN",
+    "correct": true,
+    "time": 0.81
+  },
+  {
+    "id": 33,
+    "name": "Join with high resource pressure \u2014 eager to merge",
+    "category": "JOIN",
+    "expected": "JOIN",
+    "prediction": "JOIN",
+    "correct": true,
+    "time": 0.81
+  },
+  {
+    "id": 34,
+    "name": "Join \u2014 both branches complete, clear dependency",
+    "category": "JOIN",
+    "expected": "JOIN",
+    "prediction": "JOIN",
+    "correct": true,
+    "time": 0.81
+  },
+  {
+    "id": 35,
+    "name": "Anomaly \u2014 policy violation detected",
+    "category": "META",
+    "expected": "META",
+    "prediction": "VIOLATION",
+    "correct": false,
+    "time": 0.99
+  },
+  {
+    "id": 36,
+    "name": "Anomaly \u2014 external dependency unavailable",
+    "category": "META",
+    "expected": "META",
+    "prediction": "RETRY",
+    "correct": false,
+    "time": 0.91
+  },
+  {
+    "id": 37,
+    "name": "Anomaly \u2014 unsupported request type",
+    "category": "META",
+    "expected": "META",
+    "prediction": "NEXT",
+    "correct": false,
+    "time": 0.83
+  },
+  {
+    "id": 38,
+    "name": "Anomaly \u2014 data integrity failure",
+    "category": "META",
+    "expected": "META",
+    "prediction": "NEXT",
+    "correct": false,
+    "time": 0.83
+  },
+  {
+    "id": 39,
+    "name": "Anomaly \u2014 security concern mid-workflow",
+    "category": "META",
+    "expected": "META",
+    "prediction": "NEXT",
+    "correct": false,
+    "time": 0.86
+  },
+  {
+    "id": 40,
+    "name": "Anomaly \u2014 contradictory outcomes from parallel branches",
+    "category": "META",
+    "expected": "META",
+    "prediction": "RETRY",
+    "correct": false,
+    "time": 0.94
+  },
+  {
+    "id": 41,
+    "name": "PB-N1: Fork opportunity blocked by high resource pressure",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "FORK",
+    "correct": false,
+    "time": 0.97
+  },
+  {
+    "id": 42,
+    "name": "PB-N2: Fork opportunity blocked by high uncertainty",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "FORK",
+    "correct": false,
+    "time": 0.89
+  },
+  {
+    "id": 43,
+    "name": "PB-N3: Fork opportunity blocked \u2014 already in parallel",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "FORK",
+    "correct": false,
+    "time": 0.89
+  },
+  {
+    "id": 44,
+    "name": "PB-N4: Fork opportunity blocked \u2014 high cost budget consumed",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "FORK",
+    "correct": false,
+    "time": 0.92
+  },
+  {
+    "id": 45,
+    "name": "PB-N5: Join topology present but no active parallel \u2014 just advance",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "JOIN",
+    "correct": false,
+    "time": 0.76
+  },
+  {
+    "id": 46,
+    "name": "PB-N6: Retry exhausted \u2014 failure but max retries hit \u2192 advance",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "RETRY",
+    "correct": false,
+    "time": 0.9
+  },
+  {
+    "id": 47,
+    "name": "PB-N7: Forkable + join_ready present simultaneously \u2014 prioritize join",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "JOIN",
+    "correct": false,
+    "time": 0.81
+  },
+  {
+    "id": 48,
+    "name": "PB-N8: High SLA pressure forces skip of optional parallel",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "FORK",
+    "correct": false,
+    "time": 0.9
+  },
+  {
+    "id": 49,
+    "name": "PB-R1: Failure on first attempt \u2014 low pressure, retry obvious",
+    "category": "RETRY",
+    "expected": "RETRY",
+    "prediction": "RETRY",
+    "correct": true,
+    "time": 0.91
+  },
+  {
+    "id": 50,
+    "name": "PB-R2: Retry under moderate SLA pressure \u2014 still worthwhile",
+    "category": "RETRY",
+    "expected": "RETRY",
+    "prediction": "RETRY",
+    "correct": true,
+    "time": 0.93
+  },
+  {
+    "id": 51,
+    "name": "PB-R3: Retry inside parallel branch \u2014 failure doesn't block join",
+    "category": "RETRY",
+    "expected": "RETRY",
+    "prediction": "RETRY",
+    "correct": true,
+    "time": 0.89
+  },
+  {
+    "id": 52,
+    "name": "PB-R4: Retry after soft failure \u2014 data quality issue",
+    "category": "RETRY",
+    "expected": "RETRY",
+    "prediction": "RETRY",
+    "correct": true,
+    "time": 0.92
+  },
+  {
+    "id": 53,
+    "name": "PB-F1: Classic fork \u2014 low pressure, actors independent, no parallel yet",
+    "category": "FORK",
+    "expected": "FORK",
+    "prediction": "FORK",
+    "correct": true,
+    "time": 0.95
+  },
+  {
+    "id": 54,
+    "name": "PB-F2: Fork after successful checkpoint \u2014 resources freed",
+    "category": "FORK",
+    "expected": "FORK",
+    "prediction": "FORK",
+    "correct": true,
+    "time": 0.9
+  },
+  {
+    "id": 55,
+    "name": "PB-F3: Fork with three branches \u2014 all actors different",
+    "category": "FORK",
+    "expected": "FORK",
+    "prediction": "FORK",
+    "correct": true,
+    "time": 0.9
+  },
+  {
+    "id": 56,
+    "name": "PB-F4: Fork opportunity \u2014 moderate goal progress, low risk",
+    "category": "FORK",
+    "expected": "FORK",
+    "prediction": "FORK",
+    "correct": true,
+    "time": 0.94
+  },
+  {
+    "id": 57,
+    "name": "PB-J1: Classic join \u2014 parallel active, join_ready, both branches done",
+    "category": "JOIN",
+    "expected": "JOIN",
+    "prediction": "JOIN",
+    "correct": true,
+    "time": 0.84
+  },
+  {
+    "id": 58,
+    "name": "PB-J2: Join under SLA pressure \u2014 merge quickly",
+    "category": "JOIN",
+    "expected": "JOIN",
+    "prediction": "JOIN",
+    "correct": true,
+    "time": 0.82
+  },
+  {
+    "id": 59,
+    "name": "PB-J3: Join at high parallel depth \u2014 reduce complexity",
+    "category": "JOIN",
+    "expected": "JOIN",
+    "prediction": "JOIN",
+    "correct": true,
+    "time": 0.83
+  },
+  {
+    "id": 60,
+    "name": "PB-J4: Join with post-retry convergence \u2014 one branch recovered",
+    "category": "JOIN",
+    "expected": "JOIN",
+    "prediction": "JOIN",
+    "correct": true,
+    "time": 0.82
+  },
+  {
+    "id": 61,
+    "name": "PB-M1: Very high uncertainty + anomaly outcome \u2192 clear META",
+    "category": "META",
+    "expected": "META",
+    "prediction": "NEXT",
+    "correct": false,
+    "time": 0.84
+  },
+  {
+    "id": 62,
+    "name": "PB-M2: Missing context \u2014 no eligible node can satisfy requirement",
+    "category": "META",
+    "expected": "META",
+    "prediction": "NEXT",
+    "correct": false,
+    "time": 0.88
+  },
+  {
+    "id": 63,
+    "name": "PB-M3: Repeated failures + high uncertainty \u2192 META not RETRY",
+    "category": "META",
+    "expected": "META",
+    "prediction": "RETRY",
+    "correct": false,
+    "time": 0.91
+  },
+  {
+    "id": 64,
+    "name": "PB-M4: Regulatory compliance flag \u2014 must halt for review",
+    "category": "META",
+    "expected": "META",
+    "prediction": "NEXT",
+    "correct": false,
+    "time": 0.83
+  },
+  {
+    "id": 65,
+    "name": "PB-X1: Ambiguous outcome \u2014 moderate uncertainty \u2192 NEXT (not META)",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "NEXT",
+    "correct": true,
+    "time": 0.81
+  },
+  {
+    "id": 66,
+    "name": "PB-X2: Success after many retries \u2014 just advance (not retry again)",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "NEXT",
+    "correct": true,
+    "time": 0.69
+  },
+  {
+    "id": 67,
+    "name": "PB-X3: Multiple eligible including terminator \u2014 mid-progress, keep going",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "CONTINUE",
+    "correct": false,
+    "time": 0.94
+  },
+  {
+    "id": 68,
+    "name": "PB-X4: Low progress + terminator available \u2192 still advance (too early to quit)",
+    "category": "NEXT",
+    "expected": "NEXT",
+    "prediction": "NEXT",
+    "correct": true,
+    "time": 0.82
+  },
+  {
+    "id": 69,
+    "name": "PB-R5: Retry with uncertainty decreasing \u2014 transient issue resolving",
+    "category": "RETRY",
+    "expected": "RETRY",
+    "prediction": "RETRY",
+    "correct": true,
+    "time": 0.9
+  },
+  {
+    "id": 70,
+    "name": "PB-R6: Retry in parallel branch \u2014 failure doesn't impact other branch",
+    "category": "RETRY",
+    "expected": "RETRY",
+    "prediction": "RETRY",
+    "correct": true,
+    "time": 0.88
+  },
+  {
+    "id": 71,
+    "name": "PB-F5: Fork at workflow start \u2014 fresh resources, multiple entry paths",
+    "category": "FORK",
+    "expected": "FORK",
+    "prediction": "FORK",
+    "correct": true,
+    "time": 0.93
+  },
+  {
+    "id": 72,
+    "name": "PB-F6: Fork after successful resolution \u2014 reward parallelism",
+    "category": "FORK",
+    "expected": "FORK",
+    "prediction": "FORK",
+    "correct": true,
+    "time": 0.93
+  },
+  {
+    "id": 73,
+    "name": "PB-J5: Join with cost pressure \u2014 merge to stop spending",
+    "category": "JOIN",
+    "expected": "JOIN",
+    "prediction": "JOIN",
+    "correct": true,
+    "time": 0.8
+  },
+  {
+    "id": 74,
+    "name": "PB-J6: Join at end of workflow \u2014 final merge before terminal",
+    "category": "JOIN",
+    "expected": "JOIN",
+    "prediction": "JOIN",
+    "correct": true,
+    "time": 0.8
+  },
+  {
+    "id": 75,
+    "name": "PB-J7: Join with low resource pressure \u2014 clean merge",
+    "category": "JOIN",
+    "expected": "JOIN",
+    "prediction": "JOIN",
+    "correct": true,
+    "time": 0.81
+  },
+  {
+    "id": 76,
+    "name": "PB-M5: Timeout escalation \u2014 multiple timeouts indicate systemic issue",
+    "category": "META",
+    "expected": "META",
+    "prediction": "RETRY",
+    "correct": false,
+    "time": 0.88
+  }
+]