Upload local-intelligence eval results for 69d27a3e074a09ff9be109a9

eval/local_intelligence/results/jobs/69d27a3e074a09ff9be109a9/nousresearch-hermes-3-llama-3-1-8b.json

@@ -0,0 +1,1067 @@
+{
+  "summary": {
+    "provider": "hf-transformers",
+    "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+    "cards": 24,
+    "passed": 7,
+    "pass_rate": 0.2917,
+    "average_score": 0.4167,
+    "median_latency_ms": 2874.14,
+    "exact_parse_rate": 0.875,
+    "normalized_parse_rate": 0.0,
+    "parse_failure_rate": 0.125
+  },
+  "rows": [
+    {
+      "card_id": "memo-auto-title",
+      "title": "Memo Auto-Title (<1s)",
+      "tier": "ship-soon",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 1105.89,
+      "raw_text": "{\n  \"title\": \"Book Dentist & Email Form\"\n}",
+      "grade": {
+        "passed": true,
+        "score": 1.0,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": null,
+        "assertions": [
+          {
+            "assertion": "title is not \"Voice Memo\"",
+            "passed": true,
+            "details": "title='Book Dentist & Email Form'"
+          },
+          {
+            "assertion": "title length <= 48",
+            "passed": true,
+            "details": "len(title)=25"
+          },
+          {
+            "assertion": "title contains either \"dentist\" or \"insurance\"",
+            "passed": true,
+            "details": "title='book dentist & email form'"
+          }
+        ],
+        "parsed": {
+          "title": "Book Dentist & Email Form"
+        }
+      }
+    },
+    {
+      "card_id": "memo-type-detection",
+      "title": "Memo Type Detection",
+      "tier": "ship-soon",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 1024.1,
+      "raw_text": "{\n  \"primaryType\": \"journal\",\n  \"confidence\": 0.85,\n  \"needsReview\": false\n}",
+      "grade": {
+        "passed": false,
+        "score": 0.6667,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": "task",
+        "assertions": [
+          {
+            "assertion": "primaryType is either \"journal\" or \"idea\"",
+            "passed": true,
+            "details": "primaryType='journal'"
+          },
+          {
+            "assertion": "confidence < 0.8",
+            "passed": false,
+            "details": "confidence=0.85"
+          },
+          {
+            "assertion": "needsReview can be true when uncertainty is high",
+            "passed": true,
+            "details": "needsReview=False"
+          }
+        ],
+        "parsed": {
+          "primaryType": "journal",
+          "confidence": 0.85,
+          "needsReview": false
+        }
+      }
+    },
+    {
+      "card_id": "action-item-extraction",
+      "title": "Action Item Extraction With Confidence",
+      "tier": "ship-soon",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 2228.12,
+      "raw_text": "[\n  {\n    \"text\": \"Clean up the old docs folder\",\n    \"owner\": \"Arach\",\n    \"dueDateISO\": null,\n    \"confidence\": 0.95,\n    \"evidence\": \"Maybe clean up the old docs folder sometime.\"\n  }\n]",
+      "grade": {
+        "passed": false,
+        "score": 0.6667,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": "task",
+        "assertions": [
+          {
+            "assertion": "items length >= 1",
+            "passed": true,
+            "details": "items=1"
+          },
+          {
+            "assertion": "item confidence < 0.75 for vague tasks",
+            "passed": false,
+            "details": "items=[{'text': 'Clean up the old docs folder', 'owner': 'Arach', 'dueDateISO': None, 'confidence': 0.95, 'evidence': 'Maybe clean up the old docs folder sometime.'}]"
+          },
+          {
+            "assertion": "evidence field is always present",
+            "passed": true,
+            "details": "items=[{'text': 'Clean up the old docs folder', 'owner': 'Arach', 'dueDateISO': None, 'confidence': 0.95, 'evidence': 'Maybe clean up the old docs folder sometime.'}]"
+          }
+        ],
+        "parsed": {
+          "items": [
+            {
+              "text": "Clean up the old docs folder",
+              "owner": "Arach",
+              "dueDateISO": null,
+              "confidence": 0.95,
+              "evidence": "Maybe clean up the old docs folder sometime."
+            }
+          ]
+        }
+      }
+    },
+    {
+      "card_id": "transcript-cleanup-presets",
+      "title": "Transcript Cleanup Presets",
+      "tier": "ship-soon",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 799.28,
+      "raw_text": "* Fixed 3 out of 5 export bugs\n* Spent 4 hours on bug fixing",
+      "grade": {
+        "passed": false,
+        "score": 0.0,
+        "parse_error": "Expecting value: line 1 column 1 (char 0)",
+        "parse_mode": "failed",
+        "format_recovered": false,
+        "failure_kind": "parse",
+        "assertions": [
+          {
+            "assertion": "output contains number 4",
+            "passed": false,
+            "details": "output was not valid JSON"
+          },
+          {
+            "assertion": "output contains phrase 3 of 5",
+            "passed": false,
+            "details": "output was not valid JSON"
+          },
+          {
+            "assertion": "filler words are not introduced",
+            "passed": false,
+            "details": "output was not valid JSON"
+          }
+        ],
+        "parsed": null
+      }
+    },
+    {
+      "card_id": "what-matters-summary",
+      "title": "Top-3 What Matters Summary",
+      "tier": "ship-soon",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 5511.21,
+      "raw_text": "{\n  \"summary\": [\n    {\n      \"point\": \"The transcript contains four decisions.\",\n      \"evidence\": \"Decision one. Decision two. Decision three. Decision four.\",\n      \"priority\": 100\n    },\n    {\n      \"point\": \"No additional context or details are provided about the nature of the decisions.\",\n      \"evidence\": \"No additional context or details are provided.\",\n      \"priority\": 80\n    },\n    {\n      \"point\": \"The decisions are listed in sequential order, suggesting they may be part of a series or process.\",\n      \"evidence\": \"Decision one. Decision two. Decision three. Decision four.\",\n      \"priority\": 90\n    }\n  ]\n}",
+      "grade": {
+        "passed": true,
+        "score": 1.0,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": null,
+        "assertions": [
+          {
+            "assertion": "topPoints length equals 3",
+            "passed": true,
+            "details": "topPoints=3"
+          },
+          {
+            "assertion": "priority values are numeric",
+            "passed": true,
+            "details": "topPoints=[{'point': 'The transcript contains four decisions.', 'priority': 100, 'evidence': 'Decision one. Decision two. Decision three. Decision four.'}, {'point': 'No additional context or details are provided about the nature of the decisions.', 'priority': 80, 'evidence': 'No additional context or details are provided.'}, {'point': 'The decisions are listed in sequential order, suggesting they may be part of a series or process.', 'priority': 90, 'evidence': 'Decision one. Decision two. Decision three. Decision four.'}]"
+          },
+          {
+            "assertion": "each point includes evidence",
+            "passed": true,
+            "details": "topPoints=[{'point': 'The transcript contains four decisions.', 'priority': 100, 'evidence': 'Decision one. Decision two. Decision three. Decision four.'}, {'point': 'No additional context or details are provided about the nature of the decisions.', 'priority': 80, 'evidence': 'No additional context or details are provided.'}, {'point': 'The decisions are listed in sequential order, suggesting they may be part of a series or process.', 'priority': 90, 'evidence': 'Decision one. Decision two. Decision three. Decision four.'}]"
+          }
+        ],
+        "parsed": {
+          "summary": [
+            {
+              "point": "The transcript contains four decisions.",
+              "evidence": "Decision one. Decision two. Decision three. Decision four.",
+              "priority": 100
+            },
+            {
+              "point": "No additional context or details are provided about the nature of the decisions.",
+              "evidence": "No additional context or details are provided.",
+              "priority": 80
+            },
+            {
+              "point": "The decisions are listed in sequential order, suggesting they may be part of a series or process.",
+              "evidence": "Decision one. Decision two. Decision three. Decision four.",
+              "priority": 90
+            }
+          ],
+          "topPoints": [
+            {
+              "point": "The transcript contains four decisions.",
+              "priority": 100,
+              "evidence": "Decision one. Decision two. Decision three. Decision four."
+            },
+            {
+              "point": "No additional context or details are provided about the nature of the decisions.",
+              "priority": 80,
+              "evidence": "No additional context or details are provided."
+            },
+            {
+              "point": "The decisions are listed in sequential order, suggesting they may be part of a series or process.",
+              "priority": 90,
+              "evidence": "Decision one. Decision two. Decision three. Decision four."
+            }
+          ]
+        }
+      }
+    },
+    {
+      "card_id": "calendar-intent-detection",
+      "title": "Calendar Intent Detection",
+      "tier": "ship-soon",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 2143.84,
+      "raw_text": "{\n  \"intents\": {\n    \"date_time\": {\n      \"value\": \"2026-04-05T14:00:00-04:00\",\n      \"confidence\": 1.0,\n      \"needsConfirmation\": false\n    }\n  }\n}",
+      "grade": {
+        "passed": false,
+        "score": 0.0,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": "task",
+        "assertions": [
+          {
+            "assertion": "startAtISO date is 2026-04-05",
+            "passed": false,
+            "details": "startAtISO=''"
+          },
+          {
+            "assertion": "timezone offset in startAtISO is -04:00",
+            "passed": false,
+            "details": "startAtISO=''"
+          },
+          {
+            "assertion": "event title mentions Sam",
+            "passed": false,
+            "details": "title=''"
+          }
+        ],
+        "parsed": {
+          "intents": {
+            "date_time": {
+              "value": "2026-04-05T14:00:00-04:00",
+              "confidence": 1.0,
+              "needsConfirmation": false
+            }
+          }
+        }
+      }
+    },
+    {
+      "card_id": "reminder-normalization",
+      "title": "Reminder Phrasing Normalization",
+      "tier": "ship-soon",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 2410.13,
+      "raw_text": "{\n  \"reminder\": \"Review new card design options\",\n  \"dueDateISO\": null,\n  \"dueDate\": \"soon\",\n  \"followUpQuestion\": \"When would be a good time to explore new card designs?\",\n  \"context\": \"Maybe look into new card design at some point.\"\n}",
+      "grade": {
+        "passed": false,
+        "score": 0.6667,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": "task",
+        "assertions": [
+          {
+            "assertion": "reminderText starts with a verb",
+            "passed": false,
+            "details": "reminderText=''"
+          },
+          {
+            "assertion": "reminderText does not contain \"maybe\"",
+            "passed": true,
+            "details": "reminderText=''"
+          },
+          {
+            "assertion": "followUpQuestion exists when due date is missing",
+            "passed": true,
+            "details": "followUpQuestion='When would be a good time to explore new card designs?'"
+          }
+        ],
+        "parsed": {
+          "reminder": "Review new card design options",
+          "dueDateISO": null,
+          "dueDate": "soon",
+          "followUpQuestion": "When would be a good time to explore new card designs?",
+          "context": "Maybe look into new card design at some point."
+        }
+      }
+    },
+    {
+      "card_id": "private-redaction-pass",
+      "title": "Private Redaction Pass",
+      "tier": "ship-soon",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 4367.95,
+      "raw_text": "{\n  \"redacted\": \"SSN [REDACTED_TYPE:SSN] and account [REDCATED_TYPE:ACCOUNT_NUMBER:00391299] should never leave device.\",\n  \"redacted_entities\": [\n    {\n      \"text\": \"SSN 555-44-9012\",\n      \"type\": \"SSN\",\n      \"confidence\": 1.0\n    },\n    {\n      \"text\": \"00391299\",\n      \"type\": \"ACCOUNT_NUMBER\",\n      \"confidence\": 1.0\n    }\n  ]\n}",
+      "grade": {
+        "passed": true,
+        "score": 1.0,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": null,
+        "assertions": [
+          {
+            "assertion": "redactedTranscript does not contain 555-44-9012",
+            "passed": true,
+            "details": "redactedTranscript=''"
+          },
+          {
+            "assertion": "redactedTranscript does not contain 00391299",
+            "passed": true,
+            "details": "redactedTranscript=''"
+          },
+          {
+            "assertion": "entities length >= 2",
+            "passed": true,
+            "details": "entities=[{'value': None, 'type': 'SSN', 'confidence': 1.0}, {'value': None, 'type': 'ACCOUNT_NUMBER', 'confidence': 1.0}]"
+          }
+        ],
+        "parsed": {
+          "redacted": "SSN [REDACTED_TYPE:SSN] and account [REDCATED_TYPE:ACCOUNT_NUMBER:00391299] should never leave device.",
+          "redacted_entities": [
+            {
+              "text": "SSN 555-44-9012",
+              "type": "SSN",
+              "confidence": 1.0
+            },
+            {
+              "text": "00391299",
+              "type": "ACCOUNT_NUMBER",
+              "confidence": 1.0
+            }
+          ],
+          "entities": [
+            {
+              "value": null,
+              "type": "SSN",
+              "confidence": 1.0
+            },
+            {
+              "value": null,
+              "type": "ACCOUNT_NUMBER",
+              "confidence": 1.0
+            }
+          ]
+        }
+      }
+    },
+    {
+      "card_id": "writing-style-memory",
+      "title": "Per-User Writing Style Memory",
+      "tier": "local-first",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 575.76,
+      "raw_text": "{\n  \"response\": \"Try a new onboarding flow.\"\n}",
+      "grade": {
+        "passed": true,
+        "score": 1.0,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": null,
+        "assertions": [
+          {
+            "assertion": "styledSummary does not contain \"utilize\"",
+            "passed": true,
+            "details": "styledSummary=''"
+          },
+          {
+            "assertion": "output preserves onboarding meaning",
+            "passed": true,
+            "details": "{\"response\": \"Try a new onboarding flow.\"}"
+          }
+        ],
+        "parsed": {
+          "response": "Try a new onboarding flow."
+        }
+      }
+    },
+    {
+      "card_id": "similar-memo-recall",
+      "title": "Find Similar Memo (Semantic Recall)",
+      "tier": "local-first",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 4365.49,
+      "raw_text": "[\n  {\n    \"memoId\": \"a\",\n    \"relevanceScore\": 0.8,\n    \"rationale\": \"This memo directly addresses the topic of cancellation policy, which is most relevant to the current need to revisit the cancellation policy and trials.\"\n  },\n  {\n    \"memoId\": \"b\",\n    \"relevanceScore\": 0.2,\n    \"rationale\": \"While this memo mentions discussing a policy page, it does not specifically mention the cancellation policy or trials, making it less relevant to the current need.\"\n  }\n]",
+      "grade": {
+        "passed": false,
+        "score": 0.0,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": "task",
+        "assertions": [
+          {
+            "assertion": "memoId \"a\" ranks first",
+            "passed": false,
+            "details": "first=None"
+          },
+          {
+            "assertion": "top match relevance > 0.8",
+            "passed": false,
+            "details": "relevance=None"
+          }
+        ],
+        "parsed": [
+          {
+            "memoId": "a",
+            "relevanceScore": 0.8,
+            "rationale": "This memo directly addresses the topic of cancellation policy, which is most relevant to the current need to revisit the cancellation policy and trials."
+          },
+          {
+            "memoId": "b",
+            "relevanceScore": 0.2,
+            "rationale": "While this memo mentions discussing a policy page, it does not specifically mention the cancellation policy or trials, making it less relevant to the current need."
+          }
+        ]
+      }
+    },
+    {
+      "card_id": "project-clustering",
+      "title": "Project Clustering From Voice History",
+      "tier": "local-first",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 3590.36,
+      "raw_text": "[\n  {\n    \"clusterId\": \"cluster1\",\n    \"title\": \"Sync Queue Issues\",\n    \"memoIds\": [\n      \"a1\",\n      \"a2\"\n    ],\n    \"confidence\": 1.0\n  },\n  {\n    \"clusterId\": \"cluster2\",\n    \"title\": \"Miscellaneous\",\n    \"memoIds\": [\n      \"b1\"\n    ],\n    \"confidence\": 0.5\n  }\n]",
+      "grade": {
+        "passed": true,
+        "score": 1.0,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": null,
+        "assertions": [
+          {
+            "assertion": "a1 and a2 share a clusterId",
+            "passed": true,
+            "details": "mapping={'a1': 'cluster1', 'a2': 'cluster1', 'b1': 'cluster2'}"
+          },
+          {
+            "assertion": "b1 may be separate or misc",
+            "passed": true,
+            "details": "non-blocking assertion"
+          }
+        ],
+        "parsed": {
+          "clusters": [
+            {
+              "clusterId": "cluster1",
+              "title": "Sync Queue Issues",
+              "memoIds": [
+                "a1",
+                "a2"
+              ],
+              "confidence": 1.0
+            },
+            {
+              "clusterId": "cluster2",
+              "title": "Miscellaneous",
+              "memoIds": [
+                "b1"
+              ],
+              "confidence": 0.5
+            }
+          ]
+        }
+      }
+    },
+    {
+      "card_id": "next-step-suggestion-engine",
+      "title": "Next-Step Suggestion Engine",
+      "tier": "local-first",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 386.99,
+      "raw_text": "{\n  \"action\": \"schedule\"\n}",
+      "grade": {
+        "passed": false,
+        "score": 0.5,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": "task",
+        "assertions": [
+          {
+            "assertion": "action is either \"extract-tasks\" or \"schedule\"",
+            "passed": true,
+            "details": "action='schedule'"
+          },
+          {
+            "assertion": "reason references explicit deadline",
+            "passed": false,
+            "details": "reason=''"
+          }
+        ],
+        "parsed": {
+          "action": "schedule"
+        }
+      }
+    },
+    {
+      "card_id": "follow-up-question-generator",
+      "title": "Follow-Up Question Generator",
+      "tier": "local-first",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 839.7,
+      "raw_text": "Here is a clarifying question:\nWhat specific aspects of the onboarding process would you like to improve?",
+      "grade": {
+        "passed": false,
+        "score": 0.0,
+        "parse_error": "Expecting value: line 1 column 1 (char 0)",
+        "parse_mode": "failed",
+        "format_recovered": false,
+        "failure_kind": "parse",
+        "assertions": [
+          {
+            "assertion": "questions length equals 1",
+            "passed": false,
+            "details": "output was not valid JSON"
+          },
+          {
+            "assertion": "question ends with ?",
+            "passed": false,
+            "details": "output was not valid JSON"
+          }
+        ],
+        "parsed": null
+      }
+    },
+    {
+      "card_id": "hybrid-model-router",
+      "title": "Hybrid Local/Cloud Router",
+      "tier": "local-first",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 4901.52,
+      "raw_text": "{\n  \"requestId\": \"12345\",\n  \"userId\": \"user123\",\n  \"timestamp\": \"2023-04-24T12:34:56.789Z\",\n  \"language\": \"en\",\n  \"device\": {\n    \"type\": \"smartphone\",\n    \"os\": \"iOS\",\n    \"version\": \"15.7\"\n  },\n  \"location\": {\n    \"country\": \"US\",\n    \"region\": \"California\",\n    \"city\": \"San Francisco\"\n  },\n  \"privacyLevel\": \"high\",\n  \"query\": \"What is the weather like in San Francisco today?\"\n}",
+      "grade": {
+        "passed": false,
+        "score": 0.0,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": "task",
+        "assertions": [
+          {
+            "assertion": "route equals \"cloud\"",
+            "passed": false,
+            "details": "route=None"
+          },
+          {
+            "assertion": "reason mentions long context",
+            "passed": false,
+            "details": "reason=''"
+          }
+        ],
+        "parsed": {
+          "requestId": "12345",
+          "userId": "user123",
+          "timestamp": "2023-04-24T12:34:56.789Z",
+          "language": "en",
+          "device": {
+            "type": "smartphone",
+            "os": "iOS",
+            "version": "15.7"
+          },
+          "location": {
+            "country": "US",
+            "region": "California",
+            "city": "San Francisco"
+          },
+          "privacyLevel": "high",
+          "query": "What is the weather like in San Francisco today?"
+        }
+      }
+    },
+    {
+      "card_id": "context-packet-builder",
+      "title": "Context Packet Builder",
+      "tier": "local-first",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 7359.98,
+      "raw_text": "{\n  \"decisions\": [],\n  \"tasks\": [\n    \"Research and define the roles and responsibilities for each position\",\n    \"Create a job description for each position\",\n    \"Set a hiring timeline and process\",\n    \"Develop a budget for hiring and onboarding\",\n    \"Outline the launch plan, including key milestones and deadlines\",\n    \"Draft initial documentation for the product or service\",\n    \"Identify key support channels and processes\",\n    \"Research and select pricing models and tiers\",\n    \"Set up analytics tracking and reporting\",\n    \"Assign tasks to team members\"\n  ],\n  \"questions\": [\n    \"What is the target launch date?\",\n    \"How many positions need to be filled?\",\n    \"What is the budget for each position?\",\n    \"What documentation needs to be completed before launch?\",\n    \"What support channels will be available?\",\n    \"How will pricing be communicated to customers?\",\n    \"What metrics will be tracked in analytics?\"\n  ]\n}",
+      "grade": {
+        "passed": false,
+        "score": 0.0,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": "task",
+        "assertions": [
+          {
+            "assertion": "tokenEstimate <= 120",
+            "passed": false,
+            "details": "tokenEstimate=None"
+          },
+          {
+            "assertion": "packet has at least one decision or openQuestion field",
+            "passed": false,
+            "details": "packet={'decisions': [], 'tasks': ['Research and define the roles and responsibilities for each position', 'Create a job description for each position', 'Set a hiring timeline and process', 'Develop a budget for hiring and onboarding', 'Outline the launch plan, including key milestones and deadlines', 'Draft initial documentation for the product or service', 'Identify key support channels and processes', 'Research and select pricing models and tiers', 'Set up analytics tracking and reporting', 'Assign tasks to team members'], 'questions': ['What is the target launch date?', 'How many positions need to be filled?', 'What is the budget for each position?', 'What documentation needs to be completed before launch?', 'What support channels will be available?', 'How will pricing be communicated to customers?', 'What metrics will be tracked in analytics?']}"
+          }
+        ],
+        "parsed": {
+          "decisions": [],
+          "tasks": [
+            "Research and define the roles and responsibilities for each position",
+            "Create a job description for each position",
+            "Set a hiring timeline and process",
+            "Develop a budget for hiring and onboarding",
+            "Outline the launch plan, including key milestones and deadlines",
+            "Draft initial documentation for the product or service",
+            "Identify key support channels and processes",
+            "Research and select pricing models and tiers",
+            "Set up analytics tracking and reporting",
+            "Assign tasks to team members"
+          ],
+          "questions": [
+            "What is the target launch date?",
+            "How many positions need to be filled?",
+            "What is the budget for each position?",
+            "What documentation needs to be completed before launch?",
+            "What support channels will be available?",
+            "How will pricing be communicated to customers?",
+            "What metrics will be tracked in analytics?"
+          ]
+        }
+      }
+    },
+    {
+      "card_id": "conversation-to-checklist",
+      "title": "Conversation To Checklist Transformer",
+      "tier": "local-first",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 5068.97,
+      "raw_text": "[\n  {\n    \"id\": 1,\n    \"text\": \"Export data\",\n    \"order\": 1,\n    \"dependsOn\": [],\n    \"owner\": \"Arach\"\n  },\n  {\n    \"id\": 2,\n    \"text\": \"Verify CSV\",\n    \"order\": 2,\n    \"dependsOn\": [\n      1\n    ],\n    \"owner\": \"Arach\"\n  },\n  {\n    \"id\": 3,\n    \"text\": \"Email finance\",\n    \"order\": 3,\n    \"dependsOn\": [\n      2\n    ],\n    \"owner\": \"Arach\"\n  }\n]",
+      "grade": {
+        "passed": false,
+        "score": 0.0,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": "task",
+        "assertions": [
+          {
+            "assertion": "order values are 1,2,3",
+            "passed": false,
+            "details": "orders=[]"
+          },
+          {
+            "assertion": "item 2 depends on item 1",
+            "passed": false,
+            "details": "dependsOn=[]"
+          },
+          {
+            "assertion": "item 3 depends on item 2 or item 1",
+            "passed": false,
+            "details": "dependsOn=[]"
+          }
+        ],
+        "parsed": [
+          {
+            "id": 1,
+            "text": "Export data",
+            "order": 1,
+            "dependsOn": [],
+            "owner": "Arach"
+          },
+          {
+            "id": 2,
+            "text": "Verify CSV",
+            "order": 2,
+            "dependsOn": [
+              1
+            ],
+            "owner": "Arach"
+          },
+          {
+            "id": 3,
+            "text": "Email finance",
+            "order": 3,
+            "dependsOn": [
+              2
+            ],
+            "owner": "Arach"
+          }
+        ]
+      }
+    },
+    {
+      "card_id": "personal-knowledge-graph",
+      "title": "Personal Knowledge Graph From Voice",
+      "tier": "big-swings",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 7171.11,
+      "raw_text": "{\n  \"nodes\": [\n    {\n      \"id\": \"n1\",\n      \"type\": \"person\",\n      \"name\": \"Chris\"\n    },\n    {\n      \"id\": \"n2\",\n      \"type\": \"person\",\n      \"name\": \"Assistant\"\n    },\n    {\n      \"id\": \"n3\",\n      \"type\": \"decision\",\n      \"name\": \"Pricing\"\n    }\n  ],\n  \"edges\": [\n    {\n      \"id\": \"e1\",\n      \"type\": \"asked\",\n      \"sourceNodeId\": \"n2\",\n      \"targetNodeId\": \"n1\",\n      \"evidenceMemoId\": \"a\"\n    },\n    {\n      \"id\": \"e2\",\n      \"type\": \"confirmed\",\n      \"sourceNodeId\": \"n1\",\n      \"targetNodeId\": \"n3\",\n      \"evidenceMemoId\": \"b\"\n    }\n  ]\n}",
+      "grade": {
+        "passed": false,
+        "score": 0.0,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": "task",
+        "assertions": [
+          {
+            "assertion": "only one node labeled Chris",
+            "passed": false,
+            "details": "count=0"
+          },
+          {
+            "assertion": "edges reference merged Chris node id",
+            "passed": false,
+            "details": "chris_ids=set()"
+          }
+        ],
+        "parsed": {
+          "nodes": [
+            {
+              "id": "n1",
+              "type": "person",
+              "name": "Chris"
+            },
+            {
+              "id": "n2",
+              "type": "person",
+              "name": "Assistant"
+            },
+            {
+              "id": "n3",
+              "type": "decision",
+              "name": "Pricing"
+            }
+          ],
+          "edges": [
+            {
+              "id": "e1",
+              "type": "asked",
+              "sourceNodeId": "n2",
+              "targetNodeId": "n1",
+              "evidenceMemoId": "a"
+            },
+            {
+              "id": "e2",
+              "type": "confirmed",
+              "sourceNodeId": "n1",
+              "targetNodeId": "n3",
+              "evidenceMemoId": "b"
+            }
+          ]
+        }
+      }
+    },
+    {
+      "card_id": "catch-me-up-daily-brief",
+      "title": "Catch Me Up Daily Brief",
+      "tier": "big-swings",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 3344.99,
+      "raw_text": "{\n  \"wins\": [\n    \"- Completed tasks\",\n    \"- Progress on open tasks\"\n  ],\n  \"risks\": [\n    \"- Tight deadline for finalizing budget\",\n    \"- Potential changes to project scope\"\n  ],\n  \"nextActions\": [\n    \"- Focus on finalizing budget by EOD\",\n    \"- Schedule meeting with team to discuss project scope\",\n    \"- Communicate any changes in project scope to stakeholders\"\n  ]\n}",
+      "grade": {
+        "passed": true,
+        "score": 1.0,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": null,
+        "assertions": [
+          {
+            "assertion": "wins section exists",
+            "passed": true,
+            "details": "wins=['- Completed tasks', '- Progress on open tasks']"
+          },
+          {
+            "assertion": "risks section exists",
+            "passed": true,
+            "details": "risks=['- Tight deadline for finalizing budget', '- Potential changes to project scope']"
+          },
+          {
+            "assertion": "nextActions section exists",
+            "passed": true,
+            "details": "nextActions=['- Focus on finalizing budget by EOD', '- Schedule meeting with team to discuss project scope', '- Communicate any changes in project scope to stakeholders']"
+          }
+        ],
+        "parsed": {
+          "wins": [
+            "- Completed tasks",
+            "- Progress on open tasks"
+          ],
+          "risks": [
+            "- Tight deadline for finalizing budget",
+            "- Potential changes to project scope"
+          ],
+          "nextActions": [
+            "- Focus on finalizing budget by EOD",
+            "- Schedule meeting with team to discuss project scope",
+            "- Communicate any changes in project scope to stakeholders"
+          ]
+        }
+      }
+    },
+    {
+      "card_id": "contradiction-drift-detection",
+      "title": "Contradiction And Drift Detection",
+      "tier": "big-swings",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 2110.42,
+      "raw_text": "[\n  {\n    \"severity\": \"high\",\n    \"conflictingDecisionId\": \"d9\",\n    \"explanation\": \"The new memo contradicts the prior decision to hold the quarterly review on June 20 by moving it to June 2.\"\n  }\n]",
+      "grade": {
+        "passed": false,
+        "score": 0.0,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": "task",
+        "assertions": [
+          {
+            "assertion": "conflicts length >= 1",
+            "passed": false,
+            "details": "conflicts=[]"
+          },
+          {
+            "assertion": "first conflict severity equals high",
+            "passed": false,
+            "details": "severity=None"
+          }
+        ],
+        "parsed": [
+          {
+            "severity": "high",
+            "conflictingDecisionId": "d9",
+            "explanation": "The new memo contradicts the prior decision to hold the quarterly review on June 20 by moving it to June 2."
+          }
+        ]
+      }
+    },
+    {
+      "card_id": "momentum-scoring",
+      "title": "Momentum Scoring",
+      "tier": "big-swings",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 275.48,
+      "raw_text": "{\"score\": 0}",
+      "grade": {
+        "passed": false,
+        "score": 0.0,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": "task",
+        "assertions": [
+          {
+            "assertion": "momentumScore < 50",
+            "passed": false,
+            "details": "momentumScore=None"
+          },
+          {
+            "assertion": "unresolvedHotspots includes \"Fix onboarding bug\"",
+            "passed": false,
+            "details": "unresolvedHotspots=[]"
+          }
+        ],
+        "parsed": {
+          "score": 0
+        }
+      }
+    },
+    {
+      "card_id": "local-agent-loop",
+      "title": "Local Agent Loop For Memo Processing",
+      "tier": "big-swings",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 9602.36,
+      "raw_text": "{\n  \"steps\": [\n    {\n      \"name\": \"classify\",\n      \"input\": \"Maybe move something around at some point.\",\n      \"output\": \"classifications\": [\n        {\n          \"category\": \"planning\",\n          \"confidence\": 0.95\n        }\n      ]\n    },\n    {\n      \"name\": \"extract\",\n      \"input\": \"Maybe move something around at some point.\",\n      \"output\": \"extracts\": [\n        {\n          \"text\": \"move something\",\n          \"confidence\": 0.9\n        },\n        {\n          \"text\": \"at some point\",\n          \"confidence\": 0.95\n        }\n      ]\n    },\n    {\n      \"name\": \"summarize\",\n      \"input\": \"Maybe move something around at some point.\",\n      \"output\": \"summary\": \"The task is to potentially relocate an object at a future time.\"\n    },\n    {\n      \"name\": \"route\",\n      \"input\": \"The task is to potentially relocate an object at a future time.\",\n      \"output\": \"routes\": [\n        {\n          \"destination\": \"planning\",\n          \"confidence\": 0.98\n        }\n      ]\n    },\n    {\n      \"name\": \"verify\",",
+      "grade": {
+        "passed": false,
+        "score": 0.0,
+        "parse_error": "Expecting ',' delimiter: line 6 column 34 (char 141)",
+        "parse_mode": "failed",
+        "format_recovered": false,
+        "failure_kind": "parse",
+        "assertions": [
+          {
+            "assertion": "verify.passed can be false for vague memo",
+            "passed": false,
+            "details": "output was not valid JSON"
+          },
+          {
+            "assertion": "finalStatus equals \"needs-review\" when verify fails",
+            "passed": false,
+            "details": "output was not valid JSON"
+          }
+        ],
+        "parsed": null
+      }
+    },
+    {
+      "card_id": "meeting-live-structure",
+      "title": "Meeting Mode Live Rolling Structure",
+      "tier": "big-swings",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 2779.77,
+      "raw_text": "{\n  \"agenda\": [],\n  \"decisions\": [\n    {\n      \"text\": \"Keep beta closed for now.\",\n      \"chunkIndex\": 2\n    },\n    {\n      \"text\": \"Keep beta closed for now.\",\n      \"chunkIndex\": 8\n    }\n  ],\n  \"openQuestions\": [],\n  \"actionItems\": []\n}",
+      "grade": {
+        "passed": false,
+        "score": 0.0,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": "task",
+        "assertions": [
+          {
+            "assertion": "updatedState.decisions length stays 1",
+            "passed": false,
+            "details": "decisions=[]"
+          },
+          {
+            "assertion": "existing decision remains unchanged",
+            "passed": false,
+            "details": "decisions='[]'"
+          }
+        ],
+        "parsed": {
+          "agenda": [],
+          "decisions": [
+            {
+              "text": "Keep beta closed for now.",
+              "chunkIndex": 2
+            },
+            {
+              "text": "Keep beta closed for now.",
+              "chunkIndex": 8
+            }
+          ],
+          "openQuestions": [],
+          "actionItems": []
+        }
+      }
+    },
+    {
+      "card_id": "intent-trend-alerts",
+      "title": "Intent Trend Alerts",
+      "tier": "big-swings",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 2968.5,
+      "raw_text": "{\n  \"alert\": true,\n  \"intent\": \"Refactor UI\",\n  \"mentionCount\": 2,\n  \"completionCount\": 0,\n  \"suggestedIntervention\": \"Schedule a team meeting to discuss prioritizing the refactoring of the UI based on the recent increase in mentions. Assign specific team members to own the effort and establish a timeline for completion.\"\n}",
+      "grade": {
+        "passed": true,
+        "score": 1.0,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": null,
+        "assertions": [
+          {
+            "assertion": "alerts length equals 0",
+            "passed": true,
+            "details": "alerts=[]"
+          }
+        ],
+        "parsed": {
+          "alert": true,
+          "intent": "Refactor UI",
+          "mentionCount": 2,
+          "completionCount": 0,
+          "suggestedIntervention": "Schedule a team meeting to discuss prioritizing the refactoring of the UI based on the recent increase in mentions. Assign specific team members to own the effort and establish a timeline for completion."
+        }
+      }
+    },
+    {
+      "card_id": "voice-os-command-layer",
+      "title": "Voice-Based Operating Layer",
+      "tier": "big-swings",
+      "provider": "hf-transformers",
+      "model": "NousResearch/Hermes-3-Llama-3.1-8B",
+      "latency_ms": 3532.16,
+      "raw_text": "{\n  \"steps\": [\n    {\n      \"app\": \"finder\",\n      \"action\": \"open\",\n      \"args\": \"/Users/username/exports\"\n    },\n    {\n      \"app\": \"finder\",\n      \"action\": \"delete\",\n      \"args\": \"/Users/username/exports\",\n      \"requiresConfirmation\": true\n    },\n    {\n      \"app\": \"finder\",\n      \"action\": \"emptyTrash\"\n    }\n  ]\n}",
+      "grade": {
+        "passed": false,
+        "score": 0.5,
+        "parse_error": null,
+        "parse_mode": "exact",
+        "format_recovered": false,
+        "failure_kind": "task",
+        "assertions": [
+          {
+            "assertion": "requiresConfirmation equals true",
+            "passed": false,
+            "details": "requiresConfirmation=None"
+          },
+          {
+            "assertion": "at least one step action is marked destructive",
+            "passed": true,
+            "details": "steps=[{'app': 'finder', 'action': 'open', 'args': '/Users/username/exports'}, {'app': 'finder', 'action': 'delete', 'args': '/Users/username/exports', 'requiresConfirmation': True}, {'app': 'finder', 'action': 'emptyTrash'}]"
+          }
+        ],
+        "parsed": {
+          "steps": [
+            {
+              "app": "finder",
+              "action": "open",
+              "args": "/Users/username/exports"
+            },
+            {
+              "app": "finder",
+              "action": "delete",
+              "args": "/Users/username/exports",
+              "requiresConfirmation": true
+            },
+            {
+              "app": "finder",
+              "action": "emptyTrash"
+            }
+          ]
+        }
+      }
+    }
+  ]
+}