diff --git "a/artifacts/evaluation/latest/iab_cross_vertical_behavior_lock_regression.json" "b/artifacts/evaluation/latest/iab_cross_vertical_behavior_lock_regression.json" --- "a/artifacts/evaluation/latest/iab_cross_vertical_behavior_lock_regression.json" +++ "b/artifacts/evaluation/latest/iab_cross_vertical_behavior_lock_regression.json" @@ -1,21 +1,21 @@ { "by_status": { "must_fix": { - "failed": 0, - "passed": 90, + "failed": 88, + "passed": 2, "total": 90 } }, - "cases_path": "/Users/manikumargouni/Desktop/AdMesh/protocol/agentic-intent-classifier/examples/iab_cross_vertical_behavior_lock_cases.json", + "cases_path": "/content/agentic-intent-classifier/examples/iab_cross_vertical_behavior_lock_cases.json", "count": 90, - "failed": 0, - "passed": 90, + "failed": 88, + "passed": 2, "results": [ { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Travel", - "model_output.classification.iab_content.tier2.label": "Travel Type" + "model_output.classification.iab_content.mapping_mode": "exact", + "model_output.classification.iab_content.tier1.label": "Automotive", + "model_output.classification.iab_content.tier2.label": "Auto Rentals" }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -23,9 +23,25 @@ "model_output.classification.iab_content.tier2.label": "Travel Type" }, "id": "auto-buying-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "Automotive", + "expected": "Travel", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": "Auto Rentals", + "expected": "Travel Type", + "path": "model_output.classification.iab_content.tier2.label" + } + ], "notes": "Cross-vertical easy IAB mapping case for Automotive > Auto Buying and Selling.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Which car should I buy for commuting?" }, @@ -70,7 +86,7 @@ "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", "model_output.classification.iab_content.tier1.label": "Technology & Computing", "model_output.classification.iab_content.tier2.label": "Computing", - "model_output.classification.iab_content.tier3.label": "Software and Applications" + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -79,9 +95,15 @@ "model_output.classification.iab_content.tier3.label": "Software and Applications" }, "id": "sales-crm-easy", - "mismatches": [], + "mismatches": [ + { + "actual": null, + "expected": "Software and Applications", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical easy IAB mapping case for Business and Finance > Business > Sales.", - "pass": true, + "pass": false, "status": "must_fix", "text": "What is CRM software?" }, @@ -89,8 +111,8 @@ "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", "model_output.classification.iab_content.tier1.label": "Technology & Computing", - "model_output.classification.iab_content.tier2.label": "Computing", - "model_output.classification.iab_content.tier3.label": "Internet" + "model_output.classification.iab_content.tier2.label": null, + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -99,9 +121,20 @@ "model_output.classification.iab_content.tier3.label": "Internet" }, "id": "sales-crm-medium", - "mismatches": [], + "mismatches": [ + { + "actual": null, + "expected": "Computing", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Internet", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Business and Finance > Business > Sales.", - "pass": true, + "pass": false, "status": "must_fix", "text": "HubSpot vs Zoho for a small team" }, @@ -110,7 +143,7 @@ "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", "model_output.classification.iab_content.tier1.label": "Business and Finance", "model_output.classification.iab_content.tier2.label": "Business", - "model_output.classification.iab_content.tier3.label": "Sales" + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -119,18 +152,24 @@ "model_output.classification.iab_content.tier3.label": "Sales" }, "id": "sales-crm-hard", - "mismatches": [], + "mismatches": [ + { + "actual": null, + "expected": "Sales", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical hard IAB mapping case for Business and Finance > Business > Sales.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Need software to manage leads and pipeline for a startup sales team" }, { "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Technology & Computing", - "model_output.classification.iab_content.tier2.label": "Computing", - "model_output.classification.iab_content.tier3.label": "Internet" + "model_output.classification.iab_content.tier1.label": "Hobbies & Interests", + "model_output.classification.iab_content.tier2.label": null, + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -139,17 +178,33 @@ "model_output.classification.iab_content.tier3.label": "Internet" }, "id": "marketing-tools-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "Hobbies & Interests", + "expected": "Technology & Computing", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": null, + "expected": "Computing", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Internet", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical easy IAB mapping case for Business and Finance > Business > Marketing and Advertising.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Best SEO tools for content teams" }, { "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Business and Finance", - "model_output.classification.iab_content.tier2.label": "Business" + "model_output.classification.iab_content.tier1.label": "Careers", + "model_output.classification.iab_content.tier2.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -157,18 +212,29 @@ "model_output.classification.iab_content.tier2.label": "Business" }, "id": "marketing-tools-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "Careers", + "expected": "Business and Finance", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": null, + "expected": "Business", + "path": "model_output.classification.iab_content.tier2.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Business and Finance > Business > Marketing and Advertising.", - "pass": true, + "pass": false, "status": "must_fix", "text": "How should I compare ad attribution platforms?" }, { "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Technology & Computing", - "model_output.classification.iab_content.tier2.label": "Computing", - "model_output.classification.iab_content.tier3.label": "Software and Applications" + "model_output.classification.iab_content.tier1.label": "Personal Finance", + "model_output.classification.iab_content.tier2.label": null, + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -177,9 +243,25 @@ "model_output.classification.iab_content.tier3.label": "Software and Applications" }, "id": "marketing-tools-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "Personal Finance", + "expected": "Technology & Computing", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": null, + "expected": "Computing", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Software and Applications", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical hard IAB mapping case for Business and Finance > Business > Marketing and Advertising.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Need software to measure channel performance across paid and organic campaigns" }, @@ -188,7 +270,7 @@ "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", "model_output.classification.iab_content.tier1.label": "Technology & Computing", "model_output.classification.iab_content.tier2.label": "Computing", - "model_output.classification.iab_content.tier3.label": "Internet" + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -197,17 +279,23 @@ "model_output.classification.iab_content.tier3.label": "Internet" }, "id": "business-it-easy", - "mismatches": [], + "mismatches": [ + { + "actual": null, + "expected": "Internet", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical easy IAB mapping case for Business and Finance > Business > Business I.T..", - "pass": true, + "pass": false, "status": "must_fix", "text": "How do I reset my work password?" }, { "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Careers", - "model_output.classification.iab_content.tier2.label": "Job Search" + "model_output.classification.iab_content.tier1.label": "Personal Finance", + "model_output.classification.iab_content.tier2.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -215,18 +303,29 @@ "model_output.classification.iab_content.tier2.label": "Job Search" }, "id": "business-it-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "Personal Finance", + "expected": "Careers", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": null, + "expected": "Job Search", + "path": "model_output.classification.iab_content.tier2.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Business and Finance > Business > Business I.T..", - "pass": true, + "pass": false, "status": "must_fix", "text": "My employees keep getting locked out of their accounts" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Technology & Computing", "model_output.classification.iab_content.tier2.label": "Computing", - "model_output.classification.iab_content.tier3.label": "Internet" + "model_output.classification.iab_content.tier3.label": "Information and Network Security" }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -235,15 +334,26 @@ "model_output.classification.iab_content.tier3.label": "Internet" }, "id": "business-it-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": "Information and Network Security", + "expected": "Internet", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical hard IAB mapping case for Business and Finance > Business > Business I.T..", - "pass": true, + "pass": false, "status": "must_fix", "text": "Need identity and access software for login, permissions, and account security" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Food & Drink", "model_output.classification.iab_content.tier2.label": "Dining Out" }, @@ -253,17 +363,23 @@ "model_output.classification.iab_content.tier2.label": "Dining Out" }, "id": "dining-out-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical easy IAB mapping case for Food & Drink > Dining Out.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Book a table for six tonight" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Food & Drink", - "model_output.classification.iab_content.tier2.label": "Dining Out" + "model_output.classification.iab_content.mapping_mode": "exact", + "model_output.classification.iab_content.tier1.label": "Attractions", + "model_output.classification.iab_content.tier2.label": "Bars & Restaurants" }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -271,15 +387,31 @@ "model_output.classification.iab_content.tier2.label": "Dining Out" }, "id": "dining-out-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "Attractions", + "expected": "Food & Drink", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": "Bars & Restaurants", + "expected": "Dining Out", + "path": "model_output.classification.iab_content.tier2.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Food & Drink > Dining Out.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Good restaurants for a client dinner downtown" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Food & Drink", "model_output.classification.iab_content.tier2.label": "Dining Out" }, @@ -289,15 +421,21 @@ "model_output.classification.iab_content.tier2.label": "Dining Out" }, "id": "dining-out-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical hard IAB mapping case for Food & Drink > Dining Out.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Need a place to eat tonight where I can make a reservation online" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Food & Drink" }, "expected": { @@ -305,15 +443,21 @@ "model_output.classification.iab_content.tier1.label": "Food & Drink" }, "id": "alcoholic-beverages-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical easy IAB mapping case for Food & Drink > Alcoholic Beverages.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Which whiskey cocktail should I order?" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Food & Drink" }, "expected": { @@ -321,15 +465,21 @@ "model_output.classification.iab_content.tier1.label": "Food & Drink" }, "id": "alcoholic-beverages-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical medium IAB mapping case for Food & Drink > Alcoholic Beverages.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Best vodka drinks for beginners" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Food & Drink" }, "expected": { @@ -337,31 +487,43 @@ "model_output.classification.iab_content.tier1.label": "Food & Drink" }, "id": "alcoholic-beverages-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical hard IAB mapping case for Food & Drink > Alcoholic Beverages.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Want a spirit-forward drink recommendation, not a restaurant suggestion" }, { "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Technology & Computing" + "model_output.classification.iab_content.tier1.label": "Sports" }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", "model_output.classification.iab_content.tier1.label": "Technology & Computing" }, "id": "artificial-intelligence-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "Sports", + "expected": "Technology & Computing", + "path": "model_output.classification.iab_content.tier1.label" + } + ], "notes": "Cross-vertical easy IAB mapping case for Technology & Computing > Artificial Intelligence.", - "pass": true, + "pass": false, "status": "must_fix", "text": "What is intent classification in NLP?" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Education", "model_output.classification.iab_content.tier2.label": "Language Learning" }, @@ -371,15 +533,21 @@ "model_output.classification.iab_content.tier2.label": "Language Learning" }, "id": "artificial-intelligence-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical medium IAB mapping case for Technology & Computing > Artificial Intelligence.", - "pass": true, + "pass": false, "status": "must_fix", "text": "How do large language models handle text classification?" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Education", "model_output.classification.iab_content.tier2.label": "Language Learning" }, @@ -389,17 +557,23 @@ "model_output.classification.iab_content.tier2.label": "Language Learning" }, "id": "artificial-intelligence-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical hard IAB mapping case for Technology & Computing > Artificial Intelligence.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Need the machine learning concept behind language understanding, not software to buy" }, { "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Business and Finance", - "model_output.classification.iab_content.tier2.label": "Business" + "model_output.classification.iab_content.tier1.label": "Technology & Computing", + "model_output.classification.iab_content.tier2.label": "Computing" }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -407,9 +581,20 @@ "model_output.classification.iab_content.tier2.label": "Business" }, "id": "software-apps-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "Technology & Computing", + "expected": "Business and Finance", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": "Computing", + "expected": "Business", + "path": "model_output.classification.iab_content.tier2.label" + } + ], "notes": "Cross-vertical easy IAB mapping case for Technology & Computing > Computing > Computer Software and Applications.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Best workflow software for a small operations team" }, @@ -418,8 +603,8 @@ "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", "model_output.classification.iab_content.tier1.label": "Technology & Computing", "model_output.classification.iab_content.tier2.label": "Computing", - "model_output.classification.iab_content.tier3.label": "Internet", - "model_output.classification.iab_content.tier4.label": "Cloud Computing" + "model_output.classification.iab_content.tier3.label": null, + "model_output.classification.iab_content.tier4.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -429,18 +614,29 @@ "model_output.classification.iab_content.tier4.label": "Cloud Computing" }, "id": "software-apps-medium", - "mismatches": [], + "mismatches": [ + { + "actual": null, + "expected": "Internet", + "path": "model_output.classification.iab_content.tier3.label" + }, + { + "actual": null, + "expected": "Cloud Computing", + "path": "model_output.classification.iab_content.tier4.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Technology & Computing > Computing > Computer Software and Applications.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Need project management software for a distributed team" }, { "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Technology & Computing", - "model_output.classification.iab_content.tier2.label": "Computing", - "model_output.classification.iab_content.tier3.label": "Software and Applications" + "model_output.classification.iab_content.tier1.label": "Business and Finance", + "model_output.classification.iab_content.tier2.label": null, + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -449,19 +645,35 @@ "model_output.classification.iab_content.tier3.label": "Software and Applications" }, "id": "software-apps-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "Business and Finance", + "expected": "Technology & Computing", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": null, + "expected": "Computing", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Software and Applications", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical hard IAB mapping case for Technology & Computing > Computing > Computer Software and Applications.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Looking for a business software platform to organize internal workflows" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Technology & Computing", - "model_output.classification.iab_content.tier2.label": "Computing", - "model_output.classification.iab_content.tier3.label": "Software and Applications", - "model_output.classification.iab_content.tier4.label": "Communication" + "model_output.classification.iab_content.mapping_mode": "exact", + "model_output.classification.iab_content.tier1.label": "Careers", + "model_output.classification.iab_content.tier2.label": "Remote Working", + "model_output.classification.iab_content.tier3.label": null, + "model_output.classification.iab_content.tier4.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -471,9 +683,35 @@ "model_output.classification.iab_content.tier4.label": "Communication" }, "id": "communication-software-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "Careers", + "expected": "Technology & Computing", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": "Remote Working", + "expected": "Computing", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Software and Applications", + "path": "model_output.classification.iab_content.tier3.label" + }, + { + "actual": null, + "expected": "Communication", + "path": "model_output.classification.iab_content.tier4.label" + } + ], "notes": "Cross-vertical easy IAB mapping case for Technology & Computing > Computing > Computer Software and Applications > Communication.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Best communication software for remote teams" }, @@ -482,8 +720,8 @@ "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", "model_output.classification.iab_content.tier1.label": "Technology & Computing", "model_output.classification.iab_content.tier2.label": "Computing", - "model_output.classification.iab_content.tier3.label": "Software and Applications", - "model_output.classification.iab_content.tier4.label": "Communication" + "model_output.classification.iab_content.tier3.label": null, + "model_output.classification.iab_content.tier4.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -493,19 +731,30 @@ "model_output.classification.iab_content.tier4.label": "Communication" }, "id": "communication-software-medium", - "mismatches": [], + "mismatches": [ + { + "actual": null, + "expected": "Software and Applications", + "path": "model_output.classification.iab_content.tier3.label" + }, + { + "actual": null, + "expected": "Communication", + "path": "model_output.classification.iab_content.tier4.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Technology & Computing > Computing > Computer Software and Applications > Communication.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Slack vs Teams for internal messaging" }, { "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Technology & Computing", - "model_output.classification.iab_content.tier2.label": "Computing", - "model_output.classification.iab_content.tier3.label": "Software and Applications", - "model_output.classification.iab_content.tier4.label": "Communication" + "model_output.classification.iab_content.tier1.label": "Careers", + "model_output.classification.iab_content.tier2.label": null, + "model_output.classification.iab_content.tier3.label": null, + "model_output.classification.iab_content.tier4.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -515,9 +764,30 @@ "model_output.classification.iab_content.tier4.label": "Communication" }, "id": "communication-software-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "Careers", + "expected": "Technology & Computing", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": null, + "expected": "Computing", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Software and Applications", + "path": "model_output.classification.iab_content.tier3.label" + }, + { + "actual": null, + "expected": "Communication", + "path": "model_output.classification.iab_content.tier4.label" + } + ], "notes": "Cross-vertical hard IAB mapping case for Technology & Computing > Computing > Computer Software and Applications > Communication.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Need a workplace chat tool for cross-functional collaboration" }, @@ -526,8 +796,8 @@ "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", "model_output.classification.iab_content.tier1.label": "Technology & Computing", "model_output.classification.iab_content.tier2.label": "Computing", - "model_output.classification.iab_content.tier3.label": "Internet", - "model_output.classification.iab_content.tier4.label": "Web Hosting" + "model_output.classification.iab_content.tier3.label": null, + "model_output.classification.iab_content.tier4.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -537,9 +807,20 @@ "model_output.classification.iab_content.tier4.label": "Web Hosting" }, "id": "web-hosting-easy", - "mismatches": [], + "mismatches": [ + { + "actual": null, + "expected": "Internet", + "path": "model_output.classification.iab_content.tier3.label" + }, + { + "actual": null, + "expected": "Web Hosting", + "path": "model_output.classification.iab_content.tier4.label" + } + ], "notes": "Cross-vertical easy IAB mapping case for Technology & Computing > Computing > Internet > Web Hosting.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Vercel vs Netlify for website hosting" }, @@ -548,8 +829,8 @@ "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", "model_output.classification.iab_content.tier1.label": "Technology & Computing", "model_output.classification.iab_content.tier2.label": "Computing", - "model_output.classification.iab_content.tier3.label": "Internet", - "model_output.classification.iab_content.tier4.label": "Web Hosting" + "model_output.classification.iab_content.tier3.label": null, + "model_output.classification.iab_content.tier4.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -559,9 +840,20 @@ "model_output.classification.iab_content.tier4.label": "Web Hosting" }, "id": "web-hosting-medium", - "mismatches": [], + "mismatches": [ + { + "actual": null, + "expected": "Internet", + "path": "model_output.classification.iab_content.tier3.label" + }, + { + "actual": null, + "expected": "Web Hosting", + "path": "model_output.classification.iab_content.tier4.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Technology & Computing > Computing > Internet > Web Hosting.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Best hosting platform for a startup website" }, @@ -570,8 +862,8 @@ "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", "model_output.classification.iab_content.tier1.label": "Technology & Computing", "model_output.classification.iab_content.tier2.label": "Computing", - "model_output.classification.iab_content.tier3.label": "Internet", - "model_output.classification.iab_content.tier4.label": "Web Hosting" + "model_output.classification.iab_content.tier3.label": null, + "model_output.classification.iab_content.tier4.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -581,9 +873,20 @@ "model_output.classification.iab_content.tier4.label": "Web Hosting" }, "id": "web-hosting-hard", - "mismatches": [], + "mismatches": [ + { + "actual": null, + "expected": "Internet", + "path": "model_output.classification.iab_content.tier3.label" + }, + { + "actual": null, + "expected": "Web Hosting", + "path": "model_output.classification.iab_content.tier4.label" + } + ], "notes": "Cross-vertical hard IAB mapping case for Technology & Computing > Computing > Internet > Web Hosting.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Need a managed hosting provider to deploy and run our marketing site" }, @@ -591,8 +894,8 @@ "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", "model_output.classification.iab_content.tier1.label": "Technology & Computing", - "model_output.classification.iab_content.tier2.label": "Computing", - "model_output.classification.iab_content.tier3.label": "Laptops" + "model_output.classification.iab_content.tier2.label": null, + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -601,15 +904,26 @@ "model_output.classification.iab_content.tier3.label": "Laptops" }, "id": "laptops-easy", - "mismatches": [], + "mismatches": [ + { + "actual": null, + "expected": "Computing", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Laptops", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical easy IAB mapping case for Technology & Computing > Computing > Laptops.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Which laptop should I buy for college?" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Technology & Computing", "model_output.classification.iab_content.tier2.label": "Computing", "model_output.classification.iab_content.tier3.label": "Laptops" @@ -621,9 +935,15 @@ "model_output.classification.iab_content.tier3.label": "Laptops" }, "id": "laptops-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical medium IAB mapping case for Technology & Computing > Computing > Laptops.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Best laptop for work and study under 1200" }, @@ -631,8 +951,8 @@ "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", "model_output.classification.iab_content.tier1.label": "Technology & Computing", - "model_output.classification.iab_content.tier2.label": "Consumer Electronics", - "model_output.classification.iab_content.tier3.label": "Smartphones" + "model_output.classification.iab_content.tier2.label": "Computing", + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -641,9 +961,20 @@ "model_output.classification.iab_content.tier3.label": "Smartphones" }, "id": "laptops-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "Computing", + "expected": "Consumer Electronics", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Smartphones", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical hard IAB mapping case for Technology & Computing > Computing > Laptops.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Need a portable computer with good battery life for everyday work" }, @@ -653,7 +984,7 @@ "model_output.classification.iab_content.tier1.label": "Technology & Computing", "model_output.classification.iab_content.tier2.label": "Computing", "model_output.classification.iab_content.tier3.label": "Software and Applications", - "model_output.classification.iab_content.tier4.label": "Photo Editing Software" + "model_output.classification.iab_content.tier4.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -663,9 +994,15 @@ "model_output.classification.iab_content.tier4.label": "Photo Editing Software" }, "id": "desktops-easy", - "mismatches": [], + "mismatches": [ + { + "actual": null, + "expected": "Photo Editing Software", + "path": "model_output.classification.iab_content.tier4.label" + } + ], "notes": "Cross-vertical easy IAB mapping case for Technology & Computing > Computing > Desktops.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Best desktop for video editing" }, @@ -674,7 +1011,7 @@ "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", "model_output.classification.iab_content.tier1.label": "Technology & Computing", "model_output.classification.iab_content.tier2.label": "Computing", - "model_output.classification.iab_content.tier3.label": "Desktops" + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -683,15 +1020,21 @@ "model_output.classification.iab_content.tier3.label": "Desktops" }, "id": "desktops-medium", - "mismatches": [], + "mismatches": [ + { + "actual": null, + "expected": "Desktops", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Technology & Computing > Computing > Desktops.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Which desktop computer should I buy for a home office?" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Technology & Computing", "model_output.classification.iab_content.tier2.label": "Computing", "model_output.classification.iab_content.tier3.label": "Desktops" @@ -703,15 +1046,21 @@ "model_output.classification.iab_content.tier3.label": "Desktops" }, "id": "desktops-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical hard IAB mapping case for Technology & Computing > Computing > Desktops.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Need a desktop PC with strong performance for creative work" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Technology & Computing", "model_output.classification.iab_content.tier2.label": "Consumer Electronics" }, @@ -721,18 +1070,24 @@ "model_output.classification.iab_content.tier2.label": "Consumer Electronics" }, "id": "smartphones-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical easy IAB mapping case for Technology & Computing > Consumer Electronics > Smartphones.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Best phone with a good camera under 700" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Technology & Computing", "model_output.classification.iab_content.tier2.label": "Consumer Electronics", - "model_output.classification.iab_content.tier3.label": "Smartphones" + "model_output.classification.iab_content.tier3.label": "Wearable Technology" }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -741,18 +1096,29 @@ "model_output.classification.iab_content.tier3.label": "Smartphones" }, "id": "smartphones-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": "Wearable Technology", + "expected": "Smartphones", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Technology & Computing > Consumer Electronics > Smartphones.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Should I buy an iPhone or Pixel this year?" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Technology & Computing", "model_output.classification.iab_content.tier2.label": "Consumer Electronics", - "model_output.classification.iab_content.tier3.label": "Smartphones" + "model_output.classification.iab_content.tier3.label": "Wearable Technology" }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -761,18 +1127,29 @@ "model_output.classification.iab_content.tier3.label": "Smartphones" }, "id": "smartphones-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": "Wearable Technology", + "expected": "Smartphones", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical hard IAB mapping case for Technology & Computing > Consumer Electronics > Smartphones.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Need a new smartphone with strong battery life and a clean software experience" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Style & Fashion", - "model_output.classification.iab_content.tier2.label": "Women's Fashion", - "model_output.classification.iab_content.tier3.label": "Women's Shoes and Footwear" + "model_output.classification.iab_content.tier2.label": "Designer Clothing", + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -781,18 +1158,34 @@ "model_output.classification.iab_content.tier3.label": "Women's Shoes and Footwear" }, "id": "style-fashion-parent-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": "Designer Clothing", + "expected": "Women's Fashion", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Women's Shoes and Footwear", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical easy IAB mapping case for Style & Fashion.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Best shoes under 100 dollars" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Style & Fashion", - "model_output.classification.iab_content.tier2.label": "Women's Fashion", - "model_output.classification.iab_content.tier3.label": "Women's Clothing" + "model_output.classification.iab_content.tier2.label": "Designer Clothing", + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -801,18 +1194,34 @@ "model_output.classification.iab_content.tier3.label": "Women's Clothing" }, "id": "style-fashion-parent-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": "Designer Clothing", + "expected": "Women's Fashion", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Women's Clothing", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Style & Fashion.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Affordable fashion accessories for everyday wear" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Style & Fashion", - "model_output.classification.iab_content.tier2.label": "Women's Fashion", - "model_output.classification.iab_content.tier3.label": "Women's Shoes and Footwear" + "model_output.classification.iab_content.tier2.label": "Designer Clothing", + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -821,18 +1230,34 @@ "model_output.classification.iab_content.tier3.label": "Women's Shoes and Footwear" }, "id": "style-fashion-parent-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": "Designer Clothing", + "expected": "Women's Fashion", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Women's Shoes and Footwear", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical hard IAB mapping case for Style & Fashion.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Need style recommendations for clothing and footwear without a specific brand in mind" }, { "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Style & Fashion", - "model_output.classification.iab_content.tier2.label": "Women's Fashion", - "model_output.classification.iab_content.tier3.label": "Women's Clothing" + "model_output.classification.iab_content.tier1.label": "Sports", + "model_output.classification.iab_content.tier2.label": null, + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -841,18 +1266,34 @@ "model_output.classification.iab_content.tier3.label": "Women's Clothing" }, "id": "womens-shoes-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "Sports", + "expected": "Style & Fashion", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": null, + "expected": "Women's Fashion", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Women's Clothing", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical easy IAB mapping case for Style & Fashion > Women's Fashion > Women's Shoes and Footwear.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Best women's running shoes under 100 dollars" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Style & Fashion", - "model_output.classification.iab_content.tier2.label": "Women's Fashion", - "model_output.classification.iab_content.tier3.label": "Women's Clothing" + "model_output.classification.iab_content.mapping_mode": "exact", + "model_output.classification.iab_content.tier1.label": "Sports", + "model_output.classification.iab_content.tier2.label": "Walking", + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -861,15 +1302,36 @@ "model_output.classification.iab_content.tier3.label": "Women's Clothing" }, "id": "womens-shoes-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "Sports", + "expected": "Style & Fashion", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": "Walking", + "expected": "Women's Fashion", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Women's Clothing", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Style & Fashion > Women's Fashion > Women's Shoes and Footwear.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Comfortable women's sneakers for walking all day" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Style & Fashion", "model_output.classification.iab_content.tier2.label": "Women's Fashion", "model_output.classification.iab_content.tier3.label": "Women's Shoes and Footwear" @@ -881,9 +1343,15 @@ "model_output.classification.iab_content.tier3.label": "Women's Shoes and Footwear" }, "id": "womens-shoes-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical hard IAB mapping case for Style & Fashion > Women's Fashion > Women's Shoes and Footwear.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Need women's footwear for commuting that looks polished but feels comfortable" }, @@ -891,8 +1359,8 @@ "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", "model_output.classification.iab_content.tier1.label": "Style & Fashion", - "model_output.classification.iab_content.tier2.label": "Men's Fashion", - "model_output.classification.iab_content.tier3.label": "Men's Clothing" + "model_output.classification.iab_content.tier2.label": null, + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -901,18 +1369,29 @@ "model_output.classification.iab_content.tier3.label": "Men's Clothing" }, "id": "mens-shoes-easy", - "mismatches": [], + "mismatches": [ + { + "actual": null, + "expected": "Men's Fashion", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Men's Clothing", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical easy IAB mapping case for Style & Fashion > Men's Fashion > Men's Shoes and Footwear.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Best men's sneakers for daily wear" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Style & Fashion", "model_output.classification.iab_content.tier2.label": "Men's Fashion", - "model_output.classification.iab_content.tier3.label": "Men's Clothing" + "model_output.classification.iab_content.tier3.label": "Men's Shoes and Footwear" }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -921,9 +1400,20 @@ "model_output.classification.iab_content.tier3.label": "Men's Clothing" }, "id": "mens-shoes-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": "Men's Shoes and Footwear", + "expected": "Men's Clothing", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Style & Fashion > Men's Fashion > Men's Shoes and Footwear.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Good men's dress shoes for office use" }, @@ -931,8 +1421,8 @@ "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", "model_output.classification.iab_content.tier1.label": "Style & Fashion", - "model_output.classification.iab_content.tier2.label": "Men's Fashion", - "model_output.classification.iab_content.tier3.label": "Men's Shoes and Footwear" + "model_output.classification.iab_content.tier2.label": null, + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -941,17 +1431,28 @@ "model_output.classification.iab_content.tier3.label": "Men's Shoes and Footwear" }, "id": "mens-shoes-hard", - "mismatches": [], + "mismatches": [ + { + "actual": null, + "expected": "Men's Fashion", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Men's Shoes and Footwear", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical hard IAB mapping case for Style & Fashion > Men's Fashion > Men's Shoes and Footwear.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Need men's footwear that works for workdays and weekend walking" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Real Estate", - "model_output.classification.iab_content.tier2.label": "Hotel Properties" + "model_output.classification.iab_content.mapping_mode": "exact", + "model_output.classification.iab_content.tier1.label": "Travel", + "model_output.classification.iab_content.tier2.label": "Travel Type" }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -959,15 +1460,31 @@ "model_output.classification.iab_content.tier2.label": "Hotel Properties" }, "id": "hotels-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "Travel", + "expected": "Real Estate", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": "Travel Type", + "expected": "Hotel Properties", + "path": "model_output.classification.iab_content.tier2.label" + } + ], "notes": "Cross-vertical easy IAB mapping case for Travel > Travel Type > Hotels and Motels.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Need a hotel in Chicago for two nights" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Travel", "model_output.classification.iab_content.tier2.label": "Travel Type", "model_output.classification.iab_content.tier3.label": "Hotels and Motels" @@ -979,17 +1496,23 @@ "model_output.classification.iab_content.tier3.label": "Hotels and Motels" }, "id": "hotels-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical medium IAB mapping case for Travel > Travel Type > Hotels and Motels.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Best hotels near Times Square for a weekend trip" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Travel", - "model_output.classification.iab_content.tier2.label": "Travel Type" + "model_output.classification.iab_content.tier2.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -997,15 +1520,26 @@ "model_output.classification.iab_content.tier2.label": "Travel Type" }, "id": "hotels-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": null, + "expected": "Travel Type", + "path": "model_output.classification.iab_content.tier2.label" + } + ], "notes": "Cross-vertical hard IAB mapping case for Travel > Travel Type > Hotels and Motels.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Looking for a place to stay during a work trip, not general travel advice" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Real Estate", "model_output.classification.iab_content.tier2.label": "Apartments" }, @@ -1015,15 +1549,21 @@ "model_output.classification.iab_content.tier2.label": "Apartments" }, "id": "real-estate-rentals-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical easy IAB mapping case for Real Estate > Real Estate Renting and Leasing.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Apartments for rent near downtown Austin" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Real Estate", "model_output.classification.iab_content.tier2.label": "Apartments" }, @@ -1033,18 +1573,24 @@ "model_output.classification.iab_content.tier2.label": "Apartments" }, "id": "real-estate-rentals-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical medium IAB mapping case for Real Estate > Real Estate Renting and Leasing.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Best neighborhoods to lease a two-bedroom apartment in Seattle" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Personal Finance", - "model_output.classification.iab_content.tier2.label": "Personal Debt", - "model_output.classification.iab_content.tier3.label": "Home Financing" + "model_output.classification.iab_content.mapping_mode": "exact", + "model_output.classification.iab_content.tier1.label": "Real Estate", + "model_output.classification.iab_content.tier2.label": "Real Estate Renting and Leasing", + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -1053,18 +1599,39 @@ "model_output.classification.iab_content.tier3.label": "Home Financing" }, "id": "real-estate-rentals-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "Real Estate", + "expected": "Personal Finance", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": "Real Estate Renting and Leasing", + "expected": "Personal Debt", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Home Financing", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical hard IAB mapping case for Real Estate > Real Estate Renting and Leasing.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Need rental listings for a short move, not home-buying advice" }, { "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Business and Finance", - "model_output.classification.iab_content.tier2.label": "Business", - "model_output.classification.iab_content.tier3.label": "Green Solutions" + "model_output.classification.iab_content.tier1.label": "Healthy Living", + "model_output.classification.iab_content.tier2.label": null, + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -1073,18 +1640,34 @@ "model_output.classification.iab_content.tier3.label": "Green Solutions" }, "id": "running-and-jogging-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "Healthy Living", + "expected": "Business and Finance", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": null, + "expected": "Business", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Green Solutions", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical easy IAB mapping case for Healthy Living > Fitness and Exercise > Running and Jogging.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Best running plan for a first 10k" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Healthy Living", - "model_output.classification.iab_content.tier2.label": "Fitness and Exercise", - "model_output.classification.iab_content.tier3.label": "Running and Jogging" + "model_output.classification.iab_content.mapping_mode": "exact", + "model_output.classification.iab_content.tier1.label": "Sports", + "model_output.classification.iab_content.tier2.label": "Walking", + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -1093,18 +1676,39 @@ "model_output.classification.iab_content.tier3.label": "Running and Jogging" }, "id": "running-and-jogging-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "Sports", + "expected": "Healthy Living", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": "Walking", + "expected": "Fitness and Exercise", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Running and Jogging", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Healthy Living > Fitness and Exercise > Running and Jogging.", - "pass": true, + "pass": false, "status": "must_fix", "text": "How should I train for a half marathon as a beginner?" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Healthy Living", "model_output.classification.iab_content.tier2.label": "Fitness and Exercise", - "model_output.classification.iab_content.tier3.label": "Running and Jogging" + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -1113,15 +1717,26 @@ "model_output.classification.iab_content.tier3.label": "Running and Jogging" }, "id": "running-and-jogging-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": null, + "expected": "Running and Jogging", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical hard IAB mapping case for Healthy Living > Fitness and Exercise > Running and Jogging.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Need guidance on building a weekly jogging routine without getting injured" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Sports", "model_output.classification.iab_content.tier2.label": "Soccer" }, @@ -1131,15 +1746,21 @@ "model_output.classification.iab_content.tier2.label": "Soccer" }, "id": "soccer-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical easy IAB mapping case for Sports > Soccer.", - "pass": true, + "pass": false, "status": "must_fix", "text": "How do offside rules work in soccer?" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Sports", "model_output.classification.iab_content.tier2.label": "Soccer" }, @@ -1149,15 +1770,21 @@ "model_output.classification.iab_content.tier2.label": "Soccer" }, "id": "soccer-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical medium IAB mapping case for Sports > Soccer.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Best soccer drills for beginner players" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Sports" }, "expected": { @@ -1165,17 +1792,23 @@ "model_output.classification.iab_content.tier1.label": "Sports" }, "id": "soccer-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical hard IAB mapping case for Sports > Soccer.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Need help understanding football tactics for the Premier League, not fantasy sports" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Books and Literature", - "model_output.classification.iab_content.tier2.label": "Fiction" + "model_output.classification.iab_content.mapping_mode": "exact", + "model_output.classification.iab_content.tier1.label": "Genres", + "model_output.classification.iab_content.tier2.label": "Fantasy" }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -1183,15 +1816,31 @@ "model_output.classification.iab_content.tier2.label": "Fiction" }, "id": "fiction-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "Genres", + "expected": "Books and Literature", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": "Fantasy", + "expected": "Fiction", + "path": "model_output.classification.iab_content.tier2.label" + } + ], "notes": "Cross-vertical easy IAB mapping case for Books and Literature > Fiction.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Recommend a good fantasy novel to read" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Books and Literature", "model_output.classification.iab_content.tier2.label": "Fiction" }, @@ -1201,15 +1850,21 @@ "model_output.classification.iab_content.tier2.label": "Fiction" }, "id": "fiction-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical medium IAB mapping case for Books and Literature > Fiction.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Best fiction books for a long flight" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Books and Literature" }, "expected": { @@ -1217,15 +1872,21 @@ "model_output.classification.iab_content.tier1.label": "Books and Literature" }, "id": "fiction-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical hard IAB mapping case for Books and Literature > Fiction.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Looking for a character-driven novel, not comics or poetry" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Home & Garden", "model_output.classification.iab_content.tier2.label": "Remodeling & Construction" }, @@ -1235,18 +1896,24 @@ "model_output.classification.iab_content.tier2.label": "Remodeling & Construction" }, "id": "home-improvement-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical easy IAB mapping case for Home & Garden > Home Improvement.", - "pass": true, + "pass": false, "status": "must_fix", "text": "How much does a kitchen remodel usually cost?" }, { "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Style & Fashion", - "model_output.classification.iab_content.tier2.label": "Personal Care", - "model_output.classification.iab_content.tier3.label": "Bath and Shower" + "model_output.classification.iab_content.tier1.label": "Home & Garden", + "model_output.classification.iab_content.tier2.label": null, + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -1255,17 +1922,33 @@ "model_output.classification.iab_content.tier3.label": "Bath and Shower" }, "id": "home-improvement-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "Home & Garden", + "expected": "Style & Fashion", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": null, + "expected": "Personal Care", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Bath and Shower", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Home & Garden > Home Improvement.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Best tools for a DIY bathroom renovation" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Home & Garden", - "model_output.classification.iab_content.tier2.label": "Interior Decorating" + "model_output.classification.iab_content.mapping_mode": "exact", + "model_output.classification.iab_content.tier1.label": "Real Estate", + "model_output.classification.iab_content.tier2.label": "Houses" }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -1273,17 +1956,33 @@ "model_output.classification.iab_content.tier2.label": "Interior Decorating" }, "id": "home-improvement-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "Real Estate", + "expected": "Home & Garden", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": "Houses", + "expected": "Interior Decorating", + "path": "model_output.classification.iab_content.tier2.label" + } + ], "notes": "Cross-vertical hard IAB mapping case for Home & Garden > Home Improvement.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Need practical advice for upgrading an older house, not interior decor inspiration" }, { "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Education", - "model_output.classification.iab_content.tier2.label": "Language Learning" + "model_output.classification.iab_content.tier1.label": "Technology & Computing", + "model_output.classification.iab_content.tier2.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -1291,15 +1990,26 @@ "model_output.classification.iab_content.tier2.label": "Language Learning" }, "id": "online-education-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "Technology & Computing", + "expected": "Education", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": null, + "expected": "Language Learning", + "path": "model_output.classification.iab_content.tier2.label" + } + ], "notes": "Cross-vertical easy IAB mapping case for Education > Online Education.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Best online courses for learning Python" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Careers", "model_output.classification.iab_content.tier2.label": "Remote Working" }, @@ -1309,31 +2019,43 @@ "model_output.classification.iab_content.tier2.label": "Remote Working" }, "id": "online-education-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical medium IAB mapping case for Education > Online Education.", - "pass": true, + "pass": false, "status": "must_fix", "text": "What are good platforms for remote professional classes?" }, { "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Careers" + "model_output.classification.iab_content.tier1.label": "Healthy Living" }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", "model_output.classification.iab_content.tier1.label": "Careers" }, "id": "online-education-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "Healthy Living", + "expected": "Careers", + "path": "model_output.classification.iab_content.tier1.label" + } + ], "notes": "Cross-vertical hard IAB mapping case for Education > Online Education.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Need internet-based training options I can finish after work hours" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Education", "model_output.classification.iab_content.tier2.label": "College Education" }, @@ -1343,19 +2065,25 @@ "model_output.classification.iab_content.tier2.label": "College Education" }, "id": "postgraduate-education-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical easy IAB mapping case for Education > College Education > Postgraduate Education.", - "pass": true, + "pass": false, "status": "must_fix", "text": "best universities to study masters" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Education", "model_output.classification.iab_content.tier2.label": "College Education", "model_output.classification.iab_content.tier3.label": "Postgraduate Education", - "model_output.classification.iab_content.tier4.label": "Professional School" + "model_output.classification.iab_content.tier4.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -1365,15 +2093,26 @@ "model_output.classification.iab_content.tier4.label": "Professional School" }, "id": "postgraduate-education-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": null, + "expected": "Professional School", + "path": "model_output.classification.iab_content.tier4.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Education > College Education > Postgraduate Education.", - "pass": true, + "pass": false, "status": "must_fix", "text": "which graduate schools have strong data science programs" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Education", "model_output.classification.iab_content.tier2.label": "College Education" }, @@ -1383,9 +2122,15 @@ "model_output.classification.iab_content.tier2.label": "College Education" }, "id": "postgraduate-education-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical hard IAB mapping case for Education > College Education > Postgraduate Education.", - "pass": true, + "pass": false, "status": "must_fix", "text": "need postgraduate options for a master's degree, not short online courses" }, @@ -1394,7 +2139,7 @@ "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", "model_output.classification.iab_content.tier1.label": "Medical Health", "model_output.classification.iab_content.tier2.label": "Diseases and Conditions", - "model_output.classification.iab_content.tier3.label": "Allergies" + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -1403,19 +2148,25 @@ "model_output.classification.iab_content.tier3.label": "Allergies" }, "id": "medical-health-easy", - "mismatches": [], + "mismatches": [ + { + "actual": null, + "expected": "Allergies", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical easy IAB mapping case for Medical Health.", - "pass": true, + "pass": false, "status": "must_fix", "text": "what do these allergy symptoms mean" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Medical Health", "model_output.classification.iab_content.tier2.label": "Diseases and Conditions", "model_output.classification.iab_content.tier3.label": "Injuries", - "model_output.classification.iab_content.tier4.label": "First Aid" + "model_output.classification.iab_content.tier4.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -1425,18 +2176,29 @@ "model_output.classification.iab_content.tier4.label": "First Aid" }, "id": "medical-health-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": null, + "expected": "First Aid", + "path": "model_output.classification.iab_content.tier4.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Medical Health.", - "pass": true, + "pass": false, "status": "must_fix", "text": "when should i see a doctor for persistent knee pain" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Healthy Living", - "model_output.classification.iab_content.tier2.label": "Wellness", - "model_output.classification.iab_content.tier3.label": "Physical Therapy" + "model_output.classification.iab_content.mapping_mode": "exact", + "model_output.classification.iab_content.tier1.label": "Medical Health", + "model_output.classification.iab_content.tier2.label": "Surgery", + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -1445,15 +2207,36 @@ "model_output.classification.iab_content.tier3.label": "Physical Therapy" }, "id": "medical-health-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "Medical Health", + "expected": "Healthy Living", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": "Surgery", + "expected": "Wellness", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Physical Therapy", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical hard IAB mapping case for Medical Health.", - "pass": true, + "pass": false, "status": "must_fix", "text": "need medical advice about symptoms, not wellness or fitness tips" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Careers", "model_output.classification.iab_content.tier2.label": "Remote Working" }, @@ -1463,9 +2246,15 @@ "model_output.classification.iab_content.tier2.label": "Remote Working" }, "id": "careers-job-search-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical easy IAB mapping case for Careers > Job Search.", - "pass": true, + "pass": false, "status": "must_fix", "text": "best remote jobs for data analysts" }, @@ -1474,7 +2263,7 @@ "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", "model_output.classification.iab_content.tier1.label": "Business and Finance", "model_output.classification.iab_content.tier2.label": "Business", - "model_output.classification.iab_content.tier3.label": "Sales" + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -1483,17 +2272,23 @@ "model_output.classification.iab_content.tier3.label": "Sales" }, "id": "careers-job-search-medium", - "mismatches": [], + "mismatches": [ + { + "actual": null, + "expected": "Sales", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Careers > Job Search.", - "pass": true, + "pass": false, "status": "must_fix", "text": "where should i look for product manager openings" }, { "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Careers", - "model_output.classification.iab_content.tier2.label": "Job Search" + "model_output.classification.iab_content.tier1.label": "Genres", + "model_output.classification.iab_content.tier2.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -1501,17 +2296,28 @@ "model_output.classification.iab_content.tier2.label": "Job Search" }, "id": "careers-job-search-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "Genres", + "expected": "Careers", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": null, + "expected": "Job Search", + "path": "model_output.classification.iab_content.tier2.label" + } + ], "notes": "Cross-vertical hard IAB mapping case for Careers > Job Search.", - "pass": true, + "pass": false, "status": "must_fix", "text": "need help finding a new role and preparing for interviews" }, { "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Food & Drink", - "model_output.classification.iab_content.tier2.label": "Food Movements" + "model_output.classification.iab_content.tier1.label": "Personal Celebrations & Life Events", + "model_output.classification.iab_content.tier2.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -1519,18 +2325,29 @@ "model_output.classification.iab_content.tier2.label": "Food Movements" }, "id": "personal-finance-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "Personal Celebrations & Life Events", + "expected": "Food & Drink", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": null, + "expected": "Food Movements", + "path": "model_output.classification.iab_content.tier2.label" + } + ], "notes": "Cross-vertical easy IAB mapping case for Personal Finance > Financial Planning.", - "pass": true, + "pass": false, "status": "must_fix", "text": "how much should i save each month" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Business and Finance", - "model_output.classification.iab_content.tier2.label": "Economy", - "model_output.classification.iab_content.tier3.label": "Financial Reform" + "model_output.classification.iab_content.mapping_mode": "exact", + "model_output.classification.iab_content.tier1.label": "Personal Finance", + "model_output.classification.iab_content.tier2.label": null, + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -1539,15 +2356,36 @@ "model_output.classification.iab_content.tier3.label": "Financial Reform" }, "id": "personal-finance-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "Personal Finance", + "expected": "Business and Finance", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": null, + "expected": "Economy", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Financial Reform", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Personal Finance > Financial Planning.", - "pass": true, + "pass": false, "status": "must_fix", "text": "best budgeting approach for a growing family" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Personal Finance", "model_output.classification.iab_content.tier2.label": "Retirement Planning" }, @@ -1557,15 +2395,21 @@ "model_output.classification.iab_content.tier2.label": "Retirement Planning" }, "id": "personal-finance-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical hard IAB mapping case for Personal Finance > Financial Planning.", - "pass": true, + "pass": false, "status": "must_fix", "text": "need help planning savings and retirement, not business finance advice" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Family and Relationships", "model_output.classification.iab_content.tier2.label": "Parenting" }, @@ -1575,17 +2419,23 @@ "model_output.classification.iab_content.tier2.label": "Parenting" }, "id": "parenting-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical easy IAB mapping case for Family and Relationships > Parenting.", - "pass": true, + "pass": false, "status": "must_fix", "text": "tips for parenting a toddler" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Family and Relationships", - "model_output.classification.iab_content.tier2.label": "Parenting" + "model_output.classification.iab_content.mapping_mode": "exact", + "model_output.classification.iab_content.tier1.label": "Hobbies & Interests", + "model_output.classification.iab_content.tier2.label": "Content Production" }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -1593,15 +2443,31 @@ "model_output.classification.iab_content.tier2.label": "Parenting" }, "id": "parenting-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "Hobbies & Interests", + "expected": "Family and Relationships", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": "Content Production", + "expected": "Parenting", + "path": "model_output.classification.iab_content.tier2.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Family and Relationships > Parenting.", - "pass": true, + "pass": false, "status": "must_fix", "text": "how do i help my teenager spend less time online" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Family and Relationships", "model_output.classification.iab_content.tier2.label": "Parenting", "model_output.classification.iab_content.tier3.label": "Special Needs Kids" @@ -1613,15 +2479,21 @@ "model_output.classification.iab_content.tier3.label": "Special Needs Kids" }, "id": "parenting-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical hard IAB mapping case for Family and Relationships > Parenting.", - "pass": true, + "pass": false, "status": "must_fix", "text": "need parenting advice for a child starting preschool" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Home & Garden" }, "expected": { @@ -1629,17 +2501,23 @@ "model_output.classification.iab_content.tier1.label": "Home & Garden" }, "id": "gardening-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical easy IAB mapping case for Home & Garden > Gardening.", - "pass": true, + "pass": false, "status": "must_fix", "text": "best plants for a small balcony garden" }, { "actual": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Home & Garden", - "model_output.classification.iab_content.tier2.label": "Gardening" + "model_output.classification.iab_content.tier1.label": "Food & Drink", + "model_output.classification.iab_content.tier2.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -1647,15 +2525,26 @@ "model_output.classification.iab_content.tier2.label": "Gardening" }, "id": "gardening-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "Food & Drink", + "expected": "Home & Garden", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": null, + "expected": "Gardening", + "path": "model_output.classification.iab_content.tier2.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Home & Garden > Gardening.", - "pass": true, + "pass": false, "status": "must_fix", "text": "how often should i water tomato plants" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Home & Garden" }, "expected": { @@ -1663,15 +2552,21 @@ "model_output.classification.iab_content.tier1.label": "Home & Garden" }, "id": "gardening-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical hard IAB mapping case for Home & Garden > Gardening.", - "pass": true, + "pass": false, "status": "must_fix", "text": "need gardening advice for a shady backyard, not interior decor ideas" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Entertainment", "model_output.classification.iab_content.tier2.label": "Movies" }, @@ -1681,18 +2576,24 @@ "model_output.classification.iab_content.tier2.label": "Movies" }, "id": "movies-easy", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + } + ], "notes": "Cross-vertical easy IAB mapping case for Entertainment > Movies.", - "pass": true, + "pass": false, "status": "must_fix", "text": "What movie should we watch tonight?" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", - "model_output.classification.iab_content.tier1.label": "Video Gaming", - "model_output.classification.iab_content.tier2.label": "Video Game Genres", - "model_output.classification.iab_content.tier3.label": "Horror Video Games" + "model_output.classification.iab_content.mapping_mode": "exact", + "model_output.classification.iab_content.tier1.label": "Genres", + "model_output.classification.iab_content.tier2.label": "Horror", + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -1701,18 +2602,39 @@ "model_output.classification.iab_content.tier3.label": "Horror Video Games" }, "id": "movies-medium", - "mismatches": [], + "mismatches": [ + { + "actual": "Genres", + "expected": "Video Gaming", + "path": "model_output.classification.iab_content.tier1.label" + }, + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": "Horror", + "expected": "Video Game Genres", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Horror Video Games", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical medium IAB mapping case for Entertainment > Movies.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Best thriller movies from the last few years" }, { "actual": { - "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", + "model_output.classification.iab_content.mapping_mode": "exact", "model_output.classification.iab_content.tier1.label": "Entertainment", - "model_output.classification.iab_content.tier2.label": "Music", - "model_output.classification.iab_content.tier3.label": "Soundtracks, TV and Showtunes" + "model_output.classification.iab_content.tier2.label": "Movies", + "model_output.classification.iab_content.tier3.label": null }, "expected": { "model_output.classification.iab_content.mapping_mode": "nearest_equivalent", @@ -1721,9 +2643,25 @@ "model_output.classification.iab_content.tier3.label": "Soundtracks, TV and Showtunes" }, "id": "movies-hard", - "mismatches": [], + "mismatches": [ + { + "actual": "exact", + "expected": "nearest_equivalent", + "path": "model_output.classification.iab_content.mapping_mode" + }, + { + "actual": "Movies", + "expected": "Music", + "path": "model_output.classification.iab_content.tier2.label" + }, + { + "actual": null, + "expected": "Soundtracks, TV and Showtunes", + "path": "model_output.classification.iab_content.tier3.label" + } + ], "notes": "Cross-vertical hard IAB mapping case for Entertainment > Movies.", - "pass": true, + "pass": false, "status": "must_fix", "text": "Looking for film recommendations, not TV shows or music" }