{ "schema": "coreai-encoder-fixtures/1", "model": "convaiinnovations/laya", "model_sha": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "subfolder": "multilingual", "window": 256, "temperature": [ 1.0, 1.0, 1.0 ], "temperature_by_options": {}, "answers": "the official laya 0.3.4 Agent.predict, batched per fixture, max_len = window (frozen LiteRT-lane fixture)", "oracle": "the official DecisionModel on each row alone, batch 1, right-padded to this window, CPU fp32", "fixture_sha256": "98d0e591826c708375191425f4b475d1fa3f997051172ee24b91e9a52f46c83f", "oracle_outputs_sha256": "208e3cddf263dbbc57bfaa278ef3213e81082de7a7cfa9b7b3001e9b862a12cf", "rows": [ { "row_id": "ML_A01/department", "fixture_id": "ML_A01", "question_id": "department", "family": "A", "language": "EN", "primary_language": "EN", "question": { "type": "choice", "instructions": "Which department should handle this request?", "criteria": { "billing": "invoices, payments, refunds", "technical": "bugs, outages, system errors", "sales": "pricing, new contracts", "other": "everything else" } }, "qtype": 0, "K": 4, "sequence_length": 68, "sequence_ids": [ 2, 6241, 2872, 235292, 12236, 9888, 1412, 6589, 736, 3853, 235336, 1, 4, 54972, 235292, 88220, 235269, 15598, 235269, 85869, 4, 9838, 235292, 30608, 235269, 142788, 235269, 1812, 10266, 4, 7108, 235292, 25063, 235269, 888, 20078, 4, 1156, 235292, 4553, 1354, 1, 19946, 2273, 1192, 664, 56041, 11309, 824, 664, 15029, 1192, 664, 12993, 6256, 824, 664, 2168, 1192, 664, 5958, 2203, 573, 38294, 7984, 1464, 235270, 1 ], "marker_positions": [ 12, 20, 29, 36 ], "probabilities": [ 0.9999998807907104, 5.818159820591973e-08, 2.4648846519426115e-08, 6.190021650809285e-08 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "billing", "probabilities": { "billing": 1.0, "technical": 0.0, "sales": 0.0, "other": 0.0 }, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ 12.993932723999023, -3.6657912731170654, -4.524628639221191, -3.603837490081787 ], "act_logits": [ 1191.4835205078125, -1424.588623046875 ] } }, { "row_id": "ML_A01/urgency", "fixture_id": "ML_A01", "question_id": "urgency", "family": "A", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How urgent is this request?", "criteria": [ "not urgent", "soon", "critical deadline or blocking issue" ] }, "qtype": 1, "K": 3, "sequence_length": 61, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 34722, 603, 736, 3853, 235336, 1, 4, 2403, 235248, 235276, 235292, 780, 34722, 4, 2403, 235248, 235274, 235292, 4445, 4, 2403, 235248, 235284, 235292, 9523, 30699, 689, 35578, 4295, 1, 19946, 2273, 1192, 664, 56041, 11309, 824, 664, 15029, 1192, 664, 12993, 6256, 824, 664, 2168, 1192, 664, 5958, 2203, 573, 38294, 7984, 1464, 235270, 1 ], "marker_positions": [ 11, 18, 24 ], "probabilities": [ 0.025542890653014183, 0.23200185596942902, 0.7424553036689758 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.7169, "legend": { "0": "not urgent", "1": "soon", "2": "critical deadline or blocking issue" }, "probabilities": { "0": 0.0255, "1": 0.232, "2": 0.7425 }, "confidence": 0.4049, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -2.2645416259765625, -0.0581548810005188, 1.105062484741211 ], "act_logits": [ 1541.9920654296875, -1820.5230712890625 ] } }, { "row_id": "ML_A01/churn_risk", "fixture_id": "ML_A01", "question_id": "churn_risk", "family": "A", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does the user threaten to cancel or leave?" }, "qtype": 2, "K": 2, "sequence_length": 60, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 2425, 60399, 577, 12459, 689, 5077, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2273, 1192, 664, 56041, 11309, 824, 664, 15029, 1192, 664, 12993, 6256, 824, 664, 2168, 1192, 664, 5958, 2203, 573, 38294, 7984, 1464, 235270, 1 ], "marker_positions": [ 15, 25 ], "probabilities": [ 0.9958906173706055, 0.0041094026528298855 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0041, "confidence": 0.9959, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 4.8075408935546875, -0.6828190088272095 ], "act_logits": [ 1439.5357666015625, -1708.66162109375 ] } }, { "row_id": "ML_A01/refund_requested", "fixture_id": "ML_A01", "question_id": "refund_requested", "family": "A", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does the user explicitly request a refund?" }, "qtype": 2, "K": 2, "sequence_length": 59, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 2425, 34913, 3853, 476, 19745, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2273, 1192, 664, 56041, 11309, 824, 664, 15029, 1192, 664, 12993, 6256, 824, 664, 2168, 1192, 664, 5958, 2203, 573, 38294, 7984, 1464, 235270, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.14990149438381195, 0.8500984907150269 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.8501, "confidence": 0.8501, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ -1.2950031757354736, 0.4403710961341858 ], "act_logits": [ 1369.1229248046875, -1626.43798828125 ] } }, { "row_id": "ML_A02/department", "fixture_id": "ML_A02", "question_id": "department", "family": "A", "language": "EN", "primary_language": "EN", "question": { "type": "choice", "instructions": "Which department should handle this request?", "criteria": { "billing": "invoices, payments, refunds", "technical": "bugs, outages, system errors", "sales": "pricing, new contracts", "other": "everything else" } }, "qtype": 0, "K": 4, "sequence_length": 147, "sequence_ids": [ 2, 6241, 2872, 235292, 12236, 9888, 1412, 6589, 736, 3853, 235336, 1, 4, 54972, 235292, 88220, 235269, 15598, 235269, 85869, 4, 9838, 235292, 30608, 235269, 142788, 235269, 1812, 10266, 4, 7108, 235292, 25063, 235269, 888, 20078, 4, 1156, 235292, 4553, 1354, 1, 19946, 2273, 1192, 664, 56041, 11309, 824, 664, 15029, 1192, 664, 21692, 28526, 824, 664, 2168, 1192, 664, 651, 16783, 21326, 1535, 1069, 235265, 5626, 20257, 603, 28526, 3646, 235265, 1927, 736, 603, 780, 8115, 235269, 783, 877, 12459, 573, 1780, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 1464, 235270, 1 ], "marker_positions": [ 12, 20, 29, 36 ], "probabilities": [ 0.0044455742463469505, 0.9122844934463501, 0.01113373227417469, 0.07213611900806427 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "technical", "probabilities": { "billing": 0.0044, "technical": 0.9123, "sales": 0.0111, "other": 0.0721 }, "confidence": 0.7493, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -4.57979679107666, 0.7442420721054077, -3.66172456741333, -1.7931464910507202 ], "act_logits": [ 1349.5994873046875, -1610.758544921875 ] } }, { "row_id": "ML_A02/urgency", "fixture_id": "ML_A02", "question_id": "urgency", "family": "A", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How urgent is this request?", "criteria": [ "not urgent", "soon", "critical deadline or blocking issue" ] }, "qtype": 1, "K": 3, "sequence_length": 140, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 34722, 603, 736, 3853, 235336, 1, 4, 2403, 235248, 235276, 235292, 780, 34722, 4, 2403, 235248, 235274, 235292, 4445, 4, 2403, 235248, 235284, 235292, 9523, 30699, 689, 35578, 4295, 1, 19946, 2273, 1192, 664, 56041, 11309, 824, 664, 15029, 1192, 664, 21692, 28526, 824, 664, 2168, 1192, 664, 651, 16783, 21326, 1535, 1069, 235265, 5626, 20257, 603, 28526, 3646, 235265, 1927, 736, 603, 780, 8115, 235269, 783, 877, 12459, 573, 1780, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 1464, 235270, 1 ], "marker_positions": [ 11, 18, 24 ], "probabilities": [ 0.008238467387855053, 0.15517078340053558, 0.8365907073020935 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.8284, "legend": { "0": "not urgent", "1": "soon", "2": "critical deadline or blocking issue" }, "probabilities": { "0": 0.0082, "1": 0.1552, "2": 0.8366 }, "confidence": 0.565, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -2.780177116394043, 0.15553492307662964, 1.840343952178955 ], "act_logits": [ 1574.210693359375, -1862.01904296875 ] } }, { "row_id": "ML_A02/churn_risk", "fixture_id": "ML_A02", "question_id": "churn_risk", "family": "A", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does the user threaten to cancel or leave?" }, "qtype": 2, "K": 2, "sequence_length": 139, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 2425, 60399, 577, 12459, 689, 5077, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2273, 1192, 664, 56041, 11309, 824, 664, 15029, 1192, 664, 21692, 28526, 824, 664, 2168, 1192, 664, 651, 16783, 21326, 1535, 1069, 235265, 5626, 20257, 603, 28526, 3646, 235265, 1927, 736, 603, 780, 8115, 235269, 783, 877, 12459, 573, 1780, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 1464, 235270, 1 ], "marker_positions": [ 15, 25 ], "probabilities": [ 0.9835962057113647, 0.016403818503022194 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0164, "confidence": 0.9836, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 3.7650961875915527, -0.3286062479019165 ], "act_logits": [ 1208.8897705078125, -1439.7767333984375 ] } }, { "row_id": "ML_A02/refund_requested", "fixture_id": "ML_A02", "question_id": "refund_requested", "family": "A", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does the user explicitly request a refund?" }, "qtype": 2, "K": 2, "sequence_length": 138, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 2425, 34913, 3853, 476, 19745, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2273, 1192, 664, 56041, 11309, 824, 664, 15029, 1192, 664, 21692, 28526, 824, 664, 2168, 1192, 664, 651, 16783, 21326, 1535, 1069, 235265, 5626, 20257, 603, 28526, 3646, 235265, 1927, 736, 603, 780, 8115, 235269, 783, 877, 12459, 573, 1780, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 1464, 235270, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.9991276860237122, 0.0008723080973140895 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0009, "confidence": 0.9991, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 6.372121334075928, -0.671375036239624 ], "act_logits": [ 1506.6925048828125, -1792.860595703125 ] } }, { "row_id": "ML_A03/department", "fixture_id": "ML_A03", "question_id": "department", "family": "A", "language": "EN", "primary_language": "EN", "question": { "type": "choice", "instructions": "Which department should handle this request?", "criteria": { "billing": "invoices, payments, refunds", "technical": "bugs, outages, system errors", "sales": "pricing, new contracts", "other": "everything else" } }, "qtype": 0, "K": 4, "sequence_length": 256, "sequence_ids": [ 2, 6241, 2872, 235292, 12236, 9888, 1412, 6589, 736, 3853, 235336, 1, 4, 54972, 235292, 88220, 235269, 15598, 235269, 85869, 4, 9838, 235292, 30608, 235269, 142788, 235269, 1812, 10266, 4, 7108, 235292, 25063, 235269, 888, 20078, 4, 1156, 235292, 4553, 1354, 1, 19946, 2273, 1192, 664, 56041, 11309, 824, 664, 15029, 1192, 664, 9058, 4691, 824, 664, 2168, 1192, 664, 1734, 708, 14113, 476, 888, 7613, 604, 1167, 20257, 235265, 5651, 10200, 573, 3712, 604, 476, 8107, 2778, 578, 1368, 577, 9900, 476, 33436, 2351, 2788, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 1 ], "marker_positions": [ 12, 20, 29, 36 ], "probabilities": [ 0.0018838427495211363, 0.007903624325990677, 0.980435848236084, 0.009776654653251171 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "sales", "probabilities": { "billing": 0.0019, "technical": 0.0079, "sales": 0.9804, "other": 0.0098 }, "confidence": 0.9173, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:3-5", "state_right_truncated": true, "oracle": { "marker_logits": [ -3.6635069847106934, -2.229499101638794, 2.591176748275757, -2.0168230533599854 ], "act_logits": [ 1345.94482421875, -1605.2491455078125 ] } }, { "row_id": "ML_A03/urgency", "fixture_id": "ML_A03", "question_id": "urgency", "family": "A", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How urgent is this request?", "criteria": [ "not urgent", "soon", "critical deadline or blocking issue" ] }, "qtype": 1, "K": 3, "sequence_length": 256, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 34722, 603, 736, 3853, 235336, 1, 4, 2403, 235248, 235276, 235292, 780, 34722, 4, 2403, 235248, 235274, 235292, 4445, 4, 2403, 235248, 235284, 235292, 9523, 30699, 689, 35578, 4295, 1, 19946, 2273, 1192, 664, 56041, 11309, 824, 664, 15029, 1192, 664, 9058, 4691, 824, 664, 2168, 1192, 664, 1734, 708, 14113, 476, 888, 7613, 604, 1167, 20257, 235265, 5651, 10200, 573, 3712, 604, 476, 8107, 2778, 578, 1368, 577, 9900, 476, 33436, 2351, 2788, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 1 ], "marker_positions": [ 11, 18, 24 ], "probabilities": [ 0.004166151862591505, 0.2091919183731079, 0.7866419553756714 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.7825, "legend": { "0": "not urgent", "1": "soon", "2": "critical deadline or blocking issue" }, "probabilities": { "0": 0.0042, "1": 0.2092, "2": 0.7866 }, "confidence": 0.5095, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": true, "oracle": { "marker_logits": [ -2.8379783630371094, 1.0782811641693115, 2.4028022289276123 ], "act_logits": [ 1536.0687255859375, -1817.03466796875 ] } }, { "row_id": "ML_A03/churn_risk", "fixture_id": "ML_A03", "question_id": "churn_risk", "family": "A", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does the user threaten to cancel or leave?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 2425, 60399, 577, 12459, 689, 5077, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2273, 1192, 664, 56041, 11309, 824, 664, 15029, 1192, 664, 9058, 4691, 824, 664, 2168, 1192, 664, 1734, 708, 14113, 476, 888, 7613, 604, 1167, 20257, 235265, 5651, 10200, 573, 3712, 604, 476, 8107, 2778, 578, 1368, 577, 9900, 476, 33436, 2351, 2788, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1 ], "marker_positions": [ 15, 25 ], "probabilities": [ 0.9997252821922302, 0.00027469941414892673 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0003, "confidence": 0.9997, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 7.224259376525879, -0.9752992987632751 ], "act_logits": [ 1485.9893798828125, -1765.1993408203125 ] } }, { "row_id": "ML_A03/refund_requested", "fixture_id": "ML_A03", "question_id": "refund_requested", "family": "A", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does the user explicitly request a refund?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 2425, 34913, 3853, 476, 19745, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2273, 1192, 664, 56041, 11309, 824, 664, 15029, 1192, 664, 9058, 4691, 824, 664, 2168, 1192, 664, 1734, 708, 14113, 476, 888, 7613, 604, 1167, 20257, 235265, 5651, 10200, 573, 3712, 604, 476, 8107, 2778, 578, 1368, 577, 9900, 476, 33436, 2351, 2788, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.9981619715690613, 0.0018380879191681743 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0018, "confidence": 0.9982, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 5.557384490966797, -0.7398056387901306 ], "act_logits": [ 1447.790771484375, -1718.219970703125 ] } }, { "row_id": "ML_A04/department", "fixture_id": "ML_A04", "question_id": "department", "family": "A", "language": "JA", "primary_language": "JA", "question": { "type": "choice", "instructions": "Which department should handle this request?", "criteria": { "billing": "invoices, payments, refunds", "technical": "bugs, outages, system errors", "sales": "pricing, new contracts", "other": "everything else" } }, "qtype": 0, "K": 4, "sequence_length": 77, "sequence_ids": [ 2, 6241, 2872, 235292, 12236, 9888, 1412, 6589, 736, 3853, 235336, 1, 4, 54972, 235292, 88220, 235269, 15598, 235269, 85869, 4, 9838, 235292, 30608, 235269, 142788, 235269, 1812, 10266, 4, 7108, 235292, 25063, 235269, 888, 20078, 4, 1156, 235292, 4553, 1354, 1, 19946, 2273, 1192, 664, 63090, 236948, 824, 664, 15029, 1192, 664, 217491, 133074, 824, 664, 2168, 1192, 664, 62737, 235425, 217491, 235400, 33341, 150432, 47644, 235362, 236631, 238082, 235432, 236572, 235854, 54934, 235362, 12990, 1 ], "marker_positions": [ 12, 20, 29, 36 ], "probabilities": [ 0.9992189407348633, 5.210045401327079e-06, 0.0007621371187269688, 1.3717257388634607e-05 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "billing", "probabilities": { "billing": 0.9992, "technical": 0.0, "sales": 0.0008, "other": 0.0 }, "confidence": 0.9953, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ 7.643063068389893, -4.521077632904053, 0.4644666314125061, -3.5530123710632324 ], "act_logits": [ 1316.449951171875, -1573.5203857421875 ] } }, { "row_id": "ML_A04/urgency", "fixture_id": "ML_A04", "question_id": "urgency", "family": "A", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How urgent is this request?", "criteria": [ "not urgent", "soon", "critical deadline or blocking issue" ] }, "qtype": 1, "K": 3, "sequence_length": 70, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 34722, 603, 736, 3853, 235336, 1, 4, 2403, 235248, 235276, 235292, 780, 34722, 4, 2403, 235248, 235274, 235292, 4445, 4, 2403, 235248, 235284, 235292, 9523, 30699, 689, 35578, 4295, 1, 19946, 2273, 1192, 664, 63090, 236948, 824, 664, 15029, 1192, 664, 217491, 133074, 824, 664, 2168, 1192, 664, 62737, 235425, 217491, 235400, 33341, 150432, 47644, 235362, 236631, 238082, 235432, 236572, 235854, 54934, 235362, 12990, 1 ], "marker_positions": [ 11, 18, 24 ], "probabilities": [ 0.010628378950059414, 0.15766942501068115, 0.8317021727561951 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.8211, "legend": { "0": "not urgent", "1": "soon", "2": "critical deadline or blocking issue" }, "probabilities": { "0": 0.0106, "1": 0.1577, "2": 0.8317 }, "confidence": 0.5514, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -2.7175726890563965, -0.020600318908691406, 1.642374038696289 ], "act_logits": [ 1561.7000732421875, -1846.804443359375 ] } }, { "row_id": "ML_A04/churn_risk", "fixture_id": "ML_A04", "question_id": "churn_risk", "family": "A", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does the user threaten to cancel or leave?" }, "qtype": 2, "K": 2, "sequence_length": 69, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 2425, 60399, 577, 12459, 689, 5077, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2273, 1192, 664, 63090, 236948, 824, 664, 15029, 1192, 664, 217491, 133074, 824, 664, 2168, 1192, 664, 62737, 235425, 217491, 235400, 33341, 150432, 47644, 235362, 236631, 238082, 235432, 236572, 235854, 54934, 235362, 12990, 1 ], "marker_positions": [ 15, 25 ], "probabilities": [ 0.9871699810028076, 0.012829991988837719 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0128, "confidence": 0.9872, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 4.319343566894531, -0.0237123966217041 ], "act_logits": [ 1322.129638671875, -1573.8560791015625 ] } }, { "row_id": "ML_A04/refund_requested", "fixture_id": "ML_A04", "question_id": "refund_requested", "family": "A", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does the user explicitly request a refund?" }, "qtype": 2, "K": 2, "sequence_length": 68, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 2425, 34913, 3853, 476, 19745, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2273, 1192, 664, 63090, 236948, 824, 664, 15029, 1192, 664, 217491, 133074, 824, 664, 2168, 1192, 664, 62737, 235425, 217491, 235400, 33341, 150432, 47644, 235362, 236631, 238082, 235432, 236572, 235854, 54934, 235362, 12990, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.010173930786550045, 0.9898260831832886 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.9898, "confidence": 0.9898, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ -3.8145623207092285, 0.7631384134292603 ], "act_logits": [ 1234.306640625, -1472.085693359375 ] } }, { "row_id": "ML_A05/department", "fixture_id": "ML_A05", "question_id": "department", "family": "A", "language": "JA", "primary_language": "JA", "question": { "type": "choice", "instructions": "Which department should handle this request?", "criteria": { "billing": "invoices, payments, refunds", "technical": "bugs, outages, system errors", "sales": "pricing, new contracts", "other": "everything else" } }, "qtype": 0, "K": 4, "sequence_length": 173, "sequence_ids": [ 2, 6241, 2872, 235292, 12236, 9888, 1412, 6589, 736, 3853, 235336, 1, 4, 54972, 235292, 88220, 235269, 15598, 235269, 85869, 4, 9838, 235292, 30608, 235269, 142788, 235269, 1812, 10266, 4, 7108, 235292, 25063, 235269, 888, 20078, 4, 1156, 235292, 4553, 1354, 1, 19946, 2273, 1192, 664, 63090, 236948, 824, 664, 15029, 1192, 664, 32452, 87735, 824, 664, 2168, 1192, 664, 27364, 235432, 32452, 218267, 24847, 40081, 235425, 236567, 146267, 235362, 137988, 44393, 233772, 235664, 21509, 9481, 235362, 235948, 26011, 40957, 235868, 236517, 235432, 85077, 9178, 235362, 189686, 29709, 119920, 9178, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 12990, 1 ], "marker_positions": [ 12, 20, 29, 36 ], "probabilities": [ 0.03208402916789055, 0.44801387190818787, 0.0884794071316719, 0.43142274022102356 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "technical", "probabilities": { "billing": 0.0321, "technical": 0.448, "sales": 0.0885, "other": 0.4314 }, "confidence": 0.2445, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -4.4264302253723145, -1.7899632453918457, -3.412019968032837, -1.8277055025100708 ], "act_logits": [ 1255.0972900390625, -1495.9267578125 ] } }, { "row_id": "ML_A05/urgency", "fixture_id": "ML_A05", "question_id": "urgency", "family": "A", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How urgent is this request?", "criteria": [ "not urgent", "soon", "critical deadline or blocking issue" ] }, "qtype": 1, "K": 3, "sequence_length": 166, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 34722, 603, 736, 3853, 235336, 1, 4, 2403, 235248, 235276, 235292, 780, 34722, 4, 2403, 235248, 235274, 235292, 4445, 4, 2403, 235248, 235284, 235292, 9523, 30699, 689, 35578, 4295, 1, 19946, 2273, 1192, 664, 63090, 236948, 824, 664, 15029, 1192, 664, 32452, 87735, 824, 664, 2168, 1192, 664, 27364, 235432, 32452, 218267, 24847, 40081, 235425, 236567, 146267, 235362, 137988, 44393, 233772, 235664, 21509, 9481, 235362, 235948, 26011, 40957, 235868, 236517, 235432, 85077, 9178, 235362, 189686, 29709, 119920, 9178, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 12990, 1 ], "marker_positions": [ 11, 18, 24 ], "probabilities": [ 0.004046465270221233, 0.17571629583835602, 0.8202372193336487 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.8162, "legend": { "0": "not urgent", "1": "soon", "2": "critical deadline or blocking issue" }, "probabilities": { "0": 0.004, "1": 0.1757, "2": 0.8202 }, "confidence": 0.5536, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -2.6476645469665527, 1.1233625411987305, 2.6640849113464355 ], "act_logits": [ 1556.85205078125, -1839.682373046875 ] } }, { "row_id": "ML_A05/churn_risk", "fixture_id": "ML_A05", "question_id": "churn_risk", "family": "A", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does the user threaten to cancel or leave?" }, "qtype": 2, "K": 2, "sequence_length": 165, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 2425, 60399, 577, 12459, 689, 5077, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2273, 1192, 664, 63090, 236948, 824, 664, 15029, 1192, 664, 32452, 87735, 824, 664, 2168, 1192, 664, 27364, 235432, 32452, 218267, 24847, 40081, 235425, 236567, 146267, 235362, 137988, 44393, 233772, 235664, 21509, 9481, 235362, 235948, 26011, 40957, 235868, 236517, 235432, 85077, 9178, 235362, 189686, 29709, 119920, 9178, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 12990, 1 ], "marker_positions": [ 15, 25 ], "probabilities": [ 0.9978975057601929, 0.0021024872548878193 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0021, "confidence": 0.9979, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 5.328619480133057, -0.8339105248451233 ], "act_logits": [ 1295.109375, -1540.0726318359375 ] } }, { "row_id": "ML_A05/refund_requested", "fixture_id": "ML_A05", "question_id": "refund_requested", "family": "A", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does the user explicitly request a refund?" }, "qtype": 2, "K": 2, "sequence_length": 164, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 2425, 34913, 3853, 476, 19745, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2273, 1192, 664, 63090, 236948, 824, 664, 15029, 1192, 664, 32452, 87735, 824, 664, 2168, 1192, 664, 27364, 235432, 32452, 218267, 24847, 40081, 235425, 236567, 146267, 235362, 137988, 44393, 233772, 235664, 21509, 9481, 235362, 235948, 26011, 40957, 235868, 236517, 235432, 85077, 9178, 235362, 189686, 29709, 119920, 9178, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 12990, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.9996463060379028, 0.00035368380486033857 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0004, "confidence": 0.9996, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 7.038372039794922, -0.9083811044692993 ], "act_logits": [ 1597.7738037109375, -1900.33642578125 ] } }, { "row_id": "ML_A06/department", "fixture_id": "ML_A06", "question_id": "department", "family": "A", "language": "JA", "primary_language": "JA", "question": { "type": "choice", "instructions": "Which department should handle this request?", "criteria": { "billing": "invoices, payments, refunds", "technical": "bugs, outages, system errors", "sales": "pricing, new contracts", "other": "everything else" } }, "qtype": 0, "K": 4, "sequence_length": 256, "sequence_ids": [ 2, 6241, 2872, 235292, 12236, 9888, 1412, 6589, 736, 3853, 235336, 1, 4, 54972, 235292, 88220, 235269, 15598, 235269, 85869, 4, 9838, 235292, 30608, 235269, 142788, 235269, 1812, 10266, 4, 7108, 235292, 25063, 235269, 888, 20078, 4, 1156, 235292, 4553, 1354, 1, 19946, 2273, 1192, 664, 63090, 236948, 824, 664, 15029, 1192, 664, 235545, 237090, 53410, 63090, 824, 664, 2168, 1192, 664, 235938, 35539, 44393, 236322, 132095, 235677, 88302, 81322, 73476, 235394, 57853, 235362, 28221, 235425, 89690, 97540, 235394, 24247, 235714, 12588, 163812, 11960, 235362, 32776, 235545, 237090, 34319, 53410, 63090, 2297, 235362, 54755, 235418, 18210, 2297, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 1 ], "marker_positions": [ 12, 20, 29, 36 ], "probabilities": [ 0.06967029720544815, 0.15413546562194824, 0.10963791608810425, 0.6665562987327576 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "other", "probabilities": { "billing": 0.0697, "technical": 0.1541, "sales": 0.1096, "other": 0.6666 }, "confidence": 0.2883, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:3-5", "state_right_truncated": true, "oracle": { "marker_logits": [ -4.225309371948242, -3.4312520027160645, -3.7719011306762695, -1.9669591188430786 ], "act_logits": [ 1192.6673583984375, -1424.5753173828125 ] } }, { "row_id": "ML_A06/urgency", "fixture_id": "ML_A06", "question_id": "urgency", "family": "A", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How urgent is this request?", "criteria": [ "not urgent", "soon", "critical deadline or blocking issue" ] }, "qtype": 1, "K": 3, "sequence_length": 256, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 34722, 603, 736, 3853, 235336, 1, 4, 2403, 235248, 235276, 235292, 780, 34722, 4, 2403, 235248, 235274, 235292, 4445, 4, 2403, 235248, 235284, 235292, 9523, 30699, 689, 35578, 4295, 1, 19946, 2273, 1192, 664, 63090, 236948, 824, 664, 15029, 1192, 664, 235545, 237090, 53410, 63090, 824, 664, 2168, 1192, 664, 235938, 35539, 44393, 236322, 132095, 235677, 88302, 81322, 73476, 235394, 57853, 235362, 28221, 235425, 89690, 97540, 235394, 24247, 235714, 12588, 163812, 11960, 235362, 32776, 235545, 237090, 34319, 53410, 63090, 2297, 235362, 54755, 235418, 18210, 2297, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 1 ], "marker_positions": [ 11, 18, 24 ], "probabilities": [ 0.019749503582715988, 0.3695833384990692, 0.6106671094894409 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.5909, "legend": { "0": "not urgent", "1": "soon", "2": "critical deadline or blocking issue" }, "probabilities": { "0": 0.0197, "1": 0.3696, "2": 0.6107 }, "confidence": 0.3204, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": true, "oracle": { "marker_logits": [ -1.4877381324768066, 1.4415098428726196, 1.9436858892440796 ], "act_logits": [ 1516.8128662109375, -1789.4168701171875 ] } }, { "row_id": "ML_A06/churn_risk", "fixture_id": "ML_A06", "question_id": "churn_risk", "family": "A", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does the user threaten to cancel or leave?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 2425, 60399, 577, 12459, 689, 5077, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2273, 1192, 664, 63090, 236948, 824, 664, 15029, 1192, 664, 235545, 237090, 53410, 63090, 824, 664, 2168, 1192, 664, 235938, 35539, 44393, 236322, 132095, 235677, 88302, 81322, 73476, 235394, 57853, 235362, 28221, 235425, 89690, 97540, 235394, 24247, 235714, 12588, 163812, 11960, 235362, 32776, 235545, 237090, 34319, 53410, 63090, 2297, 235362, 54755, 235418, 18210, 2297, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 1 ], "marker_positions": [ 15, 25 ], "probabilities": [ 0.9999998807907104, 7.986622563294077e-08 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 15.039213180541992, -1.3036998510360718 ], "act_logits": [ 1472.8133544921875, -1751.9169921875 ] } }, { "row_id": "ML_A06/refund_requested", "fixture_id": "ML_A06", "question_id": "refund_requested", "family": "A", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does the user explicitly request a refund?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 2425, 34913, 3853, 476, 19745, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2273, 1192, 664, 63090, 236948, 824, 664, 15029, 1192, 664, 235545, 237090, 53410, 63090, 824, 664, 2168, 1192, 664, 235938, 35539, 44393, 236322, 132095, 235677, 88302, 81322, 73476, 235394, 57853, 235362, 28221, 235425, 89690, 97540, 235394, 24247, 235714, 12588, 163812, 11960, 235362, 32776, 235545, 237090, 34319, 53410, 63090, 2297, 235362, 54755, 235418, 18210, 2297, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 50855, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.9999406337738037, 5.9388254157966e-05 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0001, "confidence": 0.9999, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 8.77853775024414, -0.9528175592422485 ], "act_logits": [ 1440.4320068359375, -1713.3397216796875 ] } }, { "row_id": "ML_B01/category", "fixture_id": "ML_B01", "question_id": "category", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "choice", "instructions": "Which team should handle the email in `body`?", "criteria": { "billing": "invoices, payments, refunds", "technical": "bugs, outages, integrations", "sales": "pricing, demos, new purchases", "security": "phishing, scams, account compromise", "hr": "hiring, leave, payroll", "other": "none of the above" } }, "qtype": 0, "K": 6, "sequence_length": 94, "sequence_ids": [ 2, 6241, 2872, 235292, 12236, 2970, 1412, 6589, 573, 4801, 575, 4103, 2168, 235376, 235336, 1, 4, 54972, 235292, 88220, 235269, 15598, 235269, 85869, 4, 9838, 235292, 30608, 235269, 142788, 235269, 134538, 4, 7108, 235292, 25063, 235269, 105404, 235269, 888, 29330, 4, 6206, 235292, 118602, 235269, 108105, 235269, 3185, 37031, 4, 25211, 235292, 34233, 235269, 5077, 235269, 63529, 4, 1156, 235292, 8031, 576, 573, 3131, 1, 19946, 15029, 1192, 664, 40308, 5469, 824, 664, 2168, 1192, 664, 5958, 5527, 476, 5469, 576, 1167, 6838, 42529, 19515, 664, 2273, 1192, 664, 5041, 11309, 12990, 1 ], "marker_positions": [ 16, 24, 32, 41, 50, 58 ], "probabilities": [ 0.9999996423721313, 7.615844310748798e-08, 6.101717531237227e-08, 5.7436555067624795e-08, 5.730524321734265e-08, 1.5779764339640678e-07 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "billing", "probabilities": { "billing": 1.0, "technical": 0.0, "sales": 0.0, "security": 0.0, "hr": 0.0, "other": 0.0 }, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": false, "oracle": { "marker_logits": [ 12.085796356201172, -4.304623126983643, -4.5262861251831055, -4.586759567260742, -4.589051246643066, -3.5761218070983887 ], "act_logits": [ 1255.5592041015625, -1500.880859375 ] } }, { "row_id": "ML_B01/is_spam", "fixture_id": "ML_B01", "question_id": "is_spam", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Is this email unsolicited spam or bulk marketing?" }, "qtype": 2, "K": 2, "sequence_length": 62, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 736, 4801, 183367, 31967, 689, 19717, 8924, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 15029, 1192, 664, 40308, 5469, 824, 664, 2168, 1192, 664, 5958, 5527, 476, 5469, 576, 1167, 6838, 42529, 19515, 664, 2273, 1192, 664, 5041, 11309, 12990, 1 ], "marker_positions": [ 15, 25 ], "probabilities": [ 0.99994957447052, 5.043886631028727e-05 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0001, "confidence": 0.9999, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 8.284006118774414, -1.6106913089752197 ], "act_logits": [ 1521.6600341796875, -1809.433837890625 ] } }, { "row_id": "ML_B01/is_phishing", "fixture_id": "ML_B01", "question_id": "is_phishing", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Is this email a phishing or scam attempt to steal money, credentials, or personal data?", "criteria": { "true": "phishing, scam, or fraud", "false": "a legitimate email" } }, "qtype": 2, "K": 2, "sequence_length": 68, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 736, 4801, 476, 118602, 689, 37853, 8697, 577, 32261, 3370, 235269, 36983, 235269, 689, 3749, 1423, 235336, 1, 4, 1566, 235292, 476, 33957, 4801, 4, 1382, 235292, 118602, 235269, 37853, 235269, 689, 19367, 1, 19946, 15029, 1192, 664, 40308, 5469, 824, 664, 2168, 1192, 664, 5958, 5527, 476, 5469, 576, 1167, 6838, 42529, 19515, 664, 2273, 1192, 664, 5041, 11309, 12990, 1 ], "marker_positions": [ 24, 30 ], "probabilities": [ 0.9999817609786987, 1.8181413906859234e-05 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 6.686473846435547, -4.228619575500488 ], "act_logits": [ 1484.0855712890625, -1767.619140625 ] } }, { "row_id": "ML_B01/urgency", "fixture_id": "ML_B01", "question_id": "urgency", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How urgent is the request in `body`?", "criteria": [ "no time pressure", "needs attention soon", "blocking issue or hard deadline" ] }, "qtype": 1, "K": 3, "sequence_length": 70, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 34722, 603, 573, 3853, 575, 4103, 2168, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 793, 1069, 5601, 4, 2403, 235248, 235274, 235292, 4026, 6137, 4445, 4, 2403, 235248, 235284, 235292, 35578, 4295, 689, 2811, 30699, 1, 19946, 15029, 1192, 664, 40308, 5469, 824, 664, 2168, 1192, 664, 5958, 5527, 476, 5469, 576, 1167, 6838, 42529, 19515, 664, 2273, 1192, 664, 5041, 11309, 12990, 1 ], "marker_positions": [ 15, 23, 31 ], "probabilities": [ 0.0042721182107925415, 0.20295366644859314, 0.7927742004394531 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.7885, "legend": { "0": "no time pressure", "1": "needs attention soon", "2": "blocking issue or hard deadline" }, "probabilities": { "0": 0.0043, "1": 0.203, "2": 0.7928 }, "confidence": 0.5166, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -3.4125170707702637, 0.44835132360458374, 1.8109116554260254 ], "act_logits": [ 1491.8404541015625, -1760.91015625 ] } }, { "row_id": "ML_B01/needs_reply", "fixture_id": "ML_B01", "question_id": "needs_reply", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does the sender expect a reply?" }, "qtype": 2, "K": 2, "sequence_length": 60, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 14695, 5246, 476, 7260, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 15029, 1192, 664, 40308, 5469, 824, 664, 2168, 1192, 664, 5958, 5527, 476, 5469, 576, 1167, 6838, 42529, 19515, 664, 2273, 1192, 664, 5041, 11309, 12990, 1 ], "marker_positions": [ 13, 23 ], "probabilities": [ 0.9260594844818115, 0.07394052296876907 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0739, "confidence": 0.9261, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 2.2682528495788574, -0.2594255805015564 ], "act_logits": [ 1314.0147705078125, -1562.2159423828125 ] } }, { "row_id": "ML_B02/category", "fixture_id": "ML_B02", "question_id": "category", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "choice", "instructions": "Which team should handle the email in `body`?", "criteria": { "billing": "invoices, payments, refunds", "technical": "bugs, outages, integrations", "sales": "pricing, demos, new purchases", "security": "phishing, scams, account compromise", "hr": "hiring, leave, payroll", "other": "none of the above" } }, "qtype": 0, "K": 6, "sequence_length": 168, "sequence_ids": [ 2, 6241, 2872, 235292, 12236, 2970, 1412, 6589, 573, 4801, 575, 4103, 2168, 235376, 235336, 1, 4, 54972, 235292, 88220, 235269, 15598, 235269, 85869, 4, 9838, 235292, 30608, 235269, 142788, 235269, 134538, 4, 7108, 235292, 25063, 235269, 105404, 235269, 888, 29330, 4, 6206, 235292, 118602, 235269, 108105, 235269, 3185, 37031, 4, 25211, 235292, 34233, 235269, 5077, 235269, 63529, 4, 1156, 235292, 8031, 576, 573, 3131, 1, 19946, 15029, 1192, 664, 14414, 11892, 824, 664, 2168, 1192, 664, 651, 1635, 11892, 736, 5764, 235265, 1448, 1476, 577, 11885, 573, 15162, 1794, 14610, 235265, 5651, 7260, 675, 573, 2351, 7161, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 19515, 664, 2273, 1192, 664, 5041, 11309, 12990, 1 ], "marker_positions": [ 16, 24, 32, 41, 50, 58 ], "probabilities": [ 0.00534707959741354, 0.03833889588713646, 0.016684329137206078, 0.0024674036540091038, 0.020952537655830383, 0.9162098169326782 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "other", "probabilities": { "billing": 0.0053, "technical": 0.0383, "sales": 0.0167, "security": 0.0025, "hr": 0.021, "other": 0.9162 }, "confidence": 0.7783, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": false, "oracle": { "marker_logits": [ -3.2603039741516113, -1.2903892993927002, -2.1223843097686768, -4.033688545227051, -1.8945940732955933, 1.883391261100769 ], "act_logits": [ 1325.621337890625, -1579.927490234375 ] } }, { "row_id": "ML_B02/is_spam", "fixture_id": "ML_B02", "question_id": "is_spam", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Is this email unsolicited spam or bulk marketing?" }, "qtype": 2, "K": 2, "sequence_length": 136, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 736, 4801, 183367, 31967, 689, 19717, 8924, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 15029, 1192, 664, 14414, 11892, 824, 664, 2168, 1192, 664, 651, 1635, 11892, 736, 5764, 235265, 1448, 1476, 577, 11885, 573, 15162, 1794, 14610, 235265, 5651, 7260, 675, 573, 2351, 7161, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 19515, 664, 2273, 1192, 664, 5041, 11309, 12990, 1 ], "marker_positions": [ 15, 25 ], "probabilities": [ 1.0, 5.6502305056937985e-09 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 17.62608528137207, -1.3654841184616089 ], "act_logits": [ 1476.7271728515625, -1756.705810546875 ] } }, { "row_id": "ML_B02/is_phishing", "fixture_id": "ML_B02", "question_id": "is_phishing", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Is this email a phishing or scam attempt to steal money, credentials, or personal data?", "criteria": { "true": "phishing, scam, or fraud", "false": "a legitimate email" } }, "qtype": 2, "K": 2, "sequence_length": 142, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 736, 4801, 476, 118602, 689, 37853, 8697, 577, 32261, 3370, 235269, 36983, 235269, 689, 3749, 1423, 235336, 1, 4, 1566, 235292, 476, 33957, 4801, 4, 1382, 235292, 118602, 235269, 37853, 235269, 689, 19367, 1, 19946, 15029, 1192, 664, 14414, 11892, 824, 664, 2168, 1192, 664, 651, 1635, 11892, 736, 5764, 235265, 1448, 1476, 577, 11885, 573, 15162, 1794, 14610, 235265, 5651, 7260, 675, 573, 2351, 7161, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 19515, 664, 2273, 1192, 664, 5041, 11309, 12990, 1 ], "marker_positions": [ 24, 30 ], "probabilities": [ 1.0, 1.652857295475485e-09 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 16.734619140625, -3.4861414432525635 ], "act_logits": [ 1460.8197021484375, -1739.0811767578125 ] } }, { "row_id": "ML_B02/urgency", "fixture_id": "ML_B02", "question_id": "urgency", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How urgent is the request in `body`?", "criteria": [ "no time pressure", "needs attention soon", "blocking issue or hard deadline" ] }, "qtype": 1, "K": 3, "sequence_length": 144, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 34722, 603, 573, 3853, 575, 4103, 2168, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 793, 1069, 5601, 4, 2403, 235248, 235274, 235292, 4026, 6137, 4445, 4, 2403, 235248, 235284, 235292, 35578, 4295, 689, 2811, 30699, 1, 19946, 15029, 1192, 664, 14414, 11892, 824, 664, 2168, 1192, 664, 651, 1635, 11892, 736, 5764, 235265, 1448, 1476, 577, 11885, 573, 15162, 1794, 14610, 235265, 5651, 7260, 675, 573, 2351, 7161, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 19515, 664, 2273, 1192, 664, 5041, 11309, 12990, 1 ], "marker_positions": [ 15, 23, 31 ], "probabilities": [ 0.031963981688022614, 0.36337810754776, 0.6046579480171204 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.5727, "legend": { "0": "no time pressure", "1": "needs attention soon", "2": "blocking issue or hard deadline" }, "probabilities": { "0": 0.032, "1": 0.3634, "2": 0.6047 }, "confidence": 0.2881, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -0.9312545657157898, 1.4995801448822021, 2.0087990760803223 ], "act_logits": [ 1546.5059814453125, -1825.812255859375 ] } }, { "row_id": "ML_B02/needs_reply", "fixture_id": "ML_B02", "question_id": "needs_reply", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does the sender expect a reply?" }, "qtype": 2, "K": 2, "sequence_length": 134, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 14695, 5246, 476, 7260, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 15029, 1192, 664, 14414, 11892, 824, 664, 2168, 1192, 664, 651, 1635, 11892, 736, 5764, 235265, 1448, 1476, 577, 11885, 573, 15162, 1794, 14610, 235265, 5651, 7260, 675, 573, 2351, 7161, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 19515, 664, 2273, 1192, 664, 5041, 11309, 12990, 1 ], "marker_positions": [ 13, 23 ], "probabilities": [ 0.7370258569717407, 0.2629741132259369 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.263, "confidence": 0.737, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 1.1615427732467651, 0.1309746503829956 ], "act_logits": [ 1334.1756591796875, -1582.86767578125 ] } }, { "row_id": "ML_B03/category", "fixture_id": "ML_B03", "question_id": "category", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "choice", "instructions": "Which team should handle the email in `body`?", "criteria": { "billing": "invoices, payments, refunds", "technical": "bugs, outages, integrations", "sales": "pricing, demos, new purchases", "security": "phishing, scams, account compromise", "hr": "hiring, leave, payroll", "other": "none of the above" } }, "qtype": 0, "K": 6, "sequence_length": 256, "sequence_ids": [ 2, 6241, 2872, 235292, 12236, 2970, 1412, 6589, 573, 4801, 575, 4103, 2168, 235376, 235336, 1, 4, 54972, 235292, 88220, 235269, 15598, 235269, 85869, 4, 9838, 235292, 30608, 235269, 142788, 235269, 134538, 4, 7108, 235292, 25063, 235269, 105404, 235269, 888, 29330, 4, 6206, 235292, 118602, 235269, 108105, 235269, 3185, 37031, 4, 25211, 235292, 34233, 235269, 5077, 235269, 63529, 4, 1156, 235292, 8031, 576, 573, 3131, 1, 19946, 15029, 1192, 664, 26440, 11968, 824, 664, 2168, 1192, 664, 5958, 10508, 4270, 573, 15162, 5077, 11968, 919, 1125, 11559, 235265, 1448, 708, 20553, 573, 79623, 1780, 578, 1134, 13849, 476, 7260, 736, 2788, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 1 ], "marker_positions": [ 16, 24, 32, 41, 50, 58 ], "probabilities": [ 0.0022935073357075453, 0.008254819549620152, 0.005009915214031935, 0.0023293793201446533, 0.6720939874649048, 0.310018390417099 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "hr", "probabilities": { "billing": 0.0023, "technical": 0.0083, "sales": 0.005, "security": 0.0023, "hr": 0.6721, "other": 0.31 }, "confidence": 0.5957, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": true, "oracle": { "marker_logits": [ -4.592365264892578, -3.311650276184082, -3.8110275268554688, -4.576846122741699, 1.0879511833190918, 0.31418412923812866 ], "act_logits": [ 1292.8934326171875, -1543.367431640625 ] } }, { "row_id": "ML_B03/is_spam", "fixture_id": "ML_B03", "question_id": "is_spam", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Is this email unsolicited spam or bulk marketing?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 736, 4801, 183367, 31967, 689, 19717, 8924, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 15029, 1192, 664, 26440, 11968, 824, 664, 2168, 1192, 664, 5958, 10508, 4270, 573, 15162, 5077, 11968, 919, 1125, 11559, 235265, 1448, 708, 20553, 573, 79623, 1780, 578, 1134, 13849, 476, 7260, 736, 2788, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 476, 1889, 576, 13333, 1 ], "marker_positions": [ 15, 25 ], "probabilities": [ 1.0, 6.086532944316048e-10 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 17.633541107177734, -3.5862317085266113 ], "act_logits": [ 1427.3902587890625, -1699.12646484375 ] } }, { "row_id": "ML_B03/is_phishing", "fixture_id": "ML_B03", "question_id": "is_phishing", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Is this email a phishing or scam attempt to steal money, credentials, or personal data?", "criteria": { "true": "phishing, scam, or fraud", "false": "a legitimate email" } }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 736, 4801, 476, 118602, 689, 37853, 8697, 577, 32261, 3370, 235269, 36983, 235269, 689, 3749, 1423, 235336, 1, 4, 1566, 235292, 476, 33957, 4801, 4, 1382, 235292, 118602, 235269, 37853, 235269, 689, 19367, 1, 19946, 15029, 1192, 664, 26440, 11968, 824, 664, 2168, 1192, 664, 5958, 10508, 4270, 573, 15162, 5077, 11968, 919, 1125, 11559, 235265, 1448, 708, 20553, 573, 79623, 1780, 578, 1134, 13849, 476, 7260, 736, 2788, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 1 ], "marker_positions": [ 24, 30 ], "probabilities": [ 1.0, 1.7969147003693564e-11 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 17.68414306640625, -7.058221340179443 ], "act_logits": [ 1418.8341064453125, -1691.11572265625 ] } }, { "row_id": "ML_B03/urgency", "fixture_id": "ML_B03", "question_id": "urgency", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How urgent is the request in `body`?", "criteria": [ "no time pressure", "needs attention soon", "blocking issue or hard deadline" ] }, "qtype": 1, "K": 3, "sequence_length": 256, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 34722, 603, 573, 3853, 575, 4103, 2168, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 793, 1069, 5601, 4, 2403, 235248, 235274, 235292, 4026, 6137, 4445, 4, 2403, 235248, 235284, 235292, 35578, 4295, 689, 2811, 30699, 1, 19946, 15029, 1192, 664, 26440, 11968, 824, 664, 2168, 1192, 664, 5958, 10508, 4270, 573, 15162, 5077, 11968, 919, 1125, 11559, 235265, 1448, 708, 20553, 573, 79623, 1780, 578, 1134, 13849, 476, 7260, 736, 2788, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 1 ], "marker_positions": [ 15, 23, 31 ], "probabilities": [ 0.02179609052836895, 0.6046504378318787, 0.37355345487594604 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.3518, "legend": { "0": "no time pressure", "1": "needs attention soon", "2": "blocking issue or hard deadline" }, "probabilities": { "0": 0.0218, "1": 0.6047, "2": 0.3736 }, "confidence": 0.3124, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": true, "oracle": { "marker_logits": [ -1.4005411863327026, 1.9223793745040894, 1.4407893419265747 ], "act_logits": [ 1525.621826171875, -1799.9793701171875 ] } }, { "row_id": "ML_B03/needs_reply", "fixture_id": "ML_B03", "question_id": "needs_reply", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does the sender expect a reply?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 14695, 5246, 476, 7260, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 15029, 1192, 664, 26440, 11968, 824, 664, 2168, 1192, 664, 5958, 10508, 4270, 573, 15162, 5077, 11968, 919, 1125, 11559, 235265, 1448, 708, 20553, 573, 79623, 1780, 578, 1134, 13849, 476, 7260, 736, 2788, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 476, 1889, 576, 13333, 235269, 578, 1 ], "marker_positions": [ 13, 23 ], "probabilities": [ 0.46634986996650696, 0.5336501002311707 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.5337, "confidence": 0.5337, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 0.10422414541244507, 0.2390286922454834 ], "act_logits": [ 1247.007568359375, -1482.67138671875 ] } }, { "row_id": "ML_B04/category", "fixture_id": "ML_B04", "question_id": "category", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "choice", "instructions": "Which team should handle the email in `body`?", "criteria": { "billing": "invoices, payments, refunds", "technical": "bugs, outages, integrations", "sales": "pricing, demos, new purchases", "security": "phishing, scams, account compromise", "hr": "hiring, leave, payroll", "other": "none of the above" } }, "qtype": 0, "K": 6, "sequence_length": 98, "sequence_ids": [ 2, 6241, 2872, 235292, 12236, 2970, 1412, 6589, 573, 4801, 575, 4103, 2168, 235376, 235336, 1, 4, 54972, 235292, 88220, 235269, 15598, 235269, 85869, 4, 9838, 235292, 30608, 235269, 142788, 235269, 134538, 4, 7108, 235292, 25063, 235269, 105404, 235269, 888, 29330, 4, 6206, 235292, 118602, 235269, 108105, 235269, 3185, 37031, 4, 25211, 235292, 34233, 235269, 5077, 235269, 63529, 4, 1156, 235292, 8031, 576, 573, 3131, 1, 19946, 15029, 1192, 664, 94898, 61863, 824, 664, 2168, 1192, 664, 27428, 208345, 235341, 15759, 1490, 235341, 17254, 7984, 577, 5035, 861, 17777, 41409, 664, 2273, 1192, 664, 5041, 11309, 12990, 1 ], "marker_positions": [ 16, 24, 32, 41, 50, 58 ], "probabilities": [ 0.480670303106308, 0.004561949986964464, 0.5030978322029114, 0.0022389197256416082, 0.0015982812037691474, 0.00783265009522438 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "sales", "probabilities": { "billing": 0.4807, "technical": 0.0046, "sales": 0.5031, "security": 0.0022, "hr": 0.0016, "other": 0.0078 }, "confidence": 0.5623, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": false, "oracle": { "marker_logits": [ 1.7357804775238037, -2.9216599464416504, 1.7813801765441895, -3.6334097385406494, -3.9704785346984863, -2.3811049461364746 ], "act_logits": [ 1186.1778564453125, -1418.166015625 ] } }, { "row_id": "ML_B04/is_spam", "fixture_id": "ML_B04", "question_id": "is_spam", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Is this email unsolicited spam or bulk marketing?" }, "qtype": 2, "K": 2, "sequence_length": 66, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 736, 4801, 183367, 31967, 689, 19717, 8924, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 15029, 1192, 664, 94898, 61863, 824, 664, 2168, 1192, 664, 27428, 208345, 235341, 15759, 1490, 235341, 17254, 7984, 577, 5035, 861, 17777, 41409, 664, 2273, 1192, 664, 5041, 11309, 12990, 1 ], "marker_positions": [ 15, 25 ], "probabilities": [ 0.00030877991230227053, 0.9996912479400635 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.9997, "confidence": 0.9997, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ -4.810556411743164, 3.2720165252685547 ], "act_logits": [ 1124.3245849609375, -1343.67578125 ] } }, { "row_id": "ML_B04/is_phishing", "fixture_id": "ML_B04", "question_id": "is_phishing", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Is this email a phishing or scam attempt to steal money, credentials, or personal data?", "criteria": { "true": "phishing, scam, or fraud", "false": "a legitimate email" } }, "qtype": 2, "K": 2, "sequence_length": 72, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 736, 4801, 476, 118602, 689, 37853, 8697, 577, 32261, 3370, 235269, 36983, 235269, 689, 3749, 1423, 235336, 1, 4, 1566, 235292, 476, 33957, 4801, 4, 1382, 235292, 118602, 235269, 37853, 235269, 689, 19367, 1, 19946, 15029, 1192, 664, 94898, 61863, 824, 664, 2168, 1192, 664, 27428, 208345, 235341, 15759, 1490, 235341, 17254, 7984, 577, 5035, 861, 17777, 41409, 664, 2273, 1192, 664, 5041, 11309, 12990, 1 ], "marker_positions": [ 24, 30 ], "probabilities": [ 0.00013583360123448074, 0.9998641014099121 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.9999, "confidence": 0.9999, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ -6.850337982177734, 2.0536048412323 ], "act_logits": [ 1234.9774169921875, -1474.3128662109375 ] } }, { "row_id": "ML_B04/urgency", "fixture_id": "ML_B04", "question_id": "urgency", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How urgent is the request in `body`?", "criteria": [ "no time pressure", "needs attention soon", "blocking issue or hard deadline" ] }, "qtype": 1, "K": 3, "sequence_length": 74, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 34722, 603, 573, 3853, 575, 4103, 2168, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 793, 1069, 5601, 4, 2403, 235248, 235274, 235292, 4026, 6137, 4445, 4, 2403, 235248, 235284, 235292, 35578, 4295, 689, 2811, 30699, 1, 19946, 15029, 1192, 664, 94898, 61863, 824, 664, 2168, 1192, 664, 27428, 208345, 235341, 15759, 1490, 235341, 17254, 7984, 577, 5035, 861, 17777, 41409, 664, 2273, 1192, 664, 5041, 11309, 12990, 1 ], "marker_positions": [ 15, 23, 31 ], "probabilities": [ 0.010461467318236828, 0.18218450248241425, 0.8073540329933167 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.7969, "legend": { "0": "no time pressure", "1": "needs attention soon", "2": "blocking issue or hard deadline" }, "probabilities": { "0": 0.0105, "1": 0.1822, "2": 0.8074 }, "confidence": 0.5169, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -2.615556240081787, 0.24176543951034546, 1.730507493019104 ], "act_logits": [ 1493.8917236328125, -1762.85400390625 ] } }, { "row_id": "ML_B04/needs_reply", "fixture_id": "ML_B04", "question_id": "needs_reply", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does the sender expect a reply?" }, "qtype": 2, "K": 2, "sequence_length": 64, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 14695, 5246, 476, 7260, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 15029, 1192, 664, 94898, 61863, 824, 664, 2168, 1192, 664, 27428, 208345, 235341, 15759, 1490, 235341, 17254, 7984, 577, 5035, 861, 17777, 41409, 664, 2273, 1192, 664, 5041, 11309, 12990, 1 ], "marker_positions": [ 13, 23 ], "probabilities": [ 0.180540531873703, 0.8194594383239746 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.8195, "confidence": 0.8195, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ -0.9993331432342529, 0.5133565068244934 ], "act_logits": [ 1187.255126953125, -1414.77294921875 ] } }, { "row_id": "ML_B05/category", "fixture_id": "ML_B05", "question_id": "category", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "choice", "instructions": "Which team should handle the email in `body`?", "criteria": { "billing": "invoices, payments, refunds", "technical": "bugs, outages, integrations", "sales": "pricing, demos, new purchases", "security": "phishing, scams, account compromise", "hr": "hiring, leave, payroll", "other": "none of the above" } }, "qtype": 0, "K": 6, "sequence_length": 168, "sequence_ids": [ 2, 6241, 2872, 235292, 12236, 2970, 1412, 6589, 573, 4801, 575, 4103, 2168, 235376, 235336, 1, 4, 54972, 235292, 88220, 235269, 15598, 235269, 85869, 4, 9838, 235292, 30608, 235269, 142788, 235269, 134538, 4, 7108, 235292, 25063, 235269, 105404, 235269, 888, 29330, 4, 6206, 235292, 118602, 235269, 108105, 235269, 3185, 37031, 4, 25211, 235292, 34233, 235269, 5077, 235269, 63529, 4, 1156, 235292, 8031, 576, 573, 3131, 1, 19946, 15029, 1192, 664, 35730, 3684, 824, 664, 2168, 1192, 664, 6922, 3185, 877, 3387, 3646, 235265, 12956, 675, 861, 8003, 578, 12708, 3409, 7544, 235265, 2390, 780, 3764, 573, 1707, 11309, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 19515, 664, 2273, 1192, 664, 5041, 11309, 12990, 1 ], "marker_positions": [ 16, 24, 32, 41, 50, 58 ], "probabilities": [ 0.11239219456911087, 0.06029629334807396, 0.019321400672197342, 0.343994140625, 0.13820922374725342, 0.32578667998313904 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "security", "probabilities": { "billing": 0.1124, "technical": 0.0603, "sales": 0.0193, "security": 0.344, "hr": 0.1382, "other": 0.3258 }, "confidence": 0.1644, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": false, "oracle": { "marker_logits": [ -1.463880181312561, -2.0866036415100098, -3.224661350250244, -0.3452500104904175, -1.257105827331543, -0.3996320366859436 ], "act_logits": [ 1313.8597412109375, -1566.2047119140625 ] } }, { "row_id": "ML_B05/is_spam", "fixture_id": "ML_B05", "question_id": "is_spam", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Is this email unsolicited spam or bulk marketing?" }, "qtype": 2, "K": 2, "sequence_length": 136, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 736, 4801, 183367, 31967, 689, 19717, 8924, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 15029, 1192, 664, 35730, 3684, 824, 664, 2168, 1192, 664, 6922, 3185, 877, 3387, 3646, 235265, 12956, 675, 861, 8003, 578, 12708, 3409, 7544, 235265, 2390, 780, 3764, 573, 1707, 11309, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 19515, 664, 2273, 1192, 664, 5041, 11309, 12990, 1 ], "marker_positions": [ 15, 25 ], "probabilities": [ 1.0, 3.9159750997441733e-08 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 15.869136810302734, -1.1864798069000244 ], "act_logits": [ 1551.3343505859375, -1845.933349609375 ] } }, { "row_id": "ML_B05/is_phishing", "fixture_id": "ML_B05", "question_id": "is_phishing", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Is this email a phishing or scam attempt to steal money, credentials, or personal data?", "criteria": { "true": "phishing, scam, or fraud", "false": "a legitimate email" } }, "qtype": 2, "K": 2, "sequence_length": 142, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 736, 4801, 476, 118602, 689, 37853, 8697, 577, 32261, 3370, 235269, 36983, 235269, 689, 3749, 1423, 235336, 1, 4, 1566, 235292, 476, 33957, 4801, 4, 1382, 235292, 118602, 235269, 37853, 235269, 689, 19367, 1, 19946, 15029, 1192, 664, 35730, 3684, 824, 664, 2168, 1192, 664, 6922, 3185, 877, 3387, 3646, 235265, 12956, 675, 861, 8003, 578, 12708, 3409, 7544, 235265, 2390, 780, 3764, 573, 1707, 11309, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 19515, 664, 2273, 1192, 664, 5041, 11309, 12990, 1 ], "marker_positions": [ 24, 30 ], "probabilities": [ 0.9992222785949707, 0.0007777583668939769 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0008, "confidence": 0.9992, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 5.770097732543945, -1.3882192373275757 ], "act_logits": [ 1449.6429443359375, -1724.1787109375 ] } }, { "row_id": "ML_B05/urgency", "fixture_id": "ML_B05", "question_id": "urgency", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How urgent is the request in `body`?", "criteria": [ "no time pressure", "needs attention soon", "blocking issue or hard deadline" ] }, "qtype": 1, "K": 3, "sequence_length": 144, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 34722, 603, 573, 3853, 575, 4103, 2168, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 793, 1069, 5601, 4, 2403, 235248, 235274, 235292, 4026, 6137, 4445, 4, 2403, 235248, 235284, 235292, 35578, 4295, 689, 2811, 30699, 1, 19946, 15029, 1192, 664, 35730, 3684, 824, 664, 2168, 1192, 664, 6922, 3185, 877, 3387, 3646, 235265, 12956, 675, 861, 8003, 578, 12708, 3409, 7544, 235265, 2390, 780, 3764, 573, 1707, 11309, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 19515, 664, 2273, 1192, 664, 5041, 11309, 12990, 1 ], "marker_positions": [ 15, 23, 31 ], "probabilities": [ 0.008980668149888515, 0.19867490231990814, 0.7923443913459778 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.7834, "legend": { "0": "no time pressure", "1": "needs attention soon", "2": "blocking issue or hard deadline" }, "probabilities": { "0": 0.009, "1": 0.1987, "2": 0.7923 }, "confidence": 0.5013, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -2.1389479637145996, 0.9576479196548462, 2.3409740924835205 ], "act_logits": [ 1489.6641845703125, -1760.4595947265625 ] } }, { "row_id": "ML_B05/needs_reply", "fixture_id": "ML_B05", "question_id": "needs_reply", "family": "B", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does the sender expect a reply?" }, "qtype": 2, "K": 2, "sequence_length": 134, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 14695, 5246, 476, 7260, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 15029, 1192, 664, 35730, 3684, 824, 664, 2168, 1192, 664, 6922, 3185, 877, 3387, 3646, 235265, 12956, 675, 861, 8003, 578, 12708, 3409, 7544, 235265, 2390, 780, 3764, 573, 1707, 11309, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 19515, 664, 2273, 1192, 664, 5041, 11309, 12990, 1 ], "marker_positions": [ 13, 23 ], "probabilities": [ 0.0727592259645462, 0.9272407293319702 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.9272, "confidence": 0.9272, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ -2.070378065109253, 0.4746796488761902 ], "act_logits": [ 1266.7015380859375, -1506.843505859375 ] } }, { "row_id": "ML_B06/category", "fixture_id": "ML_B06", "question_id": "category", "family": "B", "language": "JA", "primary_language": "JA", "question": { "type": "choice", "instructions": "Which team should handle the email in `body`?", "criteria": { "billing": "invoices, payments, refunds", "technical": "bugs, outages, integrations", "sales": "pricing, demos, new purchases", "security": "phishing, scams, account compromise", "hr": "hiring, leave, payroll", "other": "none of the above" } }, "qtype": 0, "K": 6, "sequence_length": 256, "sequence_ids": [ 2, 6241, 2872, 235292, 12236, 2970, 1412, 6589, 573, 4801, 575, 4103, 2168, 235376, 235336, 1, 4, 54972, 235292, 88220, 235269, 15598, 235269, 85869, 4, 9838, 235292, 30608, 235269, 142788, 235269, 134538, 4, 7108, 235292, 25063, 235269, 105404, 235269, 888, 29330, 4, 6206, 235292, 118602, 235269, 108105, 235269, 3185, 37031, 4, 25211, 235292, 34233, 235269, 5077, 235269, 63529, 4, 1156, 235292, 8031, 576, 573, 3131, 1, 19946, 15029, 1192, 664, 133074, 224664, 150694, 824, 664, 2168, 1192, 664, 161809, 237848, 58671, 133074, 236128, 235432, 187928, 236173, 200744, 235362, 137108, 55495, 26685, 175442, 236514, 75214, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 1 ], "marker_positions": [ 16, 24, 32, 41, 50, 58 ], "probabilities": [ 0.9999997615814209, 3.5833910061455754e-08, 6.076863456883075e-08, 3.51834721357136e-08, 3.680872140421343e-08, 7.952222347284987e-08 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "billing", "probabilities": { "billing": 1.0, "technical": 0.0, "sales": 0.0, "security": 0.0, "hr": 0.0, "other": 0.0 }, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": true, "oracle": { "marker_logits": [ 12.587414741516113, -4.55695915222168, -4.028779029846191, -4.5752763748168945, -4.530119895935059, -3.759817123413086 ], "act_logits": [ 1221.5692138671875, -1461.58837890625 ] } }, { "row_id": "ML_B06/is_spam", "fixture_id": "ML_B06", "question_id": "is_spam", "family": "B", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Is this email unsolicited spam or bulk marketing?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 736, 4801, 183367, 31967, 689, 19717, 8924, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 15029, 1192, 664, 133074, 224664, 150694, 824, 664, 2168, 1192, 664, 161809, 237848, 58671, 133074, 236128, 235432, 187928, 236173, 200744, 235362, 137108, 55495, 26685, 175442, 236514, 75214, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 50855, 72368, 235362, 78546, 235372, 63090, 238518, 7711, 44393, 235372, 174612, 235410, 53479, 235372, 130073, 9301, 235432, 90518, 22031, 235362, 235987, 23176, 151978, 219689, 236943, 14848, 62865, 1 ], "marker_positions": [ 15, 25 ], "probabilities": [ 1.0, 5.1957282226622326e-11 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 17.737506866455078, -5.943093776702881 ], "act_logits": [ 1363.92333984375, -1626.54736328125 ] } }, { "row_id": "ML_B06/is_phishing", "fixture_id": "ML_B06", "question_id": "is_phishing", "family": "B", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Is this email a phishing or scam attempt to steal money, credentials, or personal data?", "criteria": { "true": "phishing, scam, or fraud", "false": "a legitimate email" } }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 736, 4801, 476, 118602, 689, 37853, 8697, 577, 32261, 3370, 235269, 36983, 235269, 689, 3749, 1423, 235336, 1, 4, 1566, 235292, 476, 33957, 4801, 4, 1382, 235292, 118602, 235269, 37853, 235269, 689, 19367, 1, 19946, 15029, 1192, 664, 133074, 224664, 150694, 824, 664, 2168, 1192, 664, 161809, 237848, 58671, 133074, 236128, 235432, 187928, 236173, 200744, 235362, 137108, 55495, 26685, 175442, 236514, 75214, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 50855, 72368, 235362, 78546, 235372, 63090, 238518, 7711, 44393, 235372, 174612, 235410, 53479, 235372, 130073, 9301, 235432, 90518, 22031, 235362, 235987, 1 ], "marker_positions": [ 24, 30 ], "probabilities": [ 1.0, 1.8650408908221783e-12 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 17.732009887695312, -9.275724411010742 ], "act_logits": [ 1327.414794921875, -1584.4556884765625 ] } }, { "row_id": "ML_B06/urgency", "fixture_id": "ML_B06", "question_id": "urgency", "family": "B", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How urgent is the request in `body`?", "criteria": [ "no time pressure", "needs attention soon", "blocking issue or hard deadline" ] }, "qtype": 1, "K": 3, "sequence_length": 256, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 34722, 603, 573, 3853, 575, 4103, 2168, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 793, 1069, 5601, 4, 2403, 235248, 235274, 235292, 4026, 6137, 4445, 4, 2403, 235248, 235284, 235292, 35578, 4295, 689, 2811, 30699, 1, 19946, 15029, 1192, 664, 133074, 224664, 150694, 824, 664, 2168, 1192, 664, 161809, 237848, 58671, 133074, 236128, 235432, 187928, 236173, 200744, 235362, 137108, 55495, 26685, 175442, 236514, 75214, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 50855, 72368, 235362, 78546, 235372, 63090, 238518, 7711, 44393, 235372, 174612, 235410, 53479, 235372, 130073, 9301, 235432, 90518, 22031, 1 ], "marker_positions": [ 15, 23, 31 ], "probabilities": [ 0.00234546372666955, 0.28496667742729187, 0.7126878499984741 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.7103, "legend": { "0": "no time pressure", "1": "needs attention soon", "2": "blocking issue or hard deadline" }, "probabilities": { "0": 0.0023, "1": 0.285, "2": 0.7127 }, "confidence": 0.4417, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": true, "oracle": { "marker_logits": [ -3.3854634761810303, 1.4144268035888672, 2.33109712600708 ], "act_logits": [ 1545.9593505859375, -1824.982666015625 ] } }, { "row_id": "ML_B06/needs_reply", "fixture_id": "ML_B06", "question_id": "needs_reply", "family": "B", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does the sender expect a reply?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 14695, 5246, 476, 7260, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 15029, 1192, 664, 133074, 224664, 150694, 824, 664, 2168, 1192, 664, 161809, 237848, 58671, 133074, 236128, 235432, 187928, 236173, 200744, 235362, 137108, 55495, 26685, 175442, 236514, 75214, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 50855, 72368, 235362, 78546, 235372, 63090, 238518, 7711, 44393, 235372, 174612, 235410, 53479, 235372, 130073, 9301, 235432, 90518, 22031, 235362, 235987, 23176, 151978, 219689, 236943, 14848, 62865, 235362, 81625, 1 ], "marker_positions": [ 13, 23 ], "probabilities": [ 0.15296564996242523, 0.8470343351364136 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.847, "confidence": 0.847, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ -1.3824729919433594, 0.3290550112724304 ], "act_logits": [ 1230.62890625, -1464.4969482421875 ] } }, { "row_id": "ML_B07/category", "fixture_id": "ML_B07", "question_id": "category", "family": "B", "language": "JA", "primary_language": "JA", "question": { "type": "choice", "instructions": "Which team should handle the email in `body`?", "criteria": { "billing": "invoices, payments, refunds", "technical": "bugs, outages, integrations", "sales": "pricing, demos, new purchases", "security": "phishing, scams, account compromise", "hr": "hiring, leave, payroll", "other": "none of the above" } }, "qtype": 0, "K": 6, "sequence_length": 97, "sequence_ids": [ 2, 6241, 2872, 235292, 12236, 2970, 1412, 6589, 573, 4801, 575, 4103, 2168, 235376, 235336, 1, 4, 54972, 235292, 88220, 235269, 15598, 235269, 85869, 4, 9838, 235292, 30608, 235269, 142788, 235269, 134538, 4, 7108, 235292, 25063, 235269, 105404, 235269, 888, 29330, 4, 6206, 235292, 118602, 235269, 108105, 235269, 3185, 37031, 4, 25211, 235292, 34233, 235269, 5077, 235269, 63529, 4, 1156, 235292, 8031, 576, 573, 3131, 1, 19946, 15029, 1192, 664, 54182, 195377, 9481, 824, 664, 2168, 1192, 664, 27364, 235425, 54182, 195377, 9481, 235362, 26685, 119920, 9178, 235362, 824, 664, 2273, 1192, 664, 108368, 236948, 12990, 1 ], "marker_positions": [ 16, 24, 32, 41, 50, 58 ], "probabilities": [ 0.027459673583507538, 0.1427200883626938, 0.03673175722360611, 0.24406760931015015, 0.3703951835632324, 0.17862564325332642 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "hr", "probabilities": { "billing": 0.0275, "technical": 0.1427, "sales": 0.0367, "security": 0.2441, "hr": 0.3704, "other": 0.1786 }, "confidence": 0.153, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": false, "oracle": { "marker_logits": [ -2.9959418773651123, -1.3477773666381836, -2.7050209045410156, -0.8112195730209351, -0.39408963918685913, -1.123368501663208 ], "act_logits": [ 1305.0589599609375, -1552.883056640625 ] } }, { "row_id": "ML_B07/is_spam", "fixture_id": "ML_B07", "question_id": "is_spam", "family": "B", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Is this email unsolicited spam or bulk marketing?" }, "qtype": 2, "K": 2, "sequence_length": 65, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 736, 4801, 183367, 31967, 689, 19717, 8924, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 15029, 1192, 664, 54182, 195377, 9481, 824, 664, 2168, 1192, 664, 27364, 235425, 54182, 195377, 9481, 235362, 26685, 119920, 9178, 235362, 824, 664, 2273, 1192, 664, 108368, 236948, 12990, 1 ], "marker_positions": [ 15, 25 ], "probabilities": [ 0.06950114667415619, 0.9304988384246826 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.9305, "confidence": 0.9305, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ -2.4526093006134033, 0.14176851511001587 ], "act_logits": [ 1159.447265625, -1378.8519287109375 ] } }, { "row_id": "ML_B07/is_phishing", "fixture_id": "ML_B07", "question_id": "is_phishing", "family": "B", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Is this email a phishing or scam attempt to steal money, credentials, or personal data?", "criteria": { "true": "phishing, scam, or fraud", "false": "a legitimate email" } }, "qtype": 2, "K": 2, "sequence_length": 71, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 736, 4801, 476, 118602, 689, 37853, 8697, 577, 32261, 3370, 235269, 36983, 235269, 689, 3749, 1423, 235336, 1, 4, 1566, 235292, 476, 33957, 4801, 4, 1382, 235292, 118602, 235269, 37853, 235269, 689, 19367, 1, 19946, 15029, 1192, 664, 54182, 195377, 9481, 824, 664, 2168, 1192, 664, 27364, 235425, 54182, 195377, 9481, 235362, 26685, 119920, 9178, 235362, 824, 664, 2273, 1192, 664, 108368, 236948, 12990, 1 ], "marker_positions": [ 24, 30 ], "probabilities": [ 0.45053812861442566, 0.549461841583252 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.5495, "confidence": 0.5495, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ -0.715679943561554, -0.5171833038330078 ], "act_logits": [ 1232.0244140625, -1464.14453125 ] } }, { "row_id": "ML_B07/urgency", "fixture_id": "ML_B07", "question_id": "urgency", "family": "B", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How urgent is the request in `body`?", "criteria": [ "no time pressure", "needs attention soon", "blocking issue or hard deadline" ] }, "qtype": 1, "K": 3, "sequence_length": 73, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 34722, 603, 573, 3853, 575, 4103, 2168, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 793, 1069, 5601, 4, 2403, 235248, 235274, 235292, 4026, 6137, 4445, 4, 2403, 235248, 235284, 235292, 35578, 4295, 689, 2811, 30699, 1, 19946, 15029, 1192, 664, 54182, 195377, 9481, 824, 664, 2168, 1192, 664, 27364, 235425, 54182, 195377, 9481, 235362, 26685, 119920, 9178, 235362, 824, 664, 2273, 1192, 664, 108368, 236948, 12990, 1 ], "marker_positions": [ 15, 23, 31 ], "probabilities": [ 0.009285608306527138, 0.20336227118968964, 0.7873520851135254 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.7781, "legend": { "0": "no time pressure", "1": "needs attention soon", "2": "blocking issue or hard deadline" }, "probabilities": { "0": 0.0093, "1": 0.2034, "2": 0.7874 }, "confidence": 0.4943, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -1.7356622219085693, 1.3508615493774414, 2.704547643661499 ], "act_logits": [ 1603.1341552734375, -1894.369873046875 ] } }, { "row_id": "ML_B07/needs_reply", "fixture_id": "ML_B07", "question_id": "needs_reply", "family": "B", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does the sender expect a reply?" }, "qtype": 2, "K": 2, "sequence_length": 63, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 14695, 5246, 476, 7260, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 15029, 1192, 664, 54182, 195377, 9481, 824, 664, 2168, 1192, 664, 27364, 235425, 54182, 195377, 9481, 235362, 26685, 119920, 9178, 235362, 824, 664, 2273, 1192, 664, 108368, 236948, 12990, 1 ], "marker_positions": [ 13, 23 ], "probabilities": [ 0.9508446455001831, 0.04915532097220421 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0492, "confidence": 0.9508, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 2.959348440170288, -0.0030176639556884766 ], "act_logits": [ 1332.265869140625, -1582.3018798828125 ] } }, { "row_id": "ML_B08/category", "fixture_id": "ML_B08", "question_id": "category", "family": "B", "language": "mixed", "primary_language": "JA", "question": { "type": "choice", "instructions": "Which team should handle the email in `body`?", "criteria": { "billing": "invoices, payments, refunds", "technical": "bugs, outages, integrations", "sales": "pricing, demos, new purchases", "security": "phishing, scams, account compromise", "hr": "hiring, leave, payroll", "other": "none of the above" } }, "qtype": 0, "K": 6, "sequence_length": 203, "sequence_ids": [ 2, 6241, 2872, 235292, 12236, 2970, 1412, 6589, 573, 4801, 575, 4103, 2168, 235376, 235336, 1, 4, 54972, 235292, 88220, 235269, 15598, 235269, 85869, 4, 9838, 235292, 30608, 235269, 142788, 235269, 134538, 4, 7108, 235292, 25063, 235269, 105404, 235269, 888, 29330, 4, 6206, 235292, 118602, 235269, 108105, 235269, 3185, 37031, 4, 25211, 235292, 34233, 235269, 5077, 235269, 63529, 4, 1156, 235292, 8031, 576, 573, 3131, 1, 19946, 15029, 1192, 664, 173045, 41913, 235372, 26685, 824, 664, 2168, 1192, 664, 198170, 235372, 173045, 41913, 235418, 158916, 47644, 235443, 235362, 53479, 235714, 27785, 63090, 28439, 26685, 30503, 5493, 235394, 54755, 119920, 9178, 235362, 651, 25791, 11968, 603, 5628, 604, 3221, 235265, 5651, 10508, 573, 15560, 235265, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 824, 664, 2273, 1192, 664, 108368, 236948, 12990, 1 ], "marker_positions": [ 16, 24, 32, 41, 50, 58 ], "probabilities": [ 0.04178085923194885, 6.855530227767304e-05, 7.164049748098478e-05, 6.470451626228169e-05, 0.9560848474502563, 0.0019294011872261763 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "hr", "probabilities": { "billing": 0.0418, "technical": 0.0001, "sales": 0.0001, "security": 0.0001, "hr": 0.9561, "other": 0.0019 }, "confidence": 0.8942, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": false, "oracle": { "marker_logits": [ 1.8776620626449585, -4.534907341003418, -4.490889549255371, -4.592714309692383, 5.008053779602051, -1.197596788406372 ], "act_logits": [ 1284.5792236328125, -1536.4027099609375 ] } }, { "row_id": "ML_B08/is_spam", "fixture_id": "ML_B08", "question_id": "is_spam", "family": "B", "language": "mixed", "primary_language": "JA", "question": { "type": "noul", "instructions": "Is this email unsolicited spam or bulk marketing?" }, "qtype": 2, "K": 2, "sequence_length": 171, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 736, 4801, 183367, 31967, 689, 19717, 8924, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 15029, 1192, 664, 173045, 41913, 235372, 26685, 824, 664, 2168, 1192, 664, 198170, 235372, 173045, 41913, 235418, 158916, 47644, 235443, 235362, 53479, 235714, 27785, 63090, 28439, 26685, 30503, 5493, 235394, 54755, 119920, 9178, 235362, 651, 25791, 11968, 603, 5628, 604, 3221, 235265, 5651, 10508, 573, 15560, 235265, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 824, 664, 2273, 1192, 664, 108368, 236948, 12990, 1 ], "marker_positions": [ 15, 25 ], "probabilities": [ 1.0, 4.013923637913308e-10 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 17.009784698486328, -4.626298427581787 ], "act_logits": [ 1407.521484375, -1676.2918701171875 ] } }, { "row_id": "ML_B08/is_phishing", "fixture_id": "ML_B08", "question_id": "is_phishing", "family": "B", "language": "mixed", "primary_language": "JA", "question": { "type": "noul", "instructions": "Is this email a phishing or scam attempt to steal money, credentials, or personal data?", "criteria": { "true": "phishing, scam, or fraud", "false": "a legitimate email" } }, "qtype": 2, "K": 2, "sequence_length": 177, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 736, 4801, 476, 118602, 689, 37853, 8697, 577, 32261, 3370, 235269, 36983, 235269, 689, 3749, 1423, 235336, 1, 4, 1566, 235292, 476, 33957, 4801, 4, 1382, 235292, 118602, 235269, 37853, 235269, 689, 19367, 1, 19946, 15029, 1192, 664, 173045, 41913, 235372, 26685, 824, 664, 2168, 1192, 664, 198170, 235372, 173045, 41913, 235418, 158916, 47644, 235443, 235362, 53479, 235714, 27785, 63090, 28439, 26685, 30503, 5493, 235394, 54755, 119920, 9178, 235362, 651, 25791, 11968, 603, 5628, 604, 3221, 235265, 5651, 10508, 573, 15560, 235265, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 824, 664, 2273, 1192, 664, 108368, 236948, 12990, 1 ], "marker_positions": [ 24, 30 ], "probabilities": [ 1.0, 2.927889528048233e-11 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 17.459320068359375, -6.794838905334473 ], "act_logits": [ 1350.8082275390625, -1610.20263671875 ] } }, { "row_id": "ML_B08/urgency", "fixture_id": "ML_B08", "question_id": "urgency", "family": "B", "language": "mixed", "primary_language": "JA", "question": { "type": "score", "instructions": "How urgent is the request in `body`?", "criteria": [ "no time pressure", "needs attention soon", "blocking issue or hard deadline" ] }, "qtype": 1, "K": 3, "sequence_length": 179, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 34722, 603, 573, 3853, 575, 4103, 2168, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 793, 1069, 5601, 4, 2403, 235248, 235274, 235292, 4026, 6137, 4445, 4, 2403, 235248, 235284, 235292, 35578, 4295, 689, 2811, 30699, 1, 19946, 15029, 1192, 664, 173045, 41913, 235372, 26685, 824, 664, 2168, 1192, 664, 198170, 235372, 173045, 41913, 235418, 158916, 47644, 235443, 235362, 53479, 235714, 27785, 63090, 28439, 26685, 30503, 5493, 235394, 54755, 119920, 9178, 235362, 651, 25791, 11968, 603, 5628, 604, 3221, 235265, 5651, 10508, 573, 15560, 235265, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 824, 664, 2273, 1192, 664, 108368, 236948, 12990, 1 ], "marker_positions": [ 15, 23, 31 ], "probabilities": [ 0.09902609884738922, 0.34860479831695557, 0.5523691177368164 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.4533, "legend": { "0": "no time pressure", "1": "needs attention soon", "2": "blocking issue or hard deadline" }, "probabilities": { "0": 0.099, "1": 0.3486, "2": 0.5524 }, "confidence": 0.1588, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ 0.201202392578125, 1.4597575664520264, 1.9200351238250732 ], "act_logits": [ 1632.4935302734375, -1925.72119140625 ] } }, { "row_id": "ML_B08/needs_reply", "fixture_id": "ML_B08", "question_id": "needs_reply", "family": "B", "language": "mixed", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does the sender expect a reply?" }, "qtype": 2, "K": 2, "sequence_length": 169, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 14695, 5246, 476, 7260, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 15029, 1192, 664, 173045, 41913, 235372, 26685, 824, 664, 2168, 1192, 664, 198170, 235372, 173045, 41913, 235418, 158916, 47644, 235443, 235362, 53479, 235714, 27785, 63090, 28439, 26685, 30503, 5493, 235394, 54755, 119920, 9178, 235362, 651, 25791, 11968, 603, 5628, 604, 3221, 235265, 5651, 10508, 573, 15560, 235265, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 824, 664, 2273, 1192, 664, 108368, 236948, 12990, 1 ], "marker_positions": [ 13, 23 ], "probabilities": [ 0.9659482836723328, 0.03405166044831276 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0341, "confidence": 0.9659, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 3.1492505073547363, -0.19598102569580078 ], "act_logits": [ 1220.3465576171875, -1452.4661865234375 ] } }, { "row_id": "ML_B09/category", "fixture_id": "ML_B09", "question_id": "category", "family": "B", "language": "JA", "primary_language": "JA", "question": { "type": "choice", "instructions": "Which team should handle the email in `body`?", "criteria": { "billing": "invoices, payments, refunds", "technical": "bugs, outages, integrations", "sales": "pricing, demos, new purchases", "security": "phishing, scams, account compromise", "hr": "hiring, leave, payroll", "other": "none of the above" } }, "qtype": 0, "K": 6, "sequence_length": 256, "sequence_ids": [ 2, 6241, 2872, 235292, 12236, 2970, 1412, 6589, 573, 4801, 575, 4103, 2168, 235376, 235336, 1, 4, 54972, 235292, 88220, 235269, 15598, 235269, 85869, 4, 9838, 235292, 30608, 235269, 142788, 235269, 134538, 4, 7108, 235292, 25063, 235269, 105404, 235269, 888, 29330, 4, 6206, 235292, 118602, 235269, 108105, 235269, 3185, 37031, 4, 25211, 235292, 34233, 235269, 5077, 235269, 63529, 4, 1156, 235292, 8031, 576, 573, 3131, 1, 19946, 15029, 1192, 664, 183272, 215890, 53410, 53871, 824, 664, 2168, 1192, 664, 203670, 235362, 235811, 9301, 183272, 236061, 235683, 235425, 77033, 45263, 235362, 71323, 21632, 2603, 235394, 154040, 235414, 26808, 47221, 125784, 60391, 235855, 200744, 235362, 76927, 7820, 150852, 78217, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 1 ], "marker_positions": [ 16, 24, 32, 41, 50, 58 ], "probabilities": [ 0.04751085862517357, 0.00674290070310235, 0.9081122875213623, 0.006666676606982946, 0.009019587188959122, 0.021947748959064484 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "sales", "probabilities": { "billing": 0.0475, "technical": 0.0067, "sales": 0.9081, "security": 0.0067, "hr": 0.009, "other": 0.0219 }, "confidence": 0.7624, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": true, "oracle": { "marker_logits": [ -2.622711420059204, -4.575179576873779, 0.3276981711387634, -4.586550235748291, -4.284272193908691, -3.395005941390991 ], "act_logits": [ 1211.922607421875, -1450.45361328125 ] } }, { "row_id": "ML_B09/is_spam", "fixture_id": "ML_B09", "question_id": "is_spam", "family": "B", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Is this email unsolicited spam or bulk marketing?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 736, 4801, 183367, 31967, 689, 19717, 8924, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 15029, 1192, 664, 183272, 215890, 53410, 53871, 824, 664, 2168, 1192, 664, 203670, 235362, 235811, 9301, 183272, 236061, 235683, 235425, 77033, 45263, 235362, 71323, 21632, 2603, 235394, 154040, 235414, 26808, 47221, 125784, 60391, 235855, 200744, 235362, 76927, 7820, 150852, 78217, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 50855, 72368, 235362, 78546, 235372, 63090, 238518, 7711, 44393, 235372, 174612, 235410, 53479, 235372, 1 ], "marker_positions": [ 15, 25 ], "probabilities": [ 0.09084034711122513, 0.9091596603393555 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.9092, "confidence": 0.9092, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ -1.9216532707214355, 0.38176411390304565 ], "act_logits": [ 1093.986572265625, -1305.408935546875 ] } }, { "row_id": "ML_B09/is_phishing", "fixture_id": "ML_B09", "question_id": "is_phishing", "family": "B", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Is this email a phishing or scam attempt to steal money, credentials, or personal data?", "criteria": { "true": "phishing, scam, or fraud", "false": "a legitimate email" } }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 736, 4801, 476, 118602, 689, 37853, 8697, 577, 32261, 3370, 235269, 36983, 235269, 689, 3749, 1423, 235336, 1, 4, 1566, 235292, 476, 33957, 4801, 4, 1382, 235292, 118602, 235269, 37853, 235269, 689, 19367, 1, 19946, 15029, 1192, 664, 183272, 215890, 53410, 53871, 824, 664, 2168, 1192, 664, 203670, 235362, 235811, 9301, 183272, 236061, 235683, 235425, 77033, 45263, 235362, 71323, 21632, 2603, 235394, 154040, 235414, 26808, 47221, 125784, 60391, 235855, 200744, 235362, 76927, 7820, 150852, 78217, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 50855, 72368, 235362, 78546, 235372, 63090, 238518, 7711, 1 ], "marker_positions": [ 24, 30 ], "probabilities": [ 0.48474982380867004, 0.5152502059936523 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.5153, "confidence": 0.5153, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ -0.8350335359573364, -0.7740132808685303 ], "act_logits": [ 1166.0264892578125, -1392.017822265625 ] } }, { "row_id": "ML_B09/urgency", "fixture_id": "ML_B09", "question_id": "urgency", "family": "B", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How urgent is the request in `body`?", "criteria": [ "no time pressure", "needs attention soon", "blocking issue or hard deadline" ] }, "qtype": 1, "K": 3, "sequence_length": 256, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 34722, 603, 573, 3853, 575, 4103, 2168, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 793, 1069, 5601, 4, 2403, 235248, 235274, 235292, 4026, 6137, 4445, 4, 2403, 235248, 235284, 235292, 35578, 4295, 689, 2811, 30699, 1, 19946, 15029, 1192, 664, 183272, 215890, 53410, 53871, 824, 664, 2168, 1192, 664, 203670, 235362, 235811, 9301, 183272, 236061, 235683, 235425, 77033, 45263, 235362, 71323, 21632, 2603, 235394, 154040, 235414, 26808, 47221, 125784, 60391, 235855, 200744, 235362, 76927, 7820, 150852, 78217, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 50855, 72368, 235362, 78546, 235372, 63090, 1 ], "marker_positions": [ 15, 23, 31 ], "probabilities": [ 0.0036920388229191303, 0.16828875243663788, 0.8280192017555237 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.8243, "legend": { "0": "no time pressure", "1": "needs attention soon", "2": "blocking issue or hard deadline" }, "probabilities": { "0": 0.0037, "1": 0.1683, "2": 0.828 }, "confidence": 0.566, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": true, "oracle": { "marker_logits": [ -3.971696376800537, -0.15219396352767944, 1.4411616325378418 ], "act_logits": [ 1508.3917236328125, -1781.46240234375 ] } }, { "row_id": "ML_B09/needs_reply", "fixture_id": "ML_B09", "question_id": "needs_reply", "family": "B", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does the sender expect a reply?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 14695, 5246, 476, 7260, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 15029, 1192, 664, 183272, 215890, 53410, 53871, 824, 664, 2168, 1192, 664, 203670, 235362, 235811, 9301, 183272, 236061, 235683, 235425, 77033, 45263, 235362, 71323, 21632, 2603, 235394, 154040, 235414, 26808, 47221, 125784, 60391, 235855, 200744, 235362, 76927, 7820, 150852, 78217, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 50855, 72368, 235362, 78546, 235372, 63090, 238518, 7711, 44393, 235372, 174612, 235410, 53479, 235372, 130073, 9301, 1 ], "marker_positions": [ 13, 23 ], "probabilities": [ 0.15685179829597473, 0.8431482315063477 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.8431, "confidence": 0.8431, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ -1.172334909439087, 0.5095069408416748 ], "act_logits": [ 1123.768798828125, -1341.2845458984375 ] } }, { "row_id": "ML_B10/category", "fixture_id": "ML_B10", "question_id": "category", "family": "B", "language": "JA", "primary_language": "JA", "question": { "type": "choice", "instructions": "Which team should handle the email in `body`?", "criteria": { "billing": "invoices, payments, refunds", "technical": "bugs, outages, integrations", "sales": "pricing, demos, new purchases", "security": "phishing, scams, account compromise", "hr": "hiring, leave, payroll", "other": "none of the above" } }, "qtype": 0, "K": 6, "sequence_length": 100, "sequence_ids": [ 2, 6241, 2872, 235292, 12236, 2970, 1412, 6589, 573, 4801, 575, 4103, 2168, 235376, 235336, 1, 4, 54972, 235292, 88220, 235269, 15598, 235269, 85869, 4, 9838, 235292, 30608, 235269, 142788, 235269, 134538, 4, 7108, 235292, 25063, 235269, 105404, 235269, 888, 29330, 4, 6206, 235292, 118602, 235269, 108105, 235269, 3185, 37031, 4, 25211, 235292, 34233, 235269, 5077, 235269, 63529, 4, 1156, 235292, 8031, 576, 573, 3131, 1, 19946, 15029, 1192, 664, 236242, 236879, 235372, 76407, 26685, 824, 664, 2168, 1192, 664, 77915, 55495, 202318, 235432, 54755, 177917, 18152, 235432, 56520, 9178, 235362, 824, 664, 2273, 1192, 664, 108368, 236948, 12990, 1 ], "marker_positions": [ 16, 24, 32, 41, 50, 58 ], "probabilities": [ 3.322208431200124e-05, 0.0011501569533720613, 3.295572241768241e-05, 0.9985621571540833, 0.00012765228166244924, 9.389079787069932e-05 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "security", "probabilities": { "billing": 0.0, "technical": 0.0012, "sales": 0.0, "security": 0.9986, "hr": 0.0001, "other": 0.0001 }, "confidence": 0.9933, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": false, "oracle": { "marker_logits": [ -4.567408561706543, -1.0229690074920654, -4.57545804977417, 5.743450164794922, -3.221313953399658, -3.5284905433654785 ], "act_logits": [ 1195.670654296875, -1430.00341796875 ] } }, { "row_id": "ML_B10/is_spam", "fixture_id": "ML_B10", "question_id": "is_spam", "family": "B", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Is this email unsolicited spam or bulk marketing?" }, "qtype": 2, "K": 2, "sequence_length": 68, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 736, 4801, 183367, 31967, 689, 19717, 8924, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 15029, 1192, 664, 236242, 236879, 235372, 76407, 26685, 824, 664, 2168, 1192, 664, 77915, 55495, 202318, 235432, 54755, 177917, 18152, 235432, 56520, 9178, 235362, 824, 664, 2273, 1192, 664, 108368, 236948, 12990, 1 ], "marker_positions": [ 15, 25 ], "probabilities": [ 0.014126763679087162, 0.9858732223510742 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.9859, "confidence": 0.9859, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ -3.5126492977142334, 0.7328076958656311 ], "act_logits": [ 1114.7484130859375, -1329.301025390625 ] } }, { "row_id": "ML_B10/is_phishing", "fixture_id": "ML_B10", "question_id": "is_phishing", "family": "B", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Is this email a phishing or scam attempt to steal money, credentials, or personal data?", "criteria": { "true": "phishing, scam, or fraud", "false": "a legitimate email" } }, "qtype": 2, "K": 2, "sequence_length": 74, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 736, 4801, 476, 118602, 689, 37853, 8697, 577, 32261, 3370, 235269, 36983, 235269, 689, 3749, 1423, 235336, 1, 4, 1566, 235292, 476, 33957, 4801, 4, 1382, 235292, 118602, 235269, 37853, 235269, 689, 19367, 1, 19946, 15029, 1192, 664, 236242, 236879, 235372, 76407, 26685, 824, 664, 2168, 1192, 664, 77915, 55495, 202318, 235432, 54755, 177917, 18152, 235432, 56520, 9178, 235362, 824, 664, 2273, 1192, 664, 108368, 236948, 12990, 1 ], "marker_positions": [ 24, 30 ], "probabilities": [ 0.00015953407273627818, 0.999840497970581 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.9998, "confidence": 0.9998, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ -6.86907434463501, 1.874018907546997 ], "act_logits": [ 1114.8673095703125, -1331.8216552734375 ] } }, { "row_id": "ML_B10/urgency", "fixture_id": "ML_B10", "question_id": "urgency", "family": "B", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How urgent is the request in `body`?", "criteria": [ "no time pressure", "needs attention soon", "blocking issue or hard deadline" ] }, "qtype": 1, "K": 3, "sequence_length": 76, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 34722, 603, 573, 3853, 575, 4103, 2168, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 793, 1069, 5601, 4, 2403, 235248, 235274, 235292, 4026, 6137, 4445, 4, 2403, 235248, 235284, 235292, 35578, 4295, 689, 2811, 30699, 1, 19946, 15029, 1192, 664, 236242, 236879, 235372, 76407, 26685, 824, 664, 2168, 1192, 664, 77915, 55495, 202318, 235432, 54755, 177917, 18152, 235432, 56520, 9178, 235362, 824, 664, 2273, 1192, 664, 108368, 236948, 12990, 1 ], "marker_positions": [ 15, 23, 31 ], "probabilities": [ 0.0052702296525239944, 0.151950865983963, 0.8427788615226746 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.8375, "legend": { "0": "no time pressure", "1": "needs attention soon", "2": "blocking issue or hard deadline" }, "probabilities": { "0": 0.0053, "1": 0.152, "2": 0.8428 }, "confidence": 0.583, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -2.953580856323242, 0.4079021215438843, 2.121049642562866 ], "act_logits": [ 1566.1395263671875, -1851.35791015625 ] } }, { "row_id": "ML_B10/needs_reply", "fixture_id": "ML_B10", "question_id": "needs_reply", "family": "B", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does the sender expect a reply?" }, "qtype": 2, "K": 2, "sequence_length": 66, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 14695, 5246, 476, 7260, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 15029, 1192, 664, 236242, 236879, 235372, 76407, 26685, 824, 664, 2168, 1192, 664, 77915, 55495, 202318, 235432, 54755, 177917, 18152, 235432, 56520, 9178, 235362, 824, 664, 2273, 1192, 664, 108368, 236948, 12990, 1 ], "marker_positions": [ 13, 23 ], "probabilities": [ 0.09917406737804413, 0.9008259177207947 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.9008, "confidence": 0.9008, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ -1.5301761627197266, 0.6762590408325195 ], "act_logits": [ 1206.9561767578125, -1437.26318359375 ] } }, { "row_id": "ML_C01/intent", "fixture_id": "ML_C01", "question_id": "intent", "family": "C", "language": "EN", "primary_language": "EN", "question": { "type": "choice", "instructions": "What does the customer want in `message`?", "criteria": { "refund": "money returned or a duplicate charge reversed", "technical_help": "a bug, outage or integration problem", "billing_question": "a question about an invoice, plan or payment method", "information": "general information, pricing or how-to", "cancellation": "wants to cancel or downgrade", "other": "none of the other options fits" } }, "qtype": 0, "K": 6, "sequence_length": 190, "sequence_ids": [ 2, 6241, 2872, 235292, 2439, 1721, 573, 6650, 1938, 575, 4103, 1529, 235376, 235336, 1, 4, 19745, 235292, 3370, 8496, 689, 476, 38294, 6256, 33201, 4, 9838, 235298, 12180, 235292, 476, 11004, 235269, 119132, 689, 16827, 3210, 4, 54972, 235298, 13252, 235292, 476, 2872, 1105, 671, 42529, 235269, 1780, 689, 7984, 2370, 4, 2113, 235292, 3311, 2113, 235269, 25063, 689, 1368, 235290, 511, 4, 29027, 235292, 8507, 577, 12459, 689, 115214, 4, 1156, 235292, 8031, 576, 573, 1156, 4561, 17508, 1, 19946, 1529, 1192, 664, 5958, 19745, 573, 38294, 7984, 235265, 9105, 692, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 1464, 235270, 1 ], "marker_positions": [ 15, 25, 37, 52, 63, 71 ], "probabilities": [ 0.9988259673118591, 8.437082578893751e-05, 0.00011986755998805165, 0.0003147715760860592, 0.00010285976895829663, 0.000552027951925993 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "refund", "probabilities": { "refund": 0.9988, "technical_help": 0.0001, "billing_question": 0.0001, "information": 0.0003, "cancellation": 0.0001, "other": 0.0006 }, "confidence": 0.994, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": false, "oracle": { "marker_logits": [ 4.816737174987793, -4.562373161315918, -4.211200714111328, -3.245743751525879, -4.364231109619141, -2.6839988231658936 ], "act_logits": [ 1266.8946533203125, -1511.560791015625 ] } }, { "row_id": "ML_C01/is_urgent", "fixture_id": "ML_C01", "question_id": "is_urgent", "family": "C", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `message` communicate time pressure or a deadline?" }, "qtype": 2, "K": 2, "sequence_length": 145, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 1529, 235376, 18708, 1069, 5601, 689, 476, 30699, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 5958, 19745, 573, 38294, 7984, 235265, 9105, 692, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 1464, 235270, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9990623593330383, 0.0009376285597681999 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0009, "confidence": 0.9991, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 5.908827781677246, -1.0623912811279297 ], "act_logits": [ 1335.25341796875, -1585.5758056640625 ] } }, { "row_id": "ML_C01/frustration", "fixture_id": "ML_C01", "question_id": "frustration", "family": "C", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How frustrated does the customer sound in `message`?", "criteria": [ "calm and neutral", "concerned but civil", "clearly annoyed", "very angry or using strong language" ] }, "qtype": 1, "K": 4, "sequence_length": 160, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 46867, 1721, 573, 6650, 4835, 575, 4103, 1529, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 15142, 578, 17120, 4, 2403, 235248, 235274, 235292, 10695, 901, 11180, 4, 2403, 235248, 235284, 235292, 8886, 61058, 4, 2403, 235248, 235304, 235292, 1508, 19456, 689, 2177, 3779, 5255, 1, 19946, 1529, 1192, 664, 5958, 19745, 573, 38294, 7984, 235265, 9105, 692, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 1464, 235270, 1 ], "marker_positions": [ 16, 24, 32, 39 ], "probabilities": [ 0.30782410502433777, 0.2851684093475342, 0.2739376127719879, 0.13306991755962372 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.2323, "legend": { "0": "calm and neutral", "1": "concerned but civil", "2": "clearly annoyed", "3": "very angry or using strong language" }, "probabilities": { "0": 0.3078, "1": 0.2852, "2": 0.2739, "3": 0.1331 }, "confidence": 0.0308, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ 1.7140815258026123, 1.637632966041565, 1.5974537134170532, 0.8754277229309082 ], "act_logits": [ 1470.1024169921875, -1734.470947265625 ] } }, { "row_id": "ML_C01/refund_requested", "fixture_id": "ML_C01", "question_id": "refund_requested", "family": "C", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does the customer ask for money back?" }, "qtype": 2, "K": 2, "sequence_length": 142, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 6650, 5291, 604, 3370, 1355, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 5958, 19745, 573, 38294, 7984, 235265, 9105, 692, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 1464, 235270, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.32823431491851807, 0.6717656850814819 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.6718, "confidence": 0.6718, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ -0.9856004118919373, -0.26941895484924316 ], "act_logits": [ 1303.399169921875, -1547.32958984375 ] } }, { "row_id": "ML_C01/churn_risk", "fixture_id": "ML_C01", "question_id": "churn_risk", "family": "C", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `message` suggest the customer may leave for a competitor or cancel?" }, "qtype": 2, "K": 2, "sequence_length": 149, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 1529, 235376, 9337, 573, 6650, 1249, 5077, 604, 476, 62420, 689, 12459, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 5958, 19745, 573, 38294, 7984, 235265, 9105, 692, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 1464, 235270, 1 ], "marker_positions": [ 21, 31 ], "probabilities": [ 0.9991003274917603, 0.0008997417171485722 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0009, "confidence": 0.9991, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 6.173697471618652, -0.8388054370880127 ], "act_logits": [ 1553.9373779296875, -1842.3868408203125 ] } }, { "row_id": "ML_C02/intent", "fixture_id": "ML_C02", "question_id": "intent", "family": "C", "language": "EN", "primary_language": "EN", "question": { "type": "choice", "instructions": "What does the customer want in `message`?", "criteria": { "refund": "money returned or a duplicate charge reversed", "technical_help": "a bug, outage or integration problem", "billing_question": "a question about an invoice, plan or payment method", "information": "general information, pricing or how-to", "cancellation": "wants to cancel or downgrade", "other": "none of the other options fits" } }, "qtype": 0, "K": 6, "sequence_length": 256, "sequence_ids": [ 2, 6241, 2872, 235292, 2439, 1721, 573, 6650, 1938, 575, 4103, 1529, 235376, 235336, 1, 4, 19745, 235292, 3370, 8496, 689, 476, 38294, 6256, 33201, 4, 9838, 235298, 12180, 235292, 476, 11004, 235269, 119132, 689, 16827, 3210, 4, 54972, 235298, 13252, 235292, 476, 2872, 1105, 671, 42529, 235269, 1780, 689, 7984, 2370, 4, 2113, 235292, 3311, 2113, 235269, 25063, 689, 1368, 235290, 511, 4, 29027, 235292, 8507, 577, 12459, 689, 115214, 4, 1156, 235292, 8031, 576, 573, 1156, 4561, 17508, 1, 19946, 1529, 1192, 664, 651, 16783, 919, 8520, 11594, 235265, 590, 1144, 21343, 1861, 783, 1476, 573, 15162, 14610, 235265, 5651, 1707, 917, 11885, 665, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 1 ], "marker_positions": [ 15, 25, 37, 52, 63, 71 ], "probabilities": [ 0.7606508731842041, 0.09137894213199615, 0.004147833678871393, 0.009211607277393341, 0.008902090601623058, 0.1257086992263794 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "refund", "probabilities": { "refund": 0.7607, "technical_help": 0.0914, "billing_question": 0.0041, "information": 0.0092, "cancellation": 0.0089, "other": 0.1257 }, "confidence": 0.5561, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": true, "oracle": { "marker_logits": [ 0.6876354217529297, -1.4315237998962402, -4.523953437805176, -3.7260749340057373, -3.76025390625, -1.1125719547271729 ], "act_logits": [ 1359.00927734375, -1621.4041748046875 ] } }, { "row_id": "ML_C02/is_urgent", "fixture_id": "ML_C02", "question_id": "is_urgent", "family": "C", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `message` communicate time pressure or a deadline?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 1529, 235376, 18708, 1069, 5601, 689, 476, 30699, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 651, 16783, 919, 8520, 11594, 235265, 590, 1144, 21343, 1861, 783, 1476, 573, 15162, 14610, 235265, 5651, 1707, 917, 11885, 665, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 476, 1889, 576, 13333, 235269, 578, 573, 3221, 798, 1281, 476, 5469, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9976873397827148, 0.00231272098608315 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0023, "confidence": 0.9977, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 5.67744779586792, -0.3895690441131592 ], "act_logits": [ 1365.7689208984375, -1620.596923828125 ] } }, { "row_id": "ML_C02/frustration", "fixture_id": "ML_C02", "question_id": "frustration", "family": "C", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How frustrated does the customer sound in `message`?", "criteria": [ "calm and neutral", "concerned but civil", "clearly annoyed", "very angry or using strong language" ] }, "qtype": 1, "K": 4, "sequence_length": 256, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 46867, 1721, 573, 6650, 4835, 575, 4103, 1529, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 15142, 578, 17120, 4, 2403, 235248, 235274, 235292, 10695, 901, 11180, 4, 2403, 235248, 235284, 235292, 8886, 61058, 4, 2403, 235248, 235304, 235292, 1508, 19456, 689, 2177, 3779, 5255, 1, 19946, 1529, 1192, 664, 651, 16783, 919, 8520, 11594, 235265, 590, 1144, 21343, 1861, 783, 1476, 573, 15162, 14610, 235265, 5651, 1707, 917, 11885, 665, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 1 ], "marker_positions": [ 16, 24, 32, 39 ], "probabilities": [ 0.042319945991039276, 0.3292026221752167, 0.3235820531845093, 0.3048953711986542 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.8911, "legend": { "0": "calm and neutral", "1": "concerned but civil", "2": "clearly annoyed", "3": "very angry or using strong language" }, "probabilities": { "0": 0.0423, "1": 0.3292, "2": 0.3236, "3": 0.3049 }, "confidence": 0.115, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": true, "oracle": { "marker_logits": [ -0.8616895079612732, 1.1897258758544922, 1.1725051403045654, 1.113020896911621 ], "act_logits": [ 1472.712158203125, -1737.73583984375 ] } }, { "row_id": "ML_C02/refund_requested", "fixture_id": "ML_C02", "question_id": "refund_requested", "family": "C", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does the customer ask for money back?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 6650, 5291, 604, 3370, 1355, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 651, 16783, 919, 8520, 11594, 235265, 590, 1144, 21343, 1861, 783, 1476, 573, 15162, 14610, 235265, 5651, 1707, 917, 11885, 665, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 476, 1889, 576, 13333, 235269, 578, 573, 3221, 798, 1281, 476, 5469, 2346, 11117, 573, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.9999996423721313, 3.3850830050141667e-07 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 14.319570541381836, -0.5791460275650024 ], "act_logits": [ 1517.1932373046875, -1800.5621337890625 ] } }, { "row_id": "ML_C02/churn_risk", "fixture_id": "ML_C02", "question_id": "churn_risk", "family": "C", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `message` suggest the customer may leave for a competitor or cancel?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 1529, 235376, 9337, 573, 6650, 1249, 5077, 604, 476, 62420, 689, 12459, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 651, 16783, 919, 8520, 11594, 235265, 590, 1144, 21343, 1861, 783, 1476, 573, 15162, 14610, 235265, 5651, 1707, 917, 11885, 665, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 476, 1889, 576, 13333, 235269, 578, 573, 3221, 1 ], "marker_positions": [ 21, 31 ], "probabilities": [ 0.9993972778320312, 0.0006026736809872091 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0006, "confidence": 0.9994, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 7.036383628845215, -0.3771480321884155 ], "act_logits": [ 1506.8572998046875, -1786.0225830078125 ] } }, { "row_id": "ML_C03/intent", "fixture_id": "ML_C03", "question_id": "intent", "family": "C", "language": "EN", "primary_language": "EN", "question": { "type": "choice", "instructions": "What does the customer want in `message`?", "criteria": { "refund": "money returned or a duplicate charge reversed", "technical_help": "a bug, outage or integration problem", "billing_question": "a question about an invoice, plan or payment method", "information": "general information, pricing or how-to", "cancellation": "wants to cancel or downgrade", "other": "none of the other options fits" } }, "qtype": 0, "K": 6, "sequence_length": 108, "sequence_ids": [ 2, 6241, 2872, 235292, 2439, 1721, 573, 6650, 1938, 575, 4103, 1529, 235376, 235336, 1, 4, 19745, 235292, 3370, 8496, 689, 476, 38294, 6256, 33201, 4, 9838, 235298, 12180, 235292, 476, 11004, 235269, 119132, 689, 16827, 3210, 4, 54972, 235298, 13252, 235292, 476, 2872, 1105, 671, 42529, 235269, 1780, 689, 7984, 2370, 4, 2113, 235292, 3311, 2113, 235269, 25063, 689, 1368, 235290, 511, 4, 29027, 235292, 8507, 577, 12459, 689, 115214, 4, 1156, 235292, 8031, 576, 573, 1156, 4561, 17508, 1, 19946, 1529, 1192, 664, 235285, 1144, 61058, 674, 736, 42529, 2076, 919, 780, 1125, 11601, 1452, 2149, 11922, 235265, 6822, 1498, 573, 3051, 4740, 3595, 235270, 1 ], "marker_positions": [ 15, 25, 37, 52, 63, 71 ], "probabilities": [ 6.60849863720614e-08, 6.526625639935446e-08, 0.9999995231628418, 6.504529181938778e-08, 6.718420308970963e-08, 8.647825211482996e-08 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "billing_question", "probabilities": { "refund": 0.0, "technical_help": 0.0, "billing_question": 1.0, "information": 0.0, "cancellation": 0.0, "other": 0.0 }, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": false, "oracle": { "marker_logits": [ -4.568935394287109, -4.581402778625488, 11.963388442993164, -4.584794044494629, -4.5524396896362305, -4.299984455108643 ], "act_logits": [ 1457.5955810546875, -1738.21875 ] } }, { "row_id": "ML_C03/is_urgent", "fixture_id": "ML_C03", "question_id": "is_urgent", "family": "C", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `message` communicate time pressure or a deadline?" }, "qtype": 2, "K": 2, "sequence_length": 63, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 1529, 235376, 18708, 1069, 5601, 689, 476, 30699, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 235285, 1144, 61058, 674, 736, 42529, 2076, 919, 780, 1125, 11601, 1452, 2149, 11922, 235265, 6822, 1498, 573, 3051, 4740, 3595, 235270, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9990077614784241, 0.0009922912577167153 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.001, "confidence": 0.999, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 6.108315467834473, -0.8061851263046265 ], "act_logits": [ 1417.796142578125, -1682.6029052734375 ] } }, { "row_id": "ML_C03/frustration", "fixture_id": "ML_C03", "question_id": "frustration", "family": "C", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How frustrated does the customer sound in `message`?", "criteria": [ "calm and neutral", "concerned but civil", "clearly annoyed", "very angry or using strong language" ] }, "qtype": 1, "K": 4, "sequence_length": 78, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 46867, 1721, 573, 6650, 4835, 575, 4103, 1529, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 15142, 578, 17120, 4, 2403, 235248, 235274, 235292, 10695, 901, 11180, 4, 2403, 235248, 235284, 235292, 8886, 61058, 4, 2403, 235248, 235304, 235292, 1508, 19456, 689, 2177, 3779, 5255, 1, 19946, 1529, 1192, 664, 235285, 1144, 61058, 674, 736, 42529, 2076, 919, 780, 1125, 11601, 1452, 2149, 11922, 235265, 6822, 1498, 573, 3051, 4740, 3595, 235270, 1 ], "marker_positions": [ 16, 24, 32, 39 ], "probabilities": [ 0.02382860891520977, 0.23940551280975342, 0.4634254574775696, 0.2733404040336609 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.9863, "legend": { "0": "calm and neutral", "1": "concerned but civil", "2": "clearly annoyed", "3": "very angry or using strong language" }, "probabilities": { "0": 0.0238, "1": 0.2394, "2": 0.4634, "3": 0.2733 }, "confidence": 0.176, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -2.1821129322052, 0.12515980005264282, 0.7856466174125671, 0.25771939754486084 ], "act_logits": [ 1553.8446044921875, -1836.414306640625 ] } }, { "row_id": "ML_C03/refund_requested", "fixture_id": "ML_C03", "question_id": "refund_requested", "family": "C", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does the customer ask for money back?" }, "qtype": 2, "K": 2, "sequence_length": 60, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 6650, 5291, 604, 3370, 1355, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 235285, 1144, 61058, 674, 736, 42529, 2076, 919, 780, 1125, 11601, 1452, 2149, 11922, 235265, 6822, 1498, 573, 3051, 4740, 3595, 235270, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.9926620721817017, 0.0073379697278141975 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0073, "confidence": 0.9927, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 4.2593841552734375, -0.6479434967041016 ], "act_logits": [ 1443.2989501953125, -1713.6568603515625 ] } }, { "row_id": "ML_C03/churn_risk", "fixture_id": "ML_C03", "question_id": "churn_risk", "family": "C", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `message` suggest the customer may leave for a competitor or cancel?" }, "qtype": 2, "K": 2, "sequence_length": 67, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 1529, 235376, 9337, 573, 6650, 1249, 5077, 604, 476, 62420, 689, 12459, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 235285, 1144, 61058, 674, 736, 42529, 2076, 919, 780, 1125, 11601, 1452, 2149, 11922, 235265, 6822, 1498, 573, 3051, 4740, 3595, 235270, 1 ], "marker_positions": [ 21, 31 ], "probabilities": [ 0.9996534585952759, 0.0003465220797806978 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0003, "confidence": 0.9997, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 7.021278381347656, -0.9459392428398132 ], "act_logits": [ 1552.621337890625, -1843.8658447265625 ] } }, { "row_id": "ML_C04/intent", "fixture_id": "ML_C04", "question_id": "intent", "family": "C", "language": "EN", "primary_language": "EN", "question": { "type": "choice", "instructions": "What does the customer want in `message`?", "criteria": { "refund": "money returned or a duplicate charge reversed", "technical_help": "a bug, outage or integration problem", "billing_question": "a question about an invoice, plan or payment method", "information": "general information, pricing or how-to", "cancellation": "wants to cancel or downgrade", "other": "none of the other options fits" } }, "qtype": 0, "K": 6, "sequence_length": 189, "sequence_ids": [ 2, 6241, 2872, 235292, 2439, 1721, 573, 6650, 1938, 575, 4103, 1529, 235376, 235336, 1, 4, 19745, 235292, 3370, 8496, 689, 476, 38294, 6256, 33201, 4, 9838, 235298, 12180, 235292, 476, 11004, 235269, 119132, 689, 16827, 3210, 4, 54972, 235298, 13252, 235292, 476, 2872, 1105, 671, 42529, 235269, 1780, 689, 7984, 2370, 4, 2113, 235292, 3311, 2113, 235269, 25063, 689, 1368, 235290, 511, 4, 29027, 235292, 8507, 577, 12459, 689, 115214, 4, 1156, 235292, 8031, 576, 573, 1156, 4561, 17508, 1, 19946, 1529, 1192, 664, 6006, 798, 590, 1682, 573, 10021, 5608, 235336, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 1464, 235270, 1 ], "marker_positions": [ 15, 25, 37, 52, 63, 71 ], "probabilities": [ 0.005322811659425497, 0.031273841857910156, 0.005342151504009962, 0.6996017098426819, 0.06004447117447853, 0.19841499626636505 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "information", "probabilities": { "refund": 0.0053, "technical_help": 0.0313, "billing_question": 0.0053, "information": 0.6996, "cancellation": 0.06, "other": 0.1984 }, "confidence": 0.4955, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": false, "oracle": { "marker_logits": [ -4.040946006774902, -2.2701637744903564, -4.037312984466553, 0.8375657200813293, -1.617862582206726, -0.4225844740867615 ], "act_logits": [ 1337.56103515625, -1593.033935546875 ] } }, { "row_id": "ML_C04/is_urgent", "fixture_id": "ML_C04", "question_id": "is_urgent", "family": "C", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `message` communicate time pressure or a deadline?" }, "qtype": 2, "K": 2, "sequence_length": 144, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 1529, 235376, 18708, 1069, 5601, 689, 476, 30699, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 6006, 798, 590, 1682, 573, 10021, 5608, 235336, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 1464, 235270, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9973899722099304, 0.0026099933311343193 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0026, "confidence": 0.9974, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 5.069255352020264, -0.8765396475791931 ], "act_logits": [ 1300.413330078125, -1544.25048828125 ] } }, { "row_id": "ML_C04/frustration", "fixture_id": "ML_C04", "question_id": "frustration", "family": "C", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How frustrated does the customer sound in `message`?", "criteria": [ "calm and neutral", "concerned but civil", "clearly annoyed", "very angry or using strong language" ] }, "qtype": 1, "K": 4, "sequence_length": 159, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 46867, 1721, 573, 6650, 4835, 575, 4103, 1529, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 15142, 578, 17120, 4, 2403, 235248, 235274, 235292, 10695, 901, 11180, 4, 2403, 235248, 235284, 235292, 8886, 61058, 4, 2403, 235248, 235304, 235292, 1508, 19456, 689, 2177, 3779, 5255, 1, 19946, 1529, 1192, 664, 6006, 798, 590, 1682, 573, 10021, 5608, 235336, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 1464, 235270, 1 ], "marker_positions": [ 16, 24, 32, 39 ], "probabilities": [ 0.36818575859069824, 0.27676868438720703, 0.2462489753961563, 0.1087966039776802 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.0957, "legend": { "0": "calm and neutral", "1": "concerned but civil", "2": "clearly annoyed", "3": "very angry or using strong language" }, "probabilities": { "0": 0.3682, "1": 0.2768, "2": 0.2462, "3": 0.1088 }, "confidence": 0.0551, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ 1.388568639755249, 1.103163242340088, 0.9863243103027344, 0.16946148872375488 ], "act_logits": [ 1473.393798828125, -1738.5689697265625 ] } }, { "row_id": "ML_C04/refund_requested", "fixture_id": "ML_C04", "question_id": "refund_requested", "family": "C", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does the customer ask for money back?" }, "qtype": 2, "K": 2, "sequence_length": 141, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 6650, 5291, 604, 3370, 1355, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 6006, 798, 590, 1682, 573, 10021, 5608, 235336, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 1464, 235270, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.9998195767402649, 0.0001804781350074336 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0002, "confidence": 0.9998, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 7.596555709838867, -1.0231653451919556 ], "act_logits": [ 1630.709228515625, -1936.380615234375 ] } }, { "row_id": "ML_C04/churn_risk", "fixture_id": "ML_C04", "question_id": "churn_risk", "family": "C", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `message` suggest the customer may leave for a competitor or cancel?" }, "qtype": 2, "K": 2, "sequence_length": 148, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 1529, 235376, 9337, 573, 6650, 1249, 5077, 604, 476, 62420, 689, 12459, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 6006, 798, 590, 1682, 573, 10021, 5608, 235336, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 1464, 235270, 1 ], "marker_positions": [ 21, 31 ], "probabilities": [ 0.9985054731369019, 0.001494506374001503 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0015, "confidence": 0.9985, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 5.899806022644043, -0.6046574115753174 ], "act_logits": [ 1512.0235595703125, -1793.149169921875 ] } }, { "row_id": "ML_C05/intent", "fixture_id": "ML_C05", "question_id": "intent", "family": "C", "language": "mixed", "primary_language": "EN", "question": { "type": "choice", "instructions": "What does the customer want in `message`?", "criteria": { "refund": "money returned or a duplicate charge reversed", "technical_help": "a bug, outage or integration problem", "billing_question": "a question about an invoice, plan or payment method", "information": "general information, pricing or how-to", "cancellation": "wants to cancel or downgrade", "other": "none of the other options fits" } }, "qtype": 0, "K": 6, "sequence_length": 256, "sequence_ids": [ 2, 6241, 2872, 235292, 2439, 1721, 573, 6650, 1938, 575, 4103, 1529, 235376, 235336, 1, 4, 19745, 235292, 3370, 8496, 689, 476, 38294, 6256, 33201, 4, 9838, 235298, 12180, 235292, 476, 11004, 235269, 119132, 689, 16827, 3210, 4, 54972, 235298, 13252, 235292, 476, 2872, 1105, 671, 42529, 235269, 1780, 689, 7984, 2370, 4, 2113, 235292, 3311, 2113, 235269, 25063, 689, 1368, 235290, 511, 4, 29027, 235292, 8507, 577, 12459, 689, 115214, 4, 1156, 235292, 8031, 576, 573, 1156, 4561, 17508, 1, 19946, 1529, 1192, 664, 15178, 736, 1780, 1490, 235265, 590, 1144, 59276, 1105, 573, 13315, 36931, 578, 1938, 793, 978, 12036, 235265, 17575, 235504, 47753, 235868, 236517, 206622, 119920, 9178, 235362, 142801, 133074, 235418, 236567, 14848, 13319, 235362, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 1 ], "marker_positions": [ 15, 25, 37, 52, 63, 71 ], "probabilities": [ 6.487210413297362e-08, 4.593981195455399e-09, 4.591957480926112e-09, 4.797318542415496e-09, 0.9999998807907104, 4.52647741511214e-09 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "cancellation", "probabilities": { "refund": 0.0, "technical_help": 0.0, "billing_question": 0.0, "information": 0.0, "cancellation": 1.0, "other": 0.0 }, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": true, "oracle": { "marker_logits": [ -1.9112920761108398, -4.558962345123291, -4.5594024658203125, -4.515650749206543, 14.639556884765625, -4.573764801025391 ], "act_logits": [ 1238.9412841796875, -1481.71435546875 ] } }, { "row_id": "ML_C05/is_urgent", "fixture_id": "ML_C05", "question_id": "is_urgent", "family": "C", "language": "mixed", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `message` communicate time pressure or a deadline?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 1529, 235376, 18708, 1069, 5601, 689, 476, 30699, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 15178, 736, 1780, 1490, 235265, 590, 1144, 59276, 1105, 573, 13315, 36931, 578, 1938, 793, 978, 12036, 235265, 17575, 235504, 47753, 235868, 236517, 206622, 119920, 9178, 235362, 142801, 133074, 235418, 236567, 14848, 13319, 235362, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9402788877487183, 0.05972110107541084 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0597, "confidence": 0.9403, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 2.789187431335449, 0.032696425914764404 ], "act_logits": [ 1152.2252197265625, -1373.3717041015625 ] } }, { "row_id": "ML_C05/frustration", "fixture_id": "ML_C05", "question_id": "frustration", "family": "C", "language": "mixed", "primary_language": "EN", "question": { "type": "score", "instructions": "How frustrated does the customer sound in `message`?", "criteria": [ "calm and neutral", "concerned but civil", "clearly annoyed", "very angry or using strong language" ] }, "qtype": 1, "K": 4, "sequence_length": 256, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 46867, 1721, 573, 6650, 4835, 575, 4103, 1529, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 15142, 578, 17120, 4, 2403, 235248, 235274, 235292, 10695, 901, 11180, 4, 2403, 235248, 235284, 235292, 8886, 61058, 4, 2403, 235248, 235304, 235292, 1508, 19456, 689, 2177, 3779, 5255, 1, 19946, 1529, 1192, 664, 15178, 736, 1780, 1490, 235265, 590, 1144, 59276, 1105, 573, 13315, 36931, 578, 1938, 793, 978, 12036, 235265, 17575, 235504, 47753, 235868, 236517, 206622, 119920, 9178, 235362, 142801, 133074, 235418, 236567, 14848, 13319, 235362, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 1 ], "marker_positions": [ 16, 24, 32, 39 ], "probabilities": [ 0.021269377321004868, 0.07146577537059784, 0.46377918124198914, 0.4434855878353119 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 2.3295, "legend": { "0": "calm and neutral", "1": "concerned but civil", "2": "clearly annoyed", "3": "very angry or using strong language" }, "probabilities": { "0": 0.0213, "1": 0.0715, "2": 0.4638, "3": 0.4435 }, "confidence": 0.2877, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": true, "oracle": { "marker_logits": [ -3.4414596557617188, -2.229509115219116, -0.35931921005249023, -0.40406274795532227 ], "act_logits": [ 1483.4654541015625, -1752.071533203125 ] } }, { "row_id": "ML_C05/refund_requested", "fixture_id": "ML_C05", "question_id": "refund_requested", "family": "C", "language": "mixed", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does the customer ask for money back?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 6650, 5291, 604, 3370, 1355, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 15178, 736, 1780, 1490, 235265, 590, 1144, 59276, 1105, 573, 13315, 36931, 578, 1938, 793, 978, 12036, 235265, 17575, 235504, 47753, 235868, 236517, 206622, 119920, 9178, 235362, 142801, 133074, 235418, 236567, 14848, 13319, 235362, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 476, 1889, 576, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.12087469547986984, 0.8791252374649048 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.8791, "confidence": 0.8791, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ -1.9206674098968506, 0.06350576877593994 ], "act_logits": [ 1150.3746337890625, -1371.0482177734375 ] } }, { "row_id": "ML_C05/churn_risk", "fixture_id": "ML_C05", "question_id": "churn_risk", "family": "C", "language": "mixed", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `message` suggest the customer may leave for a competitor or cancel?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 1529, 235376, 9337, 573, 6650, 1249, 5077, 604, 476, 62420, 689, 12459, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 15178, 736, 1780, 1490, 235265, 590, 1144, 59276, 1105, 573, 13315, 36931, 578, 1938, 793, 978, 12036, 235265, 17575, 235504, 47753, 235868, 236517, 206622, 119920, 9178, 235362, 142801, 133074, 235418, 236567, 14848, 13319, 235362, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 1 ], "marker_positions": [ 21, 31 ], "probabilities": [ 0.002982515376061201, 0.997017502784729 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.997, "confidence": 0.997, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ -4.523159027099609, 1.2888429164886475 ], "act_logits": [ 1197.9385986328125, -1431.333251953125 ] } }, { "row_id": "ML_C06/intent", "fixture_id": "ML_C06", "question_id": "intent", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "choice", "instructions": "What does the customer want in `message`?", "criteria": { "refund": "money returned or a duplicate charge reversed", "technical_help": "a bug, outage or integration problem", "billing_question": "a question about an invoice, plan or payment method", "information": "general information, pricing or how-to", "cancellation": "wants to cancel or downgrade", "other": "none of the other options fits" } }, "qtype": 0, "K": 6, "sequence_length": 110, "sequence_ids": [ 2, 6241, 2872, 235292, 2439, 1721, 573, 6650, 1938, 575, 4103, 1529, 235376, 235336, 1, 4, 19745, 235292, 3370, 8496, 689, 476, 38294, 6256, 33201, 4, 9838, 235298, 12180, 235292, 476, 11004, 235269, 119132, 689, 16827, 3210, 4, 54972, 235298, 13252, 235292, 476, 2872, 1105, 671, 42529, 235269, 1780, 689, 7984, 2370, 4, 2113, 235292, 3311, 2113, 235269, 25063, 689, 1368, 235290, 511, 4, 29027, 235292, 8507, 577, 12459, 689, 115214, 4, 1156, 235292, 8031, 576, 573, 1156, 4561, 17508, 1, 19946, 1529, 1192, 664, 217491, 235400, 33341, 150432, 17722, 235588, 235432, 236572, 235854, 54934, 235362, 26685, 7670, 103549, 235394, 188029, 55979, 13013, 236514, 75214, 235362, 107968, 235362, 12990, 1 ], "marker_positions": [ 15, 25, 37, 52, 63, 71 ], "probabilities": [ 0.9999805688858032, 3.8455791582237e-06, 3.841822035610676e-06, 3.6011879274155945e-06, 3.587332912502461e-06, 4.654340500565013e-06 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "refund", "probabilities": { "refund": 1.0, "technical_help": 0.0, "billing_question": 0.0, "information": 0.0, "cancellation": 0.0, "other": 0.0 }, "confidence": 0.9999, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": false, "oracle": { "marker_logits": [ 7.969280242919922, -4.499270439147949, -4.500247955322266, -4.564931869506836, -4.568785190582275, -4.308391571044922 ], "act_logits": [ 1366.9434814453125, -1634.8629150390625 ] } }, { "row_id": "ML_C06/is_urgent", "fixture_id": "ML_C06", "question_id": "is_urgent", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `message` communicate time pressure or a deadline?" }, "qtype": 2, "K": 2, "sequence_length": 65, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 1529, 235376, 18708, 1069, 5601, 689, 476, 30699, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 217491, 235400, 33341, 150432, 17722, 235588, 235432, 236572, 235854, 54934, 235362, 26685, 7670, 103549, 235394, 188029, 55979, 13013, 236514, 75214, 235362, 107968, 235362, 12990, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9956220984458923, 0.004377886187285185 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0044, "confidence": 0.9956, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 4.8495564460754395, -0.5772460699081421 ], "act_logits": [ 1402.2640380859375, -1664.0234375 ] } }, { "row_id": "ML_C06/frustration", "fixture_id": "ML_C06", "question_id": "frustration", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How frustrated does the customer sound in `message`?", "criteria": [ "calm and neutral", "concerned but civil", "clearly annoyed", "very angry or using strong language" ] }, "qtype": 1, "K": 4, "sequence_length": 80, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 46867, 1721, 573, 6650, 4835, 575, 4103, 1529, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 15142, 578, 17120, 4, 2403, 235248, 235274, 235292, 10695, 901, 11180, 4, 2403, 235248, 235284, 235292, 8886, 61058, 4, 2403, 235248, 235304, 235292, 1508, 19456, 689, 2177, 3779, 5255, 1, 19946, 1529, 1192, 664, 217491, 235400, 33341, 150432, 17722, 235588, 235432, 236572, 235854, 54934, 235362, 26685, 7670, 103549, 235394, 188029, 55979, 13013, 236514, 75214, 235362, 107968, 235362, 12990, 1 ], "marker_positions": [ 16, 24, 32, 39 ], "probabilities": [ 0.09678865224123001, 0.08811313658952713, 0.4213651716709137, 0.3937329649925232 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 2.112, "legend": { "0": "calm and neutral", "1": "concerned but civil", "2": "clearly annoyed", "3": "very angry or using strong language" }, "probabilities": { "0": 0.0968, "1": 0.0881, "2": 0.4214, "3": 0.3937 }, "confidence": 0.1551, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -1.3491123914718628, -1.4430204629898071, 0.12185782194137573, 0.05403071641921997 ], "act_logits": [ 1554.189453125, -1835.696044921875 ] } }, { "row_id": "ML_C06/refund_requested", "fixture_id": "ML_C06", "question_id": "refund_requested", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does the customer ask for money back?" }, "qtype": 2, "K": 2, "sequence_length": 62, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 6650, 5291, 604, 3370, 1355, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 217491, 235400, 33341, 150432, 17722, 235588, 235432, 236572, 235854, 54934, 235362, 26685, 7670, 103549, 235394, 188029, 55979, 13013, 236514, 75214, 235362, 107968, 235362, 12990, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.0873979851603508, 0.9126019477844238 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.9126, "confidence": 0.9126, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ -1.7205023765563965, 0.6253255009651184 ], "act_logits": [ 1237.591796875, -1474.6627197265625 ] } }, { "row_id": "ML_C06/churn_risk", "fixture_id": "ML_C06", "question_id": "churn_risk", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `message` suggest the customer may leave for a competitor or cancel?" }, "qtype": 2, "K": 2, "sequence_length": 69, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 1529, 235376, 9337, 573, 6650, 1249, 5077, 604, 476, 62420, 689, 12459, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 217491, 235400, 33341, 150432, 17722, 235588, 235432, 236572, 235854, 54934, 235362, 26685, 7670, 103549, 235394, 188029, 55979, 13013, 236514, 75214, 235362, 107968, 235362, 12990, 1 ], "marker_positions": [ 21, 31 ], "probabilities": [ 0.985255241394043, 0.014744759537279606 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0147, "confidence": 0.9853, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 3.8308677673339844, -0.37114596366882324 ], "act_logits": [ 1422.3702392578125, -1687.023193359375 ] } }, { "row_id": "ML_C07/intent", "fixture_id": "ML_C07", "question_id": "intent", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "choice", "instructions": "What does the customer want in `message`?", "criteria": { "refund": "money returned or a duplicate charge reversed", "technical_help": "a bug, outage or integration problem", "billing_question": "a question about an invoice, plan or payment method", "information": "general information, pricing or how-to", "cancellation": "wants to cancel or downgrade", "other": "none of the other options fits" } }, "qtype": 0, "K": 6, "sequence_length": 182, "sequence_ids": [ 2, 6241, 2872, 235292, 2439, 1721, 573, 6650, 1938, 575, 4103, 1529, 235376, 235336, 1, 4, 19745, 235292, 3370, 8496, 689, 476, 38294, 6256, 33201, 4, 9838, 235298, 12180, 235292, 476, 11004, 235269, 119132, 689, 16827, 3210, 4, 54972, 235298, 13252, 235292, 476, 2872, 1105, 671, 42529, 235269, 1780, 689, 7984, 2370, 4, 2113, 235292, 3311, 2113, 235269, 25063, 689, 1368, 235290, 511, 4, 29027, 235292, 8507, 577, 12459, 689, 115214, 4, 1156, 235292, 8031, 576, 573, 1156, 4561, 17508, 1, 19946, 1529, 1192, 664, 27364, 66599, 219254, 235362, 235781, 27710, 67988, 22708, 235400, 50165, 68180, 235443, 77818, 2297, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 12990, 1 ], "marker_positions": [ 15, 25, 37, 52, 63, 71 ], "probabilities": [ 0.00641329400241375, 0.008350739255547523, 0.0016458297614008188, 0.004547806456685066, 0.9570600986480713, 0.021982209756970406 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "cancellation", "probabilities": { "refund": 0.0064, "technical_help": 0.0084, "billing_question": 0.0016, "information": 0.0045, "cancellation": 0.9571, "other": 0.022 }, "confidence": 0.8698, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": false, "oracle": { "marker_logits": [ -3.189846992492676, -2.9258947372436523, -4.549979209899902, -3.533581256866455, 1.8156394958496094, -1.9579896926879883 ], "act_logits": [ 1460.3914794921875, -1743.503173828125 ] } }, { "row_id": "ML_C07/is_urgent", "fixture_id": "ML_C07", "question_id": "is_urgent", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `message` communicate time pressure or a deadline?" }, "qtype": 2, "K": 2, "sequence_length": 137, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 1529, 235376, 18708, 1069, 5601, 689, 476, 30699, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 27364, 66599, 219254, 235362, 235781, 27710, 67988, 22708, 235400, 50165, 68180, 235443, 77818, 2297, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 12990, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.999478280544281, 0.0005216951831243932 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0005, "confidence": 0.9995, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 6.395173072814941, -1.162732481956482 ], "act_logits": [ 1400.74560546875, -1665.020263671875 ] } }, { "row_id": "ML_C07/frustration", "fixture_id": "ML_C07", "question_id": "frustration", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How frustrated does the customer sound in `message`?", "criteria": [ "calm and neutral", "concerned but civil", "clearly annoyed", "very angry or using strong language" ] }, "qtype": 1, "K": 4, "sequence_length": 152, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 46867, 1721, 573, 6650, 4835, 575, 4103, 1529, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 15142, 578, 17120, 4, 2403, 235248, 235274, 235292, 10695, 901, 11180, 4, 2403, 235248, 235284, 235292, 8886, 61058, 4, 2403, 235248, 235304, 235292, 1508, 19456, 689, 2177, 3779, 5255, 1, 19946, 1529, 1192, 664, 27364, 66599, 219254, 235362, 235781, 27710, 67988, 22708, 235400, 50165, 68180, 235443, 77818, 2297, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 12990, 1 ], "marker_positions": [ 16, 24, 32, 39 ], "probabilities": [ 0.09940940141677856, 0.11555257439613342, 0.42819783091545105, 0.3568401336669922 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 2.0425, "legend": { "0": "calm and neutral", "1": "concerned but civil", "2": "clearly annoyed", "3": "very angry or using strong language" }, "probabilities": { "0": 0.0994, "1": 0.1156, "2": 0.4282, "3": 0.3568 }, "confidence": 0.1273, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -0.6560702919960022, -0.5055915713310242, 0.8042686581611633, 0.6219709515571594 ], "act_logits": [ 1568.2607421875, -1849.5162353515625 ] } }, { "row_id": "ML_C07/refund_requested", "fixture_id": "ML_C07", "question_id": "refund_requested", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does the customer ask for money back?" }, "qtype": 2, "K": 2, "sequence_length": 134, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 6650, 5291, 604, 3370, 1355, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 27364, 66599, 219254, 235362, 235781, 27710, 67988, 22708, 235400, 50165, 68180, 235443, 77818, 2297, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 12990, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.9999984502792358, 1.499915924796369e-06 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 11.92768383026123, -1.4824191331863403 ], "act_logits": [ 1610.35205078125, -1914.07568359375 ] } }, { "row_id": "ML_C07/churn_risk", "fixture_id": "ML_C07", "question_id": "churn_risk", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `message` suggest the customer may leave for a competitor or cancel?" }, "qtype": 2, "K": 2, "sequence_length": 141, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 1529, 235376, 9337, 573, 6650, 1249, 5077, 604, 476, 62420, 689, 12459, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 27364, 66599, 219254, 235362, 235781, 27710, 67988, 22708, 235400, 50165, 68180, 235443, 77818, 2297, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 12990, 1 ], "marker_positions": [ 21, 31 ], "probabilities": [ 0.9966378211975098, 0.003362109884619713 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0034, "confidence": 0.9966, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 5.030986785888672, -0.6608314514160156 ], "act_logits": [ 1401.9359130859375, -1665.0166015625 ] } }, { "row_id": "ML_C08/intent", "fixture_id": "ML_C08", "question_id": "intent", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "choice", "instructions": "What does the customer want in `message`?", "criteria": { "refund": "money returned or a duplicate charge reversed", "technical_help": "a bug, outage or integration problem", "billing_question": "a question about an invoice, plan or payment method", "information": "general information, pricing or how-to", "cancellation": "wants to cancel or downgrade", "other": "none of the other options fits" } }, "qtype": 0, "K": 6, "sequence_length": 256, "sequence_ids": [ 2, 6241, 2872, 235292, 2439, 1721, 573, 6650, 1938, 575, 4103, 1529, 235376, 235336, 1, 4, 19745, 235292, 3370, 8496, 689, 476, 38294, 6256, 33201, 4, 9838, 235298, 12180, 235292, 476, 11004, 235269, 119132, 689, 16827, 3210, 4, 54972, 235298, 13252, 235292, 476, 2872, 1105, 671, 42529, 235269, 1780, 689, 7984, 2370, 4, 2113, 235292, 3311, 2113, 235269, 25063, 689, 1368, 235290, 511, 4, 29027, 235292, 8507, 577, 12459, 689, 115214, 4, 1156, 235292, 8031, 576, 573, 1156, 4561, 17508, 1, 19946, 1529, 1192, 664, 133074, 238082, 16508, 112791, 63385, 47140, 33291, 28221, 181752, 235362, 90711, 235394, 79292, 236652, 235452, 22031, 235362, 50430, 98073, 89590, 32072, 16459, 235394, 186264, 236661, 13238, 13319, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 1 ], "marker_positions": [ 15, 25, 37, 52, 63, 71 ], "probabilities": [ 0.9997286200523376, 8.717861419427209e-06, 0.0002124706661561504, 8.807520316622686e-06, 8.154205715982243e-06, 3.319383904454298e-05 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "refund", "probabilities": { "refund": 0.9997, "technical_help": 0.0, "billing_question": 0.0002, "information": 0.0, "cancellation": 0.0, "other": 0.0 }, "confidence": 0.9985, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": true, "oracle": { "marker_logits": [ 7.153299808502197, -4.496564865112305, -1.3031339645385742, -4.48633337020874, -4.5634050369262695, -3.159574270248413 ], "act_logits": [ 1241.4285888671875, -1483.3267822265625 ] } }, { "row_id": "ML_C08/is_urgent", "fixture_id": "ML_C08", "question_id": "is_urgent", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `message` communicate time pressure or a deadline?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 1529, 235376, 18708, 1069, 5601, 689, 476, 30699, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 133074, 238082, 16508, 112791, 63385, 47140, 33291, 28221, 181752, 235362, 90711, 235394, 79292, 236652, 235452, 22031, 235362, 50430, 98073, 89590, 32072, 16459, 235394, 186264, 236661, 13238, 13319, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 50855, 72368, 235362, 78546, 235372, 63090, 238518, 7711, 44393, 235372, 174612, 235410, 53479, 235372, 130073, 9301, 235432, 90518, 22031, 235362, 235987, 23176, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9998216032981873, 0.0001784235064405948 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0002, "confidence": 0.9998, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 7.771510601043701, -0.8596608638763428 ], "act_logits": [ 1510.0538330078125, -1792.6944580078125 ] } }, { "row_id": "ML_C08/frustration", "fixture_id": "ML_C08", "question_id": "frustration", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How frustrated does the customer sound in `message`?", "criteria": [ "calm and neutral", "concerned but civil", "clearly annoyed", "very angry or using strong language" ] }, "qtype": 1, "K": 4, "sequence_length": 256, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 46867, 1721, 573, 6650, 4835, 575, 4103, 1529, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 15142, 578, 17120, 4, 2403, 235248, 235274, 235292, 10695, 901, 11180, 4, 2403, 235248, 235284, 235292, 8886, 61058, 4, 2403, 235248, 235304, 235292, 1508, 19456, 689, 2177, 3779, 5255, 1, 19946, 1529, 1192, 664, 133074, 238082, 16508, 112791, 63385, 47140, 33291, 28221, 181752, 235362, 90711, 235394, 79292, 236652, 235452, 22031, 235362, 50430, 98073, 89590, 32072, 16459, 235394, 186264, 236661, 13238, 13319, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 50855, 72368, 235362, 78546, 235372, 63090, 238518, 1 ], "marker_positions": [ 16, 24, 32, 39 ], "probabilities": [ 0.06459284573793411, 0.1788083016872406, 0.509062647819519, 0.24753624200820923 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.9395, "legend": { "0": "calm and neutral", "1": "concerned but civil", "2": "clearly annoyed", "3": "very angry or using strong language" }, "probabilities": { "0": 0.0646, "1": 0.1788, "2": 0.5091, "3": 0.2475 }, "confidence": 0.1531, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": true, "oracle": { "marker_logits": [ -1.3813018798828125, -0.36309075355529785, 0.683165967464447, -0.03784823417663574 ], "act_logits": [ 1577.806884765625, -1860.979248046875 ] } }, { "row_id": "ML_C08/refund_requested", "fixture_id": "ML_C08", "question_id": "refund_requested", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does the customer ask for money back?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 6650, 5291, 604, 3370, 1355, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 133074, 238082, 16508, 112791, 63385, 47140, 33291, 28221, 181752, 235362, 90711, 235394, 79292, 236652, 235452, 22031, 235362, 50430, 98073, 89590, 32072, 16459, 235394, 186264, 236661, 13238, 13319, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 50855, 72368, 235362, 78546, 235372, 63090, 238518, 7711, 44393, 235372, 174612, 235410, 53479, 235372, 130073, 9301, 235432, 90518, 22031, 235362, 235987, 23176, 151978, 219689, 236943, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.8083775043487549, 0.19162243604660034 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.1916, "confidence": 0.8084, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 1.2414294481277466, -0.1980729103088379 ], "act_logits": [ 1316.4764404296875, -1565.2103271484375 ] } }, { "row_id": "ML_C08/churn_risk", "fixture_id": "ML_C08", "question_id": "churn_risk", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `message` suggest the customer may leave for a competitor or cancel?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 1529, 235376, 9337, 573, 6650, 1249, 5077, 604, 476, 62420, 689, 12459, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 133074, 238082, 16508, 112791, 63385, 47140, 33291, 28221, 181752, 235362, 90711, 235394, 79292, 236652, 235452, 22031, 235362, 50430, 98073, 89590, 32072, 16459, 235394, 186264, 236661, 13238, 13319, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 50855, 72368, 235362, 78546, 235372, 63090, 238518, 7711, 44393, 235372, 174612, 235410, 53479, 235372, 130073, 9301, 235432, 90518, 1 ], "marker_positions": [ 21, 31 ], "probabilities": [ 0.9983804225921631, 0.0016195849748328328 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0016, "confidence": 0.9984, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 6.250990390777588, -0.17297500371932983 ], "act_logits": [ 1501.2713623046875, -1782.154052734375 ] } }, { "row_id": "ML_C09/intent", "fixture_id": "ML_C09", "question_id": "intent", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "choice", "instructions": "What does the customer want in `message`?", "criteria": { "refund": "money returned or a duplicate charge reversed", "technical_help": "a bug, outage or integration problem", "billing_question": "a question about an invoice, plan or payment method", "information": "general information, pricing or how-to", "cancellation": "wants to cancel or downgrade", "other": "none of the other options fits" } }, "qtype": 0, "K": 6, "sequence_length": 116, "sequence_ids": [ 2, 6241, 2872, 235292, 2439, 1721, 573, 6650, 1938, 575, 4103, 1529, 235376, 235336, 1, 4, 19745, 235292, 3370, 8496, 689, 476, 38294, 6256, 33201, 4, 9838, 235298, 12180, 235292, 476, 11004, 235269, 119132, 689, 16827, 3210, 4, 54972, 235298, 13252, 235292, 476, 2872, 1105, 671, 42529, 235269, 1780, 689, 7984, 2370, 4, 2113, 235292, 3311, 2113, 235269, 25063, 689, 1368, 235290, 511, 4, 29027, 235292, 8507, 577, 12459, 689, 115214, 4, 1156, 235292, 8031, 576, 573, 1156, 4561, 17508, 1, 19946, 1529, 1192, 664, 23631, 235441, 114618, 235410, 28221, 83706, 50637, 73452, 35570, 235394, 20712, 185327, 87735, 235362, 235811, 31150, 235868, 236517, 54934, 235362, 8740, 84438, 133074, 235418, 32910, 40004, 21509, 9481, 235362, 12990, 1 ], "marker_positions": [ 15, 25, 37, 52, 63, 71 ], "probabilities": [ 7.879260920162778e-06, 0.0009873589733615518, 8.582729606132489e-06, 7.94364223111188e-06, 0.9989787340164185, 9.556800250720698e-06 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "cancellation", "probabilities": { "refund": 0.0, "technical_help": 0.001, "billing_question": 0.0, "information": 0.0, "cancellation": 0.999, "other": 0.0 }, "confidence": 0.9954, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": false, "oracle": { "marker_logits": [ -4.5900797843933105, 0.2407206892967224, -4.504561424255371, -4.5819411277771, 7.160175323486328, -4.397060394287109 ], "act_logits": [ 1292.0897216796875, -1547.3153076171875 ] } }, { "row_id": "ML_C09/is_urgent", "fixture_id": "ML_C09", "question_id": "is_urgent", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `message` communicate time pressure or a deadline?" }, "qtype": 2, "K": 2, "sequence_length": 71, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 1529, 235376, 18708, 1069, 5601, 689, 476, 30699, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 23631, 235441, 114618, 235410, 28221, 83706, 50637, 73452, 35570, 235394, 20712, 185327, 87735, 235362, 235811, 31150, 235868, 236517, 54934, 235362, 8740, 84438, 133074, 235418, 32910, 40004, 21509, 9481, 235362, 12990, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.5231514573097229, 0.4768485128879547 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.4768, "confidence": 0.5232, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ -0.3144078254699707, -0.4070799946784973 ], "act_logits": [ 1359.9708251953125, -1618.080810546875 ] } }, { "row_id": "ML_C09/frustration", "fixture_id": "ML_C09", "question_id": "frustration", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How frustrated does the customer sound in `message`?", "criteria": [ "calm and neutral", "concerned but civil", "clearly annoyed", "very angry or using strong language" ] }, "qtype": 1, "K": 4, "sequence_length": 86, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 46867, 1721, 573, 6650, 4835, 575, 4103, 1529, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 15142, 578, 17120, 4, 2403, 235248, 235274, 235292, 10695, 901, 11180, 4, 2403, 235248, 235284, 235292, 8886, 61058, 4, 2403, 235248, 235304, 235292, 1508, 19456, 689, 2177, 3779, 5255, 1, 19946, 1529, 1192, 664, 23631, 235441, 114618, 235410, 28221, 83706, 50637, 73452, 35570, 235394, 20712, 185327, 87735, 235362, 235811, 31150, 235868, 236517, 54934, 235362, 8740, 84438, 133074, 235418, 32910, 40004, 21509, 9481, 235362, 12990, 1 ], "marker_positions": [ 16, 24, 32, 39 ], "probabilities": [ 0.022263575345277786, 0.07403847575187683, 0.5225715041160583, 0.38112643361091614 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 2.2626, "legend": { "0": "calm and neutral", "1": "concerned but civil", "2": "clearly annoyed", "3": "very angry or using strong language" }, "probabilities": { "0": 0.0223, "1": 0.074, "2": 0.5226, "3": 0.3811 }, "confidence": 0.29, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -3.8938748836517334, -2.692241907119751, -0.7380648851394653, -1.0536956787109375 ], "act_logits": [ 1490.6878662109375, -1764.31298828125 ] } }, { "row_id": "ML_C09/refund_requested", "fixture_id": "ML_C09", "question_id": "refund_requested", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does the customer ask for money back?" }, "qtype": 2, "K": 2, "sequence_length": 68, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 6650, 5291, 604, 3370, 1355, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 23631, 235441, 114618, 235410, 28221, 83706, 50637, 73452, 35570, 235394, 20712, 185327, 87735, 235362, 235811, 31150, 235868, 236517, 54934, 235362, 8740, 84438, 133074, 235418, 32910, 40004, 21509, 9481, 235362, 12990, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.800119161605835, 0.19988085329532623 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.1999, "confidence": 0.8001, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 0.5949375629425049, -0.7921019792556763 ], "act_logits": [ 1256.0919189453125, -1495.617919921875 ] } }, { "row_id": "ML_C09/churn_risk", "fixture_id": "ML_C09", "question_id": "churn_risk", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `message` suggest the customer may leave for a competitor or cancel?" }, "qtype": 2, "K": 2, "sequence_length": 75, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 1529, 235376, 9337, 573, 6650, 1249, 5077, 604, 476, 62420, 689, 12459, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 23631, 235441, 114618, 235410, 28221, 83706, 50637, 73452, 35570, 235394, 20712, 185327, 87735, 235362, 235811, 31150, 235868, 236517, 54934, 235362, 8740, 84438, 133074, 235418, 32910, 40004, 21509, 9481, 235362, 12990, 1 ], "marker_positions": [ 21, 31 ], "probabilities": [ 0.015197246335446835, 0.984802782535553 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.9848, "confidence": 0.9848, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ -3.979158878326416, 0.19216817617416382 ], "act_logits": [ 1220.734375, -1455.85986328125 ] } }, { "row_id": "ML_C10/intent", "fixture_id": "ML_C10", "question_id": "intent", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "choice", "instructions": "What does the customer want in `message`?", "criteria": { "refund": "money returned or a duplicate charge reversed", "technical_help": "a bug, outage or integration problem", "billing_question": "a question about an invoice, plan or payment method", "information": "general information, pricing or how-to", "cancellation": "wants to cancel or downgrade", "other": "none of the other options fits" } }, "qtype": 0, "K": 6, "sequence_length": 225, "sequence_ids": [ 2, 6241, 2872, 235292, 2439, 1721, 573, 6650, 1938, 575, 4103, 1529, 235376, 235336, 1, 4, 19745, 235292, 3370, 8496, 689, 476, 38294, 6256, 33201, 4, 9838, 235298, 12180, 235292, 476, 11004, 235269, 119132, 689, 16827, 3210, 4, 54972, 235298, 13252, 235292, 476, 2872, 1105, 671, 42529, 235269, 1780, 689, 7984, 2370, 4, 2113, 235292, 3311, 2113, 235269, 25063, 689, 1368, 235290, 511, 4, 29027, 235292, 8507, 577, 12459, 689, 115214, 4, 1156, 235292, 8031, 576, 573, 1156, 4561, 17508, 1, 19946, 1529, 1192, 664, 44393, 236322, 235398, 239006, 3290, 240258, 67175, 236076, 235400, 94257, 11519, 2297, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 12990, 1 ], "marker_positions": [ 15, 25, 37, 52, 63, 71 ], "probabilities": [ 0.07743897289037704, 0.0031349833589047194, 0.001894165063276887, 0.007403286173939705, 0.13660238683223724, 0.773526132106781 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "other", "probabilities": { "refund": 0.0774, "technical_help": 0.0031, "billing_question": 0.0019, "information": 0.0074, "cancellation": 0.1366, "other": 0.7735 }, "confidence": 0.5898, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": false, "oracle": { "marker_logits": [ -0.8487107753753662, -4.055574417114258, -4.559423446655273, -3.1962788105010986, -0.28113508224487305, 1.4527667760849 ], "act_logits": [ 1314.5367431640625, -1569.70751953125 ] } }, { "row_id": "ML_C10/is_urgent", "fixture_id": "ML_C10", "question_id": "is_urgent", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `message` communicate time pressure or a deadline?" }, "qtype": 2, "K": 2, "sequence_length": 180, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 1529, 235376, 18708, 1069, 5601, 689, 476, 30699, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 44393, 236322, 235398, 239006, 3290, 240258, 67175, 236076, 235400, 94257, 11519, 2297, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 12990, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9999945163726807, 5.510443315870361e-06 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 11.133159637451172, -0.9757003784179688 ], "act_logits": [ 1597.2857666015625, -1899.1728515625 ] } }, { "row_id": "ML_C10/frustration", "fixture_id": "ML_C10", "question_id": "frustration", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How frustrated does the customer sound in `message`?", "criteria": [ "calm and neutral", "concerned but civil", "clearly annoyed", "very angry or using strong language" ] }, "qtype": 1, "K": 4, "sequence_length": 195, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 46867, 1721, 573, 6650, 4835, 575, 4103, 1529, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 15142, 578, 17120, 4, 2403, 235248, 235274, 235292, 10695, 901, 11180, 4, 2403, 235248, 235284, 235292, 8886, 61058, 4, 2403, 235248, 235304, 235292, 1508, 19456, 689, 2177, 3779, 5255, 1, 19946, 1529, 1192, 664, 44393, 236322, 235398, 239006, 3290, 240258, 67175, 236076, 235400, 94257, 11519, 2297, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 12990, 1 ], "marker_positions": [ 16, 24, 32, 39 ], "probabilities": [ 0.08511137217283249, 0.0936875268816948, 0.5204501152038574, 0.3007510304450989 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 2.0368, "legend": { "0": "calm and neutral", "1": "concerned but civil", "2": "clearly annoyed", "3": "very angry or using strong language" }, "probabilities": { "0": 0.0851, "1": 0.0937, "2": 0.5205, "3": 0.3008 }, "confidence": 0.1829, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -0.2206013798713684, -0.12459683418273926, 1.590132236480713, 1.0417208671569824 ], "act_logits": [ 1537.5523681640625, -1813.101806640625 ] } }, { "row_id": "ML_C10/refund_requested", "fixture_id": "ML_C10", "question_id": "refund_requested", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does the customer ask for money back?" }, "qtype": 2, "K": 2, "sequence_length": 177, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 573, 6650, 5291, 604, 3370, 1355, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 44393, 236322, 235398, 239006, 3290, 240258, 67175, 236076, 235400, 94257, 11519, 2297, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 12990, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.9999998807907104, 8.391440076138679e-08 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 15.195626258850098, -1.097846269607544 ], "act_logits": [ 1564.44189453125, -1859.823974609375 ] } }, { "row_id": "ML_C10/churn_risk", "fixture_id": "ML_C10", "question_id": "churn_risk", "family": "C", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `message` suggest the customer may leave for a competitor or cancel?" }, "qtype": 2, "K": 2, "sequence_length": 184, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 1529, 235376, 9337, 573, 6650, 1249, 5077, 604, 476, 62420, 689, 12459, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 1529, 1192, 664, 44393, 236322, 235398, 239006, 3290, 240258, 67175, 236076, 235400, 94257, 11519, 2297, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 12990, 1 ], "marker_positions": [ 21, 31 ], "probabilities": [ 0.9999955892562866, 4.437002189661143e-06 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 11.318958282470703, -1.0065709352493286 ], "act_logits": [ 1650.5577392578125, -1961.487548828125 ] } }, { "row_id": "ML_E01/toxic", "fixture_id": "ML_E01", "question_id": "toxic", "family": "E", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Is `post` toxic: rude, disrespectful or likely to make someone leave the discussion?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 4103, 2729, 235376, 21801, 235292, 30143, 235269, 142588, 689, 5476, 577, 1501, 4630, 5077, 573, 8827, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 651, 8717, 3703, 4713, 13456, 235265, 9105, 692, 604, 11766, 573, 5642, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 476, 1889, 576, 13333, 235269, 578, 573, 3221, 798, 1281, 476, 5469, 2346, 11117, 1 ], "marker_positions": [ 24, 34 ], "probabilities": [ 0.9991046786308289, 0.0008952730568125844 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0009, "confidence": 0.9991, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 6.443716526031494, -0.5737694501876831 ], "act_logits": [ 1727.2843017578125, -2050.012451171875 ] } }, { "row_id": "ML_E01/harassment", "fixture_id": "ML_E01", "question_id": "harassment", "family": "E", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `post` target or harass a specific person?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 2729, 235376, 4408, 689, 40868, 476, 3724, 1552, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 651, 8717, 3703, 4713, 13456, 235265, 9105, 692, 604, 11766, 573, 5642, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 476, 1889, 576, 13333, 235269, 578, 573, 3221, 798, 1281, 476, 5469, 2346, 11117, 573, 3464, 235265, 714, 8467, 1049, 12184, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9999125003814697, 8.749849075684324e-05 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0001, "confidence": 0.9999, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 8.342058181762695, -1.001743197441101 ], "act_logits": [ 1582.60791015625, -1880.1279296875 ] } }, { "row_id": "ML_E01/threat", "fixture_id": "ML_E01", "question_id": "threat", "family": "E", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `post` threaten violence, harm or intimidation?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 2729, 235376, 60399, 13673, 235269, 14649, 689, 132353, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 651, 8717, 3703, 4713, 13456, 235265, 9105, 692, 604, 11766, 573, 5642, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 476, 1889, 576, 13333, 235269, 578, 573, 3221, 798, 1281, 476, 5469, 2346, 11117, 573, 3464, 235265, 714, 8467, 1049, 12184, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9997220635414124, 0.00027790505555458367 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0003, "confidence": 0.9997, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 7.437276363372803, -0.7506767511367798 ], "act_logits": [ 1655.8199462890625, -1967.740234375 ] } }, { "row_id": "ML_E01/spam", "fixture_id": "ML_E01", "question_id": "spam", "family": "E", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Is `post` spam or advertising?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 4103, 2729, 235376, 31967, 689, 16171, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 651, 8717, 3703, 4713, 13456, 235265, 9105, 692, 604, 11766, 573, 5642, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 476, 1889, 576, 13333, 235269, 578, 573, 3221, 798, 1281, 476, 5469, 2346, 11117, 573, 3464, 235265, 714, 8467, 1049, 12184, 575, 974, 2040, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 1.0, 2.9475091256614405e-09 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 17.523685455322266, -2.118622064590454 ], "act_logits": [ 1473.6944580078125, -1752.879638671875 ] } }, { "row_id": "ML_E01/severity", "fixture_id": "ML_E01", "question_id": "severity", "family": "E", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How severe is any rule-breaking in `post`?", "criteria": [ "no rule-breaking: ordinary on-topic post", "mild: rude tone or off-topic, no target", "clear violation: insults, harassment or spam aimed at someone", "severe: threats, hate speech or calls for violence" ] }, "qtype": 1, "K": 4, "sequence_length": 256, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 14586, 603, 1089, 6933, 235290, 33194, 575, 4103, 2729, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 793, 6933, 235290, 33194, 235292, 14814, 611, 235290, 16768, 2053, 4, 2403, 235248, 235274, 235292, 21870, 235292, 30143, 14533, 689, 1401, 235290, 16768, 235269, 793, 4408, 4, 2403, 235248, 235284, 235292, 3110, 25152, 235292, 103994, 235269, 53760, 689, 31967, 24362, 696, 4630, 4, 2403, 235248, 235304, 235292, 14586, 235292, 28730, 235269, 10874, 11360, 689, 9712, 604, 13673, 1, 19946, 2729, 1192, 664, 651, 8717, 3703, 4713, 13456, 235265, 9105, 692, 604, 11766, 573, 5642, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 1 ], "marker_positions": [ 17, 32, 48, 64 ], "probabilities": [ 0.03755706548690796, 0.6036877036094666, 0.11592944711446762, 0.24282585084438324 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.564, "legend": { "0": "no rule-breaking: ordinary on-topic post", "1": "mild: rude tone or off-topic, no target", "2": "clear violation: insults, harassment or spam aimed at someone", "3": "severe: threats, hate speech or calls for violence" }, "probabilities": { "0": 0.0376, "1": 0.6037, "2": 0.1159, "3": 0.2428 }, "confidence": 0.2632, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": true, "oracle": { "marker_logits": [ 1.2250536680221558, 4.002248764038086, 2.3521735668182373, 3.091536283493042 ], "act_logits": [ 1642.1954345703125, -1938.2548828125 ] } }, { "row_id": "ML_E02/toxic", "fixture_id": "ML_E02", "question_id": "toxic", "family": "E", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Is `post` toxic: rude, disrespectful or likely to make someone leave the discussion?" }, "qtype": 2, "K": 2, "sequence_length": 74, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 4103, 2729, 235376, 21801, 235292, 30143, 235269, 142588, 689, 5476, 577, 1501, 4630, 5077, 573, 8827, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 1596, 3579, 603, 1105, 9205, 29630, 235269, 901, 590, 1134, 4644, 9742, 573, 21085, 576, 20828, 235265, 11188, 5596, 1354, 4313, 11041, 573, 8203, 3595, 235270, 1 ], "marker_positions": [ 24, 34 ], "probabilities": [ 0.9957993626594543, 0.004200672265142202 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0042, "confidence": 0.9958, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 4.871048927307129, -0.5972530841827393 ], "act_logits": [ 1487.9752197265625, -1762.0577392578125 ] } }, { "row_id": "ML_E02/harassment", "fixture_id": "ML_E02", "question_id": "harassment", "family": "E", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `post` target or harass a specific person?" }, "qtype": 2, "K": 2, "sequence_length": 67, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 2729, 235376, 4408, 689, 40868, 476, 3724, 1552, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 1596, 3579, 603, 1105, 9205, 29630, 235269, 901, 590, 1134, 4644, 9742, 573, 21085, 576, 20828, 235265, 11188, 5596, 1354, 4313, 11041, 573, 8203, 3595, 235270, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9967479705810547, 0.0032520554959774017 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0033, "confidence": 0.9967, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 5.004209995269775, -0.721001386642456 ], "act_logits": [ 1473.7762451171875, -1746.54345703125 ] } }, { "row_id": "ML_E02/threat", "fixture_id": "ML_E02", "question_id": "threat", "family": "E", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `post` threaten violence, harm or intimidation?" }, "qtype": 2, "K": 2, "sequence_length": 67, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 2729, 235376, 60399, 13673, 235269, 14649, 689, 132353, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 1596, 3579, 603, 1105, 9205, 29630, 235269, 901, 590, 1134, 4644, 9742, 573, 21085, 576, 20828, 235265, 11188, 5596, 1354, 4313, 11041, 573, 8203, 3595, 235270, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9985765218734741, 0.0014234856935217977 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0014, "confidence": 0.9986, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 5.85823392868042, -0.6949891448020935 ], "act_logits": [ 1534.153076171875, -1817.5728759765625 ] } }, { "row_id": "ML_E02/spam", "fixture_id": "ML_E02", "question_id": "spam", "family": "E", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Is `post` spam or advertising?" }, "qtype": 2, "K": 2, "sequence_length": 64, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 4103, 2729, 235376, 31967, 689, 16171, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 1596, 3579, 603, 1105, 9205, 29630, 235269, 901, 590, 1134, 4644, 9742, 573, 21085, 576, 20828, 235265, 11188, 5596, 1354, 4313, 11041, 573, 8203, 3595, 235270, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.9979885816574097, 0.002011369913816452 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.002, "confidence": 0.998, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 5.377635478973389, -0.8292904496192932 ], "act_logits": [ 1432.6771240234375, -1697.889892578125 ] } }, { "row_id": "ML_E02/severity", "fixture_id": "ML_E02", "question_id": "severity", "family": "E", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How severe is any rule-breaking in `post`?", "criteria": [ "no rule-breaking: ordinary on-topic post", "mild: rude tone or off-topic, no target", "clear violation: insults, harassment or spam aimed at someone", "severe: threats, hate speech or calls for violence" ] }, "qtype": 1, "K": 4, "sequence_length": 111, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 14586, 603, 1089, 6933, 235290, 33194, 575, 4103, 2729, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 793, 6933, 235290, 33194, 235292, 14814, 611, 235290, 16768, 2053, 4, 2403, 235248, 235274, 235292, 21870, 235292, 30143, 14533, 689, 1401, 235290, 16768, 235269, 793, 4408, 4, 2403, 235248, 235284, 235292, 3110, 25152, 235292, 103994, 235269, 53760, 689, 31967, 24362, 696, 4630, 4, 2403, 235248, 235304, 235292, 14586, 235292, 28730, 235269, 10874, 11360, 689, 9712, 604, 13673, 1, 19946, 2729, 1192, 664, 1596, 3579, 603, 1105, 9205, 29630, 235269, 901, 590, 1134, 4644, 9742, 573, 21085, 576, 20828, 235265, 11188, 5596, 1354, 4313, 11041, 573, 8203, 3595, 235270, 1 ], "marker_positions": [ 17, 32, 48, 64 ], "probabilities": [ 0.01620258204638958, 0.2461077868938446, 0.27042314410209656, 0.4672665297985077 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 2.1888, "legend": { "0": "no rule-breaking: ordinary on-topic post", "1": "mild: rude tone or off-topic, no target", "2": "clear violation: insults, harassment or spam aimed at someone", "3": "severe: threats, hate speech or calls for violence" }, "probabilities": { "0": 0.0162, "1": 0.2461, "2": 0.2704, "3": 0.4673 }, "confidence": 0.1914, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -0.7382619976997375, 1.9823418855667114, 2.0765583515167236, 2.6234705448150635 ], "act_logits": [ 1493.4666748046875, -1762.2869873046875 ] } }, { "row_id": "ML_E03/toxic", "fixture_id": "ML_E03", "question_id": "toxic", "family": "E", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Is `post` toxic: rude, disrespectful or likely to make someone leave the discussion?" }, "qtype": 2, "K": 2, "sequence_length": 159, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 4103, 2729, 235376, 21801, 235292, 30143, 235269, 142588, 689, 5476, 577, 1501, 4630, 5077, 573, 8827, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 3171, 15844, 729, 77131, 578, 33126, 235265, 5651, 2701, 573, 4691, 1794, 20021, 2550, 7260, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 1464, 235270, 1 ], "marker_positions": [ 24, 34 ], "probabilities": [ 0.884459912776947, 0.11554008722305298 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.1155, "confidence": 0.8845, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 1.8677717447280884, -0.16758829355239868 ], "act_logits": [ 1384.5841064453125, -1643.0118408203125 ] } }, { "row_id": "ML_E03/harassment", "fixture_id": "ML_E03", "question_id": "harassment", "family": "E", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `post` target or harass a specific person?" }, "qtype": 2, "K": 2, "sequence_length": 152, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 2729, 235376, 4408, 689, 40868, 476, 3724, 1552, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 3171, 15844, 729, 77131, 578, 33126, 235265, 5651, 2701, 573, 4691, 1794, 20021, 2550, 7260, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 1464, 235270, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9793009757995605, 0.02069898694753647 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0207, "confidence": 0.9793, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 3.352196216583252, -0.5045583248138428 ], "act_logits": [ 1366.7115478515625, -1620.9661865234375 ] } }, { "row_id": "ML_E03/threat", "fixture_id": "ML_E03", "question_id": "threat", "family": "E", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `post` threaten violence, harm or intimidation?" }, "qtype": 2, "K": 2, "sequence_length": 152, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 2729, 235376, 60399, 13673, 235269, 14649, 689, 132353, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 3171, 15844, 729, 77131, 578, 33126, 235265, 5651, 2701, 573, 4691, 1794, 20021, 2550, 7260, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 1464, 235270, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9992371797561646, 0.0007627836312167346 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0008, "confidence": 0.9992, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 6.27931022644043, -0.8984640836715698 ], "act_logits": [ 1538.1796875, -1824.91162109375 ] } }, { "row_id": "ML_E03/spam", "fixture_id": "ML_E03", "question_id": "spam", "family": "E", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Is `post` spam or advertising?" }, "qtype": 2, "K": 2, "sequence_length": 149, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 4103, 2729, 235376, 31967, 689, 16171, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 3171, 15844, 729, 77131, 578, 33126, 235265, 5651, 2701, 573, 4691, 1794, 20021, 2550, 7260, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 1464, 235270, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.9999996423721313, 3.0443280252256955e-07 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 13.424323081970215, -1.5804928541183472 ], "act_logits": [ 1557.9083251953125, -1851.925048828125 ] } }, { "row_id": "ML_E03/severity", "fixture_id": "ML_E03", "question_id": "severity", "family": "E", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How severe is any rule-breaking in `post`?", "criteria": [ "no rule-breaking: ordinary on-topic post", "mild: rude tone or off-topic, no target", "clear violation: insults, harassment or spam aimed at someone", "severe: threats, hate speech or calls for violence" ] }, "qtype": 1, "K": 4, "sequence_length": 196, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 14586, 603, 1089, 6933, 235290, 33194, 575, 4103, 2729, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 793, 6933, 235290, 33194, 235292, 14814, 611, 235290, 16768, 2053, 4, 2403, 235248, 235274, 235292, 21870, 235292, 30143, 14533, 689, 1401, 235290, 16768, 235269, 793, 4408, 4, 2403, 235248, 235284, 235292, 3110, 25152, 235292, 103994, 235269, 53760, 689, 31967, 24362, 696, 4630, 4, 2403, 235248, 235304, 235292, 14586, 235292, 28730, 235269, 10874, 11360, 689, 9712, 604, 13673, 1, 19946, 2729, 1192, 664, 3171, 15844, 729, 77131, 578, 33126, 235265, 5651, 2701, 573, 4691, 1794, 20021, 2550, 7260, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 1464, 235270, 1 ], "marker_positions": [ 17, 32, 48, 64 ], "probabilities": [ 0.010311701335012913, 0.5911958813667297, 0.1515364646911621, 0.24695594608783722 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.6351, "legend": { "0": "no rule-breaking: ordinary on-topic post", "1": "mild: rude tone or off-topic, no target", "2": "clear violation: insults, harassment or spam aimed at someone", "3": "severe: threats, hate speech or calls for violence" }, "probabilities": { "0": 0.0103, "1": 0.5912, "2": 0.1515, "3": 0.247 }, "confidence": 0.2864, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -1.605854868888855, 2.443014144897461, 1.0816928148269653, 1.5700761079788208 ], "act_logits": [ 1552.2191162109375, -1832.458251953125 ] } }, { "row_id": "ML_E04/toxic", "fixture_id": "ML_E04", "question_id": "toxic", "family": "E", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Is `post` toxic: rude, disrespectful or likely to make someone leave the discussion?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 4103, 2729, 235376, 21801, 235292, 30143, 235269, 142588, 689, 5476, 577, 1501, 4630, 5077, 573, 8827, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 49129, 16626, 12909, 25019, 599, 9360, 12125, 235341, 8820, 3255, 235341, 8798, 3646, 235341, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 476, 1889, 576, 13333, 235269, 578, 573, 3221, 798, 1281, 476, 5469, 2346, 1 ], "marker_positions": [ 24, 34 ], "probabilities": [ 0.9982560276985168, 0.001743973814882338 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0017, "confidence": 0.9983, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 5.844656944274902, -0.5051870346069336 ], "act_logits": [ 1567.248779296875, -1858.935302734375 ] } }, { "row_id": "ML_E04/harassment", "fixture_id": "ML_E04", "question_id": "harassment", "family": "E", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `post` target or harass a specific person?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 2729, 235376, 4408, 689, 40868, 476, 3724, 1552, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 49129, 16626, 12909, 25019, 599, 9360, 12125, 235341, 8820, 3255, 235341, 8798, 3646, 235341, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 476, 1889, 576, 13333, 235269, 578, 573, 3221, 798, 1281, 476, 5469, 2346, 11117, 573, 3464, 235265, 714, 8467, 1049, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.999396562576294, 0.0006033831741660833 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0006, "confidence": 0.9994, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 6.907590866088867, -0.5047637820243835 ], "act_logits": [ 1428.956787109375, -1697.3428955078125 ] } }, { "row_id": "ML_E04/threat", "fixture_id": "ML_E04", "question_id": "threat", "family": "E", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `post` threaten violence, harm or intimidation?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 2729, 235376, 60399, 13673, 235269, 14649, 689, 132353, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 49129, 16626, 12909, 25019, 599, 9360, 12125, 235341, 8820, 3255, 235341, 8798, 3646, 235341, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 476, 1889, 576, 13333, 235269, 578, 573, 3221, 798, 1281, 476, 5469, 2346, 11117, 573, 3464, 235265, 714, 8467, 1049, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9999159574508667, 8.397913916269317e-05 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0001, "confidence": 0.9999, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 8.646700859069824, -0.7381585836410522 ], "act_logits": [ 1584.69091796875, -1883.7578125 ] } }, { "row_id": "ML_E04/spam", "fixture_id": "ML_E04", "question_id": "spam", "family": "E", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Is `post` spam or advertising?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 4103, 2729, 235376, 31967, 689, 16171, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 49129, 16626, 12909, 25019, 599, 9360, 12125, 235341, 8820, 3255, 235341, 8798, 3646, 235341, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 476, 1889, 576, 13333, 235269, 578, 573, 3221, 798, 1281, 476, 5469, 2346, 11117, 573, 3464, 235265, 714, 8467, 1049, 12184, 575, 974, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.09762740135192871, 0.9023725986480713 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.9024, "confidence": 0.9024, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ -1.4590950012207031, 0.7647746801376343 ], "act_logits": [ 1120.2122802734375, -1333.473876953125 ] } }, { "row_id": "ML_E04/severity", "fixture_id": "ML_E04", "question_id": "severity", "family": "E", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How severe is any rule-breaking in `post`?", "criteria": [ "no rule-breaking: ordinary on-topic post", "mild: rude tone or off-topic, no target", "clear violation: insults, harassment or spam aimed at someone", "severe: threats, hate speech or calls for violence" ] }, "qtype": 1, "K": 4, "sequence_length": 256, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 14586, 603, 1089, 6933, 235290, 33194, 575, 4103, 2729, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 793, 6933, 235290, 33194, 235292, 14814, 611, 235290, 16768, 2053, 4, 2403, 235248, 235274, 235292, 21870, 235292, 30143, 14533, 689, 1401, 235290, 16768, 235269, 793, 4408, 4, 2403, 235248, 235284, 235292, 3110, 25152, 235292, 103994, 235269, 53760, 689, 31967, 24362, 696, 4630, 4, 2403, 235248, 235304, 235292, 14586, 235292, 28730, 235269, 10874, 11360, 689, 9712, 604, 13673, 1, 19946, 2729, 1192, 664, 49129, 16626, 12909, 25019, 599, 9360, 12125, 235341, 8820, 3255, 235341, 8798, 3646, 235341, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1 ], "marker_positions": [ 17, 32, 48, 64 ], "probabilities": [ 0.009705462493002415, 0.267549991607666, 0.32524988055229187, 0.39749470353126526 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 2.1105, "legend": { "0": "no rule-breaking: ordinary on-topic post", "1": "mild: rude tone or off-topic, no target", "2": "clear violation: insults, harassment or spam aimed at someone", "3": "severe: threats, hate speech or calls for violence" }, "probabilities": { "0": 0.0097, "1": 0.2675, "2": 0.3252, "3": 0.3975 }, "confidence": 0.185, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": true, "oracle": { "marker_logits": [ -1.7477506399154663, 1.5688672065734863, 1.7641549110412598, 1.9647427797317505 ], "act_logits": [ 1562.3353271484375, -1844.07080078125 ] } }, { "row_id": "ML_E05/toxic", "fixture_id": "ML_E05", "question_id": "toxic", "family": "E", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Is `post` toxic: rude, disrespectful or likely to make someone leave the discussion?" }, "qtype": 2, "K": 2, "sequence_length": 75, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 4103, 2729, 235376, 21801, 235292, 30143, 235269, 142588, 689, 5476, 577, 1501, 4630, 5077, 573, 8827, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 236994, 84500, 53995, 235394, 241680, 82855, 38725, 44925, 235362, 235894, 240053, 102590, 59780, 235459, 235412, 235406, 235520, 14088, 235362, 78546, 2603, 88569, 2618, 235394, 48616, 235362, 12990, 1 ], "marker_positions": [ 24, 34 ], "probabilities": [ 0.9989843964576721, 0.0010156295029446483 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.001, "confidence": 0.999, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 6.151587009429932, -0.7396444082260132 ], "act_logits": [ 1679.0042724609375, -1987.162109375 ] } }, { "row_id": "ML_E05/harassment", "fixture_id": "ML_E05", "question_id": "harassment", "family": "E", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `post` target or harass a specific person?" }, "qtype": 2, "K": 2, "sequence_length": 68, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 2729, 235376, 4408, 689, 40868, 476, 3724, 1552, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 236994, 84500, 53995, 235394, 241680, 82855, 38725, 44925, 235362, 235894, 240053, 102590, 59780, 235459, 235412, 235406, 235520, 14088, 235362, 78546, 2603, 88569, 2618, 235394, 48616, 235362, 12990, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9993983507156372, 0.0006016621482558548 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0006, "confidence": 0.9994, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 6.152425765991211, -1.2627873420715332 ], "act_logits": [ 1567.6292724609375, -1858.516357421875 ] } }, { "row_id": "ML_E05/threat", "fixture_id": "ML_E05", "question_id": "threat", "family": "E", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `post` threaten violence, harm or intimidation?" }, "qtype": 2, "K": 2, "sequence_length": 68, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 2729, 235376, 60399, 13673, 235269, 14649, 689, 132353, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 236994, 84500, 53995, 235394, 241680, 82855, 38725, 44925, 235362, 235894, 240053, 102590, 59780, 235459, 235412, 235406, 235520, 14088, 235362, 78546, 2603, 88569, 2618, 235394, 48616, 235362, 12990, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9997432827949524, 0.0002567688934504986 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0003, "confidence": 0.9997, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 7.2068986892700195, -1.060178518295288 ], "act_logits": [ 1622.1136474609375, -1925.50830078125 ] } }, { "row_id": "ML_E05/spam", "fixture_id": "ML_E05", "question_id": "spam", "family": "E", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Is `post` spam or advertising?" }, "qtype": 2, "K": 2, "sequence_length": 65, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 4103, 2729, 235376, 31967, 689, 16171, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 236994, 84500, 53995, 235394, 241680, 82855, 38725, 44925, 235362, 235894, 240053, 102590, 59780, 235459, 235412, 235406, 235520, 14088, 235362, 78546, 2603, 88569, 2618, 235394, 48616, 235362, 12990, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.9999972581863403, 2.7631826924334746e-06 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 10.96827507019043, -1.8308484554290771 ], "act_logits": [ 1563.1827392578125, -1857.2489013671875 ] } }, { "row_id": "ML_E05/severity", "fixture_id": "ML_E05", "question_id": "severity", "family": "E", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How severe is any rule-breaking in `post`?", "criteria": [ "no rule-breaking: ordinary on-topic post", "mild: rude tone or off-topic, no target", "clear violation: insults, harassment or spam aimed at someone", "severe: threats, hate speech or calls for violence" ] }, "qtype": 1, "K": 4, "sequence_length": 112, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 14586, 603, 1089, 6933, 235290, 33194, 575, 4103, 2729, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 793, 6933, 235290, 33194, 235292, 14814, 611, 235290, 16768, 2053, 4, 2403, 235248, 235274, 235292, 21870, 235292, 30143, 14533, 689, 1401, 235290, 16768, 235269, 793, 4408, 4, 2403, 235248, 235284, 235292, 3110, 25152, 235292, 103994, 235269, 53760, 689, 31967, 24362, 696, 4630, 4, 2403, 235248, 235304, 235292, 14586, 235292, 28730, 235269, 10874, 11360, 689, 9712, 604, 13673, 1, 19946, 2729, 1192, 664, 236994, 84500, 53995, 235394, 241680, 82855, 38725, 44925, 235362, 235894, 240053, 102590, 59780, 235459, 235412, 235406, 235520, 14088, 235362, 78546, 2603, 88569, 2618, 235394, 48616, 235362, 12990, 1 ], "marker_positions": [ 17, 32, 48, 64 ], "probabilities": [ 0.1287570595741272, 0.3767409026622772, 0.16182582080364227, 0.33267614245414734 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.6984, "legend": { "0": "no rule-breaking: ordinary on-topic post", "1": "mild: rude tone or off-topic, no target", "2": "clear violation: insults, harassment or spam aimed at someone", "3": "severe: threats, hate speech or calls for violence" }, "probabilities": { "0": 0.1288, "1": 0.3767, "2": 0.1618, "3": 0.3327 }, "confidence": 0.0676, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ 1.7507189512252808, 2.824350357055664, 1.979312777519226, 2.6999611854553223 ], "act_logits": [ 1822.8585205078125, -2150.74169921875 ] } }, { "row_id": "ML_E06/toxic", "fixture_id": "ML_E06", "question_id": "toxic", "family": "E", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Is `post` toxic: rude, disrespectful or likely to make someone leave the discussion?" }, "qtype": 2, "K": 2, "sequence_length": 155, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 4103, 2729, 235376, 21801, 235292, 30143, 235269, 142588, 689, 5476, 577, 1501, 4630, 5077, 573, 8827, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 137764, 206575, 237887, 4758, 235394, 93455, 48016, 30846, 235418, 43913, 96778, 235478, 2297, 235362, 23631, 68594, 93535, 2603, 235394, 223955, 89323, 148485, 235400, 192257, 13319, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 12990, 1 ], "marker_positions": [ 24, 34 ], "probabilities": [ 0.9880874156951904, 0.01191259641200304 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0119, "confidence": 0.9881, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 4.108384132385254, -0.30979102849960327 ], "act_logits": [ 1408.1890869140625, -1670.271240234375 ] } }, { "row_id": "ML_E06/harassment", "fixture_id": "ML_E06", "question_id": "harassment", "family": "E", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `post` target or harass a specific person?" }, "qtype": 2, "K": 2, "sequence_length": 148, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 2729, 235376, 4408, 689, 40868, 476, 3724, 1552, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 137764, 206575, 237887, 4758, 235394, 93455, 48016, 30846, 235418, 43913, 96778, 235478, 2297, 235362, 23631, 68594, 93535, 2603, 235394, 223955, 89323, 148485, 235400, 192257, 13319, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 12990, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9830296635627747, 0.016970347613096237 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.017, "confidence": 0.983, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 3.8929405212402344, -0.16623151302337646 ], "act_logits": [ 1390.6646728515625, -1650.07421875 ] } }, { "row_id": "ML_E06/threat", "fixture_id": "ML_E06", "question_id": "threat", "family": "E", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `post` threaten violence, harm or intimidation?" }, "qtype": 2, "K": 2, "sequence_length": 148, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 2729, 235376, 60399, 13673, 235269, 14649, 689, 132353, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 137764, 206575, 237887, 4758, 235394, 93455, 48016, 30846, 235418, 43913, 96778, 235478, 2297, 235362, 23631, 68594, 93535, 2603, 235394, 223955, 89323, 148485, 235400, 192257, 13319, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 12990, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9996287822723389, 0.00037124811206012964 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0004, "confidence": 0.9996, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 7.024663925170898, -0.8736044764518738 ], "act_logits": [ 1606.1463623046875, -1908.6409912109375 ] } }, { "row_id": "ML_E06/spam", "fixture_id": "ML_E06", "question_id": "spam", "family": "E", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Is `post` spam or advertising?" }, "qtype": 2, "K": 2, "sequence_length": 145, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 4103, 2729, 235376, 31967, 689, 16171, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 137764, 206575, 237887, 4758, 235394, 93455, 48016, 30846, 235418, 43913, 96778, 235478, 2297, 235362, 23631, 68594, 93535, 2603, 235394, 223955, 89323, 148485, 235400, 192257, 13319, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 12990, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.9993246793746948, 0.0006753046181984246 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0007, "confidence": 0.9993, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 6.559244632720947, -0.7404273152351379 ], "act_logits": [ 1430.527099609375, -1697.535400390625 ] } }, { "row_id": "ML_E06/severity", "fixture_id": "ML_E06", "question_id": "severity", "family": "E", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How severe is any rule-breaking in `post`?", "criteria": [ "no rule-breaking: ordinary on-topic post", "mild: rude tone or off-topic, no target", "clear violation: insults, harassment or spam aimed at someone", "severe: threats, hate speech or calls for violence" ] }, "qtype": 1, "K": 4, "sequence_length": 192, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 14586, 603, 1089, 6933, 235290, 33194, 575, 4103, 2729, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 793, 6933, 235290, 33194, 235292, 14814, 611, 235290, 16768, 2053, 4, 2403, 235248, 235274, 235292, 21870, 235292, 30143, 14533, 689, 1401, 235290, 16768, 235269, 793, 4408, 4, 2403, 235248, 235284, 235292, 3110, 25152, 235292, 103994, 235269, 53760, 689, 31967, 24362, 696, 4630, 4, 2403, 235248, 235304, 235292, 14586, 235292, 28730, 235269, 10874, 11360, 689, 9712, 604, 13673, 1, 19946, 2729, 1192, 664, 137764, 206575, 237887, 4758, 235394, 93455, 48016, 30846, 235418, 43913, 96778, 235478, 2297, 235362, 23631, 68594, 93535, 2603, 235394, 223955, 89323, 148485, 235400, 192257, 13319, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 12990, 1 ], "marker_positions": [ 17, 32, 48, 64 ], "probabilities": [ 0.022569503635168076, 0.5418851971626282, 0.2580096125602722, 0.17753568291664124 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.5905, "legend": { "0": "no rule-breaking: ordinary on-topic post", "1": "mild: rude tone or off-topic, no target", "2": "clear violation: insults, harassment or spam aimed at someone", "3": "severe: threats, hate speech or calls for violence" }, "probabilities": { "0": 0.0226, "1": 0.5419, "2": 0.258, "3": 0.1775 }, "confidence": 0.2253, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -0.9703335762023926, 2.2081212997436523, 1.466064214706421, 1.0922386646270752 ], "act_logits": [ 1638.067626953125, -1934.882568359375 ] } }, { "row_id": "ML_E07/toxic", "fixture_id": "ML_E07", "question_id": "toxic", "family": "E", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Is `post` toxic: rude, disrespectful or likely to make someone leave the discussion?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 4103, 2729, 235376, 21801, 235292, 30143, 235269, 142588, 689, 5476, 577, 1501, 4630, 5077, 573, 8827, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 137764, 206575, 237887, 4758, 235394, 6866, 123293, 235432, 127574, 28070, 159039, 9178, 235590, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 50855, 72368, 235362, 78546, 235372, 63090, 238518, 7711, 44393, 235372, 174612, 235410, 53479, 235372, 130073, 9301, 235432, 90518, 22031, 235362, 235987, 23176, 151978, 219689, 236943, 14848, 62865, 235362, 81625, 1 ], "marker_positions": [ 24, 34 ], "probabilities": [ 0.9990506768226624, 0.0009493016405031085 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0009, "confidence": 0.9991, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 6.246852874755859, -0.7119805812835693 ], "act_logits": [ 1710.729248046875, -2031.5152587890625 ] } }, { "row_id": "ML_E07/harassment", "fixture_id": "ML_E07", "question_id": "harassment", "family": "E", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `post` target or harass a specific person?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 2729, 235376, 4408, 689, 40868, 476, 3724, 1552, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 137764, 206575, 237887, 4758, 235394, 6866, 123293, 235432, 127574, 28070, 159039, 9178, 235590, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 50855, 72368, 235362, 78546, 235372, 63090, 238518, 7711, 44393, 235372, 174612, 235410, 53479, 235372, 130073, 9301, 235432, 90518, 22031, 235362, 235987, 23176, 151978, 219689, 236943, 14848, 62865, 235362, 81625, 40957, 150694, 77776, 26685, 235399, 235394, 56477, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9996882677078247, 0.00031178808421827853 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0003, "confidence": 0.9997, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 6.862680912017822, -1.210193157196045 ], "act_logits": [ 1552.897216796875, -1844.3756103515625 ] } }, { "row_id": "ML_E07/threat", "fixture_id": "ML_E07", "question_id": "threat", "family": "E", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `post` threaten violence, harm or intimidation?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 2729, 235376, 60399, 13673, 235269, 14649, 689, 132353, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 137764, 206575, 237887, 4758, 235394, 6866, 123293, 235432, 127574, 28070, 159039, 9178, 235590, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 50855, 72368, 235362, 78546, 235372, 63090, 238518, 7711, 44393, 235372, 174612, 235410, 53479, 235372, 130073, 9301, 235432, 90518, 22031, 235362, 235987, 23176, 151978, 219689, 236943, 14848, 62865, 235362, 81625, 40957, 150694, 77776, 26685, 235399, 235394, 56477, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9999797344207764, 2.0247713109711185e-05 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 9.493473052978516, -1.3139750957489014 ], "act_logits": [ 1590.6973876953125, -1890.7255859375 ] } }, { "row_id": "ML_E07/spam", "fixture_id": "ML_E07", "question_id": "spam", "family": "E", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Is `post` spam or advertising?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 4103, 2729, 235376, 31967, 689, 16171, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 137764, 206575, 237887, 4758, 235394, 6866, 123293, 235432, 127574, 28070, 159039, 9178, 235590, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 50855, 72368, 235362, 78546, 235372, 63090, 238518, 7711, 44393, 235372, 174612, 235410, 53479, 235372, 130073, 9301, 235432, 90518, 22031, 235362, 235987, 23176, 151978, 219689, 236943, 14848, 62865, 235362, 81625, 40957, 150694, 77776, 26685, 235399, 235394, 56477, 235418, 66886, 100391, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 1.0, 1.817303113549329e-10 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 17.54963493347168, -4.878864288330078 ], "act_logits": [ 1446.981201171875, -1723.109375 ] } }, { "row_id": "ML_E07/severity", "fixture_id": "ML_E07", "question_id": "severity", "family": "E", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How severe is any rule-breaking in `post`?", "criteria": [ "no rule-breaking: ordinary on-topic post", "mild: rude tone or off-topic, no target", "clear violation: insults, harassment or spam aimed at someone", "severe: threats, hate speech or calls for violence" ] }, "qtype": 1, "K": 4, "sequence_length": 256, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 14586, 603, 1089, 6933, 235290, 33194, 575, 4103, 2729, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 793, 6933, 235290, 33194, 235292, 14814, 611, 235290, 16768, 2053, 4, 2403, 235248, 235274, 235292, 21870, 235292, 30143, 14533, 689, 1401, 235290, 16768, 235269, 793, 4408, 4, 2403, 235248, 235284, 235292, 3110, 25152, 235292, 103994, 235269, 53760, 689, 31967, 24362, 696, 4630, 4, 2403, 235248, 235304, 235292, 14586, 235292, 28730, 235269, 10874, 11360, 689, 9712, 604, 13673, 1, 19946, 2729, 1192, 664, 137764, 206575, 237887, 4758, 235394, 6866, 123293, 235432, 127574, 28070, 159039, 9178, 235590, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 1 ], "marker_positions": [ 17, 32, 48, 64 ], "probabilities": [ 0.01834910176694393, 0.30234456062316895, 0.2107837200164795, 0.4685226082801819 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 2.1295, "legend": { "0": "no rule-breaking: ordinary on-topic post", "1": "mild: rude tone or off-topic, no target", "2": "clear violation: insults, harassment or spam aimed at someone", "3": "severe: threats, hate speech or calls for violence" }, "probabilities": { "0": 0.0183, "1": 0.3023, "2": 0.2108, "3": 0.4685 }, "confidence": 0.1932, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": true, "oracle": { "marker_logits": [ -0.008037209510803223, 2.793949604034424, 2.4332141876220703, 3.231966733932495 ], "act_logits": [ 1683.6856689453125, -1987.3184814453125 ] } }, { "row_id": "ML_E08/toxic", "fixture_id": "ML_E08", "question_id": "toxic", "family": "E", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Is `post` toxic: rude, disrespectful or likely to make someone leave the discussion?" }, "qtype": 2, "K": 2, "sequence_length": 82, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 4103, 2729, 235376, 21801, 235292, 30143, 235269, 142588, 689, 5476, 577, 1501, 4630, 5077, 573, 8827, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 235811, 9301, 154040, 235414, 142444, 111258, 235372, 32535, 39312, 53871, 22031, 235362, 124737, 119340, 22147, 154040, 2297, 235362, 6586, 79036, 235400, 42362, 235545, 150852, 13319, 235362, 76927, 7820, 23631, 125784, 196167, 45263, 235362, 12990, 1 ], "marker_positions": [ 24, 34 ], "probabilities": [ 0.999409556388855, 0.000590427836868912 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0006, "confidence": 0.9994, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 6.896636962890625, -0.5374361276626587 ], "act_logits": [ 1749.638427734375, -2074.971923828125 ] } }, { "row_id": "ML_E08/harassment", "fixture_id": "ML_E08", "question_id": "harassment", "family": "E", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `post` target or harass a specific person?" }, "qtype": 2, "K": 2, "sequence_length": 75, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 2729, 235376, 4408, 689, 40868, 476, 3724, 1552, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 235811, 9301, 154040, 235414, 142444, 111258, 235372, 32535, 39312, 53871, 22031, 235362, 124737, 119340, 22147, 154040, 2297, 235362, 6586, 79036, 235400, 42362, 235545, 150852, 13319, 235362, 76927, 7820, 23631, 125784, 196167, 45263, 235362, 12990, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9996860027313232, 0.00031399368890561163 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0003, "confidence": 0.9997, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 7.23948860168457, -0.8263356685638428 ], "act_logits": [ 1564.9075927734375, -1858.498291015625 ] } }, { "row_id": "ML_E08/threat", "fixture_id": "ML_E08", "question_id": "threat", "family": "E", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `post` threaten violence, harm or intimidation?" }, "qtype": 2, "K": 2, "sequence_length": 75, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 2729, 235376, 60399, 13673, 235269, 14649, 689, 132353, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 235811, 9301, 154040, 235414, 142444, 111258, 235372, 32535, 39312, 53871, 22031, 235362, 124737, 119340, 22147, 154040, 2297, 235362, 6586, 79036, 235400, 42362, 235545, 150852, 13319, 235362, 76927, 7820, 23631, 125784, 196167, 45263, 235362, 12990, 1 ], "marker_positions": [ 17, 27 ], "probabilities": [ 0.9997921586036682, 0.00020780935301445425 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0002, "confidence": 0.9998, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 7.736823558807373, -0.7418581247329712 ], "act_logits": [ 1641.7996826171875, -1949.4957275390625 ] } }, { "row_id": "ML_E08/spam", "fixture_id": "ML_E08", "question_id": "spam", "family": "E", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Is `post` spam or advertising?" }, "qtype": 2, "K": 2, "sequence_length": 72, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 4103, 2729, 235376, 31967, 689, 16171, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 2729, 1192, 664, 235811, 9301, 154040, 235414, 142444, 111258, 235372, 32535, 39312, 53871, 22031, 235362, 124737, 119340, 22147, 154040, 2297, 235362, 6586, 79036, 235400, 42362, 235545, 150852, 13319, 235362, 76927, 7820, 23631, 125784, 196167, 45263, 235362, 12990, 1 ], "marker_positions": [ 14, 24 ], "probabilities": [ 0.13113577663898468, 0.8688642382621765 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.8689, "confidence": 0.8689, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ -2.1276345252990723, -0.2366809844970703 ], "act_logits": [ 1272.9012451171875, -1514.378662109375 ] } }, { "row_id": "ML_E08/severity", "fixture_id": "ML_E08", "question_id": "severity", "family": "E", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How severe is any rule-breaking in `post`?", "criteria": [ "no rule-breaking: ordinary on-topic post", "mild: rude tone or off-topic, no target", "clear violation: insults, harassment or spam aimed at someone", "severe: threats, hate speech or calls for violence" ] }, "qtype": 1, "K": 4, "sequence_length": 119, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 14586, 603, 1089, 6933, 235290, 33194, 575, 4103, 2729, 235376, 235336, 1, 4, 2403, 235248, 235276, 235292, 793, 6933, 235290, 33194, 235292, 14814, 611, 235290, 16768, 2053, 4, 2403, 235248, 235274, 235292, 21870, 235292, 30143, 14533, 689, 1401, 235290, 16768, 235269, 793, 4408, 4, 2403, 235248, 235284, 235292, 3110, 25152, 235292, 103994, 235269, 53760, 689, 31967, 24362, 696, 4630, 4, 2403, 235248, 235304, 235292, 14586, 235292, 28730, 235269, 10874, 11360, 689, 9712, 604, 13673, 1, 19946, 2729, 1192, 664, 235811, 9301, 154040, 235414, 142444, 111258, 235372, 32535, 39312, 53871, 22031, 235362, 124737, 119340, 22147, 154040, 2297, 235362, 6586, 79036, 235400, 42362, 235545, 150852, 13319, 235362, 76927, 7820, 23631, 125784, 196167, 45263, 235362, 12990, 1 ], "marker_positions": [ 17, 32, 48, 64 ], "probabilities": [ 0.026024723425507545, 0.22299052774906158, 0.42968717217445374, 0.3212975561618805 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 2.0463, "legend": { "0": "no rule-breaking: ordinary on-topic post", "1": "mild: rude tone or off-topic, no target", "2": "clear violation: insults, harassment or spam aimed at someone", "3": "severe: threats, hate speech or calls for violence" }, "probabilities": { "0": 0.026, "1": 0.223, "2": 0.4297, "3": 0.3213 }, "confidence": 0.1652, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -1.6602168083190918, 0.4878699779510498, 1.1437969207763672, 0.8531017303466797 ], "act_logits": [ 1562.4600830078125, -1844.1612548828125 ] } }, { "row_id": "ML_F01/difficulty", "fixture_id": "ML_F01", "question_id": "difficulty", "family": "F", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How hard is `request` for a language model?", "criteria": [ "trivial: a lookup or one-liner", "easy: short answer, no reasoning", "moderate: several steps", "hard: long multi-step reasoning or specialist knowledge" ] }, "qtype": 1, "K": 4, "sequence_length": 177, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 2811, 603, 4103, 3840, 235376, 604, 476, 5255, 2091, 235336, 1, 4, 2403, 235248, 235276, 235292, 49423, 235292, 476, 53053, 689, 974, 235290, 47050, 4, 2403, 235248, 235274, 235292, 3980, 235292, 3309, 3448, 235269, 793, 32346, 4, 2403, 235248, 235284, 235292, 23135, 235292, 3757, 7161, 4, 2403, 235248, 235304, 235292, 2811, 235292, 1497, 3103, 235290, 8847, 32346, 689, 25336, 5567, 1, 19946, 3840, 1192, 664, 6436, 476, 974, 235290, 703, 1411, 674, 15985, 1378, 45047, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 1464, 235270, 1 ], "marker_positions": [ 16, 29, 41, 50 ], "probabilities": [ 0.009158574976027012, 0.7793104648590088, 0.15286801755428314, 0.058662958443164825 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.261, "legend": { "0": "trivial: a lookup or one-liner", "1": "easy: short answer, no reasoning", "2": "moderate: several steps", "3": "hard: long multi-step reasoning or specialist knowledge" }, "probabilities": { "0": 0.0092, "1": 0.7793, "2": 0.1529, "3": 0.0587 }, "confidence": 0.5017, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -1.9941290616989136, 2.4495904445648193, 0.8207558393478394, -0.13701075315475464 ], "act_logits": [ 1505.3016357421875, -1777.0714111328125 ] } }, { "row_id": "ML_F01/domain", "fixture_id": "ML_F01", "question_id": "domain", "family": "F", "language": "EN", "primary_language": "EN", "question": { "type": "choice", "instructions": "What domain does `request` belong to?", "criteria": { "code": "software engineering, programming, refactoring, architecture, debugging", "math_or_logic": "mathematics, logic puzzles, proofs, complex calculation", "writing": "creative writing, essays, emails, blog posts, copywriting", "factual_lookup": "facts, definitions, trivia, history", "data_analysis": "statistics, SQL, data manipulation, metrics", "chitchat": "casual conversation, greetings, small talk" } }, "qtype": 0, "K": 6, "sequence_length": 208, "sequence_ids": [ 2, 6241, 2872, 235292, 2439, 11988, 1721, 4103, 3840, 235376, 8070, 577, 235336, 1, 4, 3409, 235292, 6815, 13299, 235269, 19991, 235269, 2279, 138397, 235269, 16333, 235269, 66439, 4, 10849, 235298, 483, 235298, 17112, 235292, 26814, 235269, 14855, 56983, 235269, 59828, 235269, 5766, 22091, 4, 6706, 235292, 12354, 6706, 235269, 54012, 235269, 28136, 235269, 7808, 10776, 235269, 5469, 24168, 4, 63914, 235298, 37897, 235292, 10275, 235269, 26303, 235269, 115195, 235269, 4281, 4, 1423, 235298, 26413, 235292, 15617, 235269, 15678, 235269, 1423, 44097, 235269, 30660, 4, 788, 6327, 482, 235292, 14044, 12836, 235269, 78107, 235269, 2301, 5063, 1, 19946, 3840, 1192, 664, 6436, 476, 974, 235290, 703, 1411, 674, 15985, 1378, 45047, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 1464, 235270, 1 ], "marker_positions": [ 14, 28, 44, 59, 71, 84 ], "probabilities": [ 0.8795849680900574, 0.04579903930425644, 0.0676170065999031, 0.001452825265005231, 0.0027787494473159313, 0.0027673093136399984 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "code", "probabilities": { "code": 0.8796, "math_or_logic": 0.0458, "writing": 0.0676, "factual_lookup": 0.0015, "data_analysis": 0.0028, "chitchat": 0.0028 }, "confidence": 0.733, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": false, "oracle": { "marker_logits": [ 3.4041402339935303, 0.44895297288894653, 0.8385494947433472, -3.001800537109375, -2.3533096313476562, -2.3574349880218506 ], "act_logits": [ 1447.9864501953125, -1726.700439453125 ] } }, { "row_id": "ML_F01/needs_tools", "fixture_id": "ML_F01", "question_id": "needs_tools", "family": "F", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does answering `request` require external tools, search or private data?" }, "qtype": 2, "K": 2, "sequence_length": 150, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 39534, 4103, 3840, 235376, 2817, 9774, 8112, 235269, 4544, 689, 4765, 1423, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 3840, 1192, 664, 6436, 476, 974, 235290, 703, 1411, 674, 15985, 1378, 45047, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 1464, 235270, 1 ], "marker_positions": [ 20, 30 ], "probabilities": [ 0.9243329167366028, 0.07566705346107483 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0757, "confidence": 0.9243, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 2.2316858768463135, -0.271043598651886 ], "act_logits": [ 1263.1280517578125, -1500.21484375 ] } }, { "row_id": "ML_F01/is_sensitive", "fixture_id": "ML_F01", "question_id": "is_sensitive", "family": "F", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `request` involve money, legal, medical or safety consequences?" }, "qtype": 2, "K": 2, "sequence_length": 150, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 3840, 235376, 22395, 3370, 235269, 7035, 235269, 6910, 689, 6514, 18255, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 3840, 1192, 664, 6436, 476, 974, 235290, 703, 1411, 674, 15985, 1378, 45047, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 1464, 235270, 1 ], "marker_positions": [ 20, 30 ], "probabilities": [ 0.9999955892562866, 4.3812674448417965e-06 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 11.516128540039062, -0.8220398426055908 ], "act_logits": [ 1567.1075439453125, -1860.3031005859375 ] } }, { "row_id": "ML_F02/difficulty", "fixture_id": "ML_F02", "question_id": "difficulty", "family": "F", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How hard is `request` for a language model?", "criteria": [ "trivial: a lookup or one-liner", "easy: short answer, no reasoning", "moderate: several steps", "hard: long multi-step reasoning or specialist knowledge" ] }, "qtype": 1, "K": 4, "sequence_length": 256, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 2811, 603, 4103, 3840, 235376, 604, 476, 5255, 2091, 235336, 1, 4, 2403, 235248, 235276, 235292, 49423, 235292, 476, 53053, 689, 974, 235290, 47050, 4, 2403, 235248, 235274, 235292, 3980, 235292, 3309, 3448, 235269, 793, 32346, 4, 2403, 235248, 235284, 235292, 23135, 235292, 3757, 7161, 4, 2403, 235248, 235304, 235292, 2811, 235292, 1497, 3103, 235290, 8847, 32346, 689, 25336, 5567, 1, 19946, 3840, 1192, 664, 73379, 674, 573, 2707, 576, 573, 1370, 552, 12091, 45047, 603, 552, 43858, 235265, 18712, 2145, 671, 66810, 9916, 578, 476, 30323, 15844, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 1 ], "marker_positions": [ 16, 29, 41, 50 ], "probabilities": [ 0.0011278061429038644, 0.1515207439661026, 0.7382821440696716, 0.10906928032636642 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.9553, "legend": { "0": "trivial: a lookup or one-liner", "1": "easy: short answer, no reasoning", "2": "moderate: several steps", "3": "hard: long multi-step reasoning or specialist knowledge" }, "probabilities": { "0": 0.0011, "1": 0.1515, "2": 0.7383, "3": 0.1091 }, "confidence": 0.4523, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": true, "oracle": { "marker_logits": [ -4.044986724853516, 0.8554622530937195, 2.4390664100646973, 0.5267229676246643 ], "act_logits": [ 1466.11376953125, -1730.5228271484375 ] } }, { "row_id": "ML_F02/domain", "fixture_id": "ML_F02", "question_id": "domain", "family": "F", "language": "EN", "primary_language": "EN", "question": { "type": "choice", "instructions": "What domain does `request` belong to?", "criteria": { "code": "software engineering, programming, refactoring, architecture, debugging", "math_or_logic": "mathematics, logic puzzles, proofs, complex calculation", "writing": "creative writing, essays, emails, blog posts, copywriting", "factual_lookup": "facts, definitions, trivia, history", "data_analysis": "statistics, SQL, data manipulation, metrics", "chitchat": "casual conversation, greetings, small talk" } }, "qtype": 0, "K": 6, "sequence_length": 256, "sequence_ids": [ 2, 6241, 2872, 235292, 2439, 11988, 1721, 4103, 3840, 235376, 8070, 577, 235336, 1, 4, 3409, 235292, 6815, 13299, 235269, 19991, 235269, 2279, 138397, 235269, 16333, 235269, 66439, 4, 10849, 235298, 483, 235298, 17112, 235292, 26814, 235269, 14855, 56983, 235269, 59828, 235269, 5766, 22091, 4, 6706, 235292, 12354, 6706, 235269, 54012, 235269, 28136, 235269, 7808, 10776, 235269, 5469, 24168, 4, 63914, 235298, 37897, 235292, 10275, 235269, 26303, 235269, 115195, 235269, 4281, 4, 1423, 235298, 26413, 235292, 15617, 235269, 15678, 235269, 1423, 44097, 235269, 30660, 4, 788, 6327, 482, 235292, 14044, 12836, 235269, 78107, 235269, 2301, 5063, 1, 19946, 3840, 1192, 664, 73379, 674, 573, 2707, 576, 573, 1370, 552, 12091, 45047, 603, 552, 43858, 235265, 18712, 2145, 671, 66810, 9916, 578, 476, 30323, 15844, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 1 ], "marker_positions": [ 14, 28, 44, 59, 71, 84 ], "probabilities": [ 0.00047362170880660415, 0.9977986216545105, 0.00029591156635433435, 0.0007269502966664732, 0.0003983334172517061, 0.00030638196039944887 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "math_or_logic", "probabilities": { "code": 0.0005, "math_or_logic": 0.9978, "writing": 0.0003, "factual_lookup": 0.0007, "data_analysis": 0.0004, "chitchat": 0.0003 }, "confidence": 0.9894, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": true, "oracle": { "marker_logits": [ -2.593637228012085, 5.059260368347168, -3.0639853477478027, -2.165188789367676, -2.7667577266693115, -3.029214382171631 ], "act_logits": [ 1302.8433837890625, -1552.035400390625 ] } }, { "row_id": "ML_F02/needs_tools", "fixture_id": "ML_F02", "question_id": "needs_tools", "family": "F", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does answering `request` require external tools, search or private data?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 39534, 4103, 3840, 235376, 2817, 9774, 8112, 235269, 4544, 689, 4765, 1423, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 3840, 1192, 664, 73379, 674, 573, 2707, 576, 573, 1370, 552, 12091, 45047, 603, 552, 43858, 235265, 18712, 2145, 671, 66810, 9916, 578, 476, 30323, 15844, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 476, 1889, 576, 13333, 235269, 578, 573, 1 ], "marker_positions": [ 20, 30 ], "probabilities": [ 0.8570093512535095, 0.1429906189441681 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.143, "confidence": 0.857, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 1.9440652132034302, 0.1533956527709961 ], "act_logits": [ 1215.0479736328125, -1443.8953857421875 ] } }, { "row_id": "ML_F02/is_sensitive", "fixture_id": "ML_F02", "question_id": "is_sensitive", "family": "F", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `request` involve money, legal, medical or safety consequences?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 3840, 235376, 22395, 3370, 235269, 7035, 235269, 6910, 689, 6514, 18255, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 3840, 1192, 664, 73379, 674, 573, 2707, 576, 573, 1370, 552, 12091, 45047, 603, 552, 43858, 235265, 18712, 2145, 671, 66810, 9916, 578, 476, 30323, 15844, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 476, 1889, 576, 13333, 235269, 578, 573, 1 ], "marker_positions": [ 20, 30 ], "probabilities": [ 0.9994576573371887, 0.0005422901595011353 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0005, "confidence": 0.9995, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 7.129208087921143, -0.3899593949317932 ], "act_logits": [ 1448.786376953125, -1720.05615234375 ] } }, { "row_id": "ML_F03/difficulty", "fixture_id": "ML_F03", "question_id": "difficulty", "family": "F", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How hard is `request` for a language model?", "criteria": [ "trivial: a lookup or one-liner", "easy: short answer, no reasoning", "moderate: several steps", "hard: long multi-step reasoning or specialist knowledge" ] }, "qtype": 1, "K": 4, "sequence_length": 97, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 2811, 603, 4103, 3840, 235376, 604, 476, 5255, 2091, 235336, 1, 4, 2403, 235248, 235276, 235292, 49423, 235292, 476, 53053, 689, 974, 235290, 47050, 4, 2403, 235248, 235274, 235292, 3980, 235292, 3309, 3448, 235269, 793, 32346, 4, 2403, 235248, 235284, 235292, 23135, 235292, 3757, 7161, 4, 2403, 235248, 235304, 235292, 2811, 235292, 1497, 3103, 235290, 8847, 32346, 689, 25336, 5567, 1, 19946, 3840, 1192, 664, 5559, 476, 3309, 235269, 9376, 29151, 577, 671, 51084, 17603, 43834, 235265, 93039, 476, 53162, 235290, 1311, 11023, 578, 1281, 3890, 235269, 40248, 5255, 1464, 235270, 1 ], "marker_positions": [ 16, 29, 41, 50 ], "probabilities": [ 0.016430413350462914, 0.8314075469970703, 0.12292878329753876, 0.029233258217573166 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.165, "legend": { "0": "trivial: a lookup or one-liner", "1": "easy: short answer, no reasoning", "2": "moderate: several steps", "3": "hard: long multi-step reasoning or specialist knowledge" }, "probabilities": { "0": 0.0164, "1": 0.8314, "2": 0.1229, "3": 0.0292 }, "confidence": 0.5802, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -1.4620829820632935, 2.4619030952453613, 0.5503878593444824, -0.8859100937843323 ], "act_logits": [ 1571.5738525390625, -1855.036865234375 ] } }, { "row_id": "ML_F03/domain", "fixture_id": "ML_F03", "question_id": "domain", "family": "F", "language": "EN", "primary_language": "EN", "question": { "type": "choice", "instructions": "What domain does `request` belong to?", "criteria": { "code": "software engineering, programming, refactoring, architecture, debugging", "math_or_logic": "mathematics, logic puzzles, proofs, complex calculation", "writing": "creative writing, essays, emails, blog posts, copywriting", "factual_lookup": "facts, definitions, trivia, history", "data_analysis": "statistics, SQL, data manipulation, metrics", "chitchat": "casual conversation, greetings, small talk" } }, "qtype": 0, "K": 6, "sequence_length": 128, "sequence_ids": [ 2, 6241, 2872, 235292, 2439, 11988, 1721, 4103, 3840, 235376, 8070, 577, 235336, 1, 4, 3409, 235292, 6815, 13299, 235269, 19991, 235269, 2279, 138397, 235269, 16333, 235269, 66439, 4, 10849, 235298, 483, 235298, 17112, 235292, 26814, 235269, 14855, 56983, 235269, 59828, 235269, 5766, 22091, 4, 6706, 235292, 12354, 6706, 235269, 54012, 235269, 28136, 235269, 7808, 10776, 235269, 5469, 24168, 4, 63914, 235298, 37897, 235292, 10275, 235269, 26303, 235269, 115195, 235269, 4281, 4, 1423, 235298, 26413, 235292, 15617, 235269, 15678, 235269, 1423, 44097, 235269, 30660, 4, 788, 6327, 482, 235292, 14044, 12836, 235269, 78107, 235269, 2301, 5063, 1, 19946, 3840, 1192, 664, 5559, 476, 3309, 235269, 9376, 29151, 577, 671, 51084, 17603, 43834, 235265, 93039, 476, 53162, 235290, 1311, 11023, 578, 1281, 3890, 235269, 40248, 5255, 1464, 235270, 1 ], "marker_positions": [ 14, 28, 44, 59, 71, 84 ], "probabilities": [ 0.04547017067670822, 0.0016235695220530033, 0.39046335220336914, 0.003536428092047572, 0.001613776315934956, 0.5572925806045532 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "chitchat", "probabilities": { "code": 0.0455, "math_or_logic": 0.0016, "writing": 0.3905, "factual_lookup": 0.0035, "data_analysis": 0.0016, "chitchat": 0.5573 }, "confidence": 0.512, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": false, "oracle": { "marker_logits": [ -1.2445505857467651, -4.576981067657471, 0.9057272672653198, -3.798489809036255, -4.583029747009277, 1.2614834308624268 ], "act_logits": [ 1321.6575927734375, -1580.097412109375 ] } }, { "row_id": "ML_F03/needs_tools", "fixture_id": "ML_F03", "question_id": "needs_tools", "family": "F", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does answering `request` require external tools, search or private data?" }, "qtype": 2, "K": 2, "sequence_length": 70, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 39534, 4103, 3840, 235376, 2817, 9774, 8112, 235269, 4544, 689, 4765, 1423, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 3840, 1192, 664, 5559, 476, 3309, 235269, 9376, 29151, 577, 671, 51084, 17603, 43834, 235265, 93039, 476, 53162, 235290, 1311, 11023, 578, 1281, 3890, 235269, 40248, 5255, 1464, 235270, 1 ], "marker_positions": [ 20, 30 ], "probabilities": [ 0.7959020137786865, 0.20409803092479706 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.2041, "confidence": 0.7959, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 1.2640810012817383, -0.09679508209228516 ], "act_logits": [ 1397.5740966796875, -1656.77978515625 ] } }, { "row_id": "ML_F03/is_sensitive", "fixture_id": "ML_F03", "question_id": "is_sensitive", "family": "F", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Does `request` involve money, legal, medical or safety consequences?" }, "qtype": 2, "K": 2, "sequence_length": 70, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 3840, 235376, 22395, 3370, 235269, 7035, 235269, 6910, 689, 6514, 18255, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 3840, 1192, 664, 5559, 476, 3309, 235269, 9376, 29151, 577, 671, 51084, 17603, 43834, 235265, 93039, 476, 53162, 235290, 1311, 11023, 578, 1281, 3890, 235269, 40248, 5255, 1464, 235270, 1 ], "marker_positions": [ 20, 30 ], "probabilities": [ 0.9985840320587158, 0.0014160366263240576 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0014, "confidence": 0.9986, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 5.881800174713135, -0.6766758561134338 ], "act_logits": [ 1576.2919921875, -1867.40380859375 ] } }, { "row_id": "ML_F04/difficulty", "fixture_id": "ML_F04", "question_id": "difficulty", "family": "F", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How hard is `request` for a language model?", "criteria": [ "trivial: a lookup or one-liner", "easy: short answer, no reasoning", "moderate: several steps", "hard: long multi-step reasoning or specialist knowledge" ] }, "qtype": 1, "K": 4, "sequence_length": 206, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 2811, 603, 4103, 3840, 235376, 604, 476, 5255, 2091, 235336, 1, 4, 2403, 235248, 235276, 235292, 49423, 235292, 476, 53053, 689, 974, 235290, 47050, 4, 2403, 235248, 235274, 235292, 3980, 235292, 3309, 3448, 235269, 793, 32346, 4, 2403, 235248, 235284, 235292, 23135, 235292, 3757, 7161, 4, 2403, 235248, 235304, 235292, 2811, 235292, 1497, 3103, 235290, 8847, 32346, 689, 25336, 5567, 1, 19946, 3840, 1192, 664, 84255, 236117, 15592, 50837, 235394, 72419, 28221, 54934, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 12990, 1 ], "marker_positions": [ 16, 29, 41, 50 ], "probabilities": [ 0.013259916566312313, 0.21728457510471344, 0.663553774356842, 0.105901800096035 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.8621, "legend": { "0": "trivial: a lookup or one-liner", "1": "easy: short answer, no reasoning", "2": "moderate: several steps", "3": "hard: long multi-step reasoning or specialist knowledge" }, "probabilities": { "0": 0.0133, "1": 0.2173, "2": 0.6636, "3": 0.1059 }, "confidence": 0.3515, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -1.88552725315094, 0.9109354019165039, 2.0273375511169434, 0.19223946332931519 ], "act_logits": [ 1559.1878662109375, -1839.4920654296875 ] } }, { "row_id": "ML_F04/domain", "fixture_id": "ML_F04", "question_id": "domain", "family": "F", "language": "JA", "primary_language": "JA", "question": { "type": "choice", "instructions": "What domain does `request` belong to?", "criteria": { "code": "software engineering, programming, refactoring, architecture, debugging", "math_or_logic": "mathematics, logic puzzles, proofs, complex calculation", "writing": "creative writing, essays, emails, blog posts, copywriting", "factual_lookup": "facts, definitions, trivia, history", "data_analysis": "statistics, SQL, data manipulation, metrics", "chitchat": "casual conversation, greetings, small talk" } }, "qtype": 0, "K": 6, "sequence_length": 237, "sequence_ids": [ 2, 6241, 2872, 235292, 2439, 11988, 1721, 4103, 3840, 235376, 8070, 577, 235336, 1, 4, 3409, 235292, 6815, 13299, 235269, 19991, 235269, 2279, 138397, 235269, 16333, 235269, 66439, 4, 10849, 235298, 483, 235298, 17112, 235292, 26814, 235269, 14855, 56983, 235269, 59828, 235269, 5766, 22091, 4, 6706, 235292, 12354, 6706, 235269, 54012, 235269, 28136, 235269, 7808, 10776, 235269, 5469, 24168, 4, 63914, 235298, 37897, 235292, 10275, 235269, 26303, 235269, 115195, 235269, 4281, 4, 1423, 235298, 26413, 235292, 15617, 235269, 15678, 235269, 1423, 44097, 235269, 30660, 4, 788, 6327, 482, 235292, 14044, 12836, 235269, 78107, 235269, 2301, 5063, 1, 19946, 3840, 1192, 664, 84255, 236117, 15592, 50837, 235394, 72419, 28221, 54934, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 12990, 1 ], "marker_positions": [ 14, 28, 44, 59, 71, 84 ], "probabilities": [ 0.0013267030008137226, 0.8892918229103088, 0.0008783818921074271, 0.0965404137969017, 0.0010724752210080624, 0.010890090838074684 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "math_or_logic", "probabilities": { "code": 0.0013, "math_or_logic": 0.8893, "writing": 0.0009, "factual_lookup": 0.0965, "data_analysis": 0.0011, "chitchat": 0.0109 }, "confidence": 0.7759, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": false, "oracle": { "marker_logits": [ -2.8441269397735596, 3.663604974746704, -3.2564940452575684, 1.4431391954421997, -3.056854724884033, -0.7389688491821289 ], "act_logits": [ 1358.7955322265625, -1619.6026611328125 ] } }, { "row_id": "ML_F04/needs_tools", "fixture_id": "ML_F04", "question_id": "needs_tools", "family": "F", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does answering `request` require external tools, search or private data?" }, "qtype": 2, "K": 2, "sequence_length": 179, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 39534, 4103, 3840, 235376, 2817, 9774, 8112, 235269, 4544, 689, 4765, 1423, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 3840, 1192, 664, 84255, 236117, 15592, 50837, 235394, 72419, 28221, 54934, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 12990, 1 ], "marker_positions": [ 20, 30 ], "probabilities": [ 0.9999995231628418, 4.4163653001305647e-07 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 13.94056224822998, -0.6922135353088379 ], "act_logits": [ 1544.4912109375, -1833.1885986328125 ] } }, { "row_id": "ML_F04/is_sensitive", "fixture_id": "ML_F04", "question_id": "is_sensitive", "family": "F", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `request` involve money, legal, medical or safety consequences?" }, "qtype": 2, "K": 2, "sequence_length": 179, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 3840, 235376, 22395, 3370, 235269, 7035, 235269, 6910, 689, 6514, 18255, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 3840, 1192, 664, 84255, 236117, 15592, 50837, 235394, 72419, 28221, 54934, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 12990, 1 ], "marker_positions": [ 20, 30 ], "probabilities": [ 0.9998950958251953, 0.00010494030721019953 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0001, "confidence": 0.9999, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 8.645485877990723, -0.5165271759033203 ], "act_logits": [ 1578.3291015625, -1873.907958984375 ] } }, { "row_id": "ML_F05/difficulty", "fixture_id": "ML_F05", "question_id": "difficulty", "family": "F", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How hard is `request` for a language model?", "criteria": [ "trivial: a lookup or one-liner", "easy: short answer, no reasoning", "moderate: several steps", "hard: long multi-step reasoning or specialist knowledge" ] }, "qtype": 1, "K": 4, "sequence_length": 256, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 2811, 603, 4103, 3840, 235376, 604, 476, 5255, 2091, 235336, 1, 4, 2403, 235248, 235276, 235292, 49423, 235292, 476, 53053, 689, 974, 235290, 47050, 4, 2403, 235248, 235274, 235292, 3980, 235292, 3309, 3448, 235269, 793, 32346, 4, 2403, 235248, 235284, 235292, 23135, 235292, 3757, 7161, 4, 2403, 235248, 235304, 235292, 2811, 235292, 1497, 3103, 235290, 8847, 32346, 689, 25336, 5567, 1, 19946, 3840, 1192, 664, 237065, 235977, 235372, 44393, 132576, 235432, 24793, 3902, 114478, 200364, 13319, 235362, 236781, 238017, 237293, 235400, 238715, 239071, 35935, 235394, 237112, 229567, 73264, 235459, 219259, 235362, 23631, 82509, 130335, 235711, 115910, 14872, 235733, 235459, 99338, 235399, 235394, 115105, 235410, 235441, 236700, 236270, 99534, 26685, 9641, 235432, 28221, 54934, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 1 ], "marker_positions": [ 16, 29, 41, 50 ], "probabilities": [ 0.0037058545276522636, 0.15862861275672913, 0.6254590749740601, 0.21220646798610687 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 2.0462, "legend": { "0": "trivial: a lookup or one-liner", "1": "easy: short answer, no reasoning", "2": "moderate: several steps", "3": "hard: long multi-step reasoning or specialist knowledge" }, "probabilities": { "0": 0.0037, "1": 0.1586, "2": 0.6255, "3": 0.2122 }, "confidence": 0.3253, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": true, "oracle": { "marker_logits": [ -3.2671589851379395, 0.48949313163757324, 1.8614137172698975, 0.7804872393608093 ], "act_logits": [ 1557.8599853515625, -1838.858154296875 ] } }, { "row_id": "ML_F05/domain", "fixture_id": "ML_F05", "question_id": "domain", "family": "F", "language": "JA", "primary_language": "JA", "question": { "type": "choice", "instructions": "What domain does `request` belong to?", "criteria": { "code": "software engineering, programming, refactoring, architecture, debugging", "math_or_logic": "mathematics, logic puzzles, proofs, complex calculation", "writing": "creative writing, essays, emails, blog posts, copywriting", "factual_lookup": "facts, definitions, trivia, history", "data_analysis": "statistics, SQL, data manipulation, metrics", "chitchat": "casual conversation, greetings, small talk" } }, "qtype": 0, "K": 6, "sequence_length": 256, "sequence_ids": [ 2, 6241, 2872, 235292, 2439, 11988, 1721, 4103, 3840, 235376, 8070, 577, 235336, 1, 4, 3409, 235292, 6815, 13299, 235269, 19991, 235269, 2279, 138397, 235269, 16333, 235269, 66439, 4, 10849, 235298, 483, 235298, 17112, 235292, 26814, 235269, 14855, 56983, 235269, 59828, 235269, 5766, 22091, 4, 6706, 235292, 12354, 6706, 235269, 54012, 235269, 28136, 235269, 7808, 10776, 235269, 5469, 24168, 4, 63914, 235298, 37897, 235292, 10275, 235269, 26303, 235269, 115195, 235269, 4281, 4, 1423, 235298, 26413, 235292, 15617, 235269, 15678, 235269, 1423, 44097, 235269, 30660, 4, 788, 6327, 482, 235292, 14044, 12836, 235269, 78107, 235269, 2301, 5063, 1, 19946, 3840, 1192, 664, 237065, 235977, 235372, 44393, 132576, 235432, 24793, 3902, 114478, 200364, 13319, 235362, 236781, 238017, 237293, 235400, 238715, 239071, 35935, 235394, 237112, 229567, 73264, 235459, 219259, 235362, 23631, 82509, 130335, 235711, 115910, 14872, 235733, 235459, 99338, 235399, 235394, 115105, 235410, 235441, 236700, 236270, 99534, 26685, 9641, 235432, 28221, 54934, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 1 ], "marker_positions": [ 14, 28, 44, 59, 71, 84 ], "probabilities": [ 0.5164615511894226, 0.04022068530321121, 0.06682515144348145, 0.10922499001026154, 0.14092674851417542, 0.1263408660888672 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "code", "probabilities": { "code": 0.5165, "math_or_logic": 0.0402, "writing": 0.0668, "factual_lookup": 0.1092, "data_analysis": 0.1409, "chitchat": 0.1263 }, "confidence": 0.2015, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": true, "oracle": { "marker_logits": [ -1.5336487293243408, -4.086267948150635, -3.57857084274292, -3.0872409343719482, -2.832409381866455, -2.941666603088379 ], "act_logits": [ 1209.0660400390625, -1445.03759765625 ] } }, { "row_id": "ML_F05/needs_tools", "fixture_id": "ML_F05", "question_id": "needs_tools", "family": "F", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does answering `request` require external tools, search or private data?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 39534, 4103, 3840, 235376, 2817, 9774, 8112, 235269, 4544, 689, 4765, 1423, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 3840, 1192, 664, 237065, 235977, 235372, 44393, 132576, 235432, 24793, 3902, 114478, 200364, 13319, 235362, 236781, 238017, 237293, 235400, 238715, 239071, 35935, 235394, 237112, 229567, 73264, 235459, 219259, 235362, 23631, 82509, 130335, 235711, 115910, 14872, 235733, 235459, 99338, 235399, 235394, 115105, 235410, 235441, 236700, 236270, 99534, 26685, 9641, 235432, 28221, 54934, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 1 ], "marker_positions": [ 20, 30 ], "probabilities": [ 0.9997901320457458, 0.0002098830882459879 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0002, "confidence": 0.9998, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 7.890449047088623, -0.5783013701438904 ], "act_logits": [ 1421.38427734375, -1688.51513671875 ] } }, { "row_id": "ML_F05/is_sensitive", "fixture_id": "ML_F05", "question_id": "is_sensitive", "family": "F", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `request` involve money, legal, medical or safety consequences?" }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 3840, 235376, 22395, 3370, 235269, 7035, 235269, 6910, 689, 6514, 18255, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 3840, 1192, 664, 237065, 235977, 235372, 44393, 132576, 235432, 24793, 3902, 114478, 200364, 13319, 235362, 236781, 238017, 237293, 235400, 238715, 239071, 35935, 235394, 237112, 229567, 73264, 235459, 219259, 235362, 23631, 82509, 130335, 235711, 115910, 14872, 235733, 235459, 99338, 235399, 235394, 115105, 235410, 235441, 236700, 236270, 99534, 26685, 9641, 235432, 28221, 54934, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 1 ], "marker_positions": [ 20, 30 ], "probabilities": [ 1.0, 3.27245004427823e-08 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0, "confidence": 1.0, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 16.438369750976562, -0.796771228313446 ], "act_logits": [ 1554.4649658203125, -1848.8409423828125 ] } }, { "row_id": "ML_F06/difficulty", "fixture_id": "ML_F06", "question_id": "difficulty", "family": "F", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How hard is `request` for a language model?", "criteria": [ "trivial: a lookup or one-liner", "easy: short answer, no reasoning", "moderate: several steps", "hard: long multi-step reasoning or specialist knowledge" ] }, "qtype": 1, "K": 4, "sequence_length": 111, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 2811, 603, 4103, 3840, 235376, 604, 476, 5255, 2091, 235336, 1, 4, 2403, 235248, 235276, 235292, 49423, 235292, 476, 53053, 689, 974, 235290, 47050, 4, 2403, 235248, 235274, 235292, 3980, 235292, 3309, 3448, 235269, 793, 32346, 4, 2403, 235248, 235284, 235292, 23135, 235292, 3757, 7161, 4, 2403, 235248, 235304, 235292, 2811, 235292, 1497, 3103, 235290, 8847, 32346, 689, 25336, 5567, 1, 19946, 3840, 1192, 664, 32789, 235362, 40574, 236923, 10893, 98332, 235398, 235394, 238908, 237932, 222216, 2618, 237518, 106658, 6248, 13383, 235362, 23586, 80506, 237222, 237022, 235425, 64845, 5493, 235394, 50837, 52854, 68594, 108156, 235443, 235362, 167622, 73459, 41024, 16508, 236847, 153209, 13319, 235362, 12990, 1 ], "marker_positions": [ 16, 29, 41, 50 ], "probabilities": [ 0.030771132558584213, 0.39459794759750366, 0.4664284288883209, 0.10820244997739792 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.6521, "legend": { "0": "trivial: a lookup or one-liner", "1": "easy: short answer, no reasoning", "2": "moderate: several steps", "3": "hard: long multi-step reasoning or specialist knowledge" }, "probabilities": { "0": 0.0308, "1": 0.3946, "2": 0.4664, "3": 0.1082 }, "confidence": 0.2279, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ -1.6404311656951904, 0.9108595252037048, 1.0780963897705078, -0.38300395011901855 ], "act_logits": [ 1608.9866943359375, -1901.387939453125 ] } }, { "row_id": "ML_F06/domain", "fixture_id": "ML_F06", "question_id": "domain", "family": "F", "language": "JA", "primary_language": "JA", "question": { "type": "choice", "instructions": "What domain does `request` belong to?", "criteria": { "code": "software engineering, programming, refactoring, architecture, debugging", "math_or_logic": "mathematics, logic puzzles, proofs, complex calculation", "writing": "creative writing, essays, emails, blog posts, copywriting", "factual_lookup": "facts, definitions, trivia, history", "data_analysis": "statistics, SQL, data manipulation, metrics", "chitchat": "casual conversation, greetings, small talk" } }, "qtype": 0, "K": 6, "sequence_length": 142, "sequence_ids": [ 2, 6241, 2872, 235292, 2439, 11988, 1721, 4103, 3840, 235376, 8070, 577, 235336, 1, 4, 3409, 235292, 6815, 13299, 235269, 19991, 235269, 2279, 138397, 235269, 16333, 235269, 66439, 4, 10849, 235298, 483, 235298, 17112, 235292, 26814, 235269, 14855, 56983, 235269, 59828, 235269, 5766, 22091, 4, 6706, 235292, 12354, 6706, 235269, 54012, 235269, 28136, 235269, 7808, 10776, 235269, 5469, 24168, 4, 63914, 235298, 37897, 235292, 10275, 235269, 26303, 235269, 115195, 235269, 4281, 4, 1423, 235298, 26413, 235292, 15617, 235269, 15678, 235269, 1423, 44097, 235269, 30660, 4, 788, 6327, 482, 235292, 14044, 12836, 235269, 78107, 235269, 2301, 5063, 1, 19946, 3840, 1192, 664, 32789, 235362, 40574, 236923, 10893, 98332, 235398, 235394, 238908, 237932, 222216, 2618, 237518, 106658, 6248, 13383, 235362, 23586, 80506, 237222, 237022, 235425, 64845, 5493, 235394, 50837, 52854, 68594, 108156, 235443, 235362, 167622, 73459, 41024, 16508, 236847, 153209, 13319, 235362, 12990, 1 ], "marker_positions": [ 14, 28, 44, 59, 71, 84 ], "probabilities": [ 0.00524566974490881, 0.001375718042254448, 0.012622932903468609, 0.0020864945836365223, 0.0013608073350042105, 0.9773083329200745 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "chitchat", "probabilities": { "code": 0.0052, "math_or_logic": 0.0014, "writing": 0.0126, "factual_lookup": 0.0021, "data_analysis": 0.0014, "chitchat": 0.9773 }, "confidence": 0.924, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:6-10", "state_right_truncated": false, "oracle": { "marker_logits": [ -3.2288095951080322, -4.567237377166748, -2.3506994247436523, -4.150728702545166, -4.578133583068848, 1.9985862970352173 ], "act_logits": [ 1413.27197265625, -1689.2178955078125 ] } }, { "row_id": "ML_F06/needs_tools", "fixture_id": "ML_F06", "question_id": "needs_tools", "family": "F", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does answering `request` require external tools, search or private data?" }, "qtype": 2, "K": 2, "sequence_length": 84, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 39534, 4103, 3840, 235376, 2817, 9774, 8112, 235269, 4544, 689, 4765, 1423, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 3840, 1192, 664, 32789, 235362, 40574, 236923, 10893, 98332, 235398, 235394, 238908, 237932, 222216, 2618, 237518, 106658, 6248, 13383, 235362, 23586, 80506, 237222, 237022, 235425, 64845, 5493, 235394, 50837, 52854, 68594, 108156, 235443, 235362, 167622, 73459, 41024, 16508, 236847, 153209, 13319, 235362, 12990, 1 ], "marker_positions": [ 20, 30 ], "probabilities": [ 0.9903912544250488, 0.009608716703951359 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0096, "confidence": 0.9904, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 4.203000068664551, -0.4324297308921814 ], "act_logits": [ 1451.0233154296875, -1720.009033203125 ] } }, { "row_id": "ML_F06/is_sensitive", "fixture_id": "ML_F06", "question_id": "is_sensitive", "family": "F", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Does `request` involve money, legal, medical or safety consequences?" }, "qtype": 2, "K": 2, "sequence_length": 84, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 11188, 4103, 3840, 235376, 22395, 3370, 235269, 7035, 235269, 6910, 689, 6514, 18255, 235336, 1, 4, 1566, 235292, 793, 235269, 573, 6218, 1721, 780, 3385, 4, 1382, 235292, 7778, 235269, 573, 6218, 12723, 1, 19946, 3840, 1192, 664, 32789, 235362, 40574, 236923, 10893, 98332, 235398, 235394, 238908, 237932, 222216, 2618, 237518, 106658, 6248, 13383, 235362, 23586, 80506, 237222, 237022, 235425, 64845, 5493, 235394, 50837, 52854, 68594, 108156, 235443, 235362, 167622, 73459, 41024, 16508, 236847, 153209, 13319, 235362, 12990, 1 ], "marker_positions": [ 20, 30 ], "probabilities": [ 0.9999057054519653, 9.426734322914854e-05 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0001, "confidence": 0.9999, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ 8.29902458190918, -0.9702560305595398 ], "act_logits": [ 1667.6905517578125, -1979.6485595703125 ] } }, { "row_id": "ML_G01/resolution", "fixture_id": "ML_G01", "question_id": "resolution", "family": "G", "language": "EN", "primary_language": "EN", "question": { "type": "choice", "instructions": "What is the current resolution status?", "criteria": [ "resolved", "pending", "unclear" ] }, "qtype": 0, "K": 3, "sequence_length": 149, "sequence_ids": [ 2, 6241, 2872, 235292, 2439, 603, 573, 2474, 11879, 4938, 235336, 1, 4, 23341, 4, 25718, 4, 40846, 1, 892, 9766, 8247, 1192, 664, 1645, 824, 664, 3312, 1192, 664, 651, 15162, 877, 780, 2174, 235265, 2551, 692, 9337, 476, 5985, 2701, 235336, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 1464, 789, 19946, 8247, 1192, 664, 105776, 824, 664, 3312, 1192, 664, 10924, 476, 5469, 235269, 1492, 2174, 573, 5469, 575, 573, 36183, 1464, 789, 19946, 8247, 1192, 664, 1645, 824, 664, 3312, 1192, 664, 3171, 6800, 235265, 1307, 4024, 7260, 603, 4880, 1464, 12765, 1 ], "marker_positions": [ 12, 14, 16 ], "probabilities": [ 0.9646861553192139, 0.01468421146273613, 0.020629649981856346 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "resolved", "probabilities": { "resolved": 0.9647, "pending": 0.0147, "unclear": 0.0206 }, "confidence": 0.8391, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ 3.542182445526123, -0.6428475379943848, -0.3028910756111145 ], "act_logits": [ 1321.6864013671875, -1573.413330078125 ] } }, { "row_id": "ML_G01/helpfulness", "fixture_id": "ML_G01", "question_id": "helpfulness", "family": "G", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How helpful was the response?", "criteria": [ "not helpful", "slightly helpful", "partly helpful", "mostly helpful", "fully helpful" ] }, "qtype": 1, "K": 5, "sequence_length": 177, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 10055, 729, 573, 3590, 235336, 1, 4, 2403, 235248, 235276, 235292, 780, 10055, 4, 2403, 235248, 235274, 235292, 9677, 10055, 4, 2403, 235248, 235284, 235292, 23313, 10055, 4, 2403, 235248, 235304, 235292, 11900, 10055, 4, 2403, 235248, 235310, 235292, 7382, 10055, 1, 892, 9766, 8247, 1192, 664, 1645, 824, 664, 3312, 1192, 664, 651, 15162, 877, 780, 2174, 235265, 2551, 692, 9337, 476, 5985, 2701, 235336, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 1464, 789, 19946, 8247, 1192, 664, 105776, 824, 664, 3312, 1192, 664, 10924, 476, 5469, 235269, 1492, 2174, 573, 5469, 575, 573, 36183, 1464, 789, 19946, 8247, 1192, 664, 1645, 824, 664, 3312, 1192, 664, 3171, 6800, 235265, 1307, 4024, 7260, 603, 4880, 1464, 12765, 1 ], "marker_positions": [ 11, 18, 25, 32, 39 ], "probabilities": [ 0.4291870594024658, 0.1402372270822525, 0.03584760054945946, 0.05028338357806206, 0.34444475173950195 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 1.7406, "legend": { "0": "not helpful", "1": "slightly helpful", "2": "partly helpful", "3": "mostly helpful", "4": "fully helpful" }, "probabilities": { "0": 0.4292, "1": 0.1402, "2": 0.0358, "3": 0.0503, "4": 0.3444 }, "confidence": 0.2076, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ 1.3169790506362915, 0.19842618703842163, -1.1656306982040405, -0.8272333145141602, 1.0970265865325928 ], "act_logits": [ 1483.6597900390625, -1752.990478515625 ] } }, { "row_id": "ML_G01/follow_up", "fixture_id": "ML_G01", "question_id": "follow_up", "family": "G", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Is another response requested?", "criteria": { "true": "the user requests another response", "false": "no further response is requested" } }, "qtype": 2, "K": 2, "sequence_length": 158, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 2550, 3590, 16350, 235336, 1, 4, 1566, 235292, 793, 4024, 3590, 603, 16350, 4, 1382, 235292, 573, 2425, 12557, 2550, 3590, 1, 892, 9766, 8247, 1192, 664, 1645, 824, 664, 3312, 1192, 664, 651, 15162, 877, 780, 2174, 235265, 2551, 692, 9337, 476, 5985, 2701, 235336, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 1464, 789, 19946, 8247, 1192, 664, 105776, 824, 664, 3312, 1192, 664, 10924, 476, 5469, 235269, 1492, 2174, 573, 5469, 575, 573, 36183, 1464, 789, 19946, 8247, 1192, 664, 1645, 824, 664, 3312, 1192, 664, 3171, 6800, 235265, 1307, 4024, 7260, 603, 4880, 1464, 12765, 1 ], "marker_positions": [ 11, 19 ], "probabilities": [ 0.2563108801841736, 0.7436891198158264 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.7437, "confidence": 0.7437, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ -0.11680668592453003, 0.9484256505966187 ], "act_logits": [ 1146.0206298828125, -1365.8480224609375 ] } }, { "row_id": "ML_G02/resolution", "fixture_id": "ML_G02", "question_id": "resolution", "family": "G", "language": "EN", "primary_language": "EN", "question": { "type": "choice", "instructions": "What is the current resolution status?", "criteria": [ "resolved", "pending", "unclear" ] }, "qtype": 0, "K": 3, "sequence_length": 256, "sequence_ids": [ 2, 6241, 2872, 235292, 2439, 603, 573, 2474, 11879, 4938, 235336, 1, 4, 23341, 4, 25718, 4, 40846, 1, 892, 9766, 8247, 1192, 664, 1645, 824, 664, 3312, 1192, 664, 5958, 2110, 736, 15162, 3868, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 476, 1889, 576, 13333, 235269, 578, 573, 3221, 798, 1281, 476, 5469, 2346, 11117, 573, 3464, 235265, 714, 8467, 1049, 12184, 575, 974, 2040, 577, 7704, 2167, 16062, 774, 1855, 20232, 235265, 4660, 573, 3221, 235269, 783, 1780, 1 ], "marker_positions": [ 12, 14, 16 ], "probabilities": [ 0.33198708295822144, 0.3086467683315277, 0.35936620831489563 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "unclear", "probabilities": { "resolved": 0.332, "pending": 0.3086, "unclear": 0.3594 }, "confidence": 0.0018, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:3-5", "state_right_truncated": true, "oracle": { "marker_logits": [ 1.9166345596313477, 1.843736171722412, 1.995881199836731 ], "act_logits": [ 1215.6866455078125, -1447.5472412109375 ] } }, { "row_id": "ML_G02/helpfulness", "fixture_id": "ML_G02", "question_id": "helpfulness", "family": "G", "language": "EN", "primary_language": "EN", "question": { "type": "score", "instructions": "How helpful was the response?", "criteria": [ "not helpful", "slightly helpful", "partly helpful", "mostly helpful", "fully helpful" ] }, "qtype": 1, "K": 5, "sequence_length": 256, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 10055, 729, 573, 3590, 235336, 1, 4, 2403, 235248, 235276, 235292, 780, 10055, 4, 2403, 235248, 235274, 235292, 9677, 10055, 4, 2403, 235248, 235284, 235292, 23313, 10055, 4, 2403, 235248, 235304, 235292, 11900, 10055, 4, 2403, 235248, 235310, 235292, 7382, 10055, 1, 892, 9766, 8247, 1192, 664, 1645, 824, 664, 3312, 1192, 664, 5958, 2110, 736, 15162, 3868, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 476, 1889, 576, 13333, 235269, 578, 573, 3221, 798, 1281, 1 ], "marker_positions": [ 11, 18, 25, 32, 39 ], "probabilities": [ 0.14402681589126587, 0.31640541553497314, 0.1155787780880928, 0.17190934717655182, 0.25207969546318054 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 2.0716, "legend": { "0": "not helpful", "1": "slightly helpful", "2": "partly helpful", "3": "mostly helpful", "4": "fully helpful" }, "probabilities": { "0": 0.144, "1": 0.3164, "2": 0.1156, "3": 0.1719, "4": 0.2521 }, "confidence": 0.0415, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": true, "oracle": { "marker_logits": [ 1.198685646057129, 1.9857102632522583, 0.9786380529403687, 1.375652551651001, 1.7584307193756104 ], "act_logits": [ 1385.4595947265625, -1633.0404052734375 ] } }, { "row_id": "ML_G02/follow_up", "fixture_id": "ML_G02", "question_id": "follow_up", "family": "G", "language": "EN", "primary_language": "EN", "question": { "type": "noul", "instructions": "Is another response requested?", "criteria": { "true": "the user requests another response", "false": "no further response is requested" } }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 2550, 3590, 16350, 235336, 1, 4, 1566, 235292, 793, 4024, 3590, 603, 16350, 4, 1382, 235292, 573, 2425, 12557, 2550, 3590, 1, 892, 9766, 8247, 1192, 664, 1645, 824, 664, 3312, 1192, 664, 5958, 2110, 736, 15162, 3868, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 476, 1889, 576, 13333, 235269, 578, 573, 3221, 798, 1281, 476, 5469, 2346, 11117, 573, 3464, 235265, 714, 8467, 1049, 12184, 575, 974, 2040, 577, 7704, 2167, 16062, 774, 1 ], "marker_positions": [ 11, 19 ], "probabilities": [ 0.9959471821784973, 0.004052863456308842 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.0041, "confidence": 0.9959, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 4.703449726104736, -0.8008229732513428 ], "act_logits": [ 1245.8668212890625, -1481.488525390625 ] } }, { "row_id": "ML_G02/label", "fixture_id": "ML_G02", "question_id": "label", "family": "G", "language": "EN", "primary_language": "EN", "question": { "type": "choice", "instructions": "Which label did the user choose?", "criteria": [ "red", "blue", "green", "yellow", "black", "white", "pink", "brown", "gray", "gold", "silver", "orange", "purple", "teal", "cyan", "indigo", "violet", "coral", "lime", "navy" ] }, "qtype": 0, "K": 20, "sequence_length": 256, "sequence_ids": [ 2, 6241, 2872, 235292, 12236, 4970, 1498, 573, 2425, 6475, 235336, 1, 4, 3118, 4, 3868, 4, 4433, 4, 8123, 4, 2656, 4, 2674, 4, 9010, 4, 8426, 4, 13676, 4, 4394, 4, 8343, 4, 10436, 4, 15811, 4, 88735, 4, 60300, 4, 87816, 4, 45093, 4, 35293, 4, 25287, 4, 23008, 1, 892, 9766, 8247, 1192, 664, 1645, 824, 664, 3312, 1192, 664, 5958, 2110, 736, 15162, 3868, 235265, 1699, 4807, 235269, 573, 20257, 15667, 476, 2301, 15173, 604, 1853, 3853, 235265, 714, 8816, 3969, 6246, 1212, 783, 1476, 235289, 1450, 8467, 10200, 1368, 573, 3661, 729, 8598, 235265, 586, 51011, 15062, 573, 15162, 575, 573, 5764, 578, 11659, 573, 2196, 611, 573, 8934, 11193, 235265, 12516, 51011, 7631, 573, 1809, 15162, 1452, 13955, 235269, 2177, 573, 1809, 7161, 235265, 714, 2970, 7177, 2149, 1160, 11113, 578, 476, 7554, 4924, 235265, 16123, 604, 573, 2351, 3221, 791, 1125, 7765, 575, 476, 8905, 24655, 712, 674, 573, 2474, 3661, 9168, 3980, 577, 1611, 235265, 1448, 791, 8001, 573, 7910, 5469, 578, 4192, 476, 3309, 4670, 23357, 1853, 2669, 235265, 714, 3661, 603, 11538, 731, 573, 1069, 576, 573, 3071, 235269, 675, 573, 1546, 6077, 17789, 696, 573, 1580, 235265, 12767, 573, 2351, 5961, 235269, 573, 1552, 13404, 736, 2599, 877, 1682, 573, 8816, 3853, 578, 573, 12412, 8467, 3584, 235265, 5651, 1281, 573, 8816, 3853, 685, 573, 1872, 2377, 576, 736, 3969, 235265, 2456, 603, 793, 5942, 6650, 2113, 575, 736, 4670, 235265, 714, 15162, 7744, 1297, 573, 20257, 11968, 578, 476, 1889, 576, 13333, 1 ], "marker_positions": [ 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50 ], "probabilities": [ 0.0004723203310277313, 0.9804604649543762, 0.0007509375573135912, 0.0002873700577765703, 0.0005817977944388986, 0.006917228456586599, 0.0011046791914850473, 0.00182013597805053, 0.0005686606746166945, 0.00034403480822220445, 0.0013274449156597257, 0.00028943587676621974, 0.0009762411937117577, 0.0006063177715986967, 0.0013495918828994036, 0.0003754637436941266, 0.0012054957915097475, 5.2781517297262326e-05, 0.00022108103439677507, 0.00028857155120931566 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "blue", "probabilities": { "red": 0.0005, "blue": 0.9805, "green": 0.0008, "yellow": 0.0003, "black": 0.0006, "white": 0.0069, "pink": 0.0011, "brown": 0.0018, "gray": 0.0006, "gold": 0.0003, "silver": 0.0013, "orange": 0.0003, "purple": 0.001, "teal": 0.0006, "cyan": 0.0013, "indigo": 0.0004, "violet": 0.0012, "coral": 0.0001, "lime": 0.0002, "navy": 0.0003 }, "confidence": 0.9524, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:11+", "state_right_truncated": true, "oracle": { "marker_logits": [ -2.3469929695129395, 5.2911272048950195, -1.8833286762237549, -2.8438801765441895, -2.1385278701782227, 0.33711960911750793, -1.4973405599594116, -0.9979844093322754, -2.161367177963257, -2.6639082431793213, -1.313639521598816, -2.836716890335083, -1.620940923690796, -2.0972461700439453, -1.2970936298370361, -2.576488971710205, -1.4100046157836914, -4.5384907722473145, -3.1061220169067383, -2.839707851409912 ], "act_logits": [ 1244.4647216796875, -1483.76318359375 ] } }, { "row_id": "ML_G03/resolution", "fixture_id": "ML_G03", "question_id": "resolution", "family": "G", "language": "JA", "primary_language": "JA", "question": { "type": "choice", "instructions": "What is the current resolution status?", "criteria": [ "resolved", "pending", "unclear" ] }, "qtype": 0, "K": 3, "sequence_length": 56, "sequence_ids": [ 2, 6241, 2872, 235292, 2439, 603, 573, 2474, 11879, 4938, 235336, 1, 4, 23341, 4, 25718, 4, 40846, 1, 892, 9766, 8247, 1192, 664, 1645, 824, 664, 3312, 1192, 664, 27364, 139578, 219254, 235362, 16406, 19946, 8247, 1192, 664, 105776, 824, 664, 3312, 1192, 664, 150694, 195315, 3548, 235394, 202094, 66599, 6248, 148495, 145049, 12765, 1 ], "marker_positions": [ 12, 14, 16 ], "probabilities": [ 0.7621758580207825, 0.13547657430171967, 0.10234759002923965 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "resolved", "probabilities": { "resolved": 0.7622, "pending": 0.1355, "unclear": 0.1023 }, "confidence": 0.3527, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ 0.9010524749755859, -0.8263261914253235, -1.1067501306533813 ], "act_logits": [ 1339.1805419921875, -1593.3751220703125 ] } }, { "row_id": "ML_G03/helpfulness", "fixture_id": "ML_G03", "question_id": "helpfulness", "family": "G", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How helpful was the response?", "criteria": [ "not helpful", "slightly helpful", "partly helpful", "mostly helpful", "fully helpful" ] }, "qtype": 1, "K": 5, "sequence_length": 84, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 10055, 729, 573, 3590, 235336, 1, 4, 2403, 235248, 235276, 235292, 780, 10055, 4, 2403, 235248, 235274, 235292, 9677, 10055, 4, 2403, 235248, 235284, 235292, 23313, 10055, 4, 2403, 235248, 235304, 235292, 11900, 10055, 4, 2403, 235248, 235310, 235292, 7382, 10055, 1, 892, 9766, 8247, 1192, 664, 1645, 824, 664, 3312, 1192, 664, 27364, 139578, 219254, 235362, 16406, 19946, 8247, 1192, 664, 105776, 824, 664, 3312, 1192, 664, 150694, 195315, 3548, 235394, 202094, 66599, 6248, 148495, 145049, 12765, 1 ], "marker_positions": [ 11, 18, 25, 32, 39 ], "probabilities": [ 0.22540076076984406, 0.19352029263973236, 0.08717377483844757, 0.15039819478988647, 0.34350696206092834 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 2.1931, "legend": { "0": "not helpful", "1": "slightly helpful", "2": "partly helpful", "3": "mostly helpful", "4": "fully helpful" }, "probabilities": { "0": 0.2254, "1": 0.1935, "2": 0.0872, "3": 0.1504, "4": 0.3435 }, "confidence": 0.0566, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": false, "oracle": { "marker_logits": [ 0.8611527681350708, 0.7086555361747742, -0.08882373571395874, 0.4565594792366028, 1.282480239868164 ], "act_logits": [ 1491.3319091796875, -1759.45849609375 ] } }, { "row_id": "ML_G03/follow_up", "fixture_id": "ML_G03", "question_id": "follow_up", "family": "G", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Is another response requested?", "criteria": { "true": "the user requests another response", "false": "no further response is requested" } }, "qtype": 2, "K": 2, "sequence_length": 65, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 2550, 3590, 16350, 235336, 1, 4, 1566, 235292, 793, 4024, 3590, 603, 16350, 4, 1382, 235292, 573, 2425, 12557, 2550, 3590, 1, 892, 9766, 8247, 1192, 664, 1645, 824, 664, 3312, 1192, 664, 27364, 139578, 219254, 235362, 16406, 19946, 8247, 1192, 664, 105776, 824, 664, 3312, 1192, 664, 150694, 195315, 3548, 235394, 202094, 66599, 6248, 148495, 145049, 12765, 1 ], "marker_positions": [ 11, 19 ], "probabilities": [ 0.41117843985557556, 0.588821530342102 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.5888, "confidence": 0.5888, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": false, "oracle": { "marker_logits": [ -0.52864670753479, -0.16955113410949707 ], "act_logits": [ 1332.6453857421875, -1584.132568359375 ] } }, { "row_id": "ML_G04/resolution", "fixture_id": "ML_G04", "question_id": "resolution", "family": "G", "language": "JA", "primary_language": "JA", "question": { "type": "choice", "instructions": "What is the current resolution status?", "criteria": [ "resolved", "pending", "unclear" ] }, "qtype": 0, "K": 3, "sequence_length": 256, "sequence_ids": [ 2, 6241, 2872, 235292, 2439, 603, 573, 2474, 11879, 4938, 235336, 1, 4, 23341, 4, 25718, 4, 40846, 1, 892, 9766, 8247, 1192, 664, 1645, 824, 664, 3312, 1192, 664, 44393, 236322, 235372, 62962, 235432, 26808, 4121, 235362, 238127, 236563, 16294, 28082, 98377, 235399, 235394, 91275, 119715, 108589, 235443, 68383, 13319, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 50855, 72368, 235362, 78546, 235372, 63090, 238518, 7711, 44393, 235372, 174612, 235410, 53479, 235372, 130073, 9301, 235432, 90518, 22031, 235362, 235987, 23176, 151978, 219689, 236943, 14848, 62865, 235362, 81625, 40957, 150694, 77776, 26685, 235399, 235394, 56477, 235418, 66886, 1 ], "marker_positions": [ 12, 14, 16 ], "probabilities": [ 0.8937832713127136, 0.011258680373430252, 0.09495817124843597 ], "act_probability": 1.0, "official_answer": { "type": "choice", "choice": "resolved", "probabilities": { "resolved": 0.8938, "pending": 0.0113, "unclear": 0.095 }, "confidence": 0.6592, "action": { "act_probability": 1.0 } }, "temperature_bucket": "choice:3-5", "state_right_truncated": true, "oracle": { "marker_logits": [ 3.332932233810425, -1.0413914918899536, 1.0909051895141602 ], "act_logits": [ 1235.0621337890625, -1472.2830810546875 ] } }, { "row_id": "ML_G04/helpfulness", "fixture_id": "ML_G04", "question_id": "helpfulness", "family": "G", "language": "JA", "primary_language": "JA", "question": { "type": "score", "instructions": "How helpful was the response?", "criteria": [ "not helpful", "slightly helpful", "partly helpful", "mostly helpful", "fully helpful" ] }, "qtype": 1, "K": 5, "sequence_length": 256, "sequence_ids": [ 2, 7716, 2872, 235292, 2250, 10055, 729, 573, 3590, 235336, 1, 4, 2403, 235248, 235276, 235292, 780, 10055, 4, 2403, 235248, 235274, 235292, 9677, 10055, 4, 2403, 235248, 235284, 235292, 23313, 10055, 4, 2403, 235248, 235304, 235292, 11900, 10055, 4, 2403, 235248, 235310, 235292, 7382, 10055, 1, 892, 9766, 8247, 1192, 664, 1645, 824, 664, 3312, 1192, 664, 44393, 236322, 235372, 62962, 235432, 26808, 4121, 235362, 238127, 236563, 16294, 28082, 98377, 235399, 235394, 91275, 119715, 108589, 235443, 68383, 13319, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 50855, 72368, 235362, 78546, 235372, 63090, 238518, 7711, 44393, 235372, 1 ], "marker_positions": [ 11, 18, 25, 32, 39 ], "probabilities": [ 0.07177174836397171, 0.1626284271478653, 0.16007547080516815, 0.2997992932796478, 0.30572509765625 ], "act_probability": 1.0, "official_answer": { "type": "score", "score": 2.6051, "legend": { "0": "not helpful", "1": "slightly helpful", "2": "partly helpful", "3": "mostly helpful", "4": "fully helpful" }, "probabilities": { "0": 0.0718, "1": 0.1626, "2": 0.1601, "3": 0.2998, "4": 0.3057 }, "confidence": 0.0673, "action": { "act_probability": 1.0 } }, "temperature_bucket": "score:3-5", "state_right_truncated": true, "oracle": { "marker_logits": [ 0.47253119945526123, 1.2905077934265137, 1.2746855020523071, 1.9021532535552979, 1.9217264652252197 ], "act_logits": [ 1645.8818359375, -1941.70654296875 ] } }, { "row_id": "ML_G04/follow_up", "fixture_id": "ML_G04", "question_id": "follow_up", "family": "G", "language": "JA", "primary_language": "JA", "question": { "type": "noul", "instructions": "Is another response requested?", "criteria": { "true": "the user requests another response", "false": "no further response is requested" } }, "qtype": 2, "K": 2, "sequence_length": 256, "sequence_ids": [ 2, 552, 6311, 2872, 235292, 2125, 2550, 3590, 16350, 235336, 1, 4, 1566, 235292, 793, 4024, 3590, 603, 16350, 4, 1382, 235292, 573, 2425, 12557, 2550, 3590, 1, 892, 9766, 8247, 1192, 664, 1645, 824, 664, 3312, 1192, 664, 44393, 236322, 235372, 62962, 235432, 26808, 4121, 235362, 238127, 236563, 16294, 28082, 98377, 235399, 235394, 91275, 119715, 108589, 235443, 68383, 13319, 235362, 201824, 11251, 235394, 221186, 26685, 11795, 32031, 80798, 9178, 235362, 53479, 73363, 111674, 55495, 62962, 98377, 235399, 235394, 238698, 48804, 235400, 88997, 53479, 74308, 23631, 114478, 96483, 235948, 14657, 235362, 18027, 16294, 28082, 235418, 238127, 189684, 90518, 2404, 203843, 2297, 235362, 44393, 236322, 6890, 235394, 100377, 127459, 27364, 235432, 231180, 100391, 22031, 235362, 53178, 27364, 235459, 215598, 236904, 235400, 26808, 235394, 55965, 26685, 235398, 62319, 150694, 235418, 88997, 171840, 235961, 19142, 235362, 128372, 62962, 235418, 40849, 187436, 237064, 22031, 235362, 191801, 62962, 235410, 53178, 11795, 235432, 237051, 30339, 26685, 15417, 6028, 235394, 40849, 2404, 138362, 7711, 153883, 28221, 235432, 237067, 111501, 235362, 26685, 2404, 235504, 40391, 237541, 235400, 49392, 14872, 12784, 235394, 235843, 3548, 235841, 235948, 235412, 17180, 235459, 190973, 235432, 235408, 235679, 36809, 235362, 55965, 67988, 22708, 6890, 235394, 105846, 96012, 235410, 201824, 27364, 235432, 49042, 26685, 3902, 153620, 235362, 53479, 235425, 145165, 36927, 11795, 235425, 237210, 82866, 6028, 235394, 26685, 237848, 58671, 32965, 7711, 236508, 235432, 83930, 5409, 235362, 116079, 96012, 146380, 3902, 50855, 72368, 235362, 78546, 235372, 63090, 238518, 7711, 44393, 235372, 174612, 235410, 53479, 235372, 130073, 9301, 235432, 90518, 22031, 235362, 235987, 23176, 151978, 219689, 236943, 14848, 62865, 235362, 81625, 1 ], "marker_positions": [ 11, 19 ], "probabilities": [ 0.7445823550224304, 0.2554176151752472 ], "act_probability": 1.0, "official_answer": { "type": "noul", "noul": 0.2554, "confidence": 0.7446, "action": { "act_probability": 1.0 } }, "temperature_bucket": "noul:2", "state_right_truncated": true, "oracle": { "marker_logits": [ 0.9034464359283447, -0.16647738218307495 ], "act_logits": [ 1203.7132568359375, -1432.9912109375 ] } } ] }