LoveMachine / evaluation.json
Koppen9029's picture
Publish LoveMachine research preview
05fbb7d verified
Raw History Blame Contribute Delete
7.39 kB
{
"matched_development": {
"episodes": 224,
"reuse": 195,
"acquire": 203,
"wait": 224,
"all_three": 176,
"matched_reference": 212
},
"controlled_real_recordings": {
"episodes": 8,
"correct_known_reference": 7,
"environment_responses": 8,
"verified_new_reference_selected": 8,
"after_response_names_expected_source": 7,
"correct_quiet": 8,
"bounded_action_emissions": 8,
"bounded_verified_references": 8,
"bounded_expected_names": 7
},
"paced_drum": {
"assembled_from_real_recordings": true,
"same_input_through_request": true,
"trials": [
{
"condition": "known",
"observed_response": false,
"verified_response": false,
"updates_during_generation": 13,
"utterances": [
{
"kind": "reuse",
"issued": 3.400358700000652,
"completed": 4.1001500000002125,
"text": "ไปŠใฎๆ˜ ๅƒใจ้Ÿณๅฃฐใง็ขบใ‹ใ‚ใ‚‰ใ‚Œใ‚‹้Ÿณๆบใฏใ€ใƒ‰ใƒฉใƒ ใฎ้Ÿณใงใ™ใ€‚"
}
]
},
{
"condition": "available",
"observed_response": true,
"verified_response": true,
"updates_during_generation": 27,
"utterances": [
{
"kind": "request",
"issued": 3.3002079000007143,
"completed": 4.050127600000451,
"text": "ไปŠ่žใ“ใˆใŸ้Ÿณใฎๅ‡บใฉใ“ใ‚ใ‚’ใ€ใ‚‚ใ†ๅฐ‘ใ—่ฟ‘ใใง่ฆ‹ใ›ใฆใใ ใ•ใ„ใ€‚"
},
{
"kind": "report",
"issued": 4.85021020000022,
"completed": 5.5002346000001126,
"text": "ไปŠใฎๆ˜ ๅƒใจ้Ÿณๅฃฐใง็ขบใ‹ใ‚ใ‚‰ใ‚Œใ‚‹้Ÿณๆบใฏใ€ใƒ‰ใƒฉใƒ ใฎ้Ÿณใงใ™ใ€‚"
}
]
},
{
"condition": "withheld",
"observed_response": false,
"verified_response": false,
"updates_during_generation": 10,
"utterances": [
{
"kind": "request",
"issued": 3.2501595999992787,
"completed": 3.80025089999981,
"text": "ไปŠ่žใ“ใˆใŸ้Ÿณใฎๅ‡บใฉใ“ใ‚ใ‚’ใ€ใ‚‚ใ†ๅฐ‘ใ—่ฟ‘ใใง่ฆ‹ใ›ใฆใใ ใ•ใ„ใ€‚"
}
]
},
{
"condition": "quiet",
"observed_response": false,
"verified_response": false,
"updates_during_generation": 0,
"utterances": []
}
]
},
"unedited_video_diagnostic": {
"dataset": "OmniPro",
"clips": 3,
"reference_events": 8,
"utterances": 2,
"utterances_specifically_describing_reference_events": 0,
"task_prompt_given": false,
"official_benchmark_score": false
},
"policy_examples": [
{
"episode": 1,
"condition": "matching history",
"features": [
1.8710153102874756,
-2.1350953578948975,
4.006110668182373,
-4.516533851623535,
0.9821213483810425,
0.7896050214767456,
0.24186863005161285,
0.5825515985488892,
0.3594391345977783,
0.37160369753837585,
0.3306118845939636,
0.4621330499649048,
0.3594391345977783
],
"recorded_action": "reuse evidence"
},
{
"episode": 1,
"condition": "missing history",
"features": [
-2.1350953578948975,
-4.516534805297852,
2.381439447402954,
-4.516533851623535,
0.9154009819030762,
0.5303648114204407,
0.3351149260997772,
0.5215769410133362,
0.3594391345977783,
0.37160369753837585,
0.3306118845939636,
0.4621330499649048,
0.3594391345977783
],
"recorded_action": "seek evidence"
},
{
"episode": 1,
"condition": "stable continuation",
"features": [
-2.1350953578948975,
-4.516534805297852,
2.381439447402954,
5.733859062194824,
0.9154009819030762,
0.5303648114204407,
0.3351149260997772,
0.5215769410133362,
0.3594391345977783,
1.0000001192092896,
1.0,
0.3594391345977783,
0.3594391345977783
],
"recorded_action": "wait"
},
{
"episode": 2,
"condition": "matching history",
"features": [
1.3189809322357178,
-5.6348066329956055,
6.953787803649902,
-5.156879425048828,
0.9990459084510803,
0.7272514700889587,
0.37180954217910767,
0.5215769410133362,
0.476666659116745,
0.2951032519340515,
0.14561410248279572,
0.4621330499649048,
0.476666659116745
],
"recorded_action": "reuse evidence"
},
{
"episode": 2,
"condition": "missing history",
"features": [
-5.156879425048828,
-5.6348066329956055,
0.47792720794677734,
-5.156879425048828,
0.6172583103179932,
0.2951032519340515,
0.1456141173839569,
0.4621330499649048,
0.476666659116745,
0.2951032519340515,
0.14561410248279572,
0.4621330499649048,
0.476666659116745
],
"recorded_action": "seek evidence"
},
{
"episode": 2,
"condition": "stable continuation",
"features": [
-5.156879425048828,
-5.6348066329956055,
0.47792720794677734,
5.705358505249023,
0.6172583103179932,
0.2951032519340515,
0.1456141173839569,
0.4621330499649048,
0.476666659116745,
1.0,
1.0,
0.476666659116745,
0.476666659116745
],
"recorded_action": "wait"
},
{
"episode": 3,
"condition": "matching history",
"features": [
4.422921657562256,
-3.669260025024414,
8.092182159423828,
-7.837890625,
0.9996942281723022,
0.9421387910842896,
0.21267522871494293,
0.4158872067928314,
0.37058016657829285,
-0.0028071217238903046,
0.21487709879875183,
0.5053750276565552,
0.37058016657829285
],
"recorded_action": "reuse evidence"
},
{
"episode": 3,
"condition": "missing history",
"features": [
-3.669260025024414,
-7.837890625,
4.168630599975586,
-7.837890625,
0.9847623109817505,
0.0372462198138237,
0.020070835947990417,
0.5215769410133362,
0.37058016657829285,
-0.0028071217238903046,
0.21487709879875183,
0.5053750276565552,
0.37058016657829285
],
"recorded_action": "seek evidence"
},
{
"episode": 3,
"condition": "stable continuation",
"features": [
-3.669260025024414,
-7.837890625,
4.168630599975586,
5.740075588226318,
0.9847623109817505,
0.0372462198138237,
0.020070835947990417,
0.5215769410133362,
0.37058016657829285,
1.0,
1.0,
0.37058016657829285,
0.37058016657829285
],
"recorded_action": "wait"
}
]
}