Next-Passage / evaluation.json
Lamapi's picture
Publish Next-Passage model
051c46f verified
Raw History Blame Contribute Delete
10.9 kB
{
"project": "Next Passage",
"profile": "full",
"seed": 73,
"device": {
"python": "3.13.16",
"platform": "Linux-6.6.159+-x86_64-with-glibc2.39",
"cpu_count": 12,
"numpy": "2.4.6",
"torch": "2.14.1+cu130",
"cuda": true,
"gpu": "NVIDIA L4",
"colab": true
},
"data": {
"type": "simulation",
"note": "Worlds generated by passage_core.make_world(seed); no external data.",
"test_seeds": [
20000,
20099
],
"densities": [
"low",
"medium",
"high"
],
"stall_probabilities": [
0.1,
0.25
],
"development_seeds": "1-999 (used while writing the code, not in the benchmark)"
},
"results": {
"overall": {
"timed": {
"episodes": 600,
"arrival_rate": 1.0,
"collision_episode_rate": 0.0,
"collisions_per_episode": 0.0,
"door_hits_per_episode": 0.0,
"unsafe_episode_rate": 0.0,
"arrival_time_mean": 30.821666666666665,
"arrival_over_static_length": 1.1283655425180663,
"replans_per_episode": 19.506666666666668,
"nodes_per_episode": 1437.6466666666668,
"ms_per_episode_python": 10.851596298343793
},
"reactive": {
"episodes": 600,
"arrival_rate": 1.0,
"collision_episode_rate": 0.5,
"collisions_per_episode": 0.9183333333333333,
"door_hits_per_episode": 0.17333333333333334,
"unsafe_episode_rate": 0.5916666666666667,
"arrival_time_mean": 34.32,
"arrival_over_static_length": 1.2595982537349353,
"replans_per_episode": 1.68,
"nodes_per_episode": 433.9866666666667,
"ms_per_episode_python": 1.3411656633365965
},
"static": {
"episodes": 600,
"arrival_rate": 1.0,
"collision_episode_rate": 0.6133333333333333,
"collisions_per_episode": 1.2216666666666667,
"door_hits_per_episode": 0.19166666666666668,
"unsafe_episode_rate": 0.7,
"arrival_time_mean": 30.113333333333333,
"arrival_over_static_length": 1.1001174710211388,
"replans_per_episode": 0.0,
"nodes_per_episode": 194.46,
"ms_per_episode_python": 0.9334195850048369
}
},
"low/stall=0.1": {
"timed": {
"episodes": 100,
"arrival_rate": 1.0,
"collision_episode_rate": 0.0,
"collisions_per_episode": 0.0,
"door_hits_per_episode": 0.0,
"unsafe_episode_rate": 0.0,
"arrival_time_mean": 29.82,
"arrival_over_static_length": 1.0895462809004304,
"replans_per_episode": 8.38,
"nodes_per_episode": 610.55,
"ms_per_episode_python": 4.061648179981603
},
"reactive": {
"episodes": 100,
"arrival_rate": 1.0,
"collision_episode_rate": 0.24,
"collisions_per_episode": 0.43,
"door_hits_per_episode": 0.17,
"unsafe_episode_rate": 0.36,
"arrival_time_mean": 33.61,
"arrival_over_static_length": 1.233392826034949,
"replans_per_episode": 1.3,
"nodes_per_episode": 386.17,
"ms_per_episode_python": 1.0644452600081422
},
"static": {
"episodes": 100,
"arrival_rate": 1.0,
"collision_episode_rate": 0.39,
"collisions_per_episode": 0.61,
"door_hits_per_episode": 0.18,
"unsafe_episode_rate": 0.53,
"arrival_time_mean": 29.56,
"arrival_over_static_length": 1.0800256884070292,
"replans_per_episode": 0.0,
"nodes_per_episode": 194.46,
"ms_per_episode_python": 0.754936149983223
}
},
"low/stall=0.25": {
"timed": {
"episodes": 100,
"arrival_rate": 1.0,
"collision_episode_rate": 0.0,
"collisions_per_episode": 0.0,
"door_hits_per_episode": 0.0,
"unsafe_episode_rate": 0.0,
"arrival_time_mean": 29.87,
"arrival_over_static_length": 1.0914604301164639,
"replans_per_episode": 17.37,
"nodes_per_episode": 934.54,
"ms_per_episode_python": 6.63758367999435
},
"reactive": {
"episodes": 100,
"arrival_rate": 1.0,
"collision_episode_rate": 0.27,
"collisions_per_episode": 0.4,
"door_hits_per_episode": 0.16,
"unsafe_episode_rate": 0.41,
"arrival_time_mean": 33.27,
"arrival_over_static_length": 1.2239181965181254,
"replans_per_episode": 1.02,
"nodes_per_episode": 356.71,
"ms_per_episode_python": 1.0321695299853673
},
"static": {
"episodes": 100,
"arrival_rate": 1.0,
"collision_episode_rate": 0.36,
"collisions_per_episode": 0.52,
"door_hits_per_episode": 0.17,
"unsafe_episode_rate": 0.48,
"arrival_time_mean": 29.56,
"arrival_over_static_length": 1.079774069752129,
"replans_per_episode": 0.0,
"nodes_per_episode": 194.46,
"ms_per_episode_python": 0.7569391700008055
}
},
"medium/stall=0.1": {
"timed": {
"episodes": 100,
"arrival_rate": 1.0,
"collision_episode_rate": 0.0,
"collisions_per_episode": 0.0,
"door_hits_per_episode": 0.0,
"unsafe_episode_rate": 0.0,
"arrival_time_mean": 30.47,
"arrival_over_static_length": 1.114301031781453,
"replans_per_episode": 14.49,
"nodes_per_episode": 1082.31,
"ms_per_episode_python": 7.966914250005175
},
"reactive": {
"episodes": 100,
"arrival_rate": 1.0,
"collision_episode_rate": 0.53,
"collisions_per_episode": 1.01,
"door_hits_per_episode": 0.2,
"unsafe_episode_rate": 0.64,
"arrival_time_mean": 34.09,
"arrival_over_static_length": 1.2461882677597556,
"replans_per_episode": 1.5,
"nodes_per_episode": 430.99,
"ms_per_episode_python": 1.324342320026517
},
"static": {
"episodes": 100,
"arrival_rate": 1.0,
"collision_episode_rate": 0.66,
"collisions_per_episode": 1.42,
"door_hits_per_episode": 0.24,
"unsafe_episode_rate": 0.74,
"arrival_time_mean": 29.91,
"arrival_over_static_length": 1.0910707461530154,
"replans_per_episode": 0.0,
"nodes_per_episode": 194.46,
"ms_per_episode_python": 0.9206604100108962
}
},
"medium/stall=0.25": {
"timed": {
"episodes": 100,
"arrival_rate": 1.0,
"collision_episode_rate": 0.0,
"collisions_per_episode": 0.0,
"door_hits_per_episode": 0.0,
"unsafe_episode_rate": 0.0,
"arrival_time_mean": 30.87,
"arrival_over_static_length": 1.1302208142868928,
"replans_per_episode": 25.3,
"nodes_per_episode": 1593.24,
"ms_per_episode_python": 12.074100600034399
},
"reactive": {
"episodes": 100,
"arrival_rate": 1.0,
"collision_episode_rate": 0.53,
"collisions_per_episode": 0.94,
"door_hits_per_episode": 0.16,
"unsafe_episode_rate": 0.61,
"arrival_time_mean": 34.73,
"arrival_over_static_length": 1.2771276768999125,
"replans_per_episode": 1.75,
"nodes_per_episode": 437.83,
"ms_per_episode_python": 1.3322608600083186
},
"static": {
"episodes": 100,
"arrival_rate": 1.0,
"collision_episode_rate": 0.65,
"collisions_per_episode": 1.28,
"door_hits_per_episode": 0.18,
"unsafe_episode_rate": 0.72,
"arrival_time_mean": 30.26,
"arrival_over_static_length": 1.105225036931732,
"replans_per_episode": 0.0,
"nodes_per_episode": 194.46,
"ms_per_episode_python": 0.9200024500296422
}
},
"high/stall=0.1": {
"timed": {
"episodes": 100,
"arrival_rate": 1.0,
"collision_episode_rate": 0.0,
"collisions_per_episode": 0.0,
"door_hits_per_episode": 0.0,
"unsafe_episode_rate": 0.0,
"arrival_time_mean": 31.69,
"arrival_over_static_length": 1.1632242208869854,
"replans_per_episode": 20.94,
"nodes_per_episode": 1917.9,
"ms_per_episode_python": 14.807957160005571
},
"reactive": {
"episodes": 100,
"arrival_rate": 1.0,
"collision_episode_rate": 0.7,
"collisions_per_episode": 1.38,
"door_hits_per_episode": 0.16,
"unsafe_episode_rate": 0.76,
"arrival_time_mean": 33.84,
"arrival_over_static_length": 1.240357567893443,
"replans_per_episode": 1.9,
"nodes_per_episode": 448.13,
"ms_per_episode_python": 1.5566964800200367
},
"static": {
"episodes": 100,
"arrival_rate": 1.0,
"collision_episode_rate": 0.84,
"collisions_per_episode": 1.88,
"door_hits_per_episode": 0.2,
"unsafe_episode_rate": 0.88,
"arrival_time_mean": 30.33,
"arrival_over_static_length": 1.1080862929280573,
"replans_per_episode": 0.0,
"nodes_per_episode": 194.46,
"ms_per_episode_python": 1.1195999200072038
}
},
"high/stall=0.25": {
"timed": {
"episodes": 100,
"arrival_rate": 1.0,
"collision_episode_rate": 0.0,
"collisions_per_episode": 0.0,
"door_hits_per_episode": 0.0,
"unsafe_episode_rate": 0.0,
"arrival_time_mean": 32.21,
"arrival_over_static_length": 1.1814404771361715,
"replans_per_episode": 30.56,
"nodes_per_episode": 2487.34,
"ms_per_episode_python": 19.56137392004166
},
"reactive": {
"episodes": 100,
"arrival_rate": 1.0,
"collision_episode_rate": 0.73,
"collisions_per_episode": 1.35,
"door_hits_per_episode": 0.19,
"unsafe_episode_rate": 0.77,
"arrival_time_mean": 36.38,
"arrival_over_static_length": 1.3366049873034263,
"replans_per_episode": 2.61,
"nodes_per_episode": 544.09,
"ms_per_episode_python": 1.7370795299711972
},
"static": {
"episodes": 100,
"arrival_rate": 1.0,
"collision_episode_rate": 0.78,
"collisions_per_episode": 1.62,
"door_hits_per_episode": 0.18,
"unsafe_episode_rate": 0.85,
"arrival_time_mean": 31.06,
"arrival_over_static_length": 1.136522991954869,
"replans_per_episode": 0.0,
"nodes_per_episode": 194.46,
"ms_per_episode_python": 1.128379409997251
}
}
},
"paired_arrival_time": {
"episodes_all_arrived": 600,
"arrival_time_timed": 30.821666666666665,
"arrival_time_reactive": 34.32,
"arrival_time_static": 30.113333333333333
},
"failures_time_aware": [],
"n_failures_time_aware": 0,
"media": {
"file": "replay.mp4",
"frames": 92,
"fps": 6,
"bytes": 99705,
"segments": [
"seed 20000",
"static planner collision: seed 20001 low stall 0.1"
]
},
"tests": {
"reload": {
"returncode": 0,
"seconds": 0.28,
"process": "fresh python -E subprocess, cwd = temporary copy of the model folder",
"passed": true,
"cases": 6,
"mismatches": []
},
"node_parity": {
"cases": 6,
"failed": 0,
"maxDiff": 0,
"results": [
{
"name": "20001-high-timed",
"maxDiff": 0
},
{
"name": "20001-high-static",
"maxDiff": 0
},
{
"name": "20002-medium-reactive",
"maxDiff": 0
},
{
"name": "20003-low-timed",
"maxDiff": 0
},
{
"name": "20004-high-timed",
"maxDiff": 0
},
{
"name": "31337-medium-timed",
"maxDiff": 0
}
],
"ms": 55,
"runtime": "node v22.23.4",
"status": "passed"
},
"browser": {
"status": "skipped",
"reason": "playwright not installed"
}
},
"notes": [
"ms_per_episode is Python wall time on the recorded device; nodes_expanded is implementation-independent."
]
}