[ { "batch_size": 1, "seq_len": 64, "baseline": { "model_name": "HRM_Baseline", "batch_size": 1, "seq_len": 64, "hidden_size": 512, "H_cycles": 2, "L_cycles": 2, "H_layers": 4, "L_layers": 4, "num_iterations": 20, "warmup_iterations": 5, "l_level_latency_mean_us": 0, "l_level_latency_min_us": 0, "l_level_latency_max_us": 0, "l_level_latency_std_us": 0, "h_level_latency_mean_us": 0, "h_level_latency_min_us": 0, "h_level_latency_max_us": 0, "h_level_latency_std_us": 0, "total_inference_latency_mean_ms": 12.233289670944213, "total_inference_latency_min_ms": 12.15283203125, "total_inference_latency_max_ms": 12.321791648864746, "total_inference_latency_std_ms": 0.05471266632900291, "throughput_samples_per_sec": 81.74416096556112, "sram_peak_mb": 0, "dram_peak_mb": 0, "total_gpu_memory_mb": 116.29345703125, "h_l_transfer_mean_us": 0, "l_h_transfer_mean_us": 0, "sram_hit_rate": 0, "triton_sram_probe_latency_us": 0, "triton_dram_probe_latency_us": 0, "gpu_utilization_pct": 23.0, "gpu_power_w": 62.07, "gpu_temperature_c": null, "h_over_l_latency_ratio": 0, "memory_efficiency": 1.0 }, "tiered": { "model_name": "HRM_Tiered", "batch_size": 1, "seq_len": 64, "hidden_size": 512, "H_cycles": 2, "L_cycles": 2, "H_layers": 4, "L_layers": 4, "num_iterations": 20, "warmup_iterations": 5, "l_level_latency_mean_us": 1864.8816029230754, "l_level_latency_min_us": 1843.9680337905884, "l_level_latency_max_us": 1886.2080574035645, "l_level_latency_std_us": 0.0, "h_level_latency_mean_us": 1861.43679022789, "h_level_latency_min_us": 1850.3680229187012, "h_level_latency_max_us": 1875.9679794311523, "h_level_latency_std_us": 0.0, "total_inference_latency_mean_ms": 11.600086498260499, "total_inference_latency_min_ms": 11.553088188171387, "total_inference_latency_max_ms": 11.630592346191406, "total_inference_latency_std_ms": 0.022018860122281114, "throughput_samples_per_sec": 86.20625373353516, "sram_peak_mb": 0.0, "dram_peak_mb": 0.0, "total_gpu_memory_mb": 132.66845703125, "h_l_transfer_mean_us": 0, "l_h_transfer_mean_us": 9.9823999684304, "sram_hit_rate": 0.0, "triton_sram_probe_latency_us": 820.2239871025085, "triton_dram_probe_latency_us": 53.247999399900436, "gpu_utilization_pct": 19.0, "gpu_power_w": 90.58, "gpu_temperature_c": null, "h_over_l_latency_ratio": 0.9981527981777578, "memory_efficiency": 0.9986629635146693 }, "speedup": 1.0545860733692518, "memory_savings_mb": -16.375, "throughput_improvement": 1.0545860733692518 }, { "batch_size": 1, "seq_len": 128, "baseline": { "model_name": "HRM_Baseline", "batch_size": 1, "seq_len": 128, "hidden_size": 512, "H_cycles": 2, "L_cycles": 2, "H_layers": 4, "L_layers": 4, "num_iterations": 20, "warmup_iterations": 5, "l_level_latency_mean_us": 0, "l_level_latency_min_us": 0, "l_level_latency_max_us": 0, "l_level_latency_std_us": 0, "h_level_latency_mean_us": 0, "h_level_latency_min_us": 0, "h_level_latency_max_us": 0, "h_level_latency_std_us": 0, "total_inference_latency_mean_ms": 12.75871524810791, "total_inference_latency_min_ms": 12.291104316711426, "total_inference_latency_max_ms": 13.331456184387207, "total_inference_latency_std_ms": 0.491103465039144, "throughput_samples_per_sec": 78.37779749401476, "sram_peak_mb": 0, "dram_peak_mb": 0, "total_gpu_memory_mb": 133.57763671875, "h_l_transfer_mean_us": 0, "l_h_transfer_mean_us": 0, "sram_hit_rate": 0, "triton_sram_probe_latency_us": 0, "triton_dram_probe_latency_us": 0, "gpu_utilization_pct": 22.0, "gpu_power_w": 91.94, "gpu_temperature_c": null, "h_over_l_latency_ratio": 0, "memory_efficiency": 1.0 }, "tiered": { "model_name": "HRM_Tiered", "batch_size": 1, "seq_len": 128, "hidden_size": 512, "H_cycles": 2, "L_cycles": 2, "H_layers": 4, "L_layers": 4, "num_iterations": 20, "warmup_iterations": 5, "l_level_latency_mean_us": 1880.560537179311, "l_level_latency_min_us": 1853.6959886550903, "l_level_latency_max_us": 1905.6639671325684, "l_level_latency_std_us": 0.0, "h_level_latency_mean_us": 1872.9376077651978, "h_level_latency_min_us": 1849.2159843444824, "h_level_latency_max_us": 1890.3039693832397, "h_level_latency_std_us": 0.0, "total_inference_latency_mean_ms": 11.684232091903686, "total_inference_latency_min_ms": 11.551487922668457, "total_inference_latency_max_ms": 11.770879745483398, "total_inference_latency_std_ms": 0.06819341259194193, "throughput_samples_per_sec": 85.58542762026497, "sram_peak_mb": 0.0, "dram_peak_mb": 0.0, "total_gpu_memory_mb": 150.07763671875, "h_l_transfer_mean_us": 0, "l_h_transfer_mean_us": 10.091200051829219, "sram_hit_rate": 0.0, "triton_sram_probe_latency_us": 47.10400104522705, "triton_dram_probe_latency_us": 38.94399851560593, "gpu_utilization_pct": 14.0, "gpu_power_w": 95.82, "gpu_temperature_c": null, "h_over_l_latency_ratio": 0.9959464589076472, "memory_efficiency": 0.9986589251294146 }, "speedup": 1.0919601003945105, "memory_savings_mb": -16.5, "throughput_improvement": 1.0919601003945105 }, { "batch_size": 8, "seq_len": 64, "baseline": { "model_name": "HRM_Baseline", "batch_size": 8, "seq_len": 64, "hidden_size": 512, "H_cycles": 2, "L_cycles": 2, "H_layers": 4, "L_layers": 4, "num_iterations": 20, "warmup_iterations": 5, "l_level_latency_mean_us": 0, "l_level_latency_min_us": 0, "l_level_latency_max_us": 0, "l_level_latency_std_us": 0, "h_level_latency_mean_us": 0, "h_level_latency_min_us": 0, "h_level_latency_max_us": 0, "h_level_latency_std_us": 0, "total_inference_latency_mean_ms": 12.226089572906494, "total_inference_latency_min_ms": 12.181280136108398, "total_inference_latency_max_ms": 12.269696235656738, "total_inference_latency_std_ms": 0.02198156350662233, "throughput_samples_per_sec": 654.3384090468568, "sram_peak_mb": 0, "dram_peak_mb": 0, "total_gpu_memory_mb": 158.31396484375, "h_l_transfer_mean_us": 0, "l_h_transfer_mean_us": 0, "sram_hit_rate": 0, "triton_sram_probe_latency_us": 0, "triton_dram_probe_latency_us": 0, "gpu_utilization_pct": 25.0, "gpu_power_w": 100.16, "gpu_temperature_c": null, "h_over_l_latency_ratio": 0, "memory_efficiency": 1.0 }, "tiered": { "model_name": "HRM_Tiered", "batch_size": 8, "seq_len": 64, "hidden_size": 512, "H_cycles": 2, "L_cycles": 2, "H_layers": 4, "L_layers": 4, "num_iterations": 20, "warmup_iterations": 5, "l_level_latency_mean_us": 1842.8634683291118, "l_level_latency_min_us": 1817.6000118255615, "l_level_latency_max_us": 1919.103980064392, "l_level_latency_std_us": 0.0, "h_level_latency_mean_us": 1826.6352117061615, "h_level_latency_min_us": 1815.551996231079, "h_level_latency_max_us": 1835.8080387115479, "h_level_latency_std_us": 0.0, "total_inference_latency_mean_ms": 11.43253436088562, "total_inference_latency_min_ms": 11.369471549987793, "total_inference_latency_max_ms": 11.523072242736816, "total_inference_latency_std_ms": 0.04659291036998755, "throughput_samples_per_sec": 699.7573545346668, "sram_peak_mb": 0.0, "dram_peak_mb": 0.0, "total_gpu_memory_mb": 175.56396484375, "h_l_transfer_mean_us": 0, "l_h_transfer_mean_us": 9.712000098079443, "sram_hit_rate": 0.0, "triton_sram_probe_latency_us": 46.08000069856644, "triton_dram_probe_latency_us": 40.95999896526337, "gpu_utilization_pct": 26.0, "gpu_power_w": 104.38, "gpu_temperature_c": null, "h_over_l_latency_ratio": 0.9911939995003188, "memory_efficiency": 0.9986813194349485 }, "speedup": 1.0694120119801154, "memory_savings_mb": -17.25, "throughput_improvement": 1.0694120119801154 }, { "batch_size": 8, "seq_len": 128, "baseline": { "model_name": "HRM_Baseline", "batch_size": 8, "seq_len": 128, "hidden_size": 512, "H_cycles": 2, "L_cycles": 2, "H_layers": 4, "L_layers": 4, "num_iterations": 20, "warmup_iterations": 5, "l_level_latency_mean_us": 0, "l_level_latency_min_us": 0, "l_level_latency_max_us": 0, "l_level_latency_std_us": 0, "h_level_latency_mean_us": 0, "h_level_latency_min_us": 0, "h_level_latency_max_us": 0, "h_level_latency_std_us": 0, "total_inference_latency_mean_ms": 12.09585280418396, "total_inference_latency_min_ms": 12.034048080444336, "total_inference_latency_max_ms": 12.1693115234375, "total_inference_latency_std_ms": 0.03280009762608858, "throughput_samples_per_sec": 661.3837097317187, "sram_peak_mb": 0, "dram_peak_mb": 0, "total_gpu_memory_mb": 186.99365234375, "h_l_transfer_mean_us": 0, "l_h_transfer_mean_us": 0, "sram_hit_rate": 0, "triton_sram_probe_latency_us": 0, "triton_dram_probe_latency_us": 0, "gpu_utilization_pct": 37.0, "gpu_power_w": 112.76, "gpu_temperature_c": null, "h_over_l_latency_ratio": 0, "memory_efficiency": 1.0 }, "tiered": { "model_name": "HRM_Tiered", "batch_size": 8, "seq_len": 128, "hidden_size": 512, "H_cycles": 2, "L_cycles": 2, "H_layers": 4, "L_layers": 4, "num_iterations": 20, "warmup_iterations": 5, "l_level_latency_mean_us": 1827.0586788654327, "l_level_latency_min_us": 1786.8800163269043, "l_level_latency_max_us": 1932.2880506515503, "l_level_latency_std_us": 0.0, "h_level_latency_mean_us": 1811.8864059448242, "h_level_latency_min_us": 1794.7200536727905, "h_level_latency_max_us": 1830.7520151138306, "h_level_latency_std_us": 0.0, "total_inference_latency_mean_ms": 11.355580854415894, "total_inference_latency_min_ms": 11.258879661560059, "total_inference_latency_max_ms": 11.465632438659668, "total_inference_latency_std_ms": 0.05902572924729382, "throughput_samples_per_sec": 704.4994089306321, "sram_peak_mb": 0.0, "dram_peak_mb": 0.0, "total_gpu_memory_mb": 204.24365234375, "h_l_transfer_mean_us": 0, "l_h_transfer_mean_us": 9.579200064763427, "sram_hit_rate": 0.0, "triton_sram_probe_latency_us": 47.16800153255463, "triton_dram_probe_latency_us": 39.93599861860275, "gpu_utilization_pct": 33.0, "gpu_power_w": 118.67, "gpu_temperature_c": null, "h_over_l_latency_ratio": 0.9916957933009408, "memory_efficiency": 0.9986882554925227 }, "speedup": 1.0651901438824412, "memory_savings_mb": -17.25, "throughput_improvement": 1.0651901438824412 }, { "batch_size": 32, "seq_len": 64, "baseline": { "model_name": "HRM_Baseline", "batch_size": 32, "seq_len": 64, "hidden_size": 512, "H_cycles": 2, "L_cycles": 2, "H_layers": 4, "L_layers": 4, "num_iterations": 20, "warmup_iterations": 5, "l_level_latency_mean_us": 0, "l_level_latency_min_us": 0, "l_level_latency_max_us": 0, "l_level_latency_std_us": 0, "h_level_latency_mean_us": 0, "h_level_latency_min_us": 0, "h_level_latency_max_us": 0, "h_level_latency_std_us": 0, "total_inference_latency_mean_ms": 12.013897466659547, "total_inference_latency_min_ms": 11.935744285583496, "total_inference_latency_max_ms": 12.108991622924805, "total_inference_latency_std_ms": 0.04052232974770088, "throughput_samples_per_sec": 2663.581913263787, "sram_peak_mb": 0, "dram_peak_mb": 0, "total_gpu_memory_mb": 228.25927734375, "h_l_transfer_mean_us": 0, "l_h_transfer_mean_us": 0, "sram_hit_rate": 0, "triton_sram_probe_latency_us": 0, "triton_dram_probe_latency_us": 0, "gpu_utilization_pct": 50.0, "gpu_power_w": 137.35, "gpu_temperature_c": null, "h_over_l_latency_ratio": 0, "memory_efficiency": 1.0 }, "tiered": { "model_name": "HRM_Tiered", "batch_size": 32, "seq_len": 64, "hidden_size": 512, "H_cycles": 2, "L_cycles": 2, "H_layers": 4, "L_layers": 4, "num_iterations": 20, "warmup_iterations": 5, "l_level_latency_mean_us": 1848.9973326524098, "l_level_latency_min_us": 1829.6960592269897, "l_level_latency_max_us": 1885.4399919509888, "l_level_latency_std_us": 0.0, "h_level_latency_mean_us": 1829.4048011302948, "h_level_latency_min_us": 1820.9600448608398, "h_level_latency_max_us": 1836.0320329666138, "h_level_latency_std_us": 0.0, "total_inference_latency_mean_ms": 11.482071924209595, "total_inference_latency_min_ms": 11.442208290100098, "total_inference_latency_max_ms": 11.52511978149414, "total_inference_latency_std_ms": 0.022921052595205035, "throughput_samples_per_sec": 2786.953453281283, "sram_peak_mb": 0.0, "dram_peak_mb": 0.0, "total_gpu_memory_mb": 247.38427734375, "h_l_transfer_mean_us": 0, "l_h_transfer_mean_us": 9.636800037696958, "sram_hit_rate": 0.0, "triton_sram_probe_latency_us": 46.08000069856644, "triton_dram_probe_latency_us": 40.95999896526337, "gpu_utilization_pct": 45.0, "gpu_power_w": 162.51, "gpu_temperature_c": null, "h_over_l_latency_ratio": 0.9894036994125842, "memory_efficiency": 0.9986952672353347 }, "speedup": 1.046317907252315, "memory_savings_mb": -19.125, "throughput_improvement": 1.046317907252315 }, { "batch_size": 32, "seq_len": 128, "baseline": { "model_name": "HRM_Baseline", "batch_size": 32, "seq_len": 128, "hidden_size": 512, "H_cycles": 2, "L_cycles": 2, "H_layers": 4, "L_layers": 4, "num_iterations": 20, "warmup_iterations": 5, "l_level_latency_mean_us": 0, "l_level_latency_min_us": 0, "l_level_latency_max_us": 0, "l_level_latency_std_us": 0, "h_level_latency_mean_us": 0, "h_level_latency_min_us": 0, "h_level_latency_max_us": 0, "h_level_latency_std_us": 0, "total_inference_latency_mean_ms": 12.027006340026855, "total_inference_latency_min_ms": 11.971232414245605, "total_inference_latency_max_ms": 12.089983940124512, "total_inference_latency_std_ms": 0.02667638763107568, "throughput_samples_per_sec": 2660.678733784433, "sram_peak_mb": 0, "dram_peak_mb": 0, "total_gpu_memory_mb": 292.13427734375, "h_l_transfer_mean_us": 0, "l_h_transfer_mean_us": 0, "sram_hit_rate": 0, "triton_sram_probe_latency_us": 0, "triton_dram_probe_latency_us": 0, "gpu_utilization_pct": 84.0, "gpu_power_w": 210.32, "gpu_temperature_c": null, "h_over_l_latency_ratio": 0, "memory_efficiency": 1.0 }, "tiered": { "model_name": "HRM_Tiered", "batch_size": 32, "seq_len": 128, "hidden_size": 512, "H_cycles": 2, "L_cycles": 2, "H_layers": 4, "L_layers": 4, "num_iterations": 20, "warmup_iterations": 5, "l_level_latency_mean_us": 1828.7717362244923, "l_level_latency_min_us": 1806.175947189331, "l_level_latency_max_us": 1862.6240491867065, "l_level_latency_std_us": 0.0, "h_level_latency_mean_us": 1824.127995967865, "h_level_latency_min_us": 1806.3360452651978, "h_level_latency_max_us": 1875.7760524749756, "h_level_latency_std_us": 0.0, "total_inference_latency_mean_ms": 11.574174451828004, "total_inference_latency_min_ms": 11.528096199035645, "total_inference_latency_max_ms": 11.638784408569336, "total_inference_latency_std_ms": 0.03262226881630495, "throughput_samples_per_sec": 2764.7760220985765, "sram_peak_mb": 0.0, "dram_peak_mb": 0.0, "total_gpu_memory_mb": 312.38427734375, "h_l_transfer_mean_us": 0, "l_h_transfer_mean_us": 9.51200001873076, "sram_hit_rate": 0.0, "triton_sram_probe_latency_us": 49.15200173854828, "triton_dram_probe_latency_us": 40.70400074124336, "gpu_utilization_pct": 78.0, "gpu_power_w": 245.59, "gpu_temperature_c": null, "h_over_l_latency_ratio": 0.9974607327067432, "memory_efficiency": 0.9987005384933674 }, "speedup": 1.0391243358291815, "memory_savings_mb": -20.25, "throughput_improvement": 1.0391243358291815 } ]