| [ |
| { |
| "batch_size": 1, |
| "seq_len": 64, |
| "baseline": { |
| "model_name": "HRM_Baseline", |
| "batch_size": 1, |
| "seq_len": 64, |
| "hidden_size": 512, |
| "H_cycles": 2, |
| "L_cycles": 2, |
| "H_layers": 4, |
| "L_layers": 4, |
| "num_iterations": 20, |
| "warmup_iterations": 5, |
| "l_level_latency_mean_us": 0, |
| "l_level_latency_min_us": 0, |
| "l_level_latency_max_us": 0, |
| "l_level_latency_std_us": 0, |
| "h_level_latency_mean_us": 0, |
| "h_level_latency_min_us": 0, |
| "h_level_latency_max_us": 0, |
| "h_level_latency_std_us": 0, |
| "total_inference_latency_mean_ms": 12.233289670944213, |
| "total_inference_latency_min_ms": 12.15283203125, |
| "total_inference_latency_max_ms": 12.321791648864746, |
| "total_inference_latency_std_ms": 0.05471266632900291, |
| "throughput_samples_per_sec": 81.74416096556112, |
| "sram_peak_mb": 0, |
| "dram_peak_mb": 0, |
| "total_gpu_memory_mb": 116.29345703125, |
| "h_l_transfer_mean_us": 0, |
| "l_h_transfer_mean_us": 0, |
| "sram_hit_rate": 0, |
| "triton_sram_probe_latency_us": 0, |
| "triton_dram_probe_latency_us": 0, |
| "gpu_utilization_pct": 23.0, |
| "gpu_power_w": 62.07, |
| "gpu_temperature_c": null, |
| "h_over_l_latency_ratio": 0, |
| "memory_efficiency": 1.0 |
| }, |
| "tiered": { |
| "model_name": "HRM_Tiered", |
| "batch_size": 1, |
| "seq_len": 64, |
| "hidden_size": 512, |
| "H_cycles": 2, |
| "L_cycles": 2, |
| "H_layers": 4, |
| "L_layers": 4, |
| "num_iterations": 20, |
| "warmup_iterations": 5, |
| "l_level_latency_mean_us": 1864.8816029230754, |
| "l_level_latency_min_us": 1843.9680337905884, |
| "l_level_latency_max_us": 1886.2080574035645, |
| "l_level_latency_std_us": 0.0, |
| "h_level_latency_mean_us": 1861.43679022789, |
| "h_level_latency_min_us": 1850.3680229187012, |
| "h_level_latency_max_us": 1875.9679794311523, |
| "h_level_latency_std_us": 0.0, |
| "total_inference_latency_mean_ms": 11.600086498260499, |
| "total_inference_latency_min_ms": 11.553088188171387, |
| "total_inference_latency_max_ms": 11.630592346191406, |
| "total_inference_latency_std_ms": 0.022018860122281114, |
| "throughput_samples_per_sec": 86.20625373353516, |
| "sram_peak_mb": 0.0, |
| "dram_peak_mb": 0.0, |
| "total_gpu_memory_mb": 132.66845703125, |
| "h_l_transfer_mean_us": 0, |
| "l_h_transfer_mean_us": 9.9823999684304, |
| "sram_hit_rate": 0.0, |
| "triton_sram_probe_latency_us": 820.2239871025085, |
| "triton_dram_probe_latency_us": 53.247999399900436, |
| "gpu_utilization_pct": 19.0, |
| "gpu_power_w": 90.58, |
| "gpu_temperature_c": null, |
| "h_over_l_latency_ratio": 0.9981527981777578, |
| "memory_efficiency": 0.9986629635146693 |
| }, |
| "speedup": 1.0545860733692518, |
| "memory_savings_mb": -16.375, |
| "throughput_improvement": 1.0545860733692518 |
| }, |
| { |
| "batch_size": 1, |
| "seq_len": 128, |
| "baseline": { |
| "model_name": "HRM_Baseline", |
| "batch_size": 1, |
| "seq_len": 128, |
| "hidden_size": 512, |
| "H_cycles": 2, |
| "L_cycles": 2, |
| "H_layers": 4, |
| "L_layers": 4, |
| "num_iterations": 20, |
| "warmup_iterations": 5, |
| "l_level_latency_mean_us": 0, |
| "l_level_latency_min_us": 0, |
| "l_level_latency_max_us": 0, |
| "l_level_latency_std_us": 0, |
| "h_level_latency_mean_us": 0, |
| "h_level_latency_min_us": 0, |
| "h_level_latency_max_us": 0, |
| "h_level_latency_std_us": 0, |
| "total_inference_latency_mean_ms": 12.75871524810791, |
| "total_inference_latency_min_ms": 12.291104316711426, |
| "total_inference_latency_max_ms": 13.331456184387207, |
| "total_inference_latency_std_ms": 0.491103465039144, |
| "throughput_samples_per_sec": 78.37779749401476, |
| "sram_peak_mb": 0, |
| "dram_peak_mb": 0, |
| "total_gpu_memory_mb": 133.57763671875, |
| "h_l_transfer_mean_us": 0, |
| "l_h_transfer_mean_us": 0, |
| "sram_hit_rate": 0, |
| "triton_sram_probe_latency_us": 0, |
| "triton_dram_probe_latency_us": 0, |
| "gpu_utilization_pct": 22.0, |
| "gpu_power_w": 91.94, |
| "gpu_temperature_c": null, |
| "h_over_l_latency_ratio": 0, |
| "memory_efficiency": 1.0 |
| }, |
| "tiered": { |
| "model_name": "HRM_Tiered", |
| "batch_size": 1, |
| "seq_len": 128, |
| "hidden_size": 512, |
| "H_cycles": 2, |
| "L_cycles": 2, |
| "H_layers": 4, |
| "L_layers": 4, |
| "num_iterations": 20, |
| "warmup_iterations": 5, |
| "l_level_latency_mean_us": 1880.560537179311, |
| "l_level_latency_min_us": 1853.6959886550903, |
| "l_level_latency_max_us": 1905.6639671325684, |
| "l_level_latency_std_us": 0.0, |
| "h_level_latency_mean_us": 1872.9376077651978, |
| "h_level_latency_min_us": 1849.2159843444824, |
| "h_level_latency_max_us": 1890.3039693832397, |
| "h_level_latency_std_us": 0.0, |
| "total_inference_latency_mean_ms": 11.684232091903686, |
| "total_inference_latency_min_ms": 11.551487922668457, |
| "total_inference_latency_max_ms": 11.770879745483398, |
| "total_inference_latency_std_ms": 0.06819341259194193, |
| "throughput_samples_per_sec": 85.58542762026497, |
| "sram_peak_mb": 0.0, |
| "dram_peak_mb": 0.0, |
| "total_gpu_memory_mb": 150.07763671875, |
| "h_l_transfer_mean_us": 0, |
| "l_h_transfer_mean_us": 10.091200051829219, |
| "sram_hit_rate": 0.0, |
| "triton_sram_probe_latency_us": 47.10400104522705, |
| "triton_dram_probe_latency_us": 38.94399851560593, |
| "gpu_utilization_pct": 14.0, |
| "gpu_power_w": 95.82, |
| "gpu_temperature_c": null, |
| "h_over_l_latency_ratio": 0.9959464589076472, |
| "memory_efficiency": 0.9986589251294146 |
| }, |
| "speedup": 1.0919601003945105, |
| "memory_savings_mb": -16.5, |
| "throughput_improvement": 1.0919601003945105 |
| }, |
| { |
| "batch_size": 8, |
| "seq_len": 64, |
| "baseline": { |
| "model_name": "HRM_Baseline", |
| "batch_size": 8, |
| "seq_len": 64, |
| "hidden_size": 512, |
| "H_cycles": 2, |
| "L_cycles": 2, |
| "H_layers": 4, |
| "L_layers": 4, |
| "num_iterations": 20, |
| "warmup_iterations": 5, |
| "l_level_latency_mean_us": 0, |
| "l_level_latency_min_us": 0, |
| "l_level_latency_max_us": 0, |
| "l_level_latency_std_us": 0, |
| "h_level_latency_mean_us": 0, |
| "h_level_latency_min_us": 0, |
| "h_level_latency_max_us": 0, |
| "h_level_latency_std_us": 0, |
| "total_inference_latency_mean_ms": 12.226089572906494, |
| "total_inference_latency_min_ms": 12.181280136108398, |
| "total_inference_latency_max_ms": 12.269696235656738, |
| "total_inference_latency_std_ms": 0.02198156350662233, |
| "throughput_samples_per_sec": 654.3384090468568, |
| "sram_peak_mb": 0, |
| "dram_peak_mb": 0, |
| "total_gpu_memory_mb": 158.31396484375, |
| "h_l_transfer_mean_us": 0, |
| "l_h_transfer_mean_us": 0, |
| "sram_hit_rate": 0, |
| "triton_sram_probe_latency_us": 0, |
| "triton_dram_probe_latency_us": 0, |
| "gpu_utilization_pct": 25.0, |
| "gpu_power_w": 100.16, |
| "gpu_temperature_c": null, |
| "h_over_l_latency_ratio": 0, |
| "memory_efficiency": 1.0 |
| }, |
| "tiered": { |
| "model_name": "HRM_Tiered", |
| "batch_size": 8, |
| "seq_len": 64, |
| "hidden_size": 512, |
| "H_cycles": 2, |
| "L_cycles": 2, |
| "H_layers": 4, |
| "L_layers": 4, |
| "num_iterations": 20, |
| "warmup_iterations": 5, |
| "l_level_latency_mean_us": 1842.8634683291118, |
| "l_level_latency_min_us": 1817.6000118255615, |
| "l_level_latency_max_us": 1919.103980064392, |
| "l_level_latency_std_us": 0.0, |
| "h_level_latency_mean_us": 1826.6352117061615, |
| "h_level_latency_min_us": 1815.551996231079, |
| "h_level_latency_max_us": 1835.8080387115479, |
| "h_level_latency_std_us": 0.0, |
| "total_inference_latency_mean_ms": 11.43253436088562, |
| "total_inference_latency_min_ms": 11.369471549987793, |
| "total_inference_latency_max_ms": 11.523072242736816, |
| "total_inference_latency_std_ms": 0.04659291036998755, |
| "throughput_samples_per_sec": 699.7573545346668, |
| "sram_peak_mb": 0.0, |
| "dram_peak_mb": 0.0, |
| "total_gpu_memory_mb": 175.56396484375, |
| "h_l_transfer_mean_us": 0, |
| "l_h_transfer_mean_us": 9.712000098079443, |
| "sram_hit_rate": 0.0, |
| "triton_sram_probe_latency_us": 46.08000069856644, |
| "triton_dram_probe_latency_us": 40.95999896526337, |
| "gpu_utilization_pct": 26.0, |
| "gpu_power_w": 104.38, |
| "gpu_temperature_c": null, |
| "h_over_l_latency_ratio": 0.9911939995003188, |
| "memory_efficiency": 0.9986813194349485 |
| }, |
| "speedup": 1.0694120119801154, |
| "memory_savings_mb": -17.25, |
| "throughput_improvement": 1.0694120119801154 |
| }, |
| { |
| "batch_size": 8, |
| "seq_len": 128, |
| "baseline": { |
| "model_name": "HRM_Baseline", |
| "batch_size": 8, |
| "seq_len": 128, |
| "hidden_size": 512, |
| "H_cycles": 2, |
| "L_cycles": 2, |
| "H_layers": 4, |
| "L_layers": 4, |
| "num_iterations": 20, |
| "warmup_iterations": 5, |
| "l_level_latency_mean_us": 0, |
| "l_level_latency_min_us": 0, |
| "l_level_latency_max_us": 0, |
| "l_level_latency_std_us": 0, |
| "h_level_latency_mean_us": 0, |
| "h_level_latency_min_us": 0, |
| "h_level_latency_max_us": 0, |
| "h_level_latency_std_us": 0, |
| "total_inference_latency_mean_ms": 12.09585280418396, |
| "total_inference_latency_min_ms": 12.034048080444336, |
| "total_inference_latency_max_ms": 12.1693115234375, |
| "total_inference_latency_std_ms": 0.03280009762608858, |
| "throughput_samples_per_sec": 661.3837097317187, |
| "sram_peak_mb": 0, |
| "dram_peak_mb": 0, |
| "total_gpu_memory_mb": 186.99365234375, |
| "h_l_transfer_mean_us": 0, |
| "l_h_transfer_mean_us": 0, |
| "sram_hit_rate": 0, |
| "triton_sram_probe_latency_us": 0, |
| "triton_dram_probe_latency_us": 0, |
| "gpu_utilization_pct": 37.0, |
| "gpu_power_w": 112.76, |
| "gpu_temperature_c": null, |
| "h_over_l_latency_ratio": 0, |
| "memory_efficiency": 1.0 |
| }, |
| "tiered": { |
| "model_name": "HRM_Tiered", |
| "batch_size": 8, |
| "seq_len": 128, |
| "hidden_size": 512, |
| "H_cycles": 2, |
| "L_cycles": 2, |
| "H_layers": 4, |
| "L_layers": 4, |
| "num_iterations": 20, |
| "warmup_iterations": 5, |
| "l_level_latency_mean_us": 1827.0586788654327, |
| "l_level_latency_min_us": 1786.8800163269043, |
| "l_level_latency_max_us": 1932.2880506515503, |
| "l_level_latency_std_us": 0.0, |
| "h_level_latency_mean_us": 1811.8864059448242, |
| "h_level_latency_min_us": 1794.7200536727905, |
| "h_level_latency_max_us": 1830.7520151138306, |
| "h_level_latency_std_us": 0.0, |
| "total_inference_latency_mean_ms": 11.355580854415894, |
| "total_inference_latency_min_ms": 11.258879661560059, |
| "total_inference_latency_max_ms": 11.465632438659668, |
| "total_inference_latency_std_ms": 0.05902572924729382, |
| "throughput_samples_per_sec": 704.4994089306321, |
| "sram_peak_mb": 0.0, |
| "dram_peak_mb": 0.0, |
| "total_gpu_memory_mb": 204.24365234375, |
| "h_l_transfer_mean_us": 0, |
| "l_h_transfer_mean_us": 9.579200064763427, |
| "sram_hit_rate": 0.0, |
| "triton_sram_probe_latency_us": 47.16800153255463, |
| "triton_dram_probe_latency_us": 39.93599861860275, |
| "gpu_utilization_pct": 33.0, |
| "gpu_power_w": 118.67, |
| "gpu_temperature_c": null, |
| "h_over_l_latency_ratio": 0.9916957933009408, |
| "memory_efficiency": 0.9986882554925227 |
| }, |
| "speedup": 1.0651901438824412, |
| "memory_savings_mb": -17.25, |
| "throughput_improvement": 1.0651901438824412 |
| }, |
| { |
| "batch_size": 32, |
| "seq_len": 64, |
| "baseline": { |
| "model_name": "HRM_Baseline", |
| "batch_size": 32, |
| "seq_len": 64, |
| "hidden_size": 512, |
| "H_cycles": 2, |
| "L_cycles": 2, |
| "H_layers": 4, |
| "L_layers": 4, |
| "num_iterations": 20, |
| "warmup_iterations": 5, |
| "l_level_latency_mean_us": 0, |
| "l_level_latency_min_us": 0, |
| "l_level_latency_max_us": 0, |
| "l_level_latency_std_us": 0, |
| "h_level_latency_mean_us": 0, |
| "h_level_latency_min_us": 0, |
| "h_level_latency_max_us": 0, |
| "h_level_latency_std_us": 0, |
| "total_inference_latency_mean_ms": 12.013897466659547, |
| "total_inference_latency_min_ms": 11.935744285583496, |
| "total_inference_latency_max_ms": 12.108991622924805, |
| "total_inference_latency_std_ms": 0.04052232974770088, |
| "throughput_samples_per_sec": 2663.581913263787, |
| "sram_peak_mb": 0, |
| "dram_peak_mb": 0, |
| "total_gpu_memory_mb": 228.25927734375, |
| "h_l_transfer_mean_us": 0, |
| "l_h_transfer_mean_us": 0, |
| "sram_hit_rate": 0, |
| "triton_sram_probe_latency_us": 0, |
| "triton_dram_probe_latency_us": 0, |
| "gpu_utilization_pct": 50.0, |
| "gpu_power_w": 137.35, |
| "gpu_temperature_c": null, |
| "h_over_l_latency_ratio": 0, |
| "memory_efficiency": 1.0 |
| }, |
| "tiered": { |
| "model_name": "HRM_Tiered", |
| "batch_size": 32, |
| "seq_len": 64, |
| "hidden_size": 512, |
| "H_cycles": 2, |
| "L_cycles": 2, |
| "H_layers": 4, |
| "L_layers": 4, |
| "num_iterations": 20, |
| "warmup_iterations": 5, |
| "l_level_latency_mean_us": 1848.9973326524098, |
| "l_level_latency_min_us": 1829.6960592269897, |
| "l_level_latency_max_us": 1885.4399919509888, |
| "l_level_latency_std_us": 0.0, |
| "h_level_latency_mean_us": 1829.4048011302948, |
| "h_level_latency_min_us": 1820.9600448608398, |
| "h_level_latency_max_us": 1836.0320329666138, |
| "h_level_latency_std_us": 0.0, |
| "total_inference_latency_mean_ms": 11.482071924209595, |
| "total_inference_latency_min_ms": 11.442208290100098, |
| "total_inference_latency_max_ms": 11.52511978149414, |
| "total_inference_latency_std_ms": 0.022921052595205035, |
| "throughput_samples_per_sec": 2786.953453281283, |
| "sram_peak_mb": 0.0, |
| "dram_peak_mb": 0.0, |
| "total_gpu_memory_mb": 247.38427734375, |
| "h_l_transfer_mean_us": 0, |
| "l_h_transfer_mean_us": 9.636800037696958, |
| "sram_hit_rate": 0.0, |
| "triton_sram_probe_latency_us": 46.08000069856644, |
| "triton_dram_probe_latency_us": 40.95999896526337, |
| "gpu_utilization_pct": 45.0, |
| "gpu_power_w": 162.51, |
| "gpu_temperature_c": null, |
| "h_over_l_latency_ratio": 0.9894036994125842, |
| "memory_efficiency": 0.9986952672353347 |
| }, |
| "speedup": 1.046317907252315, |
| "memory_savings_mb": -19.125, |
| "throughput_improvement": 1.046317907252315 |
| }, |
| { |
| "batch_size": 32, |
| "seq_len": 128, |
| "baseline": { |
| "model_name": "HRM_Baseline", |
| "batch_size": 32, |
| "seq_len": 128, |
| "hidden_size": 512, |
| "H_cycles": 2, |
| "L_cycles": 2, |
| "H_layers": 4, |
| "L_layers": 4, |
| "num_iterations": 20, |
| "warmup_iterations": 5, |
| "l_level_latency_mean_us": 0, |
| "l_level_latency_min_us": 0, |
| "l_level_latency_max_us": 0, |
| "l_level_latency_std_us": 0, |
| "h_level_latency_mean_us": 0, |
| "h_level_latency_min_us": 0, |
| "h_level_latency_max_us": 0, |
| "h_level_latency_std_us": 0, |
| "total_inference_latency_mean_ms": 12.027006340026855, |
| "total_inference_latency_min_ms": 11.971232414245605, |
| "total_inference_latency_max_ms": 12.089983940124512, |
| "total_inference_latency_std_ms": 0.02667638763107568, |
| "throughput_samples_per_sec": 2660.678733784433, |
| "sram_peak_mb": 0, |
| "dram_peak_mb": 0, |
| "total_gpu_memory_mb": 292.13427734375, |
| "h_l_transfer_mean_us": 0, |
| "l_h_transfer_mean_us": 0, |
| "sram_hit_rate": 0, |
| "triton_sram_probe_latency_us": 0, |
| "triton_dram_probe_latency_us": 0, |
| "gpu_utilization_pct": 84.0, |
| "gpu_power_w": 210.32, |
| "gpu_temperature_c": null, |
| "h_over_l_latency_ratio": 0, |
| "memory_efficiency": 1.0 |
| }, |
| "tiered": { |
| "model_name": "HRM_Tiered", |
| "batch_size": 32, |
| "seq_len": 128, |
| "hidden_size": 512, |
| "H_cycles": 2, |
| "L_cycles": 2, |
| "H_layers": 4, |
| "L_layers": 4, |
| "num_iterations": 20, |
| "warmup_iterations": 5, |
| "l_level_latency_mean_us": 1828.7717362244923, |
| "l_level_latency_min_us": 1806.175947189331, |
| "l_level_latency_max_us": 1862.6240491867065, |
| "l_level_latency_std_us": 0.0, |
| "h_level_latency_mean_us": 1824.127995967865, |
| "h_level_latency_min_us": 1806.3360452651978, |
| "h_level_latency_max_us": 1875.7760524749756, |
| "h_level_latency_std_us": 0.0, |
| "total_inference_latency_mean_ms": 11.574174451828004, |
| "total_inference_latency_min_ms": 11.528096199035645, |
| "total_inference_latency_max_ms": 11.638784408569336, |
| "total_inference_latency_std_ms": 0.03262226881630495, |
| "throughput_samples_per_sec": 2764.7760220985765, |
| "sram_peak_mb": 0.0, |
| "dram_peak_mb": 0.0, |
| "total_gpu_memory_mb": 312.38427734375, |
| "h_l_transfer_mean_us": 0, |
| "l_h_transfer_mean_us": 9.51200001873076, |
| "sram_hit_rate": 0.0, |
| "triton_sram_probe_latency_us": 49.15200173854828, |
| "triton_dram_probe_latency_us": 40.70400074124336, |
| "gpu_utilization_pct": 78.0, |
| "gpu_power_w": 245.59, |
| "gpu_temperature_c": null, |
| "h_over_l_latency_ratio": 0.9974607327067432, |
| "memory_efficiency": 0.9987005384933674 |
| }, |
| "speedup": 1.0391243358291815, |
| "memory_savings_mb": -20.25, |
| "throughput_improvement": 1.0391243358291815 |
| } |
| ] |
