Upload benchmarks/bench_12_full_pipeline_combos.json with huggingface_hub

benchmarks/bench_12_full_pipeline_combos.json

@@ -0,0 +1,340 @@
+{
+  "benchmark": "full_pipeline_combos",
+  "timestamp": "2026-03-19T12:50:46.264173+00:00",
+  "device": "cuda",
+  "gpu": "NVIDIA L4",
+  "n_gpus": 1,
+  "pipelines": {
+    "nano_diff_p75_q4": {
+      "name": "nano_diff_p75_q4",
+      "config": {
+        "variant": "nano",
+        "language_model": "Qwen/Qwen2.5-0.5B",
+        "lora_rank": 32,
+        "action_head_type": "diffusion"
+      },
+      "model": {
+        "total_params_m": 967.9,
+        "trainable_params_m": 495.6
+      },
+      "training": {
+        "steps": 30,
+        "loss_start": 2.8391,
+        "loss_end": 1.665,
+        "loss_reduction_pct": 41.4,
+        "speed_steps_per_s": 1.5
+      },
+      "pruning": {
+        "ratio": 0.75,
+        "layers_before": 24,
+        "layers_after": 15,
+        "layers_removed": [
+          2,
+          3,
+          6,
+          10,
+          13,
+          19,
+          22,
+          23,
+          24
+        ],
+        "params_after_m": 830.8
+      },
+      "quantization": {
+        "error": "'FORGEStudent' object has no attribute 'get'"
+      },
+      "inference": {
+        "fp32_p50_ms": 99.22,
+        "fp32_fps": 10.0,
+        "fp16_p50_ms": 83.25,
+        "fp16_fps": 12.0,
+        "fp16_speedup": 1.19,
+        "gpu_mem_gb": 11.67
+      },
+      "timings": {
+        "build_s": 6.0,
+        "train_s": 19.8,
+        "prune_s": 20.2,
+        "quantize_s": 124.7
+      },
+      "total_time_s": 170.7
+    },
+    "nano_flow_p50_q4": {
+      "name": "nano_flow_p50_q4",
+      "config": {
+        "variant": "nano",
+        "language_model": "Qwen/Qwen2.5-0.5B",
+        "lora_rank": 32,
+        "action_head_type": "flow"
+      },
+      "model": {
+        "total_params_m": 967.9,
+        "trainable_params_m": 495.6
+      },
+      "training": {
+        "steps": 30,
+        "loss_start": 7.8611,
+        "loss_end": 1.8667,
+        "loss_reduction_pct": 76.3,
+        "speed_steps_per_s": 1.5
+      },
+      "pruning": {
+        "ratio": 0.5,
+        "layers_before": 24,
+        "layers_after": 9,
+        "layers_removed": [
+          2,
+          3,
+          7,
+          8,
+          9,
+          10,
+          12,
+          14,
+          17,
+          18,
+          20,
+          21,
+          22,
+          23,
+          24
+        ],
+        "params_after_m": 739.3
+      },
+      "quantization": {
+        "error": "'FORGEStudent' object has no attribute 'get'"
+      },
+      "inference": {
+        "fp32_p50_ms": 70.83,
+        "fp32_fps": 14.1,
+        "fp16_p50_ms": 132.02,
+        "fp16_fps": 7.8,
+        "fp16_speedup": 0.56,
+        "gpu_mem_gb": 11.36
+      },
+      "timings": {
+        "build_s": 2.9,
+        "train_s": 19.5,
+        "prune_s": 19.1,
+        "quantize_s": 124.2
+      },
+      "total_time_s": 165.7
+    },
+    "nano_lora64_p90_q4": {
+      "name": "nano_lora64_p90_q4",
+      "config": {
+        "variant": "nano",
+        "language_model": "Qwen/Qwen2.5-0.5B",
+        "lora_rank": 64,
+        "action_head_type": "diffusion"
+      },
+      "model": {
+        "total_params_m": 972.3,
+        "trainable_params_m": 500.0
+      },
+      "training": {
+        "steps": 30,
+        "loss_start": 9.6341,
+        "loss_end": 1.3188,
+        "loss_reduction_pct": 86.3,
+        "speed_steps_per_s": 1.4
+      },
+      "pruning": {
+        "ratio": 0.9,
+        "layers_before": 24,
+        "layers_after": 18,
+        "layers_removed": [
+          5,
+          8,
+          9,
+          18,
+          19,
+          23
+        ],
+        "params_after_m": 880.8
+      },
+      "quantization": {
+        "error": "'FORGEStudent' object has no attribute 'get'"
+      },
+      "inference": {
+        "fp32_p50_ms": 109.74,
+        "fp32_fps": 9.1,
+        "fp16_p50_ms": 88.12,
+        "fp16_fps": 11.2,
+        "fp16_speedup": 1.23,
+        "gpu_mem_gb": 15.69
+      },
+      "timings": {
+        "build_s": 5.3,
+        "train_s": 21.3,
+        "prune_s": 19.8,
+        "quantize_s": 129.1
+      },
+      "total_time_s": 175.5
+    },
+    "nano_diff_p75_q8": {
+      "name": "nano_diff_p75_q8",
+      "config": {
+        "variant": "nano",
+        "language_model": "Qwen/Qwen2.5-0.5B",
+        "lora_rank": 32,
+        "action_head_type": "diffusion"
+      },
+      "model": {
+        "total_params_m": 967.9,
+        "trainable_params_m": 495.6
+      },
+      "training": {
+        "steps": 30,
+        "loss_start": 13.6661,
+        "loss_end": 1.0559,
+        "loss_reduction_pct": 92.3,
+        "speed_steps_per_s": 1.5
+      },
+      "pruning": {
+        "ratio": 0.75,
+        "layers_before": 24,
+        "layers_after": 15,
+        "layers_removed": [
+          3,
+          4,
+          8,
+          11,
+          12,
+          13,
+          16,
+          19,
+          24
+        ],
+        "params_after_m": 830.8
+      },
+      "quantization": {
+        "error": "'FORGEStudent' object has no attribute 'get'"
+      },
+      "inference": {
+        "fp32_p50_ms": 100.25,
+        "fp32_fps": 10.0,
+        "fp16_p50_ms": 86.84,
+        "fp16_fps": 11.3,
+        "fp16_speedup": 1.14,
+        "gpu_mem_gb": 11.7
+      },
+      "timings": {
+        "build_s": 3.1,
+        "train_s": 19.7,
+        "prune_s": 19.8,
+        "quantize_s": 129.2
+      },
+      "total_time_s": 171.8
+    },
+    "nano_flow_lora64_p60_q4": {
+      "name": "nano_flow_lora64_p60_q4",
+      "config": {
+        "variant": "nano",
+        "language_model": "Qwen/Qwen2.5-0.5B",
+        "lora_rank": 64,
+        "action_head_type": "flow"
+      },
+      "model": {
+        "total_params_m": 972.3,
+        "trainable_params_m": 500.0
+      },
+      "training": {
+        "steps": 30,
+        "loss_start": 16.2305,
+        "loss_end": 3.939,
+        "loss_reduction_pct": 75.7,
+        "speed_steps_per_s": 1.5
+      },
+      "pruning": {
+        "ratio": 0.6,
+        "layers_before": 24,
+        "layers_after": 11,
+        "layers_removed": [
+          2,
+          4,
+          6,
+          7,
+          9,
+          10,
+          11,
+          16,
+          18,
+          19,
+          20,
+          22,
+          24
+        ],
+        "params_after_m": 774.1
+      },
+      "quantization": {
+        "error": "'FORGEStudent' object has no attribute 'get'"
+      },
+      "inference": {
+        "fp32_p50_ms": 78.17,
+        "fp32_fps": 12.7,
+        "fp16_p50_ms": 69.98,
+        "fp16_fps": 14.1,
+        "fp16_speedup": 1.11,
+        "gpu_mem_gb": 11.54
+      },
+      "timings": {
+        "build_s": 3.1,
+        "train_s": 19.8,
+        "prune_s": 19.3,
+        "quantize_s": 125.4
+      },
+      "total_time_s": 167.6
+    },
+    "nano_diff_noprune_q8": {
+      "name": "nano_diff_noprune_q8",
+      "config": {
+        "variant": "nano",
+        "language_model": "Qwen/Qwen2.5-0.5B",
+        "lora_rank": 32,
+        "action_head_type": "diffusion"
+      },
+      "model": {
+        "total_params_m": 967.9,
+        "trainable_params_m": 495.6
+      },
+      "training": {
+        "steps": 30,
+        "loss_start": 3.97,
+        "loss_end": 1.6131,
+        "loss_reduction_pct": 59.4,
+        "speed_steps_per_s": 1.5
+      },
+      "pruning": {
+        "ratio": 1.0,
+        "layers_before": 24,
+        "layers_after": 21,
+        "layers_removed": [
+          10,
+          14,
+          16
+        ],
+        "params_after_m": 922.2
+      },
+      "quantization": {
+        "error": "'FORGEStudent' object has no attribute 'get'"
+      },
+      "inference": {
+        "fp32_p50_ms": 123.55,
+        "fp32_fps": 8.1,
+        "fp16_p50_ms": 90.81,
+        "fp16_fps": 11.0,
+        "fp16_speedup": 1.37,
+        "gpu_mem_gb": 12.04
+      },
+      "timings": {
+        "build_s": 3.0,
+        "train_s": 19.6,
+        "prune_s": 19.8,
+        "quantize_s": 124.8
+      },
+      "total_time_s": 167.2
+    }
+  }
+}