lane TP throughput benchmarks

analysis/registry_2026-10-25/bench_Qwen3-32B-FP8.json

@@ -0,0 +1,24 @@
+{
+ "model": "Qwen/Qwen3-32B-FP8",
+ "thinking": false,
+ "gpu": "NVIDIA RTX 5880 Ada Generation",
+ "n_gpus": 1,
+ "n_prompts": 256,
+ "max_tokens": 4096,
+ "gpu_mem_util": 0.9,
+ "decode": {
+  "generated_tokens": 257555,
+  "seconds": 677.8804216384888,
+  "tok_per_s": 379.94164129636596,
+  "capped": 6,
+  "mean_len": 1006.07421875
+ },
+ "prefill": {
+  "prompt_tokens": 523387,
+  "seconds": 220.94462728500366,
+  "tok_per_s": 2368.860498811162,
+  "mean_len": 2044.48046875
+ },
+ "kv_cache_blocks": 2294,
+ "block_size": 16
+}

analysis/registry_2026-10-25/bench_Qwen3-32B-FP8.log

@@ -0,0 +1,64 @@
+INFO 10-25 12:38:29 [arg_utils.py:602] HF_HUB_OFFLINE is True, replace model_id [Qwen/Qwen3-32B-FP8] to model_path [/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-32B-FP8/snapshots/aa55da1ecc13d006e8b8e4f54579b1ea8c3db2df]
+INFO 10-25 12:38:29 [utils.py:261] non-default args: {'max_model_len': 8192, 'disable_log_stats': True, 'model': '/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-32B-FP8/snapshots/aa55da1ecc13d006e8b8e4f54579b1ea8c3db2df'}
+INFO 10-25 12:38:29 [model.py:541] Resolved architecture: Qwen3ForCausalLM
+INFO 10-25 12:38:29 [model.py:1561] Using max model len 8192
+INFO 10-25 12:38:32 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=8192.
+INFO 10-25 12:38:32 [vllm.py:624] Asynchronous scheduling is enabled.
+(EngineCore_DP0 pid=784894) INFO 10-25 12:38:32 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-32B-FP8/snapshots/aa55da1ecc13d006e8b8e4f54579b1ea8c3db2df', speculative_config=None, tokenizer='/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-32B-FP8/snapshots/aa55da1ecc13d006e8b8e4f54579b1ea8c3db2df', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=fp8, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-32B-FP8/snapshots/aa55da1ecc13d006e8b8e4f54579b1ea8c3db2df, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['+quant_fp8', 'none', '+quant_fp8'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}
+(EngineCore_DP0 pid=784894) INFO 10-25 12:38:33 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://10.127.105.43:45483 backend=nccl
+(EngineCore_DP0 pid=784894) INFO 10-25 12:38:33 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
+(EngineCore_DP0 pid=784894) INFO 10-25 12:38:34 [gpu_model_runner.py:4033] Starting to load model /mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-32B-FP8/snapshots/aa55da1ecc13d006e8b8e4f54579b1ea8c3db2df...
+(EngineCore_DP0 pid=784894) INFO 10-25 12:38:35 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
+(EngineCore_DP0 pid=784894) 
+(EngineCore_DP0 pid=784894) 
+(EngineCore_DP0 pid=784894) 
+(EngineCore_DP0 pid=784894) 
+(EngineCore_DP0 pid=784894) 
+(EngineCore_DP0 pid=784894) 
+(EngineCore_DP0 pid=784894) 
+(EngineCore_DP0 pid=784894) 
+(EngineCore_DP0 pid=784894) 
+(EngineCore_DP0 pid=784894) 
+(EngineCore_DP0 pid=784894) INFO 10-25 12:39:39 [default_loader.py:291] Loading weights took 63.41 seconds
+(EngineCore_DP0 pid=784894) INFO 10-25 12:39:39 [gpu_model_runner.py:4130] Model loading took 31.98 GiB memory and 64.632279 seconds
+(EngineCore_DP0 pid=784894) INFO 10-25 12:39:51 [backends.py:812] Using cache directory: /home/vlad.smirnov/.cache/vllm/torch_compile_cache/5153e5930e/rank_0_0/backbone for vLLM's torch.compile
+(EngineCore_DP0 pid=784894) INFO 10-25 12:39:51 [backends.py:872] Dynamo bytecode transform time: 11.82 s
+(EngineCore_DP0 pid=784894) WARNING 10-25 12:39:56 [fp8_utils.py:1175] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at /mnt/data/users/vlad.smirnov/conda_envs/rl-uhead/lib/python3.11/site-packages/vllm/model_executor/layers/quantization/utils/configs/N=10240,K=5120,device_name=NVIDIA_RTX_5880_Ada_Generation,dtype=fp8_w8a8,block_shape=[128,128].json
+(EngineCore_DP0 pid=784894) WARNING 10-25 12:39:56 [fp8_utils.py:1175] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at /mnt/data/users/vlad.smirnov/conda_envs/rl-uhead/lib/python3.11/site-packages/vllm/model_executor/layers/quantization/utils/configs/N=5120,K=8192,device_name=NVIDIA_RTX_5880_Ada_Generation,dtype=fp8_w8a8,block_shape=[128,128].json
+(EngineCore_DP0 pid=784894) WARNING 10-25 12:39:56 [fp8_utils.py:1175] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at /mnt/data/users/vlad.smirnov/conda_envs/rl-uhead/lib/python3.11/site-packages/vllm/model_executor/layers/quantization/utils/configs/N=51200,K=5120,device_name=NVIDIA_RTX_5880_Ada_Generation,dtype=fp8_w8a8,block_shape=[128,128].json
+(EngineCore_DP0 pid=784894) WARNING 10-25 12:39:56 [fp8_utils.py:1175] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at /mnt/data/users/vlad.smirnov/conda_envs/rl-uhead/lib/python3.11/site-packages/vllm/model_executor/layers/quantization/utils/configs/N=5120,K=25600,device_name=NVIDIA_RTX_5880_Ada_Generation,dtype=fp8_w8a8,block_shape=[128,128].json
+(EngineCore_DP0 pid=784894) INFO 10-25 12:39:59 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 8192) from the cache, took 3.251 s
+(EngineCore_DP0 pid=784894) INFO 10-25 12:39:59 [monitor.py:34] torch.compile takes 15.07 s in total
+(EngineCore_DP0 pid=784894) INFO 10-25 12:40:01 [gpu_worker.py:356] Available KV cache memory: 8.96 GiB
+(EngineCore_DP0 pid=784894) INFO 10-25 12:40:01 [kv_cache_utils.py:1307] GPU KV cache size: 36,704 tokens
+(EngineCore_DP0 pid=784894) INFO 10-25 12:40:01 [kv_cache_utils.py:1312] Maximum concurrency for 8,192 tokens per request: 4.48x
+(EngineCore_DP0 pid=784894) 
+(EngineCore_DP0 pid=784894) 
+(EngineCore_DP0 pid=784894) INFO 10-25 12:40:11 [gpu_model_runner.py:5063] Graph capturing finished in 11 secs, took 1.12 GiB
+(EngineCore_DP0 pid=784894) INFO 10-25 12:40:11 [core.py:272] init engine (profile, create kv cache, warmup model) took 31.90 seconds
+INFO 10-25 12:40:12 [llm.py:343] Supported tasks: ['generate']
+{
+ "model": "Qwen/Qwen3-32B-FP8",
+ "thinking": false,
+ "gpu": "NVIDIA RTX 5880 Ada Generation",
+ "n_gpus": 1,
+ "n_prompts": 256,
+ "max_tokens": 4096,
+ "gpu_mem_util": 0.9,
+ "decode": {
+  "generated_tokens": 257555,
+  "seconds": 677.8804216384888,
+  "tok_per_s": 379.94164129636596,
+  "capped": 6,
+  "mean_len": 1006.07421875
+ },
+ "prefill": {
+  "prompt_tokens": 523387,
+  "seconds": 220.94462728500366,
+  "tok_per_s": 2368.860498811162,
+  "mean_len": 2044.48046875
+ },
+ "kv_cache_blocks": 2294,
+ "block_size": 16
+}
+[W1025 12:55:13.820569422 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator())

analysis/registry_2026-10-25/bench_Qwen3-32B-FP8.provenance.json

@@ -0,0 +1,73 @@
+{
+ "started_at": "2026-10-25T12:55:12+04:00",
+ "host": "ws006598",
+ "user": "vlad.smirnov",
+ "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-ood",
+ "argv": [
+  "/mnt/data/users/vlad.smirnov/rl-uhead-ood/rl_controller/scripts/throughput_bench.py",
+  "--model",
+  "Qwen/Qwen3-32B-FP8",
+  "--pool-dir",
+  "workdir/selector/pool_mth_k8_chat",
+  "--out",
+  "workdir/registry/bench_Qwen3-32B-FP8.json"
+ ],
+ "git": {
+  "commit": "0bfea921dff632984bf84f24836c4588eb47a6a4",
+  "branch": "HEAD",
+  "subject": "Add throughput_bench and lane TP: measured decode/prefill throughput per model for the registry",
+  "dirty": false,
+  "dirty_files": []
+ },
+ "gpus": {
+  "visible": "1",
+  "devices": [
+   {
+    "index": "0",
+    "name": "NVIDIA RTX 5880 Ada Generation",
+    "memory": "49140 MiB",
+    "driver": "580.178.04",
+    "uuid": "GPU-eb08932b-fbcd-41ce-e1e2-6fd7b02de161"
+   },
+   {
+    "index": "1",
+    "name": "NVIDIA RTX 5880 Ada Generation",
+    "memory": "49140 MiB",
+    "driver": "580.178.04",
+    "uuid": "GPU-cc1e8db2-7843-a8e9-8191-088167fb3499"
+   }
+  ]
+ },
+ "packages": {
+  "python": "3.11.15",
+  "torch": "2.9.1",
+  "vllm": "0.15.1",
+  "transformers": "4.57.6",
+  "speculators": "0.4.0.dev49",
+  "peft": null,
+  "accelerate": "1.13.0",
+  "numpy": "2.2.6",
+  "huggingface_hub": "0.36.2",
+  "openai": "2.32.0",
+  "wandb": "0.25.1",
+  "torch_cuda": "12.8"
+ },
+ "env": {
+  "CUDA_VISIBLE_DEVICES": "1",
+  "HF_HOME": "/mnt/data/users/vlad.smirnov/hf_cache",
+  "HF_HUB_OFFLINE": "1",
+  "TRANSFORMERS_OFFLINE": "1",
+  "NCCL_P2P_DISABLE": "1",
+  "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True"
+ },
+ "models": [
+  {
+   "repo": "Qwen/Qwen3-32B-FP8",
+   "revision": "aa55da1ecc13d006e8b8e4f54579b1ea8c3db2df",
+   "path": "/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-32B-FP8/snapshots/aa55da1ecc13d006e8b8e4f54579b1ea8c3db2df"
+  }
+ ],
+ "files": {},
+ "finished_at": "2026-10-25T12:55:12+04:00",
+ "wall_secs": 0.1
+}

analysis/registry_2026-10-25/bench_Qwen3-32B-FP8_think.json

@@ -0,0 +1,24 @@
+{
+ "model": "Qwen/Qwen3-32B-FP8",
+ "thinking": true,
+ "gpu": "NVIDIA RTX 5880 Ada Generation",
+ "n_gpus": 1,
+ "n_prompts": 256,
+ "max_tokens": 4096,
+ "gpu_mem_util": 0.9,
+ "decode": {
+  "generated_tokens": 885894,
+  "seconds": 3450.798579454422,
+  "tok_per_s": 256.72144566028584,
+  "capped": 129,
+  "mean_len": 3460.5234375
+ },
+ "prefill": {
+  "prompt_tokens": 522363,
+  "seconds": 220.4746448993683,
+  "tok_per_s": 2369.2656370460345,
+  "mean_len": 2040.48046875
+ },
+ "kv_cache_blocks": 2294,
+ "block_size": 16
+}

analysis/registry_2026-10-25/bench_Qwen3-32B-FP8_think.log

@@ -0,0 +1,64 @@
+INFO 10-25 12:55:17 [arg_utils.py:602] HF_HUB_OFFLINE is True, replace model_id [Qwen/Qwen3-32B-FP8] to model_path [/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-32B-FP8/snapshots/aa55da1ecc13d006e8b8e4f54579b1ea8c3db2df]
+INFO 10-25 12:55:17 [utils.py:261] non-default args: {'max_model_len': 8192, 'disable_log_stats': True, 'model': '/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-32B-FP8/snapshots/aa55da1ecc13d006e8b8e4f54579b1ea8c3db2df'}
+INFO 10-25 12:55:17 [model.py:541] Resolved architecture: Qwen3ForCausalLM
+INFO 10-25 12:55:17 [model.py:1561] Using max model len 8192
+INFO 10-25 12:55:18 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=8192.
+INFO 10-25 12:55:18 [vllm.py:624] Asynchronous scheduling is enabled.
+(EngineCore_DP0 pid=790900) INFO 10-25 12:55:19 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-32B-FP8/snapshots/aa55da1ecc13d006e8b8e4f54579b1ea8c3db2df', speculative_config=None, tokenizer='/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-32B-FP8/snapshots/aa55da1ecc13d006e8b8e4f54579b1ea8c3db2df', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=fp8, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-32B-FP8/snapshots/aa55da1ecc13d006e8b8e4f54579b1ea8c3db2df, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['+quant_fp8', 'none', '+quant_fp8'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}
+(EngineCore_DP0 pid=790900) INFO 10-25 12:55:19 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://10.127.105.43:34899 backend=nccl
+(EngineCore_DP0 pid=790900) INFO 10-25 12:55:19 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
+(EngineCore_DP0 pid=790900) INFO 10-25 12:55:20 [gpu_model_runner.py:4033] Starting to load model /mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-32B-FP8/snapshots/aa55da1ecc13d006e8b8e4f54579b1ea8c3db2df...
+(EngineCore_DP0 pid=790900) INFO 10-25 12:55:20 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
+(EngineCore_DP0 pid=790900) 
+(EngineCore_DP0 pid=790900) 
+(EngineCore_DP0 pid=790900) 
+(EngineCore_DP0 pid=790900) 
+(EngineCore_DP0 pid=790900) 
+(EngineCore_DP0 pid=790900) 
+(EngineCore_DP0 pid=790900) 
+(EngineCore_DP0 pid=790900) 
+(EngineCore_DP0 pid=790900) 
+(EngineCore_DP0 pid=790900) 
+(EngineCore_DP0 pid=790900) INFO 10-25 12:55:27 [default_loader.py:291] Loading weights took 6.20 seconds
+(EngineCore_DP0 pid=790900) INFO 10-25 12:55:27 [gpu_model_runner.py:4130] Model loading took 31.98 GiB memory and 6.792689 seconds
+(EngineCore_DP0 pid=790900) INFO 10-25 12:55:35 [backends.py:812] Using cache directory: /home/vlad.smirnov/.cache/vllm/torch_compile_cache/5153e5930e/rank_0_0/backbone for vLLM's torch.compile
+(EngineCore_DP0 pid=790900) INFO 10-25 12:55:35 [backends.py:872] Dynamo bytecode transform time: 8.07 s
+(EngineCore_DP0 pid=790900) WARNING 10-25 12:55:40 [fp8_utils.py:1175] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at /mnt/data/users/vlad.smirnov/conda_envs/rl-uhead/lib/python3.11/site-packages/vllm/model_executor/layers/quantization/utils/configs/N=10240,K=5120,device_name=NVIDIA_RTX_5880_Ada_Generation,dtype=fp8_w8a8,block_shape=[128,128].json
+(EngineCore_DP0 pid=790900) WARNING 10-25 12:55:40 [fp8_utils.py:1175] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at /mnt/data/users/vlad.smirnov/conda_envs/rl-uhead/lib/python3.11/site-packages/vllm/model_executor/layers/quantization/utils/configs/N=5120,K=8192,device_name=NVIDIA_RTX_5880_Ada_Generation,dtype=fp8_w8a8,block_shape=[128,128].json
+(EngineCore_DP0 pid=790900) WARNING 10-25 12:55:40 [fp8_utils.py:1175] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at /mnt/data/users/vlad.smirnov/conda_envs/rl-uhead/lib/python3.11/site-packages/vllm/model_executor/layers/quantization/utils/configs/N=51200,K=5120,device_name=NVIDIA_RTX_5880_Ada_Generation,dtype=fp8_w8a8,block_shape=[128,128].json
+(EngineCore_DP0 pid=790900) WARNING 10-25 12:55:40 [fp8_utils.py:1175] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at /mnt/data/users/vlad.smirnov/conda_envs/rl-uhead/lib/python3.11/site-packages/vllm/model_executor/layers/quantization/utils/configs/N=5120,K=25600,device_name=NVIDIA_RTX_5880_Ada_Generation,dtype=fp8_w8a8,block_shape=[128,128].json
+(EngineCore_DP0 pid=790900) INFO 10-25 12:55:43 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 8192) from the cache, took 3.118 s
+(EngineCore_DP0 pid=790900) INFO 10-25 12:55:43 [monitor.py:34] torch.compile takes 11.19 s in total
+(EngineCore_DP0 pid=790900) INFO 10-25 12:55:45 [gpu_worker.py:356] Available KV cache memory: 8.96 GiB
+(EngineCore_DP0 pid=790900) INFO 10-25 12:55:45 [kv_cache_utils.py:1307] GPU KV cache size: 36,704 tokens
+(EngineCore_DP0 pid=790900) INFO 10-25 12:55:45 [kv_cache_utils.py:1312] Maximum concurrency for 8,192 tokens per request: 4.48x
+(EngineCore_DP0 pid=790900) 
+(EngineCore_DP0 pid=790900) 
+(EngineCore_DP0 pid=790900) INFO 10-25 12:55:55 [gpu_model_runner.py:5063] Graph capturing finished in 11 secs, took 1.12 GiB
+(EngineCore_DP0 pid=790900) INFO 10-25 12:55:55 [core.py:272] init engine (profile, create kv cache, warmup model) took 28.11 seconds
+INFO 10-25 12:55:56 [llm.py:343] Supported tasks: ['generate']
+{
+ "model": "Qwen/Qwen3-32B-FP8",
+ "thinking": true,
+ "gpu": "NVIDIA RTX 5880 Ada Generation",
+ "n_gpus": 1,
+ "n_prompts": 256,
+ "max_tokens": 4096,
+ "gpu_mem_util": 0.9,
+ "decode": {
+  "generated_tokens": 885894,
+  "seconds": 3450.798579454422,
+  "tok_per_s": 256.72144566028584,
+  "capped": 129,
+  "mean_len": 3460.5234375
+ },
+ "prefill": {
+  "prompt_tokens": 522363,
+  "seconds": 220.4746448993683,
+  "tok_per_s": 2369.2656370460345,
+  "mean_len": 2040.48046875
+ },
+ "kv_cache_blocks": 2294,
+ "block_size": 16
+}
+[W1025 13:57:10.310955141 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator())

analysis/registry_2026-10-25/bench_Qwen3-32B-FP8_think.provenance.json

@@ -0,0 +1,74 @@
+{
+ "started_at": "2026-10-25T13:57:09+04:00",
+ "host": "ws006598",
+ "user": "vlad.smirnov",
+ "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-ood",
+ "argv": [
+  "/mnt/data/users/vlad.smirnov/rl-uhead-ood/rl_controller/scripts/throughput_bench.py",
+  "--model",
+  "Qwen/Qwen3-32B-FP8",
+  "--thinking",
+  "--pool-dir",
+  "workdir/selector/pool_mth_k8_chat",
+  "--out",
+  "workdir/registry/bench_Qwen3-32B-FP8_think.json"
+ ],
+ "git": {
+  "commit": "0bfea921dff632984bf84f24836c4588eb47a6a4",
+  "branch": "HEAD",
+  "subject": "Add throughput_bench and lane TP: measured decode/prefill throughput per model for the registry",
+  "dirty": false,
+  "dirty_files": []
+ },
+ "gpus": {
+  "visible": "1",
+  "devices": [
+   {
+    "index": "0",
+    "name": "NVIDIA RTX 5880 Ada Generation",
+    "memory": "49140 MiB",
+    "driver": "580.178.04",
+    "uuid": "GPU-eb08932b-fbcd-41ce-e1e2-6fd7b02de161"
+   },
+   {
+    "index": "1",
+    "name": "NVIDIA RTX 5880 Ada Generation",
+    "memory": "49140 MiB",
+    "driver": "580.178.04",
+    "uuid": "GPU-cc1e8db2-7843-a8e9-8191-088167fb3499"
+   }
+  ]
+ },
+ "packages": {
+  "python": "3.11.15",
+  "torch": "2.9.1",
+  "vllm": "0.15.1",
+  "transformers": "4.57.6",
+  "speculators": "0.4.0.dev49",
+  "peft": null,
+  "accelerate": "1.13.0",
+  "numpy": "2.2.6",
+  "huggingface_hub": "0.36.2",
+  "openai": "2.32.0",
+  "wandb": "0.25.1",
+  "torch_cuda": "12.8"
+ },
+ "env": {
+  "CUDA_VISIBLE_DEVICES": "1",
+  "HF_HOME": "/mnt/data/users/vlad.smirnov/hf_cache",
+  "HF_HUB_OFFLINE": "1",
+  "TRANSFORMERS_OFFLINE": "1",
+  "NCCL_P2P_DISABLE": "1",
+  "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True"
+ },
+ "models": [
+  {
+   "repo": "Qwen/Qwen3-32B-FP8",
+   "revision": "aa55da1ecc13d006e8b8e4f54579b1ea8c3db2df",
+   "path": "/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-32B-FP8/snapshots/aa55da1ecc13d006e8b8e4f54579b1ea8c3db2df"
+  }
+ ],
+ "files": {},
+ "finished_at": "2026-10-25T13:57:09+04:00",
+ "wall_secs": 0.0
+}

analysis/registry_2026-10-25/bench_Qwen3-8B.json

@@ -0,0 +1,24 @@
+{
+ "model": "Qwen/Qwen3-8B",
+ "thinking": false,
+ "gpu": "NVIDIA RTX 5880 Ada Generation",
+ "n_gpus": 1,
+ "n_prompts": 256,
+ "max_tokens": 4096,
+ "gpu_mem_util": 0.9,
+ "decode": {
+  "generated_tokens": 284294,
+  "seconds": 180.27869725227356,
+  "tok_per_s": 1576.9694608019731,
+  "capped": 9,
+  "mean_len": 1110.5234375
+ },
+ "prefill": {
+  "prompt_tokens": 523387,
+  "seconds": 83.14631795883179,
+  "tok_per_s": 6294.7706266336945,
+  "mean_len": 2044.48046875
+ },
+ "kv_cache_blocks": 11778,
+ "block_size": 16
+}

analysis/registry_2026-10-25/bench_Qwen3-8B.log

@@ -0,0 +1,58 @@
+INFO 10-25 12:18:30 [arg_utils.py:602] HF_HUB_OFFLINE is True, replace model_id [Qwen/Qwen3-8B] to model_path [/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218]
+INFO 10-25 12:18:31 [utils.py:261] non-default args: {'max_model_len': 8192, 'disable_log_stats': True, 'model': '/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218'}
+INFO 10-25 12:18:31 [model.py:541] Resolved architecture: Qwen3ForCausalLM
+INFO 10-25 12:18:31 [model.py:1561] Using max model len 8192
+INFO 10-25 12:18:31 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=8192.
+INFO 10-25 12:18:31 [vllm.py:624] Asynchronous scheduling is enabled.
+(EngineCore_DP0 pid=777638) INFO 10-25 12:18:32 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218', speculative_config=None, tokenizer='/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}
+(EngineCore_DP0 pid=777638) INFO 10-25 12:18:34 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://10.127.105.43:57441 backend=nccl
+(EngineCore_DP0 pid=777638) INFO 10-25 12:18:35 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
+(EngineCore_DP0 pid=777638) INFO 10-25 12:18:36 [gpu_model_runner.py:4033] Starting to load model /mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218...
+(EngineCore_DP0 pid=777638) INFO 10-25 12:18:37 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
+(EngineCore_DP0 pid=777638) 
+(EngineCore_DP0 pid=777638) 
+(EngineCore_DP0 pid=777638) 
+(EngineCore_DP0 pid=777638) 
+(EngineCore_DP0 pid=777638) 
+(EngineCore_DP0 pid=777638) 
+(EngineCore_DP0 pid=777638) 
+(EngineCore_DP0 pid=777638) 
+(EngineCore_DP0 pid=777638) INFO 10-25 12:18:51 [default_loader.py:291] Loading weights took 13.54 seconds
+(EngineCore_DP0 pid=777638) INFO 10-25 12:18:51 [gpu_model_runner.py:4130] Model loading took 15.27 GiB memory and 14.957416 seconds
+(EngineCore_DP0 pid=777638) INFO 10-25 12:18:59 [backends.py:812] Using cache directory: /home/vlad.smirnov/.cache/vllm/torch_compile_cache/5250b7e167/rank_0_0/backbone for vLLM's torch.compile
+(EngineCore_DP0 pid=777638) INFO 10-25 12:18:59 [backends.py:872] Dynamo bytecode transform time: 7.77 s
+(EngineCore_DP0 pid=777638) INFO 10-25 12:19:02 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 8192) from the cache, took 1.110 s
+(EngineCore_DP0 pid=777638) INFO 10-25 12:19:02 [monitor.py:34] torch.compile takes 8.88 s in total
+(EngineCore_DP0 pid=777638) INFO 10-25 12:19:03 [gpu_worker.py:356] Available KV cache memory: 25.88 GiB
+(EngineCore_DP0 pid=777638) INFO 10-25 12:19:03 [kv_cache_utils.py:1307] GPU KV cache size: 188,448 tokens
+(EngineCore_DP0 pid=777638) INFO 10-25 12:19:03 [kv_cache_utils.py:1312] Maximum concurrency for 8,192 tokens per request: 23.00x
+(EngineCore_DP0 pid=777638) 
+(EngineCore_DP0 pid=777638) 
+(EngineCore_DP0 pid=777638) INFO 10-25 12:19:07 [gpu_model_runner.py:5063] Graph capturing finished in 4 secs, took 0.56 GiB
+(EngineCore_DP0 pid=777638) INFO 10-25 12:19:07 [core.py:272] init engine (profile, create kv cache, warmup model) took 16.17 seconds
+INFO 10-25 12:19:08 [llm.py:343] Supported tasks: ['generate']
+{
+ "model": "Qwen/Qwen3-8B",
+ "thinking": false,
+ "gpu": "NVIDIA RTX 5880 Ada Generation",
+ "n_gpus": 1,
+ "n_prompts": 256,
+ "max_tokens": 4096,
+ "gpu_mem_util": 0.9,
+ "decode": {
+  "generated_tokens": 284294,
+  "seconds": 180.27869725227356,
+  "tok_per_s": 1576.9694608019731,
+  "capped": 9,
+  "mean_len": 1110.5234375
+ },
+ "prefill": {
+  "prompt_tokens": 523387,
+  "seconds": 83.14631795883179,
+  "tok_per_s": 6294.7706266336945,
+  "mean_len": 2044.48046875
+ },
+ "kv_cache_blocks": 11778,
+ "block_size": 16
+}
+[W1025 12:23:34.955082452 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator())

analysis/registry_2026-10-25/bench_Qwen3-8B.provenance.json

@@ -0,0 +1,73 @@
+{
+ "started_at": "2026-10-25T12:23:32+04:00",
+ "host": "ws006598",
+ "user": "vlad.smirnov",
+ "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-ood",
+ "argv": [
+  "/mnt/data/users/vlad.smirnov/rl-uhead-ood/rl_controller/scripts/throughput_bench.py",
+  "--model",
+  "Qwen/Qwen3-8B",
+  "--pool-dir",
+  "workdir/selector/pool_mth_k8_chat",
+  "--out",
+  "workdir/registry/bench_Qwen3-8B.json"
+ ],
+ "git": {
+  "commit": "0bfea921dff632984bf84f24836c4588eb47a6a4",
+  "branch": "HEAD",
+  "subject": "Add throughput_bench and lane TP: measured decode/prefill throughput per model for the registry",
+  "dirty": false,
+  "dirty_files": []
+ },
+ "gpus": {
+  "visible": "1",
+  "devices": [
+   {
+    "index": "0",
+    "name": "NVIDIA RTX 5880 Ada Generation",
+    "memory": "49140 MiB",
+    "driver": "580.178.04",
+    "uuid": "GPU-eb08932b-fbcd-41ce-e1e2-6fd7b02de161"
+   },
+   {
+    "index": "1",
+    "name": "NVIDIA RTX 5880 Ada Generation",
+    "memory": "49140 MiB",
+    "driver": "580.178.04",
+    "uuid": "GPU-cc1e8db2-7843-a8e9-8191-088167fb3499"
+   }
+  ]
+ },
+ "packages": {
+  "python": "3.11.15",
+  "torch": "2.9.1",
+  "vllm": "0.15.1",
+  "transformers": "4.57.6",
+  "speculators": "0.4.0.dev49",
+  "peft": null,
+  "accelerate": "1.13.0",
+  "numpy": "2.2.6",
+  "huggingface_hub": "0.36.2",
+  "openai": "2.32.0",
+  "wandb": "0.25.1",
+  "torch_cuda": "12.8"
+ },
+ "env": {
+  "CUDA_VISIBLE_DEVICES": "1",
+  "HF_HOME": "/mnt/data/users/vlad.smirnov/hf_cache",
+  "HF_HUB_OFFLINE": "1",
+  "TRANSFORMERS_OFFLINE": "1",
+  "NCCL_P2P_DISABLE": "1",
+  "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True"
+ },
+ "models": [
+  {
+   "repo": "Qwen/Qwen3-8B",
+   "revision": "b968826d9c46dd6066d109eabc6255188de91218",
+   "path": "/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218"
+  }
+ ],
+ "files": {},
+ "finished_at": "2026-10-25T12:23:32+04:00",
+ "wall_secs": 0.2
+}

analysis/registry_2026-10-25/bench_Qwen3-8B_think.json

@@ -0,0 +1,24 @@
+{
+ "model": "Qwen/Qwen3-8B",
+ "thinking": true,
+ "gpu": "NVIDIA RTX 5880 Ada Generation",
+ "n_gpus": 1,
+ "n_prompts": 256,
+ "max_tokens": 4096,
+ "gpu_mem_util": 0.9,
+ "decode": {
+  "generated_tokens": 929161,
+  "seconds": 769.5907077789307,
+  "tok_per_s": 1207.3443592914414,
+  "capped": 160,
+  "mean_len": 3629.53515625
+ },
+ "prefill": {
+  "prompt_tokens": 522363,
+  "seconds": 83.4063332080841,
+  "tok_per_s": 6262.869735524716,
+  "mean_len": 2040.48046875
+ },
+ "kv_cache_blocks": 11778,
+ "block_size": 16
+}

analysis/registry_2026-10-25/bench_Qwen3-8B_think.log

@@ -0,0 +1,58 @@
+INFO 10-25 12:23:38 [arg_utils.py:602] HF_HUB_OFFLINE is True, replace model_id [Qwen/Qwen3-8B] to model_path [/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218]
+INFO 10-25 12:23:38 [utils.py:261] non-default args: {'max_model_len': 8192, 'disable_log_stats': True, 'model': '/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218'}
+INFO 10-25 12:23:38 [model.py:541] Resolved architecture: Qwen3ForCausalLM
+INFO 10-25 12:23:38 [model.py:1561] Using max model len 8192
+INFO 10-25 12:23:38 [scheduler.py:226] Chunked prefill is enabled with max_num_batched_tokens=8192.
+INFO 10-25 12:23:38 [vllm.py:624] Asynchronous scheduling is enabled.
+(EngineCore_DP0 pid=779544) INFO 10-25 12:23:38 [core.py:96] Initializing a V1 LLM engine (v0.15.1) with config: model='/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218', speculative_config=None, tokenizer='/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False), seed=0, served_model_name=/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::unified_kv_cache_update'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': True}, 'local_cache_dir': None, 'static_all_moe_layers': []}
+(EngineCore_DP0 pid=779544) INFO 10-25 12:23:40 [parallel_state.py:1212] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://10.127.105.43:44805 backend=nccl
+(EngineCore_DP0 pid=779544) INFO 10-25 12:23:40 [parallel_state.py:1423] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A
+(EngineCore_DP0 pid=779544) INFO 10-25 12:23:40 [gpu_model_runner.py:4033] Starting to load model /mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218...
+(EngineCore_DP0 pid=779544) INFO 10-25 12:23:41 [cuda.py:364] Using FLASH_ATTN attention backend out of potential backends: ('FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION')
+(EngineCore_DP0 pid=779544) 
+(EngineCore_DP0 pid=779544) 
+(EngineCore_DP0 pid=779544) 
+(EngineCore_DP0 pid=779544) 
+(EngineCore_DP0 pid=779544) 
+(EngineCore_DP0 pid=779544) 
+(EngineCore_DP0 pid=779544) 
+(EngineCore_DP0 pid=779544) 
+(EngineCore_DP0 pid=779544) INFO 10-25 12:23:44 [default_loader.py:291] Loading weights took 3.12 seconds
+(EngineCore_DP0 pid=779544) INFO 10-25 12:23:44 [gpu_model_runner.py:4130] Model loading took 15.27 GiB memory and 3.527412 seconds
+(EngineCore_DP0 pid=779544) INFO 10-25 12:23:48 [backends.py:812] Using cache directory: /home/vlad.smirnov/.cache/vllm/torch_compile_cache/5250b7e167/rank_0_0/backbone for vLLM's torch.compile
+(EngineCore_DP0 pid=779544) INFO 10-25 12:23:48 [backends.py:872] Dynamo bytecode transform time: 3.39 s
+(EngineCore_DP0 pid=779544) INFO 10-25 12:23:51 [backends.py:267] Directly load the compiled graph(s) for compile range (1, 8192) from the cache, took 1.106 s
+(EngineCore_DP0 pid=779544) INFO 10-25 12:23:51 [monitor.py:34] torch.compile takes 4.50 s in total
+(EngineCore_DP0 pid=779544) INFO 10-25 12:23:52 [gpu_worker.py:356] Available KV cache memory: 25.88 GiB
+(EngineCore_DP0 pid=779544) INFO 10-25 12:23:52 [kv_cache_utils.py:1307] GPU KV cache size: 188,448 tokens
+(EngineCore_DP0 pid=779544) INFO 10-25 12:23:52 [kv_cache_utils.py:1312] Maximum concurrency for 8,192 tokens per request: 23.00x
+(EngineCore_DP0 pid=779544) 
+(EngineCore_DP0 pid=779544) 
+(EngineCore_DP0 pid=779544) INFO 10-25 12:23:56 [gpu_model_runner.py:5063] Graph capturing finished in 4 secs, took 0.56 GiB
+(EngineCore_DP0 pid=779544) INFO 10-25 12:23:56 [core.py:272] init engine (profile, create kv cache, warmup model) took 11.98 seconds
+INFO 10-25 12:23:57 [llm.py:343] Supported tasks: ['generate']
+{
+ "model": "Qwen/Qwen3-8B",
+ "thinking": true,
+ "gpu": "NVIDIA RTX 5880 Ada Generation",
+ "n_gpus": 1,
+ "n_prompts": 256,
+ "max_tokens": 4096,
+ "gpu_mem_util": 0.9,
+ "decode": {
+  "generated_tokens": 929161,
+  "seconds": 769.5907077789307,
+  "tok_per_s": 1207.3443592914414,
+  "capped": 160,
+  "mean_len": 3629.53515625
+ },
+ "prefill": {
+  "prompt_tokens": 522363,
+  "seconds": 83.4063332080841,
+  "tok_per_s": 6262.869735524716,
+  "mean_len": 2040.48046875
+ },
+ "kv_cache_blocks": 11778,
+ "block_size": 16
+}
+[W1025 12:38:12.056032103 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator())

analysis/registry_2026-10-25/bench_Qwen3-8B_think.provenance.json

@@ -0,0 +1,74 @@
+{
+ "started_at": "2026-10-25T12:38:11+04:00",
+ "host": "ws006598",
+ "user": "vlad.smirnov",
+ "cwd": "/mnt/data/users/vlad.smirnov/rl-uhead-ood",
+ "argv": [
+  "/mnt/data/users/vlad.smirnov/rl-uhead-ood/rl_controller/scripts/throughput_bench.py",
+  "--model",
+  "Qwen/Qwen3-8B",
+  "--thinking",
+  "--pool-dir",
+  "workdir/selector/pool_mth_k8_chat",
+  "--out",
+  "workdir/registry/bench_Qwen3-8B_think.json"
+ ],
+ "git": {
+  "commit": "0bfea921dff632984bf84f24836c4588eb47a6a4",
+  "branch": "HEAD",
+  "subject": "Add throughput_bench and lane TP: measured decode/prefill throughput per model for the registry",
+  "dirty": false,
+  "dirty_files": []
+ },
+ "gpus": {
+  "visible": "1",
+  "devices": [
+   {
+    "index": "0",
+    "name": "NVIDIA RTX 5880 Ada Generation",
+    "memory": "49140 MiB",
+    "driver": "580.178.04",
+    "uuid": "GPU-eb08932b-fbcd-41ce-e1e2-6fd7b02de161"
+   },
+   {
+    "index": "1",
+    "name": "NVIDIA RTX 5880 Ada Generation",
+    "memory": "49140 MiB",
+    "driver": "580.178.04",
+    "uuid": "GPU-cc1e8db2-7843-a8e9-8191-088167fb3499"
+   }
+  ]
+ },
+ "packages": {
+  "python": "3.11.15",
+  "torch": "2.9.1",
+  "vllm": "0.15.1",
+  "transformers": "4.57.6",
+  "speculators": "0.4.0.dev49",
+  "peft": null,
+  "accelerate": "1.13.0",
+  "numpy": "2.2.6",
+  "huggingface_hub": "0.36.2",
+  "openai": "2.32.0",
+  "wandb": "0.25.1",
+  "torch_cuda": "12.8"
+ },
+ "env": {
+  "CUDA_VISIBLE_DEVICES": "1",
+  "HF_HOME": "/mnt/data/users/vlad.smirnov/hf_cache",
+  "HF_HUB_OFFLINE": "1",
+  "TRANSFORMERS_OFFLINE": "1",
+  "NCCL_P2P_DISABLE": "1",
+  "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True"
+ },
+ "models": [
+  {
+   "repo": "Qwen/Qwen3-8B",
+   "revision": "b968826d9c46dd6066d109eabc6255188de91218",
+   "path": "/mnt/data/users/vlad.smirnov/hf_cache/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218"
+  }
+ ],
+ "files": {},
+ "finished_at": "2026-10-25T12:38:11+04:00",
+ "wall_secs": 0.1
+}