Buckets:
| { | |
| "model_tokenizer": "Qwen/Qwen3.5-0.8B", | |
| "context": 32768, | |
| "max_prompt_tokens": 16000, | |
| "train_pool": 207, | |
| "eval": 78, | |
| "calib": 52, | |
| "train_by_set": { | |
| "train_proc": 144, | |
| "train_ts": 63 | |
| }, | |
| "eval_by_set": { | |
| "eval_proc": 36, | |
| "eval_ts": 16, | |
| "opendecisions": 16, | |
| "localllama": 10 | |
| }, | |
| "calib_by_set": { | |
| "calib_proc": 36, | |
| "calib_ts": 16 | |
| }, | |
| "train_by_type": { | |
| "choice": 123, | |
| "noul": 52, | |
| "score": 32 | |
| }, | |
| "eval_by_type": { | |
| "score": 12, | |
| "choice": 46, | |
| "noul": 20 | |
| }, | |
| "exclusions": { | |
| "calib_ts:options_out_of_range": 2, | |
| "calib_ts:procedural_copies_removed": 1500, | |
| "eval_ts:procedural_copies_removed": 1500, | |
| "localllama:gold_tie": 35, | |
| "train:overlap_with_eval_removed": 0, | |
| "train:prompt_over_16000_tokens": 1, | |
| "train_ts:options_out_of_range": 2, | |
| "train_ts:procedural_copies_removed": 120013 | |
| }, | |
| "prompt_tokens_max": { | |
| "train": 3178, | |
| "eval": 2713 | |
| } | |
| } | |
| (MainProcess pid=9367) INFO 10-30 23:51:13 [model.py:733] Resolved architecture: Qwen3_5ForConditionalGeneration | |
| (MainProcess pid=9367) INFO 10-30 23:51:13 [model.py:2152] Using max model len 32768 | |
| Fetching 12 files: 0%| | 0/12 [00:00<?, ?it/s] Fetching 12 files: 100%|██████████| 12/12 [00:00<00:00, 1106.00it/s] | |
| (MainProcess pid=9367) INFO 10-30 23:51:15 [scheduler.py:307] Chunked prefill is enabled with max_num_batched_tokens=2048. | |
| (MainProcess pid=9367) INFO 10-30 23:51:15 [config.py:688] Mamba cache mode is set to 'align' for Qwen3_5ForConditionalGeneration by default when prefix caching is enabled | |
| Parse safetensors files: 0%| | 0/1 [00:00<?, ?it/s] Parse safetensors files: 100%|██████████| 1/1 [00:00<00:00, 3.54it/s] Parse safetensors files: 100%|██████████| 1/1 [00:00<00:00, 3.53it/s] | |
| (MainProcess pid=9367) INFO 10-30 23:51:15 [kernel.py:467] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) | |
| (MainProcess pid=9367) WARNING 10-30 23:51:27 [system_utils.py:153] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized | |
| (EngineCore pid=9787) INFO 10-30 23:51:37 [core.py:133] Initializing a V1 LLM engine (v0.32.0) with config: model='Qwen/Qwen3.5-0.8B', speculative_config=None, tokenizer='Qwen/Qwen3.5-0.8B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, aux_output_config=AuxOutputConfig(enable_return_routed_experts=False, max_bytes=None), kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, custom_histogram_buckets=None, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=17, served_model_name=Qwen/Qwen3.5-0.8B, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::qwen4_exp_qsa_prepare_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'cudagraph_decoder_replay': True, 'decoder_replay_cudagraph_capture_sizes': [], 'decoder_replay_trim_threshold': 768, 'compile_sizes': [], 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_rocm_segmented_attn_autotune=False, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) | |
| (EngineCore pid=9787) INFO 10-30 23:51:39 [parallel_state.py:1906] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_bfcfa19d316446df9ef6b3d72d34c589 backend=nccl | |
| (EngineCore pid=9787) INFO 10-30 23:51:39 [parallel_state.py:2350] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A | |
| (EngineCore pid=9787) INFO 10-30 23:51:39 [gpu_worker.py:518] Using V2 Model Runner | |
| (EngineCore pid=9787) INFO 10-30 23:51:40 [model_runner.py:392] Loading model from scratch... | |
| Fetching 12 files: 0%| | 0/12 [00:00<?, ?it/s] Fetching 12 files: 100%|██████████| 12/12 [00:00<00:00, 1057.77it/s] | |
| (EngineCore pid=9787) INFO 10-30 23:51:41 [cuda.py:565] Using backend AttentionBackendEnum.FLASH_ATTN for vit attention | |
| (EngineCore pid=9787) INFO 10-30 23:51:41 [mm_encoder_attention.py:372] Using AttentionBackendEnum.FLASH_ATTN for MMEncoderAttention. | |
| (EngineCore pid=9787) INFO 10-30 23:51:41 [qwen_gdn_linear_attn.py:221] Using Triton/FLA GDN prefill kernel (requested=auto, head_k_dim=128). | |
| (EngineCore pid=9787) INFO 10-30 23:51:42 [cuda.py:506] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. | |
| (EngineCore pid=9787) INFO 10-30 23:51:42 [flash_attn.py:1117] Using FlashAttention version 2 | |
| (EngineCore pid=9787) INFO 10-30 23:51:42 [base_loader.py:82] Initializing model took 2.088368 seconds | |
| (EngineCore pid=9787) INFO 10-30 23:51:42 [weight_utils.py:1108] Filesystem type for checkpoints: XFS. Checkpoint size: 1.63 GiB. Available RAM: 197.51 GiB. | |
| (EngineCore pid=9787) INFO 10-30 23:51:42 [weight_utils.py:1131] Auto-prefetch is disabled because the filesystem (XFS) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch. | |
| Loading safetensors checkpoint shards: 0% Completed | 0/1 [00:00<?, ?it/s] | |
| (EngineCore pid=9787) WARNING 10-30 23:51:42 [utils.py:333] Attempted to load weight layers.0.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=9787) WARNING 10-30 23:51:42 [utils.py:333] Attempted to load weight layers.1.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=9787) WARNING 10-30 23:51:42 [utils.py:333] Attempted to load weight layers.10.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=9787) WARNING 10-30 23:51:42 [utils.py:333] Attempted to load weight layers.12.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=9787) WARNING 10-30 23:51:42 [utils.py:333] Attempted to load weight layers.13.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=9787) WARNING 10-30 23:51:42 [utils.py:333] Attempted to load weight layers.14.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=9787) WARNING 10-30 23:51:42 [utils.py:333] Attempted to load weight layers.16.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=9787) WARNING 10-30 23:51:42 [utils.py:333] Attempted to load weight layers.17.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=9787) WARNING 10-30 23:51:42 [utils.py:333] Attempted to load weight layers.18.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=9787) WARNING 10-30 23:51:42 [utils.py:333] Attempted to load weight layers.2.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=9787) WARNING 10-30 23:51:42 [utils.py:333] Attempted to load weight layers.20.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=9787) WARNING 10-30 23:51:42 [utils.py:333] Attempted to load weight layers.21.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=9787) WARNING 10-30 23:51:42 [utils.py:333] Attempted to load weight layers.22.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=9787) WARNING 10-30 23:51:42 [utils.py:333] Attempted to load weight layers.4.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=9787) WARNING 10-30 23:51:43 [utils.py:333] Attempted to load weight layers.5.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=9787) WARNING 10-30 23:51:43 [utils.py:333] Attempted to load weight layers.6.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=9787) WARNING 10-30 23:51:43 [utils.py:333] Attempted to load weight layers.8.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=9787) WARNING 10-30 23:51:43 [utils.py:333] Attempted to load weight layers.9.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00, 3.91it/s] | |
| Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00, 3.91it/s] | |
| (EngineCore pid=9787) INFO 10-30 23:51:43 [default_loader.py:501] Loading weights took 0.26 seconds | |
| (EngineCore pid=9787) INFO 10-30 23:51:44 [model_runner.py:424] Model loading took 1.63 GiB memory and 4.251835 seconds | |
| (EngineCore pid=9787) INFO 10-30 23:51:44 [topk_topp_sampler.py:124] Using FlashInfer for top-p & top-k sampling. | |
| (EngineCore pid=9787) INFO 10-30 23:51:44 [torch_utils.py:295] Reducing Torch threads from 128 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override. | |
| (EngineCore pid=9787) INFO 10-30 23:51:44 [interface.py:1096] Setting attention block size to 544 tokens to ensure that attention page size is >= mamba page size. | |
| (EngineCore pid=9787) INFO 10-30 23:51:44 [interface.py:1120] Padding mamba page size by 2.64% to ensure that mamba page size and attention page size are exactly equal. | |
| (EngineCore pid=9787) INFO 10-30 23:51:44 [utils.py:347] Using LBNHC KV cache layout. | |
| [transformers] Qwen3VL video processing does not apply the per-frame pixel cap the reference implementation (qwen-vl-utils) applies, so some videos cost far more tokens than they would there. In v5.22 the capped behavior will become the default and `cap_pixels_per_frame` will be removed. Pass `cap_pixels_per_frame=True` to adopt the reference behavior now, or `False` to keep the current behavior and silence this warning. | |
| (MainProcess pid=9367) INFO 10-30 23:51:47 [base.py:280] Multi-modal warmup completed in 20.124s | |
| (EngineCore pid=9787) INFO 10-30 23:51:52 [encoder_runner.py:131] Encoder cache will be initialized with a budget of 16384 tokens, and profiled with 1 image items of the maximum feature size. | |
| (EngineCore pid=9787) INFO 10-30 23:51:54 [monitor.py:40] [backbone] Starting torch.compile | |
| (EngineCore pid=9787) INFO 10-30 23:51:57 [backends.py:1104] [backbone] Using cache directory: /root/.cache/vllm/torch_compile_cache/48e96a4b67/rank_0_0/backbone for vLLM's torch.compile | |
| (EngineCore pid=9787) INFO 10-30 23:51:57 [backends.py:1166] [backbone] Dynamo bytecode transform time: 3.24 s | |
| (EngineCore pid=9787) INFO 10-30 23:52:12 [backends.py:398] [backbone] Compiling a graph for compile range (1, 2048) takes 15.05 s | |
| (EngineCore pid=9787) INFO 10-30 23:52:15 [backends.py:927] collected artifacts: 25 entries, 11 artifacts, 17073542 bytes total | |
| (EngineCore pid=9787) INFO 10-30 23:52:15 [decorators.py:743] [backbone] saved AOT compiled function to /root/.cache/vllm/torch_compile_cache/torch_aot_compile/38b27199840dd5788649ecfb9a7fd4945e985542636a6f138f09557d6ee82c7d/rank_0_0/model | |
| (EngineCore pid=9787) INFO 10-30 23:52:15 [monitor.py:62] [backbone] torch.compile took 21.06 s in total | |
| (EngineCore pid=9787) INFO 10-30 23:53:40 [monitor.py:91] [backbone] Initial profiling/warmup run took 84.86 s | |
| Capturing CUDA graphs (PIECEWISE): 0%| | 0/35 [00:00<?, ?it/s] Capturing CUDA graphs (PIECEWISE): 3%|▎ | 1/35 [00:00<00:27, 1.24it/s] Capturing CUDA graphs (PIECEWISE): 6%|▌ | 2/35 [00:01<00:24, 1.37it/s] Capturing CUDA graphs (PIECEWISE): 9%|▊ | 3/35 [00:01<00:15, 2.03it/s] Capturing CUDA graphs (PIECEWISE): 11%|█▏ | 4/35 [00:01<00:11, 2.68it/s] Capturing CUDA graphs (PIECEWISE): 14%|█▍ | 5/35 [00:02<00:09, 3.30it/s] Capturing CUDA graphs (PIECEWISE): 17%|█▋ | 6/35 [00:02<00:07, 3.88it/s] Capturing CUDA graphs (PIECEWISE): 20%|██ | 7/35 [00:02<00:06, 4.34it/s] Capturing CUDA graphs (PIECEWISE): 23%|██▎ | 8/35 [00:02<00:05, 4.86it/s] Capturing CUDA graphs (PIECEWISE): 26%|██▌ | 9/35 [00:02<00:04, 5.30it/s] Capturing CUDA graphs (PIECEWISE): 29%|██▊ | 10/35 [00:02<00:04, 5.76it/s] Capturing CUDA graphs (PIECEWISE): 31%|███▏ | 11/35 [00:02<00:03, 6.26it/s] Capturing CUDA graphs (PIECEWISE): 34%|███▍ | 12/35 [00:03<00:03, 6.69it/s] Capturing CUDA graphs (PIECEWISE): 37%|███▋ | 13/35 [00:03<00:03, 7.17it/s] Capturing CUDA graphs (PIECEWISE): 40%|████ | 14/35 [00:03<00:02, 7.76it/s] Capturing CUDA graphs (PIECEWISE): 43%|████▎ | 15/35 [00:03<00:02, 8.26it/s] Capturing CUDA graphs (PIECEWISE): 46%|████▌ | 16/35 [00:03<00:02, 8.70it/s] Capturing CUDA graphs (PIECEWISE): 49%|████▊ | 17/35 [00:04<00:06, 2.62it/s] Capturing CUDA graphs (PIECEWISE): 51%|█████▏ | 18/35 [00:04<00:05, 3.00it/s] Capturing CUDA graphs (PIECEWISE): 60%|██████ | 21/35 [00:04<00:02, 6.04it/s] Capturing CUDA graphs (PIECEWISE): 69%|██████▊ | 24/35 [00:04<00:01, 9.31it/s] Capturing CUDA graphs (PIECEWISE): 77%|███████▋ | 27/35 [00:05<00:00, 12.46it/s] Capturing CUDA graphs (PIECEWISE): 86%|████████▌ | 30/35 [00:05<00:00, 14.42it/s] Capturing CUDA graphs (PIECEWISE): 94%|█████████▍| 33/35 [00:05<00:00, 17.30it/s] Capturing CUDA graphs (PIECEWISE): 100%|██████████| 35/35 [00:05<00:00, 6.25it/s] | |
| Capturing CUDA graphs (FULL): 0%| | 0/2 [00:00<?, ?it/s] Capturing CUDA graphs (FULL): 100%|██████████| 2/2 [00:00<00:00, 35.18it/s] | |
| (EngineCore pid=9787) INFO 10-30 23:53:48 [model_runner.py:1127] Graph capturing finished in 7 secs, took 0.21 GiB | |
| (EngineCore pid=9787) INFO 10-30 23:53:49 [gpu_worker.py:759] Available KV cache memory: 65.6 GiB | |
| (EngineCore pid=9787) INFO 10-30 23:53:49 [gpu_worker.py:774] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9000 is equivalent to --gpu-memory-utilization=0.8972 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9028. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0. | |
| (EngineCore pid=9787) INFO 10-30 23:53:49 [kv_cache_utils.py:2327] GPU KV cache size: 5,153,379 tokens, Maximum concurrency for 32,768 tokens per request: 157.27x | |
| (EngineCore pid=9787) INFO 10-30 23:53:49 [kv_cache_utils.py:853] kv cache group sizes (544, 544, 544, 544) | |
| (EngineCore pid=9787) INFO 10-30 23:53:49 [kv_cache_utils.py:854] kv lcm block sizes 544 | |
| (EngineCore pid=9787) INFO 10-30 23:53:49 [kernel_warmup.py:176] JIT kernel warmup starting. | |
| (EngineCore pid=9787) INFO 10-30 23:53:49 [kernel_warmup.py:189] JIT kernel warmup finished in 0.00s. | |
| (EngineCore pid=9787) INFO 10-30 23:53:49 [qwen_triton_warmup.py:296] Warming up Qwen GDN Triton kernels for model_type=qwen3_5_text. | |
| (EngineCore pid=9787) INFO 10-30 23:53:51 [qwen_vl_triton_warmup.py:57] Warmed position embedding and vision rotary kernels on grids=[(1, 16, 16), (1, 16, 2), (1, 2, 16), (1, 2, 2)]. | |
| (EngineCore pid=9787) INFO 10-30 23:53:51 [mamba_triton_warmup.py:42] Warmed Mamba batch_memcpy_kernel. | |
| Capturing CUDA graphs (PIECEWISE): 0%| | 0/35 [00:00<?, ?it/s] Capturing CUDA graphs (PIECEWISE): 3%|▎ | 1/35 [00:00<00:07, 4.47it/s] Capturing CUDA graphs (PIECEWISE): 6%|▌ | 2/35 [00:00<00:07, 4.66it/s] Capturing CUDA graphs (PIECEWISE): 9%|▊ | 3/35 [00:00<00:07, 4.12it/s] Capturing CUDA graphs (PIECEWISE): 11%|█▏ | 4/35 [00:00<00:06, 4.49it/s] Capturing CUDA graphs (PIECEWISE): 14%|█▍ | 5/35 [00:01<00:09, 3.20it/s] Capturing CUDA graphs (PIECEWISE): 17%|█▋ | 6/35 [00:01<00:08, 3.56it/s] Capturing CUDA graphs (PIECEWISE): 20%|██ | 7/35 [00:01<00:06, 4.15it/s] Capturing CUDA graphs (PIECEWISE): 23%|██▎ | 8/35 [00:01<00:05, 4.72it/s] Capturing CUDA graphs (PIECEWISE): 26%|██▌ | 9/35 [00:02<00:04, 5.27it/s] Capturing CUDA graphs (PIECEWISE): 29%|██▊ | 10/35 [00:02<00:04, 5.83it/s] Capturing CUDA graphs (PIECEWISE): 31%|███▏ | 11/35 [00:02<00:03, 6.30it/s] Capturing CUDA graphs (PIECEWISE): 34%|███▍ | 12/35 [00:02<00:03, 6.89it/s] Capturing CUDA graphs (PIECEWISE): 37%|███▋ | 13/35 [00:02<00:02, 7.46it/s] Capturing CUDA graphs (PIECEWISE): 40%|████ | 14/35 [00:02<00:02, 7.93it/s] Capturing CUDA graphs (PIECEWISE): 43%|████▎ | 15/35 [00:02<00:02, 8.43it/s] Capturing CUDA graphs (PIECEWISE): 49%|████▊ | 17/35 [00:02<00:01, 11.06it/s] Capturing CUDA graphs (PIECEWISE): 60%|██████ | 21/35 [00:02<00:00, 17.69it/s] Capturing CUDA graphs (PIECEWISE): 71%|███████▏ | 25/35 [00:03<00:00, 22.23it/s] Capturing CUDA graphs (PIECEWISE): 83%|████████▎ | 29/35 [00:03<00:00, 25.33it/s] Capturing CUDA graphs (PIECEWISE): 94%|█████████▍| 33/35 [00:03<00:00, 27.67it/s] Capturing CUDA graphs (PIECEWISE): 100%|██████████| 35/35 [00:03<00:00, 10.30it/s] | |
| Capturing CUDA graphs (FULL): 0%| | 0/19 [00:00<?, ?it/s] Capturing CUDA graphs (FULL): 21%|██ | 4/19 [00:00<00:00, 38.72it/s] Capturing CUDA graphs (FULL): 42%|████▏ | 8/19 [00:00<00:00, 29.07it/s] Capturing CUDA graphs (FULL): 63%|██████▎ | 12/19 [00:00<00:00, 32.86it/s] Capturing CUDA graphs (FULL): 84%|████████▍ | 16/19 [00:00<00:00, 34.52it/s] Capturing CUDA graphs (FULL): 100%|██████████| 19/19 [00:00<00:00, 34.39it/s] | |
| (EngineCore pid=9787) INFO 10-30 23:54:14 [model_runner.py:1127] Graph capturing finished in 5 secs, took 0.12 GiB | |
| (EngineCore pid=9787) INFO 10-30 23:54:14 [gpu_worker.py:971] CUDA graph pool memory: 0.12 GiB (actual), 0.22 GiB (estimated), difference: 0.11 GiB (88.5%). | |
| (EngineCore pid=9787) INFO 10-30 23:54:14 [gpu_worker.py:1034] Free memory on device (78.84/79.25 GiB) on startup. Desired GPU memory utilization is (0.9, 71.33 GiB). Actual usage is 4.27 GiB for consumed memory (weights + non-torch), 1.45 GiB for peak activation, and 0.12 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=70155491226` (65.34 GiB) to fit into requested memory, or `--kv-cache-memory=78221557248` (72.85 GiB) to fully utilize gpu memory. Current kv cache memory in use is 65.6 GiB. | |
| (EngineCore pid=9787) INFO 10-30 23:54:15 [jit_monitor.py:84] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn. | |
| (EngineCore pid=9787) INFO 10-30 23:54:15 [core.py:395] init engine (profile, create kv cache, warmup model) took 151.12 s (compilation: 21.06 s) | |
| Parse safetensors files: 0%| | 0/1 [00:00<?, ?it/s] Parse safetensors files: 100%|██████████| 1/1 [00:00<00:00, 2.45it/s] Parse safetensors files: 100%|██████████| 1/1 [00:00<00:00, 2.45it/s] | |
| (EngineCore pid=9787) INFO 10-30 23:54:16 [kernel.py:467] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) | |
| (MainProcess pid=9367) WARNING 10-30 23:54:16 [input_processor.py:408] Passing raw prompts to InputProcessor is deprecated and will be removed in the future. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat(). | |
| (EngineCore pid=9787) WARNING 10-30 23:54:16 [jit_monitor.py:140] Triton kernel JIT compilation during inference: _topp_sb_stats_kernel. This causes a latency spike; consider extending warmup to cover this shape/config. | |
| (EngineCore pid=9787) WARNING 10-30 23:54:17 [jit_monitor.py:140] Triton kernel JIT compilation during inference: _topp_sb_step_kernel. This causes a latency spike; consider extending warmup to cover this shape/config. | |
| (EngineCore pid=9787) WARNING 10-30 23:54:18 [jit_monitor.py:140] Triton kernel JIT compilation during inference: _topp_sb_mask_kernel. This causes a latency spike; consider extending warmup to cover this shape/config. | |
| progress {"time": "23:55:15", "elapsed_min": 1.0, "submitted": 8, "finished": 1, "in_flight": 7, "gen_tokens": 2932, "tok_per_s": 49, "chain_tokens_median": 2932.0, "chain_tokens_p90": 2932.0, "chain_tokens_max": 2932, "parse_rate": 1.0, "looped": 0, "minutes_to_deadline": Infinity} | |
| progress {"time": "23:56:15", "elapsed_min": 2.0, "submitted": 8, "finished": 1, "in_flight": 7, "gen_tokens": 2932, "tok_per_s": 24, "chain_tokens_median": 2932.0, "chain_tokens_p90": 2932.0, "chain_tokens_max": 2932, "parse_rate": 1.0, "looped": 0, "minutes_to_deadline": Infinity} | |
| progress {"time": "23:57:15", "elapsed_min": 3.0, "submitted": 8, "finished": 1, "in_flight": 7, "gen_tokens": 2932, "tok_per_s": 16, "chain_tokens_median": 2932.0, "chain_tokens_p90": 2932.0, "chain_tokens_max": 2932, "parse_rate": 1.0, "looped": 0, "minutes_to_deadline": Infinity} | |
| (MainProcess pid=9367) INFO 10-30 23:58:13 [core_client.py:770] [shutdown] MPClient: start timeout=default | |
| (MainProcess pid=9367) INFO 10-30 23:58:13 [core_client.py:772] [shutdown] MPClient: stopping engine manager | |
| (MainProcess pid=9367) INFO 10-30 23:58:13 [utils.py:628] [shutdown] Process manager: send sigterm to process EngineCore | |
| (EngineCore pid=9787) INFO 10-30 23:58:13 [core.py:1454] [shutdown] EngineCore: trigger received signal=SIGTERM | |
| (EngineCore pid=9787) INFO 10-30 23:58:13 [core.py:1635] [shutdown] EngineCore: start mode=abort timeout=0s | |
| (EngineCore pid=9787) INFO 10-30 23:58:13 [core.py:1666] [shutdown] EngineCore: request processing complete; starting resource teardown | |
| (EngineCore pid=9787) INFO 10-30 23:58:13 [core.py:1499] [shutdown] EngineCore: exiting busy loop | |
| (MainProcess pid=9367) INFO 10-30 23:58:17 [core_client.py:774] [shutdown] MPClient: engine manager stopped | |
| (MainProcess pid=9367) INFO 10-30 23:58:17 [core_client.py:775] [shutdown] MPClient: cleaning up background resources | |
| (MainProcess pid=9367) INFO 10-30 23:58:17 [core_client.py:777] [shutdown] MPClient: complete | |
| { | |
| "items": 8, | |
| "samples": 15, | |
| "tokens_total": 455377, | |
| "tokens_median": 32334.0, | |
| "tokens_p90": 32490.0, | |
| "tokens_max": 32492, | |
| "parse_rate": 0.06666666666666667, | |
| "looped": 14, | |
| "items_with_correct_chain": 1, | |
| "majority_accuracy": 0.125, | |
| "seconds": 424.9759855270386, | |
| "gen_tok_per_s": 1071.5358408670063, | |
| "engine": { | |
| "submitted": 8, | |
| "finished": 8, | |
| "gen_tokens": 455377, | |
| "t0": 1793404455.7403855, | |
| "seconds": 237.64002227783203, | |
| "tok_per_s": 1916.2470851294786 | |
| }, | |
| "N_fit": 1 | |
| } | |
| {"to_generate": 8, "already_done": 8, "minutes_left": 164.9999997973442} | |
| (MainProcess pid=10814) INFO 10-30 23:58:31 [model.py:733] Resolved architecture: Qwen3_5ForConditionalGeneration | |
| (MainProcess pid=10814) INFO 10-30 23:58:31 [model.py:2152] Using max model len 32768 | |
| Fetching 12 files: 0%| | 0/12 [00:00<?, ?it/s] Fetching 12 files: 100%|██████████| 12/12 [00:00<00:00, 868.45it/s] | |
| (MainProcess pid=10814) INFO 10-30 23:58:34 [scheduler.py:307] Chunked prefill is enabled with max_num_batched_tokens=2048. | |
| (MainProcess pid=10814) INFO 10-30 23:58:34 [config.py:688] Mamba cache mode is set to 'align' for Qwen3_5ForConditionalGeneration by default when prefix caching is enabled | |
| Parse safetensors files: 0%| | 0/1 [00:00<?, ?it/s] Parse safetensors files: 100%|██████████| 1/1 [00:00<00:00, 4.50it/s] Parse safetensors files: 100%|██████████| 1/1 [00:00<00:00, 4.49it/s] | |
| (MainProcess pid=10814) INFO 10-30 23:58:34 [kernel.py:467] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) | |
| (MainProcess pid=10814) WARNING 10-30 23:58:47 [system_utils.py:153] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized | |
| (EngineCore pid=11241) INFO 10-30 23:58:59 [core.py:133] Initializing a V1 LLM engine (v0.32.0) with config: model='Qwen/Qwen3.5-0.8B', speculative_config=None, tokenizer='Qwen/Qwen3.5-0.8B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, aux_output_config=AuxOutputConfig(enable_return_routed_experts=False, max_bytes=None), kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, custom_histogram_buckets=None, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=17, served_model_name=Qwen/Qwen3.5-0.8B, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::qwen4_exp_qsa_prepare_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'cudagraph_decoder_replay': True, 'decoder_replay_cudagraph_capture_sizes': [], 'decoder_replay_trim_threshold': 768, 'compile_sizes': [], 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_rocm_segmented_attn_autotune=False, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) | |
| (EngineCore pid=11241) INFO 10-30 23:59:00 [parallel_state.py:1906] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_c24df85980734f59ac79edc5d3bd4abd backend=nccl | |
| (EngineCore pid=11241) INFO 10-30 23:59:00 [parallel_state.py:2350] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A | |
| (EngineCore pid=11241) INFO 10-30 23:59:00 [gpu_worker.py:518] Using V2 Model Runner | |
| (EngineCore pid=11241) INFO 10-30 23:59:02 [model_runner.py:392] Loading model from scratch... | |
| Fetching 12 files: 0%| | 0/12 [00:00<?, ?it/s] Fetching 12 files: 100%|██████████| 12/12 [00:00<00:00, 1177.95it/s] | |
| (EngineCore pid=11241) INFO 10-30 23:59:03 [cuda.py:565] Using backend AttentionBackendEnum.FLASH_ATTN for vit attention | |
| (EngineCore pid=11241) INFO 10-30 23:59:03 [mm_encoder_attention.py:372] Using AttentionBackendEnum.FLASH_ATTN for MMEncoderAttention. | |
| (EngineCore pid=11241) INFO 10-30 23:59:03 [qwen_gdn_linear_attn.py:221] Using Triton/FLA GDN prefill kernel (requested=auto, head_k_dim=128). | |
| (EngineCore pid=11241) INFO 10-30 23:59:04 [cuda.py:506] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. | |
| (EngineCore pid=11241) INFO 10-30 23:59:04 [flash_attn.py:1117] Using FlashAttention version 2 | |
| (EngineCore pid=11241) INFO 10-30 23:59:04 [base_loader.py:82] Initializing model took 2.068038 seconds | |
| (EngineCore pid=11241) INFO 10-30 23:59:04 [weight_utils.py:1108] Filesystem type for checkpoints: XFS. Checkpoint size: 1.63 GiB. Available RAM: 197.75 GiB. | |
| (EngineCore pid=11241) INFO 10-30 23:59:04 [weight_utils.py:1131] Auto-prefetch is disabled because the filesystem (XFS) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch. | |
| Loading safetensors checkpoint shards: 0% Completed | 0/1 [00:00<?, ?it/s] | |
| (EngineCore pid=11241) WARNING 10-30 23:59:04 [utils.py:333] Attempted to load weight layers.0.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=11241) WARNING 10-30 23:59:04 [utils.py:333] Attempted to load weight layers.1.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=11241) WARNING 10-30 23:59:04 [utils.py:333] Attempted to load weight layers.10.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=11241) WARNING 10-30 23:59:04 [utils.py:333] Attempted to load weight layers.12.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=11241) WARNING 10-30 23:59:04 [utils.py:333] Attempted to load weight layers.13.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=11241) WARNING 10-30 23:59:05 [utils.py:333] Attempted to load weight layers.14.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=11241) WARNING 10-30 23:59:05 [utils.py:333] Attempted to load weight layers.16.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=11241) WARNING 10-30 23:59:05 [utils.py:333] Attempted to load weight layers.17.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=11241) WARNING 10-30 23:59:05 [utils.py:333] Attempted to load weight layers.18.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=11241) WARNING 10-30 23:59:05 [utils.py:333] Attempted to load weight layers.2.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=11241) WARNING 10-30 23:59:05 [utils.py:333] Attempted to load weight layers.20.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=11241) WARNING 10-30 23:59:05 [utils.py:333] Attempted to load weight layers.21.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=11241) WARNING 10-30 23:59:05 [utils.py:333] Attempted to load weight layers.22.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=11241) WARNING 10-30 23:59:05 [utils.py:333] Attempted to load weight layers.4.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=11241) WARNING 10-30 23:59:05 [utils.py:333] Attempted to load weight layers.5.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=11241) WARNING 10-30 23:59:05 [utils.py:333] Attempted to load weight layers.6.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=11241) WARNING 10-30 23:59:05 [utils.py:333] Attempted to load weight layers.8.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=11241) WARNING 10-30 23:59:05 [utils.py:333] Attempted to load weight layers.9.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00, 3.81it/s] | |
| Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00, 3.81it/s] | |
| (EngineCore pid=11241) INFO 10-30 23:59:05 [default_loader.py:501] Loading weights took 0.27 seconds | |
| (EngineCore pid=11241) INFO 10-30 23:59:06 [model_runner.py:424] Model loading took 1.63 GiB memory and 4.163793 seconds | |
| (EngineCore pid=11241) INFO 10-30 23:59:06 [topk_topp_sampler.py:124] Using FlashInfer for top-p & top-k sampling. | |
| (EngineCore pid=11241) INFO 10-30 23:59:06 [torch_utils.py:295] Reducing Torch threads from 128 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override. | |
| (EngineCore pid=11241) INFO 10-30 23:59:06 [interface.py:1096] Setting attention block size to 544 tokens to ensure that attention page size is >= mamba page size. | |
| (EngineCore pid=11241) INFO 10-30 23:59:06 [interface.py:1120] Padding mamba page size by 2.64% to ensure that mamba page size and attention page size are exactly equal. | |
| (EngineCore pid=11241) INFO 10-30 23:59:06 [utils.py:347] Using LBNHC KV cache layout. | |
| [transformers] Qwen3VL video processing does not apply the per-frame pixel cap the reference implementation (qwen-vl-utils) applies, so some videos cost far more tokens than they would there. In v5.22 the capped behavior will become the default and `cap_pixels_per_frame` will be removed. Pass `cap_pixels_per_frame=True` to adopt the reference behavior now, or `False` to keep the current behavior and silence this warning. | |
| (MainProcess pid=10814) INFO 10-30 23:59:10 [base.py:280] Multi-modal warmup completed in 23.235s | |
| (EngineCore pid=11241) INFO 10-30 23:59:14 [encoder_runner.py:131] Encoder cache will be initialized with a budget of 16384 tokens, and profiled with 1 image items of the maximum feature size. | |
| (EngineCore pid=11241) INFO 10-30 23:59:17 [caching.py:342] reconstructed serializable fn from standalone compile artifacts. num_artifacts=11 num_submods=25 | |
| (EngineCore pid=11241) INFO 10-30 23:59:17 [decorators.py:319] [backbone] Directly load AOT compilation from path /root/.cache/vllm/torch_compile_cache/torch_aot_compile/38b27199840dd5788649ecfb9a7fd4945e985542636a6f138f09557d6ee82c7d/rank_0_0/model | |
| (EngineCore pid=11241) INFO 10-30 23:59:17 [monitor.py:62] [backbone] torch.compile took 0.88 s in total | |
| (EngineCore pid=11241) INFO 10-30 23:59:17 [monitor.py:91] [backbone] Initial profiling/warmup run took 0.18 s | |
| Capturing CUDA graphs (PIECEWISE): 0%| | 0/35 [00:00<?, ?it/s] Capturing CUDA graphs (PIECEWISE): 3%|▎ | 1/35 [00:00<00:17, 1.91it/s] Capturing CUDA graphs (PIECEWISE): 6%|▌ | 2/35 [00:00<00:11, 2.84it/s] Capturing CUDA graphs (PIECEWISE): 9%|▊ | 3/35 [00:01<00:10, 3.18it/s] Capturing CUDA graphs (PIECEWISE): 11%|█▏ | 4/35 [00:01<00:08, 3.72it/s] Capturing CUDA graphs (PIECEWISE): 14%|█▍ | 5/35 [00:01<00:07, 3.89it/s] Capturing CUDA graphs (PIECEWISE): 17%|█▋ | 6/35 [00:01<00:06, 4.40it/s] Capturing CUDA graphs (PIECEWISE): 20%|██ | 7/35 [00:01<00:05, 4.74it/s] Capturing CUDA graphs (PIECEWISE): 23%|██▎ | 8/35 [00:02<00:05, 4.58it/s] Capturing CUDA graphs (PIECEWISE): 26%|██▌ | 9/35 [00:02<00:05, 5.16it/s] Capturing CUDA graphs (PIECEWISE): 29%|██▊ | 10/35 [00:02<00:04, 5.40it/s] Capturing CUDA graphs (PIECEWISE): 31%|███▏ | 11/35 [00:02<00:04, 5.60it/s] Capturing CUDA graphs (PIECEWISE): 34%|███▍ | 12/35 [00:02<00:03, 6.26it/s] Capturing CUDA graphs (PIECEWISE): 37%|███▋ | 13/35 [00:02<00:03, 6.87it/s] Capturing CUDA graphs (PIECEWISE): 40%|████ | 14/35 [00:02<00:03, 6.01it/s] Capturing CUDA graphs (PIECEWISE): 43%|████▎ | 15/35 [00:03<00:03, 6.06it/s] Capturing CUDA graphs (PIECEWISE): 49%|████▊ | 17/35 [00:03<00:02, 8.18it/s] Capturing CUDA graphs (PIECEWISE): 57%|█████▋ | 20/35 [00:03<00:01, 12.59it/s] Capturing CUDA graphs (PIECEWISE): 66%|██████▌ | 23/35 [00:03<00:00, 16.32it/s] Capturing CUDA graphs (PIECEWISE): 74%|███████▍ | 26/35 [00:03<00:00, 19.35it/s] Capturing CUDA graphs (PIECEWISE): 83%|████████▎ | 29/35 [00:03<00:00, 21.78it/s] Capturing CUDA graphs (PIECEWISE): 91%|█████████▏| 32/35 [00:03<00:00, 23.82it/s] Capturing CUDA graphs (PIECEWISE): 100%|██████████| 35/35 [00:03<00:00, 24.00it/s] Capturing CUDA graphs (PIECEWISE): 100%|██████████| 35/35 [00:03<00:00, 8.94it/s] | |
| Capturing CUDA graphs (FULL): 0%| | 0/2 [00:00<?, ?it/s] Capturing CUDA graphs (FULL): 100%|██████████| 2/2 [00:00<00:00, 15.72it/s] Capturing CUDA graphs (FULL): 100%|██████████| 2/2 [00:00<00:00, 15.70it/s] | |
| (EngineCore pid=11241) INFO 10-30 23:59:23 [model_runner.py:1127] Graph capturing finished in 5 secs, took 0.21 GiB | |
| (EngineCore pid=11241) INFO 10-30 23:59:23 [gpu_worker.py:759] Available KV cache memory: 67.94 GiB | |
| (EngineCore pid=11241) INFO 10-30 23:59:23 [gpu_worker.py:774] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9000 is equivalent to --gpu-memory-utilization=0.8972 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9028. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0. | |
| (EngineCore pid=11241) INFO 10-30 23:59:23 [kv_cache_utils.py:2327] GPU KV cache size: 5,336,782 tokens, Maximum concurrency for 32,768 tokens per request: 162.87x | |
| (EngineCore pid=11241) INFO 10-30 23:59:23 [kv_cache_utils.py:853] kv cache group sizes (544, 544, 544, 544) | |
| (EngineCore pid=11241) INFO 10-30 23:59:23 [kv_cache_utils.py:854] kv lcm block sizes 544 | |
| (EngineCore pid=11241) INFO 10-30 23:59:23 [kernel_warmup.py:176] JIT kernel warmup starting. | |
| (EngineCore pid=11241) INFO 10-30 23:59:23 [kernel_warmup.py:189] JIT kernel warmup finished in 0.00s. | |
| (EngineCore pid=11241) INFO 10-30 23:59:23 [qwen_triton_warmup.py:296] Warming up Qwen GDN Triton kernels for model_type=qwen3_5_text. | |
| (EngineCore pid=11241) INFO 10-30 23:59:23 [qwen_vl_triton_warmup.py:57] Warmed position embedding and vision rotary kernels on grids=[(1, 16, 16), (1, 16, 2), (1, 2, 16), (1, 2, 2)]. | |
| (EngineCore pid=11241) INFO 10-30 23:59:23 [mamba_triton_warmup.py:42] Warmed Mamba batch_memcpy_kernel. | |
| Capturing CUDA graphs (PIECEWISE): 0%| | 0/35 [00:00<?, ?it/s] Capturing CUDA graphs (PIECEWISE): 3%|▎ | 1/35 [00:00<00:07, 4.55it/s] Capturing CUDA graphs (PIECEWISE): 6%|▌ | 2/35 [00:00<00:07, 4.26it/s] Capturing CUDA graphs (PIECEWISE): 9%|▊ | 3/35 [00:00<00:07, 4.44it/s] Capturing CUDA graphs (PIECEWISE): 11%|█▏ | 4/35 [00:00<00:06, 4.67it/s] Capturing CUDA graphs (PIECEWISE): 14%|█▍ | 5/35 [00:01<00:06, 4.36it/s] Capturing CUDA graphs (PIECEWISE): 17%|█▋ | 6/35 [00:01<00:06, 4.80it/s] Capturing CUDA graphs (PIECEWISE): 20%|██ | 7/35 [00:01<00:05, 5.04it/s] Capturing CUDA graphs (PIECEWISE): 23%|██▎ | 8/35 [00:01<00:05, 5.07it/s] Capturing CUDA graphs (PIECEWISE): 26%|██▌ | 9/35 [00:01<00:04, 5.60it/s] Capturing CUDA graphs (PIECEWISE): 29%|██▊ | 10/35 [00:01<00:04, 5.89it/s] Capturing CUDA graphs (PIECEWISE): 31%|███▏ | 11/35 [00:02<00:04, 5.42it/s] Capturing CUDA graphs (PIECEWISE): 34%|███▍ | 12/35 [00:02<00:03, 6.01it/s] Capturing CUDA graphs (PIECEWISE): 37%|███▋ | 13/35 [00:02<00:03, 6.72it/s] Capturing CUDA graphs (PIECEWISE): 40%|████ | 14/35 [00:02<00:03, 5.91it/s] Capturing CUDA graphs (PIECEWISE): 43%|████▎ | 15/35 [00:02<00:03, 6.49it/s] Capturing CUDA graphs (PIECEWISE): 49%|████▊ | 17/35 [00:02<00:01, 9.14it/s] Capturing CUDA graphs (PIECEWISE): 60%|██████ | 21/35 [00:02<00:00, 15.38it/s] Capturing CUDA graphs (PIECEWISE): 71%|███████▏ | 25/35 [00:03<00:00, 19.75it/s] Capturing CUDA graphs (PIECEWISE): 80%|████████ | 28/35 [00:05<00:02, 3.04it/s] Capturing CUDA graphs (PIECEWISE): 91%|█████████▏| 32/35 [00:05<00:00, 4.66it/s] Capturing CUDA graphs (PIECEWISE): 100%|██████████| 35/35 [00:06<00:00, 5.79it/s] | |
| Capturing CUDA graphs (FULL): 0%| | 0/19 [00:00<?, ?it/s] Capturing CUDA graphs (FULL): 21%|██ | 4/19 [00:00<00:00, 38.23it/s] Capturing CUDA graphs (FULL): 42%|████▏ | 8/19 [00:00<00:00, 37.46it/s] Capturing CUDA graphs (FULL): 63%|██████▎ | 12/19 [00:00<00:00, 37.63it/s] Capturing CUDA graphs (FULL): 84%|████████▍ | 16/19 [00:00<00:00, 37.51it/s] Capturing CUDA graphs (FULL): 100%|██████████| 19/19 [00:00<00:00, 37.61it/s] | |
| (EngineCore pid=11241) INFO 10-30 23:59:31 [model_runner.py:1127] Graph capturing finished in 7 secs, took 0.12 GiB | |
| (EngineCore pid=11241) INFO 10-30 23:59:31 [gpu_worker.py:971] CUDA graph pool memory: 0.12 GiB (actual), 0.22 GiB (estimated), difference: 0.11 GiB (88.5%). | |
| (EngineCore pid=11241) INFO 10-30 23:59:31 [gpu_worker.py:1034] Free memory on device (78.84/79.25 GiB) on startup. Desired GPU memory utilization is (0.9, 71.33 GiB). Actual usage is 1.94 GiB for consumed memory (weights + non-torch), 1.45 GiB for peak activation, and 0.12 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=72659490714` (67.67 GiB) to fit into requested memory, or `--kv-cache-memory=80725556736` (75.18 GiB) to fully utilize gpu memory. Current kv cache memory in use is 67.94 GiB. | |
| (EngineCore pid=11241) INFO 10-30 23:59:32 [jit_monitor.py:84] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn. | |
| (EngineCore pid=11241) INFO 10-30 23:59:32 [core.py:395] init engine (profile, create kv cache, warmup model) took 25.78 s (compilation: 0.88 s) | |
| Parse safetensors files: 0%| | 0/1 [00:00<?, ?it/s] Parse safetensors files: 100%|██████████| 1/1 [00:00<00:00, 4.74it/s] Parse safetensors files: 100%|██████████| 1/1 [00:00<00:00, 4.73it/s] | |
| (EngineCore pid=11241) INFO 10-30 23:59:32 [kernel.py:467] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) | |
| (MainProcess pid=10814) WARNING 10-30 23:59:32 [input_processor.py:408] Passing raw prompts to InputProcessor is deprecated and will be removed in the future. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat(). | |
| (EngineCore pid=11241) WARNING 10-30 23:59:35 [jit_monitor.py:140] Triton kernel JIT compilation during inference: _topp_sb_stats_kernel. This causes a latency spike; consider extending warmup to cover this shape/config. | |
| (EngineCore pid=11241) WARNING 10-30 23:59:35 [jit_monitor.py:140] Triton kernel JIT compilation during inference: _topp_sb_step_kernel. This causes a latency spike; consider extending warmup to cover this shape/config. | |
| (EngineCore pid=11241) WARNING 10-30 23:59:35 [jit_monitor.py:140] Triton kernel JIT compilation during inference: _topp_sb_mask_kernel. This causes a latency spike; consider extending warmup to cover this shape/config. | |
| progress {"time": "00:00:32", "elapsed_min": 1.0, "submitted": 8, "finished": 0, "in_flight": 8, "gen_tokens": 0, "tok_per_s": 0, "chain_tokens_median": 0.0, "chain_tokens_p90": 0.0, "chain_tokens_max": 0, "parse_rate": null, "looped": 0, "minutes_to_deadline": 163.0} | |
| progress {"time": "00:01:32", "elapsed_min": 2.0, "submitted": 8, "finished": 0, "in_flight": 8, "gen_tokens": 0, "tok_per_s": 0, "chain_tokens_median": 0.0, "chain_tokens_p90": 0.0, "chain_tokens_max": 0, "parse_rate": null, "looped": 0, "minutes_to_deadline": 162.0} | |
| progress {"time": "00:02:32", "elapsed_min": 3.0, "submitted": 8, "finished": 0, "in_flight": 8, "gen_tokens": 0, "tok_per_s": 0, "chain_tokens_median": 0.0, "chain_tokens_p90": 0.0, "chain_tokens_max": 0, "parse_rate": null, "looped": 0, "minutes_to_deadline": 161.0} | |
| progress {"time": "00:03:32", "elapsed_min": 4.0, "submitted": 8, "finished": 2, "in_flight": 6, "gen_tokens": 62496, "tok_per_s": 260, "chain_tokens_median": 31248.0, "chain_tokens_p90": 31731.2, "chain_tokens_max": 31852, "parse_rate": 0.0, "looped": 2, "minutes_to_deadline": 160.0} | |
| (MainProcess pid=10814) INFO 10-31 00:03:37 [core_client.py:770] [shutdown] MPClient: start timeout=default | |
| (MainProcess pid=10814) INFO 10-31 00:03:37 [core_client.py:772] [shutdown] MPClient: stopping engine manager | |
| (MainProcess pid=10814) INFO 10-31 00:03:37 [utils.py:628] [shutdown] Process manager: send sigterm to process EngineCore | |
| (EngineCore pid=11241) INFO 10-31 00:03:37 [core.py:1454] [shutdown] EngineCore: trigger received signal=SIGTERM | |
| (EngineCore pid=11241) INFO 10-31 00:03:37 [core.py:1635] [shutdown] EngineCore: start mode=abort timeout=0s | |
| (EngineCore pid=11241) INFO 10-31 00:03:37 [core.py:1666] [shutdown] EngineCore: request processing complete; starting resource teardown | |
| (EngineCore pid=11241) INFO 10-31 00:03:37 [core.py:1499] [shutdown] EngineCore: exiting busy loop | |
| (MainProcess pid=10814) INFO 10-31 00:03:41 [core_client.py:774] [shutdown] MPClient: engine manager stopped | |
| (MainProcess pid=10814) INFO 10-31 00:03:41 [core_client.py:775] [shutdown] MPClient: cleaning up background resources | |
| (MainProcess pid=10814) INFO 10-31 00:03:41 [core_client.py:777] [shutdown] MPClient: complete | |
| { | |
| "items": 16, | |
| "samples": 30, | |
| "tokens_total": 939556, | |
| "tokens_median": 32451.0, | |
| "tokens_p90": 32542.0, | |
| "tokens_max": 32542, | |
| "parse_rate": 0.03333333333333333, | |
| "looped": 29, | |
| "items_with_correct_chain": 1, | |
| "majority_accuracy": 0.0625, | |
| "this_run": { | |
| "submitted": 8, | |
| "finished": 8, | |
| "gen_tokens": 484179, | |
| "t0": 1793404772.4801898, | |
| "seconds": 244.83441710472107, | |
| "tok_per_s": 1977.5773591214752 | |
| }, | |
| "seconds_this_run": 309.6870105266571 | |
| } | |
| {"examples": 17, "tokens": 13290, "readout_gold_fallback": 16, "reasoner_examples": 1, "reasoner_no_correct_chain": 15} | |
| Loading weights: 0%| | 0/320 [00:00<?, ?it/s] Loading weights: 100%|██████████| 320/320 [00:00<00:00, 4118.42it/s] | |
| trainable params: 20,447,232 || all params: 772,840,256 || trainable%: 2.6457 | |
| [transformers] `causal_conv1d_fn` is falling back to its reference PyTorch implementation because `causal_conv1d` is not installed. This is correct but much slower; install `causal_conv1d` for the optimized kernel. | |
| /workspace/decision-reasoner/preflight/../train.py:145: UserWarning: Converting a tensor with requires_grad=True to a scalar may lead to unexpected behavior. | |
| Consider using tensor.detach() first. (Triggered internally at /__w/pytorch/pytorch/torch/csrc/autograd/generated/python_variable_methods.cpp:822.) | |
| stats["readout_loss"] += float(loss) | |
| {"step": 1, "of": 1, "lr": 0.0001, "grad_norm": 53.5244255065918, "examples": 17, "readout_examples": 16, "reasoner_examples": 1, "step_tokens": 13290, "chain_tokens": 2931, "micro_batches": 2, "tok_per_s": 79.36972920298288, "elapsed_min": 2.790736496448517, "eta_min": 0.0, "chain_loss": 0.31073397397994995, "readout_loss": 1.8940072925761342, "readout_argmax_matches_target": 0.5, "readout_mean_top_prob": 0.5667678061872721, "readout_target_entropy": 0.0035885395482182503, "mem_gb": 8.216796672} | |
| {"mode": "joint", "alpha": 0.5, "lr": 0.0001, "rank": 32, "lora_alpha": 64, "seed": 17, "examples": 17, "steps_done": 1, "steps_planned": 1, "stopped_at_deadline": false, "train_tokens": 13290, "train_seconds": 167.44484996795654, "tok_per_s": 79.36941627373592, "counts": {"readout_gold_fallback": 16, "reasoner_examples": 1, "reasoner_no_correct_chain": 15}, "reasoner_examples": 1, "readout_examples": 16} | |
| {"examples": 16, "tokens": 9921, "readout_gold_fallback": 16} | |
| Loading weights: 0%| | 0/320 [00:00<?, ?it/s] Loading weights: 100%|██████████| 320/320 [00:00<00:00, 3343.99it/s] | |
| trainable params: 20,447,232 || all params: 772,840,256 || trainable%: 2.6457 | |
| [transformers] `causal_conv1d_fn` is falling back to its reference PyTorch implementation because `causal_conv1d` is not installed. This is correct but much slower; install `causal_conv1d` for the optimized kernel. | |
| /workspace/decision-reasoner/preflight/../train.py:145: UserWarning: Converting a tensor with requires_grad=True to a scalar may lead to unexpected behavior. | |
| Consider using tensor.detach() first. (Triggered internally at /__w/pytorch/pytorch/torch/csrc/autograd/generated/python_variable_methods.cpp:822.) | |
| stats["readout_loss"] += float(loss) | |
| {"step": 1, "of": 1, "lr": 0.0001, "grad_norm": 56.7299919128418, "examples": 16, "readout_examples": 16, "reasoner_examples": 0, "step_tokens": 9921, "chain_tokens": 0, "micro_batches": 2, "tok_per_s": 391.0357275049694, "elapsed_min": 0.42285138765970864, "eta_min": 0.0, "chain_loss": null, "readout_loss": 1.893105556257069, "readout_argmax_matches_target": 0.5, "readout_mean_top_prob": 0.567022206261754, "readout_target_entropy": 0.0035885395482182503, "mem_gb": 4.359349248} | |
| {"mode": "readout_only", "alpha": 1.0, "lr": 0.0001, "rank": 32, "lora_alpha": 64, "seed": 17, "examples": 16, "steps_done": 1, "steps_planned": 1, "stopped_at_deadline": false, "train_tokens": 9921, "train_seconds": 25.371610164642334, "tok_per_s": 391.02760666825253, "counts": {"readout_gold_fallback": 16}, "reasoner_examples": 0, "readout_examples": 16} | |
| {"examples": 16, "tokens": 9921, "readout_gold_fallback": 16} | |
| Loading weights: 0%| | 0/320 [00:00<?, ?it/s] Loading weights: 100%|██████████| 320/320 [00:00<00:00, 3933.89it/s] | |
| trainable params: 20,447,232 || all params: 772,840,256 || trainable%: 2.6457 | |
| [transformers] `causal_conv1d_fn` is falling back to its reference PyTorch implementation because `causal_conv1d` is not installed. This is correct but much slower; install `causal_conv1d` for the optimized kernel. | |
| /workspace/decision-reasoner/preflight/../train.py:145: UserWarning: Converting a tensor with requires_grad=True to a scalar may lead to unexpected behavior. | |
| Consider using tensor.detach() first. (Triggered internally at /__w/pytorch/pytorch/torch/csrc/autograd/generated/python_variable_methods.cpp:822.) | |
| stats["readout_loss"] += float(loss) | |
| {"step": 1, "of": 1, "lr": 0.0001, "grad_norm": 56.730743408203125, "examples": 16, "readout_examples": 16, "reasoner_examples": 0, "step_tokens": 9921, "chain_tokens": 0, "micro_batches": 2, "tok_per_s": 1930.785204856064, "elapsed_min": 0.08563873370488485, "eta_min": 0.0, "chain_loss": null, "readout_loss": 1.893105556257069, "readout_argmax_matches_target": 0.5, "readout_mean_top_prob": 0.567022206261754, "readout_target_entropy": 0.0035885395482182503, "mem_gb": 4.359349248} | |
| {"mode": "readout_only", "alpha": 0.0, "lr": 0.0001, "rank": 32, "lora_alpha": 64, "seed": 17, "examples": 16, "steps_done": 1, "steps_planned": 1, "stopped_at_deadline": false, "train_tokens": 9921, "train_seconds": 5.139125347137451, "tok_per_s": 1930.484144646541, "counts": {"readout_gold_fallback": 16}, "reasoner_examples": 0, "readout_examples": 16} | |
| (MainProcess pid=13459) INFO 10-31 00:08:07 [model.py:733] Resolved architecture: Qwen3_5ForConditionalGeneration | |
| (MainProcess pid=13459) INFO 10-31 00:08:07 [model.py:2152] Using max model len 32768 | |
| Fetching 12 files: 0%| | 0/12 [00:00<?, ?it/s] Fetching 12 files: 100%|██████████| 12/12 [00:00<00:00, 1308.47it/s] | |
| (MainProcess pid=13459) INFO 10-31 00:08:10 [scheduler.py:307] Chunked prefill is enabled with max_num_batched_tokens=2048. | |
| (MainProcess pid=13459) INFO 10-31 00:08:10 [config.py:688] Mamba cache mode is set to 'align' for Qwen3_5ForConditionalGeneration by default when prefix caching is enabled | |
| Parse safetensors files: 0%| | 0/1 [00:00<?, ?it/s] Parse safetensors files: 100%|██████████| 1/1 [00:00<00:00, 4.52it/s] Parse safetensors files: 100%|██████████| 1/1 [00:00<00:00, 4.52it/s] | |
| (MainProcess pid=13459) INFO 10-31 00:08:10 [kernel.py:467] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) | |
| (MainProcess pid=13459) WARNING 10-31 00:08:21 [system_utils.py:153] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reasons: CUDA is initialized | |
| (EngineCore pid=13883) INFO 10-31 00:08:32 [core.py:133] Initializing a V1 LLM engine (v0.32.0) with config: model='Qwen/Qwen3.5-0.8B', speculative_config=None, tokenizer='Qwen/Qwen3.5-0.8B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, aux_output_config=AuxOutputConfig(enable_return_routed_experts=False, max_bytes=None), kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, custom_histogram_buckets=None, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=17, served_model_name=Qwen/Qwen3.5-0.8B, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::qwen4_exp_qsa_prepare_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'cudagraph_decoder_replay': True, 'decoder_replay_cudagraph_capture_sizes': [], 'decoder_replay_trim_threshold': 768, 'compile_sizes': [], 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 256, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']), enable_flashinfer_autotune=True, enable_rocm_segmented_attn_autotune=False, enable_cutedsl_warmup=True, enable_jit_warmup=True, moe_backend='auto', sparse_indexer_topk_backend='auto', linear_backend='auto', linear_backend_per_quant=None) | |
| (EngineCore pid=13883) INFO 10-31 00:08:34 [parallel_state.py:1906] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_b951912504d64ee3a664370a5237418e backend=nccl | |
| (EngineCore pid=13883) INFO 10-31 00:08:34 [parallel_state.py:2350] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, ETP rank 0, EP rank N/A, EPLB rank N/A | |
| (EngineCore pid=13883) INFO 10-31 00:08:34 [gpu_worker.py:518] Using V2 Model Runner | |
| (EngineCore pid=13883) INFO 10-31 00:08:35 [model_runner.py:392] Loading model from scratch... | |
| Fetching 12 files: 0%| | 0/12 [00:00<?, ?it/s] Fetching 12 files: 100%|██████████| 12/12 [00:00<00:00, 1211.18it/s] | |
| (EngineCore pid=13883) INFO 10-31 00:08:36 [cuda.py:565] Using backend AttentionBackendEnum.FLASH_ATTN for vit attention | |
| (EngineCore pid=13883) INFO 10-31 00:08:36 [mm_encoder_attention.py:372] Using AttentionBackendEnum.FLASH_ATTN for MMEncoderAttention. | |
| (EngineCore pid=13883) INFO 10-31 00:08:36 [qwen_gdn_linear_attn.py:221] Using Triton/FLA GDN prefill kernel (requested=auto, head_k_dim=128). | |
| (EngineCore pid=13883) INFO 10-31 00:08:37 [cuda.py:506] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']. | |
| (EngineCore pid=13883) INFO 10-31 00:08:37 [flash_attn.py:1117] Using FlashAttention version 2 | |
| (EngineCore pid=13883) INFO 10-31 00:08:37 [base_loader.py:82] Initializing model took 2.106401 seconds | |
| (EngineCore pid=13883) INFO 10-31 00:08:38 [weight_utils.py:1108] Filesystem type for checkpoints: XFS. Checkpoint size: 1.63 GiB. Available RAM: 196.94 GiB. | |
| (EngineCore pid=13883) INFO 10-31 00:08:38 [weight_utils.py:1131] Auto-prefetch is disabled because the filesystem (XFS) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch. | |
| Loading safetensors checkpoint shards: 0% Completed | 0/1 [00:00<?, ?it/s] | |
| (EngineCore pid=13883) WARNING 10-31 00:08:38 [utils.py:333] Attempted to load weight layers.0.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=13883) WARNING 10-31 00:08:38 [utils.py:333] Attempted to load weight layers.1.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=13883) WARNING 10-31 00:08:38 [utils.py:333] Attempted to load weight layers.10.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=13883) WARNING 10-31 00:08:38 [utils.py:333] Attempted to load weight layers.12.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=13883) WARNING 10-31 00:08:38 [utils.py:333] Attempted to load weight layers.13.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=13883) WARNING 10-31 00:08:38 [utils.py:333] Attempted to load weight layers.14.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=13883) WARNING 10-31 00:08:38 [utils.py:333] Attempted to load weight layers.16.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=13883) WARNING 10-31 00:08:38 [utils.py:333] Attempted to load weight layers.17.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=13883) WARNING 10-31 00:08:38 [utils.py:333] Attempted to load weight layers.18.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=13883) WARNING 10-31 00:08:38 [utils.py:333] Attempted to load weight layers.2.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=13883) WARNING 10-31 00:08:38 [utils.py:333] Attempted to load weight layers.20.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=13883) WARNING 10-31 00:08:38 [utils.py:333] Attempted to load weight layers.21.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=13883) WARNING 10-31 00:08:38 [utils.py:333] Attempted to load weight layers.22.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=13883) WARNING 10-31 00:08:38 [utils.py:333] Attempted to load weight layers.4.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=13883) WARNING 10-31 00:08:38 [utils.py:333] Attempted to load weight layers.5.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=13883) WARNING 10-31 00:08:38 [utils.py:333] Attempted to load weight layers.6.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=13883) WARNING 10-31 00:08:38 [utils.py:333] Attempted to load weight layers.8.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| (EngineCore pid=13883) WARNING 10-31 00:08:38 [utils.py:333] Attempted to load weight layers.9.linear_attn.norm.weight with dtype torch.float32 into parameter with dtype torch.bfloat16 | |
| Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00, 4.86it/s] | |
| Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00, 4.85it/s] | |
| (EngineCore pid=13883) INFO 10-31 00:08:38 [default_loader.py:501] Loading weights took 0.21 seconds | |
| (EngineCore pid=13883) INFO 10-31 00:08:40 [model_runner.py:424] Model loading took 1.63 GiB memory and 4.546822 seconds | |
| (EngineCore pid=13883) INFO 10-31 00:08:40 [topk_topp_sampler.py:124] Using FlashInfer for top-p & top-k sampling. | |
| (EngineCore pid=13883) INFO 10-31 00:08:40 [torch_utils.py:295] Reducing Torch threads from 128 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override. | |
| (EngineCore pid=13883) INFO 10-31 00:08:40 [interface.py:1096] Setting attention block size to 544 tokens to ensure that attention page size is >= mamba page size. | |
| (EngineCore pid=13883) INFO 10-31 00:08:40 [interface.py:1120] Padding mamba page size by 2.64% to ensure that mamba page size and attention page size are exactly equal. | |
| (EngineCore pid=13883) INFO 10-31 00:08:40 [utils.py:347] Using LBNHC KV cache layout. | |
| [transformers] Qwen3VL video processing does not apply the per-frame pixel cap the reference implementation (qwen-vl-utils) applies, so some videos cost far more tokens than they would there. In v5.22 the capped behavior will become the default and `cap_pixels_per_frame` will be removed. Pass `cap_pixels_per_frame=True` to adopt the reference behavior now, or `False` to keep the current behavior and silence this warning. | |
| (MainProcess pid=13459) INFO 10-31 00:08:42 [base.py:280] Multi-modal warmup completed in 20.803s | |
| (EngineCore pid=13883) INFO 10-31 00:08:47 [encoder_runner.py:131] Encoder cache will be initialized with a budget of 16384 tokens, and profiled with 1 image items of the maximum feature size. | |
| (EngineCore pid=13883) INFO 10-31 00:08:50 [caching.py:342] reconstructed serializable fn from standalone compile artifacts. num_artifacts=11 num_submods=25 | |
| (EngineCore pid=13883) INFO 10-31 00:08:50 [decorators.py:319] [backbone] Directly load AOT compilation from path /root/.cache/vllm/torch_compile_cache/torch_aot_compile/38b27199840dd5788649ecfb9a7fd4945e985542636a6f138f09557d6ee82c7d/rank_0_0/model | |
| (EngineCore pid=13883) INFO 10-31 00:08:50 [monitor.py:62] [backbone] torch.compile took 0.80 s in total | |
| (EngineCore pid=13883) INFO 10-31 00:08:53 [monitor.py:91] [backbone] Initial profiling/warmup run took 2.71 s | |
| Capturing CUDA graphs (PIECEWISE): 0%| | 0/35 [00:00<?, ?it/s] Capturing CUDA graphs (PIECEWISE): 3%|▎ | 1/35 [00:00<00:15, 2.18it/s] Capturing CUDA graphs (PIECEWISE): 6%|▌ | 2/35 [00:00<00:10, 3.07it/s] Capturing CUDA graphs (PIECEWISE): 9%|▊ | 3/35 [00:00<00:09, 3.42it/s] Capturing CUDA graphs (PIECEWISE): 11%|█▏ | 4/35 [00:01<00:08, 3.70it/s] Capturing CUDA graphs (PIECEWISE): 14%|█▍ | 5/35 [00:01<00:07, 3.76it/s] Capturing CUDA graphs (PIECEWISE): 17%|█▋ | 6/35 [00:01<00:06, 4.29it/s] Capturing CUDA graphs (PIECEWISE): 20%|██ | 7/35 [00:01<00:06, 4.51it/s] Capturing CUDA graphs (PIECEWISE): 23%|██▎ | 8/35 [00:02<00:05, 4.63it/s] Capturing CUDA graphs (PIECEWISE): 26%|██▌ | 9/35 [00:02<00:04, 5.22it/s] Capturing CUDA graphs (PIECEWISE): 29%|██▊ | 10/35 [00:02<00:04, 5.57it/s] Capturing CUDA graphs (PIECEWISE): 31%|███▏ | 11/35 [00:02<00:04, 5.15it/s] Capturing CUDA graphs (PIECEWISE): 34%|███▍ | 12/35 [00:02<00:03, 5.88it/s] Capturing CUDA graphs (PIECEWISE): 37%|███▋ | 13/35 [00:02<00:03, 6.56it/s] Capturing CUDA graphs (PIECEWISE): 40%|████ | 14/35 [00:02<00:03, 5.65it/s] Capturing CUDA graphs (PIECEWISE): 43%|████▎ | 15/35 [00:03<00:03, 6.32it/s] Capturing CUDA graphs (PIECEWISE): 49%|████▊ | 17/35 [00:03<00:02, 8.52it/s] Capturing CUDA graphs (PIECEWISE): 57%|█████▋ | 20/35 [00:03<00:01, 13.09it/s] Capturing CUDA graphs (PIECEWISE): 66%|██████▌ | 23/35 [00:03<00:00, 13.80it/s] Capturing CUDA graphs (PIECEWISE): 74%|███████▍ | 26/35 [00:03<00:00, 17.18it/s] Capturing CUDA graphs (PIECEWISE): 83%|████████▎ | 29/35 [00:03<00:00, 20.15it/s] Capturing CUDA graphs (PIECEWISE): 94%|█████████▍| 33/35 [00:03<00:00, 23.21it/s] Capturing CUDA graphs (PIECEWISE): 100%|██████████| 35/35 [00:03<00:00, 8.82it/s] | |
| Capturing CUDA graphs (FULL): 0%| | 0/2 [00:00<?, ?it/s] Capturing CUDA graphs (FULL): 50%|█████ | 1/2 [00:00<00:00, 8.83it/s] Capturing CUDA graphs (FULL): 100%|██████████| 2/2 [00:00<00:00, 13.33it/s] | |
| (EngineCore pid=13883) INFO 10-31 00:08:59 [model_runner.py:1127] Graph capturing finished in 5 secs, took 0.21 GiB | |
| (EngineCore pid=13883) INFO 10-31 00:08:59 [gpu_worker.py:759] Available KV cache memory: 67.94 GiB | |
| (EngineCore pid=13883) INFO 10-31 00:08:59 [gpu_worker.py:774] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9000 is equivalent to --gpu-memory-utilization=0.8972 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9028. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0. | |
| (EngineCore pid=13883) INFO 10-31 00:08:59 [kv_cache_utils.py:2327] GPU KV cache size: 5,336,782 tokens, Maximum concurrency for 32,768 tokens per request: 162.87x | |
| (EngineCore pid=13883) INFO 10-31 00:08:59 [kv_cache_utils.py:853] kv cache group sizes (544, 544, 544, 544) | |
| (EngineCore pid=13883) INFO 10-31 00:08:59 [kv_cache_utils.py:854] kv lcm block sizes 544 | |
| (EngineCore pid=13883) INFO 10-31 00:08:59 [kernel_warmup.py:176] JIT kernel warmup starting. | |
| (EngineCore pid=13883) INFO 10-31 00:08:59 [kernel_warmup.py:189] JIT kernel warmup finished in 0.00s. | |
| (EngineCore pid=13883) INFO 10-31 00:08:59 [qwen_triton_warmup.py:296] Warming up Qwen GDN Triton kernels for model_type=qwen3_5_text. | |
| (EngineCore pid=13883) INFO 10-31 00:08:59 [qwen_vl_triton_warmup.py:57] Warmed position embedding and vision rotary kernels on grids=[(1, 16, 16), (1, 16, 2), (1, 2, 16), (1, 2, 2)]. | |
| (EngineCore pid=13883) INFO 10-31 00:08:59 [mamba_triton_warmup.py:42] Warmed Mamba batch_memcpy_kernel. | |
| Capturing CUDA graphs (PIECEWISE): 0%| | 0/35 [00:00<?, ?it/s] Capturing CUDA graphs (PIECEWISE): 3%|▎ | 1/35 [00:00<00:08, 4.14it/s] Capturing CUDA graphs (PIECEWISE): 6%|▌ | 2/35 [00:00<00:07, 4.37it/s] Capturing CUDA graphs (PIECEWISE): 9%|▊ | 3/35 [00:00<00:07, 4.09it/s] Capturing CUDA graphs (PIECEWISE): 11%|█▏ | 4/35 [00:00<00:06, 4.46it/s] Capturing CUDA graphs (PIECEWISE): 14%|█▍ | 5/35 [00:01<00:07, 4.05it/s] Capturing CUDA graphs (PIECEWISE): 17%|█▋ | 6/35 [00:01<00:06, 4.56it/s] Capturing CUDA graphs (PIECEWISE): 20%|██ | 7/35 [00:01<00:07, 3.95it/s] Capturing CUDA graphs (PIECEWISE): 23%|██▎ | 8/35 [00:01<00:05, 4.54it/s] Capturing CUDA graphs (PIECEWISE): 26%|██▌ | 9/35 [00:02<00:08, 3.09it/s] Capturing CUDA graphs (PIECEWISE): 29%|██▊ | 10/35 [00:02<00:06, 3.79it/s] Capturing CUDA graphs (PIECEWISE): 31%|███▏ | 11/35 [00:02<00:05, 4.51it/s] Capturing CUDA graphs (PIECEWISE): 34%|███▍ | 12/35 [00:02<00:04, 5.18it/s] Capturing CUDA graphs (PIECEWISE): 37%|███▋ | 13/35 [00:02<00:03, 5.94it/s] Capturing CUDA graphs (PIECEWISE): 40%|████ | 14/35 [00:02<00:03, 6.75it/s] Capturing CUDA graphs (PIECEWISE): 43%|████▎ | 15/35 [00:03<00:02, 7.47it/s] Capturing CUDA graphs (PIECEWISE): 49%|████▊ | 17/35 [00:03<00:01, 10.17it/s] Capturing CUDA graphs (PIECEWISE): 60%|██████ | 21/35 [00:03<00:00, 16.75it/s] Capturing CUDA graphs (PIECEWISE): 71%|███████▏ | 25/35 [00:03<00:00, 21.46it/s] Capturing CUDA graphs (PIECEWISE): 83%|████████▎ | 29/35 [00:03<00:00, 24.88it/s] Capturing CUDA graphs (PIECEWISE): 94%|█████████▍| 33/35 [00:03<00:00, 27.53it/s] Capturing CUDA graphs (PIECEWISE): 100%|██████████| 35/35 [00:03<00:00, 9.34it/s] | |
| Capturing CUDA graphs (FULL): 0%| | 0/19 [00:00<?, ?it/s] Capturing CUDA graphs (FULL): 11%|█ | 2/19 [00:00<00:00, 18.65it/s] Capturing CUDA graphs (FULL): 32%|███▏ | 6/19 [00:00<00:00, 29.97it/s] Capturing CUDA graphs (FULL): 53%|█████▎ | 10/19 [00:00<00:00, 33.79it/s] Capturing CUDA graphs (FULL): 74%|███████▎ | 14/19 [00:00<00:00, 35.56it/s] Capturing CUDA graphs (FULL): 95%|█████████▍| 18/19 [00:00<00:00, 31.34it/s] Capturing CUDA graphs (FULL): 100%|██████████| 19/19 [00:00<00:00, 31.77it/s] | |
| (EngineCore pid=13883) INFO 10-31 00:09:05 [model_runner.py:1127] Graph capturing finished in 5 secs, took 0.12 GiB | |
| (EngineCore pid=13883) INFO 10-31 00:09:05 [gpu_worker.py:971] CUDA graph pool memory: 0.12 GiB (actual), 0.22 GiB (estimated), difference: 0.11 GiB (88.5%). | |
| (EngineCore pid=13883) INFO 10-31 00:09:05 [gpu_worker.py:1034] Free memory on device (78.84/79.25 GiB) on startup. Desired GPU memory utilization is (0.9, 71.33 GiB). Actual usage is 1.94 GiB for consumed memory (weights + non-torch), 1.45 GiB for peak activation, and 0.12 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=72659490714` (67.67 GiB) to fit into requested memory, or `--kv-cache-memory=80725556736` (75.18 GiB) to fully utilize gpu memory. Current kv cache memory in use is 67.94 GiB. | |
| (EngineCore pid=13883) INFO 10-31 00:09:05 [jit_monitor.py:84] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn. | |
| (EngineCore pid=13883) INFO 10-31 00:09:06 [core.py:395] init engine (profile, create kv cache, warmup model) took 26.15 s (compilation: 0.80 s) | |
| Parse safetensors files: 0%| | 0/1 [00:00<?, ?it/s] Parse safetensors files: 100%|██████████| 1/1 [00:00<00:00, 1.69it/s] Parse safetensors files: 100%|██████████| 1/1 [00:00<00:00, 1.69it/s] | |
| (EngineCore pid=13883) INFO 10-31 00:09:07 [kernel.py:467] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'], gelu_and_mul_sparse=['triton', 'native']) | |
| (MainProcess pid=13459) WARNING 10-31 00:09:08 [input_processor.py:408] Passing raw prompts to InputProcessor is deprecated and will be removed in the future. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat(). | |
| (EngineCore pid=13883) WARNING 10-31 00:09:08 [jit_monitor.py:140] Triton kernel JIT compilation during inference: _topk_log_softmax_kernel. This causes a latency spike; consider extending warmup to cover this shape/config. | |
| readout base: 130 items in 4s | |
| (EngineCore pid=13883) WARNING 10-31 00:09:10 [jit_monitor.py:140] Triton kernel JIT compilation during inference: _topp_sb_stats_kernel. This causes a latency spike; consider extending warmup to cover this shape/config. | |
| (EngineCore pid=13883) WARNING 10-31 00:09:10 [jit_monitor.py:140] Triton kernel JIT compilation during inference: _topp_sb_step_kernel. This causes a latency spike; consider extending warmup to cover this shape/config. | |
| (EngineCore pid=13883) WARNING 10-31 00:09:10 [jit_monitor.py:140] Triton kernel JIT compilation during inference: _topp_sb_mask_kernel. This causes a latency spike; consider extending warmup to cover this shape/config. | |
| progress {"time": "00:10:10", "elapsed_min": 1.0, "submitted": 130, "finished": 18, "in_flight": 112, "gen_tokens": 24097, "tok_per_s": 402, "chain_tokens_median": 859.0, "chain_tokens_p90": 3191.2000000000003, "chain_tokens_max": 4615, "parse_rate": 1.0, "looped": 0, "minutes_to_deadline": Infinity} | |
| progress {"time": "00:11:10", "elapsed_min": 2.0, "submitted": 130, "finished": 18, "in_flight": 112, "gen_tokens": 24097, "tok_per_s": 201, "chain_tokens_median": 859.0, "chain_tokens_p90": 3191.2000000000003, "chain_tokens_max": 4615, "parse_rate": 1.0, "looped": 0, "minutes_to_deadline": Infinity} | |
| progress {"time": "00:12:10", "elapsed_min": 3.0, "submitted": 130, "finished": 18, "in_flight": 112, "gen_tokens": 24097, "tok_per_s": 134, "chain_tokens_median": 859.0, "chain_tokens_p90": 3191.2000000000003, "chain_tokens_max": 4615, "parse_rate": 1.0, "looped": 0, "minutes_to_deadline": Infinity} | |
| progress {"time": "00:13:10", "elapsed_min": 4.0, "submitted": 130, "finished": 18, "in_flight": 112, "gen_tokens": 24097, "tok_per_s": 100, "chain_tokens_median": 859.0, "chain_tokens_p90": 3191.2000000000003, "chain_tokens_max": 4615, "parse_rate": 1.0, "looped": 0, "minutes_to_deadline": Infinity} | |
| progress {"time": "00:14:10", "elapsed_min": 5.0, "submitted": 130, "finished": 18, "in_flight": 112, "gen_tokens": 24097, "tok_per_s": 80, "chain_tokens_median": 859.0, "chain_tokens_p90": 3191.2000000000003, "chain_tokens_max": 4615, "parse_rate": 1.0, "looped": 0, "minutes_to_deadline": Infinity} | |
| progress {"time": "00:15:10", "elapsed_min": 6.0, "submitted": 130, "finished": 18, "in_flight": 112, "gen_tokens": 24097, "tok_per_s": 67, "chain_tokens_median": 859.0, "chain_tokens_p90": 3191.2000000000003, "chain_tokens_max": 4615, "parse_rate": 1.0, "looped": 0, "minutes_to_deadline": Infinity} | |
| progress {"time": "00:16:10", "elapsed_min": 7.0, "submitted": 130, "finished": 18, "in_flight": 112, "gen_tokens": 24097, "tok_per_s": 57, "chain_tokens_median": 859.0, "chain_tokens_p90": 3191.2000000000003, "chain_tokens_max": 4615, "parse_rate": 1.0, "looped": 0, "minutes_to_deadline": Infinity} | |
| progress {"time": "00:17:10", "elapsed_min": 8.0, "submitted": 130, "finished": 18, "in_flight": 112, "gen_tokens": 24097, "tok_per_s": 50, "chain_tokens_median": 859.0, "chain_tokens_p90": 3191.2000000000003, "chain_tokens_max": 4615, "parse_rate": 1.0, "looped": 0, "minutes_to_deadline": Infinity} | |
| progress {"time": "00:18:10", "elapsed_min": 9.0, "submitted": 130, "finished": 18, "in_flight": 112, "gen_tokens": 24097, "tok_per_s": 45, "chain_tokens_median": 859.0, "chain_tokens_p90": 3191.2000000000003, "chain_tokens_max": 4615, "parse_rate": 1.0, "looped": 0, "minutes_to_deadline": Infinity} | |
| progress {"time": "00:19:10", "elapsed_min": 10.0, "submitted": 130, "finished": 18, "in_flight": 112, "gen_tokens": 24097, "tok_per_s": 40, "chain_tokens_median": 859.0, "chain_tokens_p90": 3191.2000000000003, "chain_tokens_max": 4615, "parse_rate": 1.0, "looped": 0, "minutes_to_deadline": Infinity} | |
| progress {"time": "00:20:10", "elapsed_min": 11.0, "submitted": 130, "finished": 18, "in_flight": 112, "gen_tokens": 24097, "tok_per_s": 37, "chain_tokens_median": 859.0, "chain_tokens_p90": 3191.2000000000003, "chain_tokens_max": 4615, "parse_rate": 1.0, "looped": 0, "minutes_to_deadline": Infinity} | |
| progress {"time": "00:21:10", "elapsed_min": 12.0, "submitted": 130, "finished": 21, "in_flight": 109, "gen_tokens": 115851, "tok_per_s": 161, "chain_tokens_median": 1103.0, "chain_tokens_p90": 30057.0, "chain_tokens_max": 31029, "parse_rate": 0.8571428571428571, "looped": 3, "minutes_to_deadline": Infinity} | |
| reason base: {"submitted": 130, "finished": 130, "gen_tokens": 3631252, "t0": 1793405350.215, "seconds": 757.1685206890106, "tok_per_s": 4795.830651670016} | |
| reason base: 130 items, post-readout in 758s total | |
| (EngineCore pid=13883) INFO 10-31 00:23:45 [core.py:1454] [shutdown] EngineCore: trigger received signal=SIGTERM | |
| (EngineCore pid=13883) INFO 10-31 00:23:45 [core.py:1635] [shutdown] EngineCore: start mode=abort timeout=0s | |
| (EngineCore pid=13883) INFO 10-31 00:23:45 [core.py:1644] [shutdown] EngineCore: aborting in-flight requests count=1 | |
| (EngineCore pid=13883) INFO 10-31 00:23:45 [core.py:1666] [shutdown] EngineCore: request processing complete; starting resource teardown | |
| (EngineCore pid=13883) INFO 10-31 00:23:45 [core.py:1499] [shutdown] EngineCore: exiting busy loop | |
Xet Storage Details
- Size:
- 83.8 kB
- Xet hash:
- 41fdecc9027f4b8c5ebd9981df9c781588103d30f1ea356b38c6f0f42f399880
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.