Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+hf/tokenizer.json filter=lfs diff=lfs merge=lfs -text

data.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c378db65a2203aa8a78056fbcf7fd2d9f96a608110397a3ee932d74213ac7c9
+size 1492

hf/added_tokens.json

@@ -0,0 +1,28 @@
+{
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

hf/chat_template.jinja

@@ -0,0 +1,85 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set content = message.content %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is defined and message.reasoning_content is not none %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in message.content %}
+                {%- set content = message.content.split('</think>')[-1].lstrip('\n') %}
+                {%- set reasoning_content = message.content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

hf/config.json

@@ -0,0 +1,34 @@
+{
+  "architectures": [
+    "Qwen3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "eos_token_id": 151645,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 25600,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 64,
+  "model_type": "qwen3",
+  "num_attention_heads": 64,
+  "num_hidden_layers": 64,
+  "num_key_value_heads": 8,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "factor": 4.0,
+    "original_max_position_embeddings": 32768,
+    "rope_type": "yarn"
+  },
+  "rope_theta": 1000000,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.52.4",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

hf/generation_config.json

@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "temperature": 0.6,
+  "top_k": 20,
+  "top_p": 0.95,
+  "transformers_version": "4.52.4"
+}

hf/model-00001-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30e6f85e21a0e9d3483e8bb9e6010c80a8e797719552329ae795f76a26b52447
+size 4928419424

hf/model-00002-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99db49822e9f64fd5502fd0d42e2e5a6fb43df4206fe161cb1c8120e32670637
+size 4781605144

hf/model-00003-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e07a88f5427346dfe5224e4b2c81368b18a293ae92b747c1e1a0cd18411032c5
+size 4928450568

hf/model-00004-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5888990fba12830a1c24bc6cc470697284f57fa3ff9bc68abb2f27c15ec9ae08
+size 4980813680

hf/model-00005-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:493e7423c2354cdc29e88cb6bbd439b9fe0a0c5dc44c9df7f912ef36f3de6997
+size 4991315040

hf/model-00006-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:540b426e81ddf3232e20258648997d52ece0532c0d3b88742c0dcd232beedda3
+size 4949367504

hf/model-00007-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39b294addd72f12cb2c04c6ff91856a20f48ffc5e5300508ee3aa261a51b86c0
+size 4854996904

hf/model-00008-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a5971c2dc9c302a74844a6a8c0ac3c18776892563d5d07c6d473dbdd2e46d5b
+size 3565289480

hf/model-00009-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f948de65d6c0c4da9aa5c7b1b5634c10a92dd422f494c9554ffe06b9a68cd758
+size 4890335312

hf/model-00010-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b48a6226ab1fead18741840d0c6c2ff7a713a182210468b5861586b9a05656c7
+size 4785473976

hf/model-00011-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:307afed4ca7ac0aa48c4faf376bb3d654e143f2a6d4145b77416d8afc06cca2a
+size 4834025448

hf/model-00012-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0533a6fe98af4e719b356da4f5a996defcd6614f66fbd5c3787497ef0f9894e
+size 4792103184

hf/model-00013-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47c9088b8ab86b8c928cdfbb9a38e390e0a0ac716285e348e5ed4fcfd0854c70
+size 4949465424

hf/model-00014-of-00014.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31d997e89fb92893911b4214b4a51f7c6ed4ab25fccf47488ca5373b964e66e9
+size 3292667304

hf/model.safetensors.index.json

@@ -0,0 +1,714 @@
+{
+  "metadata": {
+    "total_size": 65524246528
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00009-of-00014.safetensors",
+    "model.embed_tokens.weight": "model-00010-of-00014.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00014-of-00014.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00010-of-00014.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00011-of-00014.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00010-of-00014.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00006-of-00014.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00004-of-00014.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00009-of-00014.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00006-of-00014.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00010-of-00014.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00009-of-00014.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00010-of-00014.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00010-of-00014.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00011-of-00014.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00004-of-00014.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00009-of-00014.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00014-of-00014.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00012-of-00014.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00006-of-00014.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00010-of-00014.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00010-of-00014.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00012-of-00014.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00010-of-00014.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00014.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00014.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00006-of-00014.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00006-of-00014.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00014-of-00014.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00010-of-00014.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00010-of-00014.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00014.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00011-of-00014.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00006-of-00014.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00012-of-00014.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00006-of-00014.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00004-of-00014.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00014-of-00014.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00009-of-00014.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00011-of-00014.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00014.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00012-of-00014.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00004-of-00014.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00011-of-00014.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00002-of-00014.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00014-of-00014.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00010-of-00014.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00014-of-00014.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00012-of-00014.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00009-of-00014.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00006-of-00014.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00004-of-00014.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00011-of-00014.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00009-of-00014.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00014-of-00014.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00012-of-00014.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00010-of-00014.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00011-of-00014.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00007-of-00014.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00007-of-00014.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00014.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00010-of-00014.safetensors",
+    "model.layers.32.self_attn.k_norm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.32.self_attn.q_norm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00006-of-00014.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.33.self_attn.k_norm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.33.self_attn.q_norm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00014-of-00014.safetensors",
+    "model.layers.34.self_attn.k_norm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.34.self_attn.q_norm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00012-of-00014.safetensors",
+    "model.layers.35.self_attn.k_norm.weight": "model-00004-of-00014.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.35.self_attn.q_norm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00010-of-00014.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00007-of-00014.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00014-of-00014.safetensors",
+    "model.layers.36.self_attn.k_norm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00010-of-00014.safetensors",
+    "model.layers.36.self_attn.q_norm.weight": "model-00014-of-00014.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00011-of-00014.safetensors",
+    "model.layers.37.self_attn.k_norm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.37.self_attn.q_norm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.38.self_attn.k_norm.weight": "model-00002-of-00014.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.38.self_attn.q_norm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.39.self_attn.k_norm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.39.self_attn.q_norm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00014-of-00014.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00010-of-00014.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00012-of-00014.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00014-of-00014.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00007-of-00014.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.40.self_attn.k_norm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.40.self_attn.q_norm.weight": "model-00012-of-00014.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00004-of-00014.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00002-of-00014.safetensors",
+    "model.layers.41.self_attn.k_norm.weight": "model-00009-of-00014.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.41.self_attn.q_norm.weight": "model-00011-of-00014.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.42.input_layernorm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.42.mlp.down_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.42.mlp.gate_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.42.mlp.up_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.42.post_attention_layernorm.weight": "model-00006-of-00014.safetensors",
+    "model.layers.42.self_attn.k_norm.weight": "model-00006-of-00014.safetensors",
+    "model.layers.42.self_attn.k_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.42.self_attn.o_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.42.self_attn.q_norm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.42.self_attn.q_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.42.self_attn.v_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.43.input_layernorm.weight": "model-00011-of-00014.safetensors",
+    "model.layers.43.mlp.down_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.43.mlp.gate_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.43.mlp.up_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.43.post_attention_layernorm.weight": "model-00002-of-00014.safetensors",
+    "model.layers.43.self_attn.k_norm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.43.self_attn.k_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.43.self_attn.o_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.43.self_attn.q_norm.weight": "model-00009-of-00014.safetensors",
+    "model.layers.43.self_attn.q_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.43.self_attn.v_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.44.input_layernorm.weight": "model-00010-of-00014.safetensors",
+    "model.layers.44.mlp.down_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.44.mlp.gate_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.44.mlp.up_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.44.post_attention_layernorm.weight": "model-00011-of-00014.safetensors",
+    "model.layers.44.self_attn.k_norm.weight": "model-00009-of-00014.safetensors",
+    "model.layers.44.self_attn.k_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.44.self_attn.o_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.44.self_attn.q_norm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.44.self_attn.q_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.44.self_attn.v_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.45.input_layernorm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.45.mlp.down_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.45.mlp.gate_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.45.mlp.up_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.45.post_attention_layernorm.weight": "model-00002-of-00014.safetensors",
+    "model.layers.45.self_attn.k_norm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.45.self_attn.k_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.45.self_attn.o_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.45.self_attn.q_norm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.45.self_attn.q_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.45.self_attn.v_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.46.input_layernorm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.46.mlp.down_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.46.mlp.gate_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.46.mlp.up_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.46.post_attention_layernorm.weight": "model-00014-of-00014.safetensors",
+    "model.layers.46.self_attn.k_norm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.46.self_attn.k_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.46.self_attn.o_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.46.self_attn.q_norm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.46.self_attn.q_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.46.self_attn.v_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.47.input_layernorm.weight": "model-00007-of-00014.safetensors",
+    "model.layers.47.mlp.down_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.47.mlp.gate_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.47.mlp.up_proj.weight": "model-00010-of-00014.safetensors",
+    "model.layers.47.post_attention_layernorm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.47.self_attn.k_norm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.47.self_attn.k_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.47.self_attn.o_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.47.self_attn.q_norm.weight": "model-00007-of-00014.safetensors",
+    "model.layers.47.self_attn.q_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.47.self_attn.v_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.48.input_layernorm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.48.mlp.down_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.48.mlp.gate_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.48.mlp.up_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.48.post_attention_layernorm.weight": "model-00006-of-00014.safetensors",
+    "model.layers.48.self_attn.k_norm.weight": "model-00012-of-00014.safetensors",
+    "model.layers.48.self_attn.k_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.48.self_attn.o_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.48.self_attn.q_norm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.48.self_attn.q_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.48.self_attn.v_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.49.input_layernorm.weight": "model-00012-of-00014.safetensors",
+    "model.layers.49.mlp.down_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.49.mlp.gate_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.49.mlp.up_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.49.post_attention_layernorm.weight": "model-00002-of-00014.safetensors",
+    "model.layers.49.self_attn.k_norm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.49.self_attn.k_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.49.self_attn.o_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.49.self_attn.q_norm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.49.self_attn.q_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.49.self_attn.v_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00010-of-00014.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00014.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00007-of-00014.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.50.input_layernorm.weight": "model-00012-of-00014.safetensors",
+    "model.layers.50.mlp.down_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.50.mlp.gate_proj.weight": "model-00010-of-00014.safetensors",
+    "model.layers.50.mlp.up_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.50.post_attention_layernorm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.50.self_attn.k_norm.weight": "model-00010-of-00014.safetensors",
+    "model.layers.50.self_attn.k_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.50.self_attn.o_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.50.self_attn.q_norm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.50.self_attn.q_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.50.self_attn.v_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.51.input_layernorm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.51.mlp.down_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.51.mlp.gate_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.51.mlp.up_proj.weight": "model-00010-of-00014.safetensors",
+    "model.layers.51.post_attention_layernorm.weight": "model-00012-of-00014.safetensors",
+    "model.layers.51.self_attn.k_norm.weight": "model-00009-of-00014.safetensors",
+    "model.layers.51.self_attn.k_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.51.self_attn.o_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.51.self_attn.q_norm.weight": "model-00012-of-00014.safetensors",
+    "model.layers.51.self_attn.q_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.51.self_attn.v_proj.weight": "model-00010-of-00014.safetensors",
+    "model.layers.52.input_layernorm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.52.mlp.down_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.52.mlp.gate_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.52.mlp.up_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.52.post_attention_layernorm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.52.self_attn.k_norm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.52.self_attn.k_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.52.self_attn.o_proj.weight": "model-00010-of-00014.safetensors",
+    "model.layers.52.self_attn.q_norm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.52.self_attn.q_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.52.self_attn.v_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.53.input_layernorm.weight": "model-00007-of-00014.safetensors",
+    "model.layers.53.mlp.down_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.53.mlp.gate_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.53.mlp.up_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.53.post_attention_layernorm.weight": "model-00014-of-00014.safetensors",
+    "model.layers.53.self_attn.k_norm.weight": "model-00007-of-00014.safetensors",
+    "model.layers.53.self_attn.k_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.53.self_attn.o_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.53.self_attn.q_norm.weight": "model-00014-of-00014.safetensors",
+    "model.layers.53.self_attn.q_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.53.self_attn.v_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.54.input_layernorm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.54.mlp.down_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.54.mlp.gate_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.54.mlp.up_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.54.post_attention_layernorm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.54.self_attn.k_norm.weight": "model-00011-of-00014.safetensors",
+    "model.layers.54.self_attn.k_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.54.self_attn.o_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.54.self_attn.q_norm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.54.self_attn.q_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.54.self_attn.v_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.55.input_layernorm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.55.mlp.down_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.55.mlp.gate_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.55.mlp.up_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.55.post_attention_layernorm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.55.self_attn.k_norm.weight": "model-00007-of-00014.safetensors",
+    "model.layers.55.self_attn.k_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.55.self_attn.o_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.55.self_attn.q_norm.weight": "model-00012-of-00014.safetensors",
+    "model.layers.55.self_attn.q_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.55.self_attn.v_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.56.input_layernorm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.56.mlp.down_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.56.mlp.gate_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.56.mlp.up_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.56.post_attention_layernorm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.56.self_attn.k_norm.weight": "model-00011-of-00014.safetensors",
+    "model.layers.56.self_attn.k_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.56.self_attn.o_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.56.self_attn.q_norm.weight": "model-00007-of-00014.safetensors",
+    "model.layers.56.self_attn.q_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.56.self_attn.v_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.57.input_layernorm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.57.mlp.down_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.57.mlp.gate_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.57.mlp.up_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.57.post_attention_layernorm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.57.self_attn.k_norm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.57.self_attn.k_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.57.self_attn.o_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.57.self_attn.q_norm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.57.self_attn.q_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.57.self_attn.v_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.58.input_layernorm.weight": "model-00011-of-00014.safetensors",
+    "model.layers.58.mlp.down_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.58.mlp.gate_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.58.mlp.up_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.58.post_attention_layernorm.weight": "model-00009-of-00014.safetensors",
+    "model.layers.58.self_attn.k_norm.weight": "model-00004-of-00014.safetensors",
+    "model.layers.58.self_attn.k_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.58.self_attn.o_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.58.self_attn.q_norm.weight": "model-00007-of-00014.safetensors",
+    "model.layers.58.self_attn.q_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.58.self_attn.v_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.59.input_layernorm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.59.mlp.down_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.59.mlp.gate_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.59.mlp.up_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.59.post_attention_layernorm.weight": "model-00004-of-00014.safetensors",
+    "model.layers.59.self_attn.k_norm.weight": "model-00010-of-00014.safetensors",
+    "model.layers.59.self_attn.k_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.59.self_attn.o_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.59.self_attn.q_norm.weight": "model-00007-of-00014.safetensors",
+    "model.layers.59.self_attn.q_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.59.self_attn.v_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00007-of-00014.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00002-of-00014.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.60.input_layernorm.weight": "model-00007-of-00014.safetensors",
+    "model.layers.60.mlp.down_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.60.mlp.gate_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.60.mlp.up_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.60.post_attention_layernorm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.60.self_attn.k_norm.weight": "model-00007-of-00014.safetensors",
+    "model.layers.60.self_attn.k_proj.weight": "model-00012-of-00014.safetensors",
+    "model.layers.60.self_attn.o_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.60.self_attn.q_norm.weight": "model-00003-of-00014.safetensors",
+    "model.layers.60.self_attn.q_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.60.self_attn.v_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.61.input_layernorm.weight": "model-00002-of-00014.safetensors",
+    "model.layers.61.mlp.down_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.61.mlp.gate_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.61.mlp.up_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.61.post_attention_layernorm.weight": "model-00007-of-00014.safetensors",
+    "model.layers.61.self_attn.k_norm.weight": "model-00011-of-00014.safetensors",
+    "model.layers.61.self_attn.k_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.61.self_attn.o_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.61.self_attn.q_norm.weight": "model-00006-of-00014.safetensors",
+    "model.layers.61.self_attn.q_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.61.self_attn.v_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.62.input_layernorm.weight": "model-00006-of-00014.safetensors",
+    "model.layers.62.mlp.down_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.62.mlp.gate_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.62.mlp.up_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.62.post_attention_layernorm.weight": "model-00002-of-00014.safetensors",
+    "model.layers.62.self_attn.k_norm.weight": "model-00009-of-00014.safetensors",
+    "model.layers.62.self_attn.k_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.62.self_attn.o_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.62.self_attn.q_norm.weight": "model-00009-of-00014.safetensors",
+    "model.layers.62.self_attn.q_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.62.self_attn.v_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.63.input_layernorm.weight": "model-00012-of-00014.safetensors",
+    "model.layers.63.mlp.down_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.63.mlp.gate_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.63.mlp.up_proj.weight": "model-00001-of-00014.safetensors",
+    "model.layers.63.post_attention_layernorm.weight": "model-00004-of-00014.safetensors",
+    "model.layers.63.self_attn.k_norm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.63.self_attn.k_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.63.self_attn.o_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.63.self_attn.q_norm.weight": "model-00012-of-00014.safetensors",
+    "model.layers.63.self_attn.q_proj.weight": "model-00008-of-00014.safetensors",
+    "model.layers.63.self_attn.v_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00008-of-00014.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00014-of-00014.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00005-of-00014.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00010-of-00014.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00011-of-00014.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00012-of-00014.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00007-of-00014.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00009-of-00014.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00014-of-00014.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00009-of-00014.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00004-of-00014.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00010-of-00014.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00014-of-00014.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00006-of-00014.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00007-of-00014.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00013-of-00014.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00012-of-00014.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00013-of-00014.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00005-of-00014.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00014.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00014.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00014.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00006-of-00014.safetensors",
+    "model.norm.weight": "model-00011-of-00014.safetensors"
+  }
+}

hf/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

hf/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
+size 11422654

hf/tokenizer_config.json

@@ -0,0 +1,240 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0].role == 'system' %}\n        {{- messages[0].content + '\\n\\n' }}\n    {%- endif %}\n    {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0].role == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n    {%- set index = (messages|length - 1) - loop.index0 %}\n    {%- if ns.multi_step_tool and message.role == \"user\" and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}\n        {%- set ns.multi_step_tool = false %}\n        {%- set ns.last_query_index = index %}\n    {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {%- set content = message.content %}\n        {%- set reasoning_content = '' %}\n        {%- if message.reasoning_content is defined and message.reasoning_content is not none %}\n            {%- set reasoning_content = message.reasoning_content %}\n        {%- else %}\n            {%- if '</think>' in message.content %}\n                {%- set content = message.content.split('</think>')[-1].lstrip('\\n') %}\n                {%- set reasoning_content = message.content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n            {%- endif %}\n        {%- endif %}\n        {%- if loop.index0 > ns.last_query_index %}\n            {%- if loop.last or (not loop.last and reasoning_content) %}\n                {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content.strip('\\n') + '\\n</think>\\n\\n' + content.lstrip('\\n') }}\n            {%- else %}\n                {{- '<|im_start|>' + message.role + '\\n' + content }}\n            {%- endif %}\n        {%- else %}\n            {{- '<|im_start|>' + message.role + '\\n' + content }}\n        {%- endif %}\n        {%- if message.tool_calls %}\n            {%- for tool_call in message.tool_calls %}\n                {%- if (loop.first and content) or (not loop.first) %}\n                    {{- '\\n' }}\n                {%- endif %}\n                {%- if tool_call.function %}\n                    {%- set tool_call = tool_call.function %}\n                {%- endif %}\n                {{- '<tool_call>\\n{\"name\": \"' }}\n                {{- tool_call.name }}\n                {{- '\", \"arguments\": ' }}\n                {%- if tool_call.arguments is string %}\n                    {{- tool_call.arguments }}\n                {%- else %}\n                    {{- tool_call.arguments | tojson }}\n                {%- endif %}\n                {{- '}\\n</tool_call>' }}\n            {%- endfor %}\n        {%- endif %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n    {%- if enable_thinking is defined and enable_thinking is false %}\n        {{- '<think>\\n\\n</think>\\n\\n' }}\n    {%- endif %}\n{%- endif %}"
+}

hf/tokenizer_config.json.bak

@@ -0,0 +1,239 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

hf_ip/lb_endpoint.txt

@@ -0,0 +1 @@
+10.45.17.59:8000

hf_ip/load_balancer.log

@@ -0,0 +1,1198 @@
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "GET /v1/models HTTP/1.1" 200 -
+10.45.190.245 - "GET /v1/models HTTP/1.1" 200 -
+10.46.50.247 - "GET /v1/models HTTP/1.1" 200 -
+10.46.17.244 - "GET /v1/models HTTP/1.1" 200 -
+10.46.50.251 - "GET /v1/models HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+1010.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+110.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1"10.45.110.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1"10.46.510.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1"10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/10.46.17.2310.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP10.46.17.23610.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 20010.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.410.46.50.247 - "POST /v1/completions HTTP/1.1" 2010.410.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 20010.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+1010.45.190.245 - "POST /v1/completions HTTP/1.1" 20010.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1"10.46.110.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 10.46.10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 2010.410.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 20010.10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+110.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -110.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+1010.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 210.4610.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 1010.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 1010.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+1010.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 20010.10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+110.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 210.4610.46.17.240 - "POST /v1/completions HTTP/1.1" 210.4610.45.190.245 - "POST /v1/completions HTTP/1.1" 10.46.10.46.50.247 - "POST /v1/completions HTTP/1.1" 10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.4510.45.190.244 - "POST /v1/completions HTTP/1.1" 210.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.4610.46.50.247 - "POST /v1/completions HTTP/1.1" 210.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.410.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 210.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.410.46.17.244 - "POST /v1/completions HTTP/1.1" 2010.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.4510.46.17.244 - "POST /v1/completions HTTP/1.1" 2010.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.410.46.17.236 - "POST /v1/completions HTTP/1.1" 2010.410.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.410.46.17.244 - "POST /v1/completions HTTP/1.1" 2010.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.10.46.50.251 - "POST /v1/completions HTTP/1.1" 20010.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.10.46.50.203 - "POST /v1/completions HTTP/1.1" 20010.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -110.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 20010.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+1010.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 2010.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+1010.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+110.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+110.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 20010.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.10.45.190.244 - "POST /v1/completions HTTP/1.1" 20010.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.10.46.17.2410.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.2410.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/10.46.17.2310.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP10.46.50.25110.46.17.236 - "POST /v1/completions HTTP10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 10.46.17.240 - "POST /v1/completions HTT10.45.190.24510.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions 10.45.190.245 - 10.46.17.244 - "POST /v1/completions H10.46.50.251 - 10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions H10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 -10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completi10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "PO10.46.50.247 - "POST /v1/completion10.45.190.245 - "P10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/110.46.50.210.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.23610.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions 10.10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 210.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "10.46.17.236 - "POST /v1/completions 10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 -10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HT10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.410.46.17.236 - "POST /v1/completions HTTP/1.1" 2010.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.210.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/110.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.210.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HT10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 10.46.17.240 - "POST /v1/completions HTT10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.24210.46.50.251 - "POST /v1/completions HTTP/1.10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.2410.46.50.203 - "POST /v1/completions HTTP/10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.210.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/10.46.17.2410.45.190.244 - "POST /v1/completions HTTP10.410.46.1710.45.190.242 - "POST /v1/completions HTT10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.410.46.50.251 - "POST /v1/completions HTTP/1.1" 2010.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.4610.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 210.4610.45.190.242 - "POST /v1/completions HTTP/1.1" 10.45.10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.10.45.190.244 - "POST /v1/completions HTTP/1.1"10.46.510.46.17.236 - "POST /v1/completions HTTP/1.1"10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.110.46.17.236 - "POST /v1/completions HTTP/1.1"10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.4610.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1"10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/110.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.2410.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.2410.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.10.46.17.10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.2510.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+110.45.190.242 - "POST /v1/completions HTTP/1.1" 200 10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.4610.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 210.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+1010.46.17.240 - "POST /v1/completions HTTP/1.1" 200 10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.210.410.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 210.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1"10.46.510.46.50.247 - "POST /v1/completions HTTP/1.1"10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.4610.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.10.46.17.240 - "POST /v1/completions HTTP/1.1" 20010.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+ -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+ -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+03 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.247 - "POST /v1/completions HTTP/1.1" 200 -
+
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.251 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.244 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.242 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.50.203 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.236 - "POST /v1/completions HTTP/1.1" 200 -
+10.46.17.240 - "POST /v1/completions HTTP/1.1" 200 -
+10.45.190.245 - "POST /v1/completions HTTP/1.1" 200 -

hf_ip/vllm_gpu0.log

@@ -0,0 +1,312 @@
+/usr/local/lib/python3.10/dist-packages/torch/utils/_pytree.py:185: FutureWarning: optree is installed but the version is too old to support PyTorch Dynamo in C++ pytree. C++ pytree support is disabled. Please consider upgrading optree using `python3 -m pip install --upgrade 'optree>=0.13.0'`.
+  warnings.warn(
+INFO 01-04 13:13:24 [__init__.py:239] Automatically detected platform cuda.
+INFO 01-04 13:13:26 [api_server.py:1043] vLLM API server version 0.8.5
+INFO 01-04 13:13:26 [api_server.py:1044] args: Namespace(host='127.0.0.1', port=8001, uvicorn_log_level='info', disable_uvicorn_access_log=False, allow_credentials=False, allowed_origins=['*'], allowed_methods=['*'], allowed_headers=['*'], api_key=None, lora_modules=None, prompt_adapters=None, chat_template=None, chat_template_content_format='auto', response_role='assistant', ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, enable_ssl_refresh=False, ssl_cert_reqs=0, root_path=None, middleware=[], return_tokens_as_token_ids=False, disable_frontend_multiprocessing=False, enable_request_id_headers=False, enable_auto_tool_choice=False, tool_call_parser=None, tool_parser_plugin='', model='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', task='auto', tokenizer=None, hf_config_path=None, skip_tokenizer_init=False, revision=None, code_revision=None, tokenizer_revision=None, tokenizer_mode='auto', trust_remote_code=True, allowed_local_media_path=None, load_format='auto', download_dir=None, model_loader_extra_config={}, use_tqdm_on_load=True, config_format=<ConfigFormat.AUTO: 'auto'>, dtype='auto', max_model_len=131072, guided_decoding_backend='auto', reasoning_parser=None, logits_processor_pattern=None, model_impl='auto', distributed_executor_backend=None, pipeline_parallel_size=1, tensor_parallel_size=1, data_parallel_size=1, enable_expert_parallel=False, max_parallel_loading_workers=None, ray_workers_use_nsight=False, disable_custom_all_reduce=False, block_size=None, gpu_memory_utilization=0.9, swap_space=4, kv_cache_dtype='auto', num_gpu_blocks_override=None, enable_prefix_caching=None, prefix_caching_hash_algo='builtin', cpu_offload_gb=0, calculate_kv_scales=False, disable_sliding_window=False, use_v2_block_manager=True, seed=None, max_logprobs=20, disable_log_stats=False, quantization=None, rope_scaling=None, rope_theta=None, hf_token=None, hf_overrides=None, enforce_eager=False, max_seq_len_to_capture=8192, tokenizer_pool_size=0, tokenizer_pool_type='ray', tokenizer_pool_extra_config={}, limit_mm_per_prompt={}, mm_processor_kwargs=None, disable_mm_preprocessor_cache=False, enable_lora=None, enable_lora_bias=False, max_loras=1, max_lora_rank=16, lora_extra_vocab_size=256, lora_dtype='auto', long_lora_scaling_factors=None, max_cpu_loras=None, fully_sharded_loras=False, enable_prompt_adapter=None, max_prompt_adapters=1, max_prompt_adapter_token=0, device='auto', speculative_config=None, ignore_patterns=[], served_model_name=['default-model'], qlora_adapter_name_or_path=None, show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, disable_async_output_proc=False, max_num_batched_tokens=None, max_num_seqs=256, max_num_partial_prefills=1, max_long_partial_prefills=1, long_prefill_token_threshold=0, num_lookahead_slots=0, scheduler_delay_factor=0.0, preemption_mode=None, num_scheduler_steps=1, multi_step_stream_outputs=True, scheduling_policy='fcfs', enable_chunked_prefill=None, disable_chunked_mm_input=False, scheduler_cls='vllm.core.scheduler.Scheduler', override_neuron_config=None, override_pooler_config=None, compilation_config=None, kv_transfer_config=None, worker_cls='auto', worker_extension_cls='', generation_config='auto', override_generation_config=None, enable_sleep_mode=False, additional_config=None, enable_reasoning=False, disable_cascade_attn=False, disable_log_requests=True, max_log_len=None, disable_fastapi_docs=False, enable_prompt_tokens_details=False, enable_server_load_tracking=False)
+INFO 01-04 13:13:33 [config.py:717] This model supports multiple tasks: {'generate', 'reward', 'embed', 'score', 'classify'}. Defaulting to 'generate'.
+INFO 01-04 13:13:33 [config.py:2003] Chunked prefill is enabled with max_num_batched_tokens=8192.
+/usr/local/lib/python3.10/dist-packages/torch/utils/_pytree.py:185: FutureWarning: optree is installed but the version is too old to support PyTorch Dynamo in C++ pytree. C++ pytree support is disabled. Please consider upgrading optree using `python3 -m pip install --upgrade 'optree>=0.13.0'`.
+  warnings.warn(
+INFO 01-04 13:13:38 [__init__.py:239] Automatically detected platform cuda.
+INFO 01-04 13:13:41 [core.py:58] Initializing a V1 LLM engine (v0.8.5) with config: model='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', speculative_config=None, tokenizer='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=131072, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto,  device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='auto', reasoning_backend=None), observability_config=ObservabilityConfig(show_hidden_metrics=False, otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=None, served_model_name=default-model, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=True, chunked_prefill_enabled=True, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs=None, pooler_config=None, compilation_config={"level":3,"custom_ops":["none"],"splitting_ops":["vllm.unified_attention","vllm.unified_attention_with_output"],"use_inductor":true,"compile_sizes":[],"use_cudagraph":true,"cudagraph_num_of_warmups":1,"cudagraph_capture_sizes":[512,504,496,488,480,472,464,456,448,440,432,424,416,408,400,392,384,376,368,360,352,344,336,328,320,312,304,296,288,280,272,264,256,248,240,232,224,216,208,200,192,184,176,168,160,152,144,136,128,120,112,104,96,88,80,72,64,56,48,40,32,24,16,8,4,2,1],"max_capture_size":512}
+WARNING 01-04 13:13:41 [utils.py:2522] Methods determine_num_available_blocks,device_config,get_cache_block_size_bytes,initialize_cache not implemented in <vllm.v1.worker.gpu_worker.Worker object at 0x7fc89ba32170>
+INFO 01-04 13:13:42 [parallel_state.py:1004] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, TP rank 0
+INFO 01-04 13:13:42 [cuda.py:221] Using Flash Attention backend on V1 engine.
+WARNING 01-04 13:13:42 [topk_topp_sampler.py:69] FlashInfer is not available. Falling back to the PyTorch-native implementation of top-p & top-k sampling. For the best performance, please install FlashInfer.
+INFO 01-04 13:13:42 [gpu_model_runner.py:1329] Starting to load model /data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf...
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+INFO 01-04 13:15:40 [loader.py:458] Loading weights took 117.89 seconds
+INFO 01-04 13:15:41 [gpu_model_runner.py:1347] Model loading took 61.0562 GiB and 118.156821 seconds
+INFO 01-04 13:15:55 [backends.py:420] Using cache directory: /root/.cache/vllm/torch_compile_cache/75e72335d9/rank_0_0 for vLLM's torch.compile
+INFO 01-04 13:15:55 [backends.py:430] Dynamo bytecode transform time: 14.60 s
+INFO 01-04 13:15:59 [backends.py:136] Cache the graph of shape None for later use
+INFO 01-04 13:16:50 [backends.py:148] Compiling a graph for general shape takes 53.42 s
+INFO 01-04 13:17:53 [monitor.py:33] torch.compile takes 68.02 s in total
+INFO 01-04 13:17:54 [kv_cache_utils.py:634] GPU KV cache size: 254,768 tokens
+INFO 01-04 13:17:54 [kv_cache_utils.py:637] Maximum concurrency for 131,072 tokens per request: 1.94x
+INFO 01-04 13:18:32 [gpu_model_runner.py:1686] Graph capturing finished in 39 secs, took 1.21 GiB
+INFO 01-04 13:18:32 [core.py:159] init engine (profile, create kv cache, warmup model) took 171.40 seconds
+INFO 01-04 13:18:32 [core_client.py:439] Core engine process 0 ready.
+WARNING 01-04 13:18:32 [config.py:1239] Default sampling parameters have been overridden by the model's Hugging Face generation config recommended from the model creator. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
+INFO 01-04 13:18:32 [serving_chat.py:118] Using default chat sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
+INFO 01-04 13:18:32 [serving_completion.py:61] Using default completion sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
+INFO 01-04 13:18:32 [api_server.py:1090] Starting vLLM API server on http://127.0.0.1:8001
+INFO 01-04 13:18:32 [launcher.py:28] Available routes are:
+INFO 01-04 13:18:32 [launcher.py:36] Route: /openapi.json, Methods: HEAD, GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /docs, Methods: HEAD, GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /docs/oauth2-redirect, Methods: HEAD, GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /redoc, Methods: HEAD, GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /health, Methods: GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /load, Methods: GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /ping, Methods: GET, POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /tokenize, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /detokenize, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/models, Methods: GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /version, Methods: GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/chat/completions, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/completions, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/embeddings, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /pooling, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /score, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/score, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/audio/transcriptions, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /rerank, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/rerank, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v2/rerank, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /invocations, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /metrics, Methods: GET
+INFO:     Started server process [1315105]
+INFO:     Waiting for application startup.
+INFO:     Application startup comINFO 01INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.INFO 01-04INFO 01-04 14:00:03 [loggers.py:111] Engine 000: Avg prompt throughput: 106.6 tokens/s, Avg generation throughput: 70.6 tokens/s, Running: 1 reqINFO 01-INFO 01-04 14:00:07 [loggers.py:111] Engine 000: Avg prINFO 01-04 INFO 01-04 14:00:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.2 tokens/s, Running: 1 reqs,INFO 01-04 14:00:15 [loggers.py:111] Engine 000: Avg prompt tINFINFO 01-04 14:00:17 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04INFO 01-04 14:00:23 [loggers.py:111] Engine 000: Avg prompt throughput: 77.3 toINFO 01-04 14:00:25 [loggers.py:111] Engine 000: Avg prompt INFO 01-04 14:00:26 [loggers.py:111] Engine 000: Avg prompt throughINFO 01-04 INFO 01-04 14:00:33 [loggers.py:111] Engine 000: Avg prompt throughput: 64.9 tokINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:00:35 [loggers.py:111] Engine 000: Avg promptINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-0INFO 01-04 14:00:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/sINFO 01-04 14:00:45 [loggers.py:111] Engine 000: Avg prompINFO 01-04 14:00:46 [loggers.py:111] Engine 000: Avg prompt througINFO 01-INFO 01-04 14:00:53 [loggers.py:111] Engine 000: Avg prompt throughput: 72.0 tokens/s, Avg generation throughput: 74.1 tokens/s, Running: 2 reINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+ININFO 01INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1INFO 01-04 INFO 01-04 14:01:03 [loggers.py:111] Engine 000: Avg prompt throughput: 70.9 tokens/s, Avg generation throughput: 72.9 tokens/s, Running: 2 reqINFO 01-04 14:01:06 [loggers.py:111] Engine 000: Avg prompt thrINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:01:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0INFOINFO 01-04 14:01:17 [loggers.py:111] Engine 000: Avg prompt tINFO 01-04 14:01:17 [loggers.py:111] Engine 000: Avg prompt thrINFO 01-04 14INFO 01-04 14:01:23 [loggers.py:111] Engine 000: Avg prompt throughput: 81.3 tokens/s, INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/INFO 01-04 14:01:27 [loggers.py:111] Engine 000: Avg prompt throughput: 81.7 INFO:     10.46.17.192:0 - "POST /v1/completions HTTPINFO 01-04 14INFO 01-04 14:01:33 [loggers.py:111] Engine 000: Avg prompt throughput: 67INFO 0INFO 01-04 14:01:37 [loggers.py:111] Engine 000: Avg prompt throughput: 110.1 tokens/s, Avg generation throughput: 106.4 tokensINFO 01-04 INFO 01-04 14:01:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 rINFO 01INFO 01-04 14:01:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 111.8 toINFO:     10.46INFO 01-04 14:01:53 [loggers.py:111] Engine 000: Avg prompt throughput: 72.8 tokeINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 20INFO INFO 01-04 14:01:57 [loggers.py:111] Engine 000: Avg prompINFO 01-04 14INFO 01-04 14:02:03 [loggers.py:111] Engine 000: Avg prompt throughput: 123.4 tokens/s, Avg generation throughput: 62.8 tokens/s, Running: 2 reqs, INFOINFO 01-04 14:02:05 [loggers.py:111] Engine 000: Avg promINFINFO 01-04 14:02:07 [loggers.py:111] Engine 000: Avg prompINFO 01-04 14:0INFO 01-04 14:02:13 [loggers.py:111] Engine 000: Avg prompt throughput: 55.0 tokens/s, Avg generation throughput: 87.8 tokens/s, Running: 2 reININFO 01-04 14:02:17 [loggers.py:111] Engine 000: Avg prompINFO 01-04 14:02:17 [loggers.py:111] Engine 000: Avg prompt througINFO 01-04 14:INFO 01-04 14:02:23 [loggers.py:111] Engine 000: Avg prompt throughput: 80.3 tokens/sINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1"INFO:   INFO 01-04 14:02:25 [loggers.py:111] Engine 000: Avg pINFO 01-04 1INFO:     10.45.190.192:0 - "POST /v1/completions HINFO 01-04 14:02INFO 01-04 14:02:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, INFO 01-04 14:02:35 [loggers.py:111] Engine 000: Avg prINFO 01-04 14:02:36 [loggers.py:111] Engine 000: Avg prompt INFO 01-04 14:02INFO 01-04 14:02:43 [loggers.py:111] Engine 000: Avg prompt throughput: 143.7 tokens/s, Avg generation throughput: 77.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache IINFO 01-04 14:0INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:02:53 [loggers.py:111] Engine 000: Avg prompt throughput: 87.8 tokens/s, Avg generation throughput: 97.0 tokens/s, Running: 2 INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:INFO 01-04 14:03:03 [loggers.py:111] Engine 000: Avg prompt throughput: 110.9 tokens/s, Avg generation throughput: 121.2 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:INFO 01-04 14:03:07 [loggers.py:111] Engine 000: Avg promINFO 01-04 14:03:09 [INFO 01-04 14:03:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 51.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.8%, Prefix caINFO:     10.45.190INFO 01-04 14:03:23 [loggers.py:111] Engine 000: Avg prompt throughput: 241.7 tokens/s, Avg generation throughput: 87.2 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.0%, Prefix cache hitINFO 01-04 1INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:37 [loggers.py:111] Engine 000: Avg prompt throughput: 126.7 tokens/s, Avg generation throughput: 82.6 tokINFO 01-04 14:03:39 INFO 01-04 14:03:43 [loggers.py:111] Engine 000: Avg prompt throughput: 139INFO 01-04 14:03:47 [loggers.py:111] Engine 000: Avg prompt throughput: 123.8 tokens/s, Avg generation throughput: 99.1 tokens/s, Running: 3 reqINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:53 [loggers.py:111] Engine 000: Avg prompt throughput:INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:57 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 1INFO 01-04 14:04:03 [loggers.py:111] Engine 000: Avg prompt throughput: 79.5 tokens/s, Avg generation throughput: 62.8 tokens/s, Running:INFO 01-04 14:04:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 94.4 tokens/s, Running: 2 reqs, WaINFO 01-04 14:04:13 [loggers.py:111] Engine 000: Avg prompt throughput: 134.8 tokens/s, Avg generation throughput: 77.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.1%, Prefix cachINFO:     10.46.17.192:0 - "POST /v1/completINFO 01-04 1INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:04:19 [loggers.INFO 01-04 14:04:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 18.0 tokens/s, RunniINFO 01-04 14:04:27 [loggers.py:111] Engine 000: Avg proINFO 01-04 14:04:29 [loggersINFO 01-04 14:04:33 [loggers.py:111] Engine 000: Avg prompt throughput: 193.7 tokens/s, Avg generation throughput: 40.1 tokens/s, RunnINFO 01-04 14:04:37 [loggers.py:111] Engine 000: Avg prompt throughput: 141.9 tokens/s, Avg generation throughput: 85.0 tokINFO 01-04 14:04:39 [loggINFO 01-04 14:04:43 [loggers.py:111] Engine 000: Avg prompt throughputINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:04:47 [loggers.py:111] Engine 000: Avg promINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:04:49 [loggeINFO 01-04 14:04:53 [loggers.py:111] Engine 000: Avg prompt throughput: 188.0 tokens/s, Avg generation throughput: 51.7 tokens/s, RuINFO 01-04 14:04:57 [loggers.py:111] Engine 000: Avg prompt throughput: 202.9 tokens/sINFO 01-04 14:05:03 [loggers.py:111] Engine 000: Avg prompt throughput: 118.1 tokens/s, Avg generation throughput: 72.5 tokens/s, RuINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.1INFO:     10.46.17.192:0 - "POST /v1/completioINFO 01-04 14:05:07 INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:13 [loggers.py:111] Engine 000: Avg prompt throughput: 208.7 tokens/s, Avg generation throughput: 77.8 tokens/s, RunnINFO 01-04 14:05:17 [loggers.py:111] Engine 000: Avg prompt throughput: 178.8 tokensINFO 01-04 14:05:23 [loggers.py:111] Engine 000: Avg prompt throughput: 155.6 tokens/s, Avg generation throughput: 74.9 tokens/s, RunnINFO 01-04 14:05:27 [loggers.py:111] Engine 000: Avg prompt throughput: 175.9 tokensINFO 01-04 14:05:33 [loggers.py:111] Engine 000: Avg prompt throughput: 164.6 tokens/s, Avg generation throughput: 105.2 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 6.4%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completioINFO 01-04 14:05:37 INFO 01-04 14:05:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 76.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.9%, Prefix cINFO 01-04 14:05:47 [loggers.py:111] Engine 000: Avg prompt throughput: 165.1 tokensINFO 01-04 14:05:53 [loggers.py:111] Engine 000: Avg prompt throughput: 332.1 tokens/s, Avg generation throughput: 80.2 tokens/s, RunnINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:5INFO 01-04 14:06:03 [loggers.py:111] Engine 000: Avg prompt throughput: 195.5 tokens/s, Avg generation throughput: 69.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cacINFO 01-04 14:06:0INFO 01-04 14:06:13 [loggers.py:111] Engine 000: Avg prompt throughput: 132.6 tokens/s, Avg generation throughput: 120.2 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix caINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:1INFO 01-04 14:06:23 [loggers.py:111] Engine 000: Avg prompt throughput: 279.3 tokens/s, Avg generation throughput: 123.9 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cache hit rate: 5.0%
+INFO:     10.46.50.192:0 - "POST /v1/completiINFO 01-04 14:06:27 [INFO 01-04 14:06:33 [loggers.py:111] Engine 000: Avg prompt throughput: 266.1 tokens/s, Avg generation throughput: 122.4 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 4.7%
+INFO:     10.46.17.192:0 - "POST /v1/completiINFO 01-04 14:06:37 [INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:43 [loggers.py:111] Engine 000: Avg prompt throughput: 248.5 tokens/s, Avg generation throughput: 102.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, PrefixINFO:     10.45.190.19INFO 01-04 14:06:53 [loggers.py:111] Engine 000: Avg prompt throughput: 365.8 tokens/s, Avg generation throughput: 112.0 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 7.4%
+INFO 01-04 14:07:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 177.2 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.1%, Prefix cache hit rate: 7.4%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 88.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.0%, Prefix cacINFO:     10.43.30INFO 01-04 14:07:23 [loggers.py:111] Engine 000: Avg prompt throughput: 507.3 tokens/s, Avg generation throughput: 87.5 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.3%, Prefix cache hit rate: 9.2%
+INFO:     10.46.50.192:0 - "POST /v1/completINFO 01-04 14:07:27 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 14:07INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:33 [loggers.py:111] Engine 000: Avg prompt throughput: 154.9 tokens/s, Avg generation throughpINFO 01-04 14:07:36 [INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO INFO 01-04 14:07:3INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:43 [loggers.py:111] Engine 000: Avg prompt throughput: 283.0 tokens/s, Avg generation throughINFO:     10.43.30INFO 0INFO 01-04 14:07:47 [loggers.py:111] Engine 000: Avg prompt throughput: 345.4 tokens/s, Avg generation throughput: 202.0 tokens/INFO 01-04 14:07:49 INFO 01-04 14:07:53 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04 14:07:57 [loggers.py:111] Engine 000: Avg prompt throughput: 199.4 tokens/s, Avg generation throughput: 105.6 tokens/s, RunINFO:     10.46.50.1INFO 01-04 14:08:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, RunnINFO 01-04 14:08:07 [loggers.py:111] Engine 000: Avg prompt throughput: 166.5 tokensINFO 01-04 14:08:13 [loggers.py:111] Engine 000: Avg prompt throughput: 218.1 tokens/s, Avg generation throughput: 84.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix caINFO 01-04 14:08:17 [loggers.py:111] Engine 000: Avg prompt throughput: 263.8 tokenINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:23 [loggers.py:111] Engine 000: Avg prompt throughput: 140.0 tokens/s, Avg generation throughput: 76.1 tokens/s, RunninINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:2INFO 01-04 14:08:33 [loggers.py:111] Engine 000: Avg prompt throughput: 342.5 tokens/s, Avg generation throughput: 71.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix caINFO 01-04 14:08:37 [loggers.py:111] Engine 000: Avg prompt throughput: 571.2 tokenINFO 01-04 14:08:43 [loggers.py:111] Engine 000: Avg prompt throughput: 226.9 tokens/s, Avg generation throughput: 77.9 tokens/s, RunniINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:47 [loggers.py:111] Engine 000: Avg prompt throughput: 284.4 tokenINFO 01-04 14:08:53 [loggers.py:111] Engine 000: Avg prompt throughput: 323.4 tokens/s, Avg generation throughput: 76.2 tokens/s, RunninINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.INFO 01-04 14:09:03 [loggers.py:111] Engine 000: Avg prompt throughput: 289.2 tokens/s, Avg generation throughput: 117.3 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.2%, Prefix cache hit rate: 7.6%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:13 [loggers.py:111] Engine 000: Avg prompt throughput: 358.6 tokens/s, Avg generation throughput: 94.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cachINFO:     10.45.1INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 74.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hINFO:     10.4INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:33 [loggers.py:111] Engine 000: Avg prompt throughput: 354.6 tokens/s, Avg generation throughput: 71.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cacheINFO 01-04 14:09INFO 01-04 14:09:43 [loggers.py:111] Engine 000: Avg prompt throughput: 771.7 tokens/s, Avg generation throughput: 85.9 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.9%, Prefix cache hit rate: 8.5%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTINFO 01-04 14:INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 99.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hiINFO:     10.INFO:     10.46.17.192:0 - "POST /v1/completions HTTPINFO 01-04 14INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:03 [loggers.py:111] Engine 000: Avg prompt throughput: 815.3 tokens/s, Avg generation throughput: 58.9 tokens/s, Running: INFO 01-04 14:10:07 [loggers.py:111] Engine 000: Avg prompt throughput: 316.7 toINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 43.2 tokens/s, Running: INFO 01-04 14:10:17 [loggers.py:111] Engine 000: Avg prompt throughput: 458.3 toINFO 01-04 14:10:23 [loggers.py:111] Engine 000: Avg prompt throughput: 378.2 tokens/s, Avg generation throughput: 39.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hINFO 01-04 14:10:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokensINFO 01-04 14:10:33 [loggers.py:111] Engine 000: Avg prompt throughput: 383.2 tokens/s, Avg generation throughput: 33.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cacINFO 01-04 14:10:37INFO 01-04 14:10:43 [loggers.py:111] Engine 000: Avg prompt throughput: 237.6 tokens/s, Avg generation throughput: 80.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 11.1%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 53.4 tokens/s, INFO 01-04 14:10:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, AvgINFO 01-04 14:11:03 [loggers.py:111] Engine 000: Avg prompt throughput: 126.0 tokens/s, Avg generation throughput: 18.9 tokens/INFO 01-04 14:11:07 [loggers.py:111] Engine 000: Avg prompt throughput: 424.9 tokens/s, Avg INFO 01-04 14:11:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.8%, PrefINFO:     10.46.17.192:0 INFO 01-04 14:11:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.9%, Prefix cache hit rate: 11.0%
+INFO 01-04 14:11:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.1%, Prefix cache hit rate: 11.0%
+INFO:     10.43.30.3:0 - "POST /v1/comINFO 01-04 14:11:47 [loggers.py:111] Engine 000: Avg prompt throughput: 405.3 tokens/s, Avg generation throughput: 26.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.7%, Prefix cache hit rate: 7.7%
+INFO 01-04 14:11:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 48.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 7.7%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:12:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 10.4 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 7.7%
+INFO 01-04 14:12:17 [loggers.py:111] Engine 000: Avg prompt throughput: 295.1 tokens/s, Avg generation throughput: 29.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: 7.5%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:12:27 [loggers.py:111] Engine 000: Avg prompt throughput: 199.6 tokens/s, Avg generation throughput: 77.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.0%, Prefix cache hit rate: 7.4%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:12:3INFO 01-04 14:12:37 [loggers.py:111] Engine 000: Avg prompt throughput: 302.2 tokens/s, Avg generation throughput: 17.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cacINFO 01-04 14:12:4INFO 01-04 14:12:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cacINFO:     10.45.19INFO:     10.46.17.192:0 - "POST /v1/completions INFO 01-04 14:12:INFO 01-04 14:12:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 31.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KINFO:     10.46.17.192:0 - "POSTINFO 01-04 14:13:INFO 01-04 14:13:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hINFO:     10.4INFO 01-04 14:13:17 [loggers.py:111] Engine 000: Avg prompt throughput: 392.7 tokens/s, Avg generation throughput: 17.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 10.0%
+INFO 01-04 14:13:27 [loggers.py:111] Engine 000: Avg prompt throughput: 464.7 tokens/s, Avg generation throughput: 45.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 9.6%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:13:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 54.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 9.6%
+INFO 01-04 14:13:47 [loggers.py:111] Engine 000: Avg prompt throughput: 818.8 tokens/s, Avg generation throughput: 43.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 9.1%
+INFO 01-04 14:13:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 92.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 9.1%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:07 [loggers.py:111] Engine 000: Avg prompt throughput: 359.1 tokens/s, Avg generation throughput: 51.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache hit rate: 8.8%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:17 [loggers.py:111] Engine 000: Avg prompt throughput: 212.3 tokens/s, Avg generation throughput: 66.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.9%, Prefix cache hit rate: 9.0%
+INFO 01-04 14:14:27 [loggers.py:111] Engine 000: Avg prompt throughput: 337.4 tokens/s, Avg generation throughput: 81.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 8.8%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:37 [loggers.py:111] Engine 000: Avg prompt throughput: 347.8 tokens/s, Avg generation throughput: 84.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 10.4%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:47 [loggers.py:111] Engine 000: Avg prompt throughput: 384.5 tokens/s, Avg generation throughput: 124.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cache hit rate: 10.1%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:57 [loggers.py:111] Engine 000: Avg prompt throughput: 886.1 tokens/s, Avg generation throughput: 61.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 10.8%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:07 [loggers.py:111] Engine 000: Avg prompt throughput: 569.5 tokens/s, Avg generation throughput: 71.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.2%, Prefix cache hit rate: 10.5%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTINFO:     10.45INFO 01-04 14:15:17 [loggers.py:111] Engine 000: Avg prompt throughput: 459.1 tokens/s, Avg generation throughput: 54.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 10.2%
+INFO:     10.45.190.192:0 - "POST /v1/completions HINFO:     10.46.INFO 01-04 14:15:27 [loggers.py:111] Engine 000: Avg prompt throughput: 458.6 tokens/s, Avg generation throughput: 48.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 12.4%
+INFO 01-04 14:15:37 [loggers.py:111] Engine 000: Avg prompt throughput: 309.6 tokens/s, Avg generation throughput: 63.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 12.2%
+INFO 01-04 14:15:47 [loggers.py:111] Engine 000: Avg prompt throughput: 525.6 tokens/s, Avg generation throughput: 119.7 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.9%, Prefix cache hit rate: 11.9%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 97.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 11.9%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:07 [loggers.py:111] Engine 000: Avg prompt throughput: 435.0 tokens/s, Avg generation throughput: 81.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 11.6%
+INFO 01-04 14:16:17 [loggers.py:111] Engine 000: Avg prompt throughput: 701.2 tokens/s, Avg generation throughput: 78.6 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.9%, Prefix cache hit rate: 11.2%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 127.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.9%, Prefix cache hit rate: 11.2%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:37 [loggers.py:111] Engine 000: Avg prompt throughput: 654.4 tokens/s, Avg generation throughput: 127.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.1%, Prefix cache hit rate: 10.8%
+INFO 01-04 14:16:47 [loggers.py:111] Engine 000: Avg prompt throughput: 675.2 tokens/s, Avg generation throughput: 106.8 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cINFO 01-04 14:16:47 [loggers.py:111] Engine 000INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:57 [loggers.py:111] Engine 000: Avg prompt throughput: 1014.1 tokens/s, Avg generation throughput: 164.8 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 10.0%, Prefix cache hit rate: 11.7%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 146.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.1%, Prefix cache hit rate: 11.7%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:17 [loggers.py:111] Engine 000: Avg prompt throughput: 316.3 tokens/s, Avg generation throughput: 92.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPUINFO 01-04 14:17:17 [loggers.py:111] Engine 000: AvgINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:27 [loggers.py:111] Engine 000: Avg prompt throughput: 556.7 tokens/s, Avg generation throughput: 106.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV INFO:     10.46.17.192:0 - "POST /v1/completionsINFO:     10.45.19INFO 01-04 14:17:33 [loggers.py:111] Engine 000: INFO 01-04 14:17:37 [loggers.py:111] Engine 000: Avg prompt throughput: 964.4 tokens/s, Avg generation throughput: 105.0 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KVINFO:     10.46.17.192:0 - "POST /v1/completions INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:47 [loggers.py:111] Engine 000: Avg prompt throughput: 483.4 tokens/s, Avg generation throughput: 120.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KVINFO:     10.43.30.3:0 - "POST /v1/completions HTINFO 01-04 14:17:57 [loggers.py:111] Engine 000: Avg prompt throughput: 410.0 tokens/s, Avg generation throughput: 106.2 tokens/s, Running: 3 reqs, Waiting: 0 reqINFO 01-04 14:17:57 [loggers.py:111] Engine 000: Avg prompINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:07 [loggers.py:111] Engine 000: Avg prompt throughput: 394.1 tokens/s, Avg generation throughput: 153.3 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.9%, Prefix cache hit rate: 13.4%
+INFO:     10.45INFO 01-04 14:18:13 [loggers.py:111] Engine 000: Avg prompt throughput: 551.9 tokens/s, Avg generation throughput: 122INFO 01-04 14:18:17 [loggers.py:111] Engine 000: Avg prompt throughput: 587.7 tokens/s, Avg generationINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:23 [loggers.py:111] Engine 000: AINFO 01-04 14:18:27 [loggers.py:111] Engine 000: Avg prompt throughput: 543.7 tokens/s, Avg generation throughput: 129.9 tokens/s, Running: 3 reqs, Waiting: 0 rINFO 01-04INFO:     10.43.30.5:0 - "POST /v1/completions HTTINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:37 [loggers.py:111] Engine 000: Avg prompt throughput: 539.9 tokens/s, Avg generation throughput: 158.9 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.4%, Prefix cache hit rate: 15.1%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:47 [loggers.py:111] Engine 000: Avg prompt throughput: 241.8 tokens/s, Avg generation throughput: 156.7 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.5%, Prefix cache hit rate: 15.0%
+INFO 01-04 14:18:57 [loggers.py:111] Engine 000: Avg prompt throughput: 315.6 tokens/s, Avg generation throughput: 144.8 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 10.4%, Prefix cache hit rate: 14.8%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:07 [loggers.py:111] Engine 000: Avg prompt throughput: 613.2 tokens/s, Avg generation throughput: 160.6 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.5%, Prefix cache hit rate: 14.7%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:17 [loggers.py:111] Engine 000: Avg prompt throughput: 282.8 tokens/s, Avg generation throughput: 153.1 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.3%, Prefix cache hit rate: 14.5%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 110.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: 14.5%
+INFO 01-04 14:19:37 [loggers.py:111] Engine 000: Avg prompt throughput: 577.7 tokens/s, Avg generation throughput: 61.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 14.2%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:47 [loggers.py:111] Engine 000: Avg prompt throughput: 498.2 tokens/s, Avg generation throughput: 83.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GINFO 01-04 14:19:53 [loggers.py:111] Engine 000: Avg prompt throughput: 451.6 tokens/s, Avg generation throughput: 93.0 INFO 01-04 14:19:57 [loggers.py:111] Engine 000: Avg prompt throughput: 1566.2 tokens/s, Avg generation throughput: 77.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg genINFO 01-04 14:20:05 [loggeINFO 01-04 14INFO 01-04 14:20:07 [loggers.py:111] Engine 000: Avg prompt throughput: 606.0 tokens/s, Avg generation throughput: 8.4 tokens/s, Running: 1 reqs, WaiINFO 01-04 14:20:13 [loggers.pyINFO 01-04 14:20:15 [logINFO 01-04 14:20:17 [loggers.py:111] Engine 000: Avg prompt throughput: 631.6 tokens/s, Avg generation throughput: 35.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, INFO:     10.43.30.5:0 - "POST INFO:     10.46.50.192:0INFO:    INFO 01-04 14:20:23 [loggers.py:1INFO 01-04 14:20:25 [logINFO 01-04 14:20:27 [loggers.py:111] Engine 000: Avg prompt throughput: 813.4 tokens/s, Avg generation throughput: 47.8 tokens/s, Running: 1 reqs, Waiting: 0 reINFO 01-04 14:20:33 [loggers.py:111INFO:     10.45.190.192:INFO 01-04 14:20:37 [loggers.py:111] Engine 000: Avg prompt throughput: 934.1 tokens/s, Avg generation throughput: 59.4 tokens/s, Running: 2 reqs, Waiting: 0 reINFO 01-04 14:20:43 [loggers.py:111] Engine 000: Avg prompt throughput: 385.3 tokens/s, Avg generation throughput: 125.2 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GINFO 01-04 14:20:45 [loggINFO 01-04 14:20:47 [loggers.py:111] Engine 000: Avg prompt throughput: 499.6 tokens/s, Avg genINFO 01-04 14:20:53 [loggers.py:111] Engine 000: Avg prompt throughput: 348.3 tokens/s, Avg generation throughput: 135.1 tokINFO:     1INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1INFO 01-04 INFO 01-04 14:20:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s,INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POSINFO:     10.43.30.3:0 - INFO 01-04 14:21:07 [loggers.py:111] Engine 000: Avg prompt throughput: 1571.0 tokens/s, Avg generation tINFO 01-04 14:21:08 [loggers.py:111] Engine 000: Avg prompt throughput: 1191.6 tokens/s, Avg generation throughput:INFO 01-04 14:21:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 89.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV caINFO 01-04 14:21:18 [loggers.py:111] Engine 00INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:27 [loggers.py:111] Engine 000: Avg prompt throughput: 1640.0 tokens/s, Avg generation thrINFO 01-04 14:21:28 [loggers.py:111] Engine 000: Avg promptIINFO 01-04 14:21:33 [loggers.py:111] Engine 000: Avg pINFO 01-04 14:21:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throuINFO 01-04 14:21:38 [loggers.py:111] Engine 000: Avg promptINFO 01-04 14:21:43 [loggers.py:111] Engine 000: AvgINFO:    INFO 01-04 14:21:47 [loggers.py:111] Engine 000: Avg promINFO 01-04 14:21:47 [loggers.py:111] Engine 000: Avg prompt throughput: 430.7 tokens/s, Avg generatioINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 2INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:57 [loggers.py:111] Engine 000: Avg prompt throughput: 741.4 tokens/s, Avg generatioINFO 01-04 14:22:03 [loggers.py:111] Engine 000: Avg prompt tINFO 01-04 14:22:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 92.8 tokens/s, Running: 2 reqs, Waiting: 0 INFO 01-04 14:22:13 [loggers.py:111] Engine 000: Avg prompt throughput: 649.5 tokens/s, Avg generation throughput: 108.5 tokensINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1INFO 01-04 14:22:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 62.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 18.0%
+INFO 01-04 14:22:27 [loggers.py:111] Engine 000: Avg prompt throughput: 1334.7 tokens/s, Avg generation throughput: 87.0 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.9%, Prefix cache hit rate: 17.4%
+INFO 01-04 14:22:37 [loggers.py:111] Engine 000: Avg prompt throughput: 639.9 tokens/s, Avg generation throughput: 131.4 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 12.0%, Prefix cache hit rate: 17.1%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:47 [loggers.py:111] Engine 000: Avg prompt throughput: 607.4 tokens/s, Avg generation throughput: 130.4 tokens/s, Running: 3 reqs, Waiting: 0 reqs,INFO 01-04 14:22:53 [loggers.py:111] Engine 000: Avg proINFO 01-04 14:22:57 [loggers.py:111] Engine 000: Avg prompt throughput: 396.3 tokens/s, Avg generation throughput: 130.5 tokens/s, Running: 4 reqs, Waiting: 0 reINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200INFOINFO 01-04 14:23:03 [loggers.py:111] Engine 000: Avg prompt throughput: 611.2 tokens/s, Avg gINFO 01-04 14:23:05 [loggers.py:11INFO 01-04 14:23:07 [loggers.py:111] Engine 000: Avg prompt throughput: 356.6 tokens/s, Avg INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:13 [loggeINFO:     10.46.17.192:0 - "POST /INFO 01-04 14:23:17 [loggers.py:111] Engine 000: Avg prompt throughput: 621.3 tokens/s, Avg generation throughput: 58.7 tokens/s, RINFO 01-04 14:23:17 [loggersINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:23 [logINFO 01-04 14:23:25 [loggers.py:11INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:27 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 14:23:27 [loggerINFO 01-04 14:23:33 [loggers.py:111] Engine 000: Avg prompt throughput: 474.7 tokens/s, Avg generation throughput: 51.2 toINFO 01-04 14:23:37 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 14:23:37 [loggerINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg INFO 01-04 14:23:45 [loggersINFO 01-04 14:23:47 [loggers.py:111] Engine 000: Avg prompt throughput: 733.8 tokens/s, Avg generatINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:53 [loggers.py:111] Engine 000: Avg prompt throughput: 414.2 tokens/s, Avg INFO 01-04 14:23:55 [loggers.pINFO 01-04 14:23:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generatINFO 01-04 14:24:03 [loggers.py:111] Engine 000: Avg prompt throughput: 587.7 tokens/s, Avg generation throughput: 73.8INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:07 [loggers.py:111] Engine 000: Avg prompt throughpuINFO 01-04 14:24:07 [loggers.py:INFO 01-04 14:24:13 [loggers.py:111] Engine 000: Avg prompt throughput: 759.1 tokens/s, Avg generation throughput: 64.INFO 01-04 14:24:17 [loggers.py:111] Engine 000: Avg prompt throughINFO 01-04 14:24:17 [loggers.py:111] Engine 000: Avg prompt throughput: 502.5 tokens/s, Avg generaINFO 01-04 14:24:23 [loggers.py:111] Engine 000: Avg INFO 01-04 14:24:27 [loggers.py:111] Engine 000: Avg prompt throughpINFO 01-04 14:24:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, WaitinINFO:     1INFO 01-04 14:24:33 [loggers.py:111] Engine 000: Avg ININFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:37 [loggers.py:111] Engine 000: Avg prompt throughput: 744.9 tokens/s, Avg generINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:43INFO 01-04 14:24:45 [loggers.py:111INFO 01-04 14:24:47 [loggers.py:111] Engine 000: Avg prompt throughpuINFO 01-04 14:24:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generaINFO 01-04 14:24:53 [loggers.py:111] Engine 000: AvgINFO 01-04 14:24:57 [loggers.py:111] Engine 000: Avg prompt throughpuINFO 01-04 14:24:57 [loggers.py:111] Engine 000: Avg prompt throughput: 197.1 tokens/s, Avg generation throughput: 47.0 tokens/s, Running: 1 reqs, WaiINFO:     10INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/ININFO 01-04 14:25:07 [loggers.py:111] Engine 000: Avg prompt throughput: 767.7 tokens/s, Avg generation throughput: 83.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.3%, Prefix cache hit rate: 15INFO 01-04 14:25:17 [loggers.py:111] Engine 000: Avg prompt throughput:INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:17 [loggers.py:111] Engine 000: Avg prompt throughput: 645.8 tokensINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+ININFO 01-04 14:25:27 [loggers.py:111] Engine 000: Avg prompt throughput: 849.6 tokens/s, Avg generation throughput: 60.4 tokens/s, Running: 2 reqs, Waiting: 0 reINFO 01-04 14:25:33 [loggers.py:111] Engine 000: Avg prINFO 01-04 14:25:37 [loggers.py:111] Engine 000: Avg prompt throughput: 698.2 tokens/s, Avg generation throughput: 21.3 tokens/s, RunniINFO 01-04 14:25:37 [loggerINFO 01-04 14:25:43 [loggers.py:111] Engine 000: Avg promINFO 01-04 14:25:47 [loggers.py:111] Engine 000: Avg prompt throughput: 555.0 tokens/s, Avg generation throughput: 66.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs,INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 118.0 tokenIINFO 01-04 14:25:57 [loggers.py:111] Engine 000: Avg prompt throughput: 761.9 tokens/s, Avg generation throughput: 46.6 tokens/s, Running: 2 reqs, Waiting: 0 rINFO 01-04 14:26:03 [loggers.py:111] Engine 000: Avg promINFO 01-04 14:26:07 [loggers.py:111] Engine 000: Avg prompt throughINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:26:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, AINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:26:17 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04 14:26:17 [loggers.py:111] Engine 000: Avg prompt throughput: 774.4 tokens/s, Avg generatINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.INFO 01-04 14:26:27 [loggers.py:111] Engine 000: Avg prompt througINFO 01-04 14:26:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.7 tokens/s, Running: 1 reqs, Waiting:INFO 01-04 14:26:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 48.3 tokens/s, RuINFO 01-04 14:26:37 [loggers.py:111] Engine 000: Avg prompt throughput: 944.1 tokens/s,INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:26:47 [loggers.py:111] Engine 000: Avg prompt throughput: 943.7 tokens/s, Avg generation throughput: 48.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 16.9%
+INFO:     10.43.30.4:0 - "INFO:     10.45.190.192:0 - "POST /v1/INFO 01-04 14:26:57 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04 14:26:57 [loggers.py:111] Engine 000: Avg prompt throughput: 552.0 tokens/s, Avg generation throughput: 47.8 tokens/s, Running: 1 reqs, Waiting:INFO 01-04 14:27:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cache hit rate: 18.0%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:27:17 [loggers.py:111] Engine 000: Avg prompt throughput: 832.9 tokens/s, Avg generation throughput: 38.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 17.7%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:27:27 [loggers.py:111] Engine 000: Avg prompt througINFO 01-04 14:27:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 56.5 tokens/s, Running: 1 reqs, WaitiINFO 01-04 14:27:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, RunnINFO 01-04 14:27:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s,INFO 01-04 14:27:57 [loggers.py:111] Engine 000: Avg prompt throughput: 1181.9 tokens/s, Avg generation throughput: 41.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.8%, Prefix cache hit rate: 17.3%
+INFO 01-04 14:28:07 [loggers.py:111] Engine 000: Avg prompt throughput: 797.1 tokens/s, Avg generation throughput: 52.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.1%, Prefix cache hit rate: 17.0%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:28:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 87.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 17.0%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:28:27 [loggers.py:111] Engine 000: Avg prompt throughput: 642.9 tokens/s, Avg generation throughput: 7.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 16.8%
+INFO 01-04 14:28:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 16.8%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:28:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 6.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.8%
+INFO 01-04 14:28:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.8%
+INFO 01-04 14:29:07 [loggers.py:111] Engine 000: Avg prompt throughput: 706.4 tokens/s, Avg generation throughput: 26.5 tokens/s, RunINFO 01-04 14:29:07 [loggers.py:111] Engine 000: Avg prompt throughput: 1099.8 tokens/INFO 01-04 14:29:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, RunninINFO 01-04 14:29:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:29:27 [loggers.py:111] Engine 000: Avg prompt throughput: 714.1 tokens/s, Avg generation throughput: 53.9 tokens/s, RunINFO 01-04 14:29:27 [loggers.py:111] Engine 000: Avg prompt throughput: 972.9 tokens/sINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:29:37 [loggers.py:111] Engine 000: Avg prompt throughINFO 01-04 14:29:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.4 tokens/s, Running: 1 reqs, WaitiINFO 01-04 14:29:47 [loggers.py:111] Engine 000: Avg prompt throughput: 1500.7 tokens/s, Avg generation throughput: 32.1 tokens/s, RuINFO 01-04 14:29:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, AINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:29:57 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04 14:29:57 [loggers.py:111] Engine 000: Avg prompt throughput: 452.7 tokens/s, Avg generation throughput: 15.5 tokens/s, Running: 1 reqs, WaitiINFO 01-04 14:30:07 [loggers.py:111] Engine 000: Avg prompt throughINFO 01-04 14:30:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, WaitingINFO 01-04 14:30:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cache hit rate: 16.6%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:30:27 [loggers.py:111] Engine 000: Avg prompt througINFO 01-04 14:30:27 [loggers.py:111] Engine 000: Avg prompt throughput: 516.8 tokens/s, Avg generation throughput: 13.5 tokens/s, Running: 1 reqs, WaiINFO 01-04 14:30:37 [loggers.py:111] Engine 000: Avg prompt throughpuINFO 01-04 14:30:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.4 tokens/s, Running: 1 reqs, WaINFO 01-04 14:30:47 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 14:30:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.2 tokens/s, Running: 1 reqs, WaitiINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:30:57 [loggers.py:111] Engine 000: Avg prompt throughput: 1098.1 tokens/s, Avg generation throughput: 51.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.0%, Prefix cache hit rate: 14.INFO 01-04 14:31:07 [loggers.py:111] Engine 000: Avg prompt throughINFO 01-04 14:31:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 49.9 tokens/s, Running: 1 reqs, WaitingINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:31:17 [loggers.py:111] Engine 000: Avg prompt throughINFO 01-04 14:31:17 [loggers.py:111] Engine 000: Avg prompt throughput: 903.7 tokens/s, Avg generation throughput: 5.7 tokens/s, Running: 1 reqs, WaitINFO 01-04 14:31:27 [loggers.py:111] Engine 000: Avg prompt throughpINFO 01-04 14:31:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.1 tokens/s, Running: 1 reqs, WaiINFO 01-04 14:31:37 [loggers.py:111] Engine 000: Avg prompt throughpuINFO 01-04 14:31:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generaINFO 01-04 14:31:45 [loggers.py:111] Engine 000: Avg pINFO 01-04 14:31:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 48.4 tokens/s, RunniINFO 01-04 14:31:47 [loggers.py:INFO 01-04 14:31:55 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.INFO 01-04 14:31:57 [loggers.py:111] Engine 000: Avg prompt throughpINFO 01-04 14:31:57 [loggers.py:1INFO 01-04 14:32:05 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.INFO 01-04 14:32:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation INFO 01-04 14:32:15 [loggers.py:111] Engine 000: Avg prompt throughput: 819.8 tokens/s, Avg generation throughput: INFO 01-04 14:32:27 [loggers.py:111] Engine 000: Avg prompt throughput: 471.4 tokens/s, Avg generation throughput: 11.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 17.0%
+INFO 01-04 14:32:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 48.0 tokens/s, RunningINFO 01-04 14:32:57 [loggers.py:111] Engine 000: Avg prompt throughput: 832.5 tokens/s, Avg generation throughput: 43.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 13.8%
+INFO 01-04 14:33:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 13.8%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:33:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 39.3 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 13.8%
+INFO 01-04 14:33:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 13.8%
+INFO 01-04 14:33:47 [loggers.py:111] Engine 000: Avg prompt throughput: 849.0 tokens/s, Avg generation throughput: 6.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 14.9%
+INFO 01-04 14:33:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cache hit rate: 14.9%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:34:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokINFO 01-04 14:34:27 [loggers.py:111] Engine 000: Avg prompt throughput: 579.9 tokens/s, Avg generation throughput: 22.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 17.3%
+INFO 01-04 14:34:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 17.3%
+INFO:     10.46.INFO 01-04 14:34:45 [loggers.py:111] Engine 000: AINFO 01-04 14:34:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 33.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV INFO 01-04 14:34:55 [loggers.py:111] Engine 000:INFO 01-04 14:34:57 [loggers.py:111] Engine 000: Avg prompt throughput: 1370.5 tokens/s, Avg generation throughput: 39.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GINFO 01-04 14:34:59 [loggers.py:111] EngineINFO 01-04 14:35:16 [loggers.py:111] Engine 000: Avg prompt throughput: 518.8 tokens/s, Avg generation throughput: 2.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 18.5%
+INFO 01-04 14:35:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 18.5%
+INFO:     10.46.50.192:0INFO 01-04 14:35:45 [loggers.py:111] Engine 000: INFO 01-04 14:36:47 [loggers.py:111] Engine 000: Avg prompt throughput: 1461.8 tokens/s, Avg generation throughput: 9.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.8%, Prefix cache hit rate: 19.9%
+INFO 01-04 14:36:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.0%, Prefix cache hit rate: 19.9%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:37:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 19.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.9%
+INFO 01-04 14:37:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPUINFO 01-04 14:37:29 [loggers.py:111] Engine 000: AvgINFO 01-04 14:38:37 [loggers.py:111] Engine 000: Avg prompt throughput: 1114.2 tokens/s, Avg generation throughput: 3.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 20.5%
+INFO 01-04 14:38:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.6%, Prefix cache hit rate: 20.5%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:38:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 2.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 20.5%
+INFO 01-04 14:39:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 20.5%
+INFO 01-04 14:40:17 [loggers.py:111] Engine 000: Avg prompt throughput: 603.6 tokens/s, Avg generation throughput: 35.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 20.3%
+INFO 01-04 14:40:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 20.3%
+INFO 01-04 14:40:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 20.3%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:40:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 30.3 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 20.3%
+INFO 01-04 14:40:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 20INFOINFO 01-04 14:41:57 [loggers.py:111] Engine 000: Avg prompt throughput: 634.9 tokens/s, Avg generation throughput: 44.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 21.0%
+INFO 01-04 14:42:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 21.0%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:42:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 4.1 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 21.0%
+INFO 01-04 14:42:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.INFO 01-04 14:42:37 [loggers.py:111] Engine 000: Avg prompt throughput: 1122.6 tokens/s, Avg generation throughput: 45.8 tokens/s, Running: 1 INFO 01-04 14:43:07 [loggers.py:111] Engine 000: Avg prompt throughput: 726.2 tokens/s, Avg generation throughput: 20.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 20INFO 01-04 14:43:46 [loggers.py:111] Engine 000: Avg prompt throughput: 639.0 tokens/s, Avg generation throughput: 34.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 18.5%
+INFO 01-04 1INFO 01-04 14:44:05 [loggers.py:111] Engine 000: Avg prompt throughput: 1187.5 tokens/s, Avg generation throughput: 40.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.8%, Prefix cache hINFO 01-04 14:4INFO 01-04 14:44:15 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.1 tokens/s, RINFO 01-04 14:44:37 [loggers.py:111] Engine 000: Avg prompt throughput: 700.7 tokens/s, Avg genINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:45:19 [loggers.py:111] Engine 000: Avg prompt throughput: 1233.9 tokens/s, Avg generation throughput: 37.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, PrefINFO 01-04 14:48:27 [loggers.py:111] Engine 000: Avg prompt throughput: 481.7 tokens/s, Avg generation throughput: 39.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 20.6%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:48:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 43.3 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 20.6%
+INFO 01-04 14:48:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 20.6%
+INFO 01-04 15:04:17 [loggers.py:111] Engine 000: Avg prompt throughput: 497.6 tokens/s, Avg generation throughput: 19.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.0%, Prefix cache hit rate: 20.5%
+INFO 01-04 15:04:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 48.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 20.5%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 15:04:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 42.1 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 20.5%
+INFO 01-04 15:04:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 20.5%
+000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 10.6%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 15:01:39 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 33.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 10.6%
+INFO 01-04 15:01:49 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 10.6%
+ration throughput: 29.4 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 15.1%
+INFO 01-04 15:01:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 15.1%
+ning: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 15.5%
+INFO 01-04 15:01:25 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 15.5%
+: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 14.7%
+INFO 01-04 14:51:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 14.7%
+INFO 01-04 14:51:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 14.7%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:51:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 2.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.7%
+IINFO 01-04 14:58:18 [loggers.py:111] Engine 000: Avg prompt throughput: 567.3 tokens/s, Avg generation throughput: 26.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 16.4%
+INFO 01-04 14:58:28 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 16.4%
+INFO 01-04 14:58:38 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 16.4%
+INFO 01-04 14:58:48 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 16.4%
+INFO 01-04 14:58:58 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 16.4%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:59:08 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 6.4 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.4%
+INFO 01-04 14:59:18 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.4%

hf_ip/vllm_gpu1.log

@@ -0,0 +1,387 @@
+/usr/local/lib/python3.10/dist-packages/torch/utils/_pytree.py:185: FutureWarning: optree is installed but the version is too old to support PyTorch Dynamo in C++ pytree. C++ pytree support is disabled. Please consider upgrading optree using `python3 -m pip install --upgrade 'optree>=0.13.0'`.
+  warnings.warn(
+INFO 01-04 13:13:27 [__init__.py:239] Automatically detected platform cuda.
+INFO 01-04 13:13:29 [api_server.py:1043] vLLM API server version 0.8.5
+INFO 01-04 13:13:29 [api_server.py:1044] args: Namespace(host='127.0.0.1', port=8002, uvicorn_log_level='info', disable_uvicorn_access_log=False, allow_credentials=False, allowed_origins=['*'], allowed_methods=['*'], allowed_headers=['*'], api_key=None, lora_modules=None, prompt_adapters=None, chat_template=None, chat_template_content_format='auto', response_role='assistant', ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, enable_ssl_refresh=False, ssl_cert_reqs=0, root_path=None, middleware=[], return_tokens_as_token_ids=False, disable_frontend_multiprocessing=False, enable_request_id_headers=False, enable_auto_tool_choice=False, tool_call_parser=None, tool_parser_plugin='', model='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', task='auto', tokenizer=None, hf_config_path=None, skip_tokenizer_init=False, revision=None, code_revision=None, tokenizer_revision=None, tokenizer_mode='auto', trust_remote_code=True, allowed_local_media_path=None, load_format='auto', download_dir=None, model_loader_extra_config={}, use_tqdm_on_load=True, config_format=<ConfigFormat.AUTO: 'auto'>, dtype='auto', max_model_len=131072, guided_decoding_backend='auto', reasoning_parser=None, logits_processor_pattern=None, model_impl='auto', distributed_executor_backend=None, pipeline_parallel_size=1, tensor_parallel_size=1, data_parallel_size=1, enable_expert_parallel=False, max_parallel_loading_workers=None, ray_workers_use_nsight=False, disable_custom_all_reduce=False, block_size=None, gpu_memory_utilization=0.9, swap_space=4, kv_cache_dtype='auto', num_gpu_blocks_override=None, enable_prefix_caching=None, prefix_caching_hash_algo='builtin', cpu_offload_gb=0, calculate_kv_scales=False, disable_sliding_window=False, use_v2_block_manager=True, seed=None, max_logprobs=20, disable_log_stats=False, quantization=None, rope_scaling=None, rope_theta=None, hf_token=None, hf_overrides=None, enforce_eager=False, max_seq_len_to_capture=8192, tokenizer_pool_size=0, tokenizer_pool_type='ray', tokenizer_pool_extra_config={}, limit_mm_per_prompt={}, mm_processor_kwargs=None, disable_mm_preprocessor_cache=False, enable_lora=None, enable_lora_bias=False, max_loras=1, max_lora_rank=16, lora_extra_vocab_size=256, lora_dtype='auto', long_lora_scaling_factors=None, max_cpu_loras=None, fully_sharded_loras=False, enable_prompt_adapter=None, max_prompt_adapters=1, max_prompt_adapter_token=0, device='auto', speculative_config=None, ignore_patterns=[], served_model_name=['default-model'], qlora_adapter_name_or_path=None, show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, disable_async_output_proc=False, max_num_batched_tokens=None, max_num_seqs=256, max_num_partial_prefills=1, max_long_partial_prefills=1, long_prefill_token_threshold=0, num_lookahead_slots=0, scheduler_delay_factor=0.0, preemption_mode=None, num_scheduler_steps=1, multi_step_stream_outputs=True, scheduling_policy='fcfs', enable_chunked_prefill=None, disable_chunked_mm_input=False, scheduler_cls='vllm.core.scheduler.Scheduler', override_neuron_config=None, override_pooler_config=None, compilation_config=None, kv_transfer_config=None, worker_cls='auto', worker_extension_cls='', generation_config='auto', override_generation_config=None, enable_sleep_mode=False, additional_config=None, enable_reasoning=False, disable_cascade_attn=False, disable_log_requests=True, max_log_len=None, disable_fastapi_docs=False, enable_prompt_tokens_details=False, enable_server_load_tracking=False)
+INFO 01-04 13:13:36 [config.py:717] This model supports multiple tasks: {'score', 'classify', 'reward', 'embed', 'generate'}. Defaulting to 'generate'.
+INFO 01-04 13:13:37 [config.py:2003] Chunked prefill is enabled with max_num_batched_tokens=8192.
+/usr/local/lib/python3.10/dist-packages/torch/utils/_pytree.py:185: FutureWarning: optree is installed but the version is too old to support PyTorch Dynamo in C++ pytree. C++ pytree support is disabled. Please consider upgrading optree using `python3 -m pip install --upgrade 'optree>=0.13.0'`.
+  warnings.warn(
+INFO 01-04 13:13:41 [__init__.py:239] Automatically detected platform cuda.
+INFO 01-04 13:13:44 [core.py:58] Initializing a V1 LLM engine (v0.8.5) with config: model='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', speculative_config=None, tokenizer='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=131072, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto,  device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='auto', reasoning_backend=None), observability_config=ObservabilityConfig(show_hidden_metrics=False, otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=None, served_model_name=default-model, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=True, chunked_prefill_enabled=True, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs=None, pooler_config=None, compilation_config={"level":3,"custom_ops":["none"],"splitting_ops":["vllm.unified_attention","vllm.unified_attention_with_output"],"use_inductor":true,"compile_sizes":[],"use_cudagraph":true,"cudagraph_num_of_warmups":1,"cudagraph_capture_sizes":[512,504,496,488,480,472,464,456,448,440,432,424,416,408,400,392,384,376,368,360,352,344,336,328,320,312,304,296,288,280,272,264,256,248,240,232,224,216,208,200,192,184,176,168,160,152,144,136,128,120,112,104,96,88,80,72,64,56,48,40,32,24,16,8,4,2,1],"max_capture_size":512}
+WARNING 01-04 13:13:44 [utils.py:2522] Methods determine_num_available_blocks,device_config,get_cache_block_size_bytes,initialize_cache not implemented in <vllm.v1.worker.gpu_worker.Worker object at 0x7fdcbc82a650>
+INFO 01-04 13:13:46 [parallel_state.py:1004] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, TP rank 0
+INFO 01-04 13:13:46 [cuda.py:221] Using Flash Attention backend on V1 engine.
+WARNING 01-04 13:13:46 [topk_topp_sampler.py:69] FlashInfer is not available. Falling back to the PyTorch-native implementation of top-p & top-k sampling. For the best performance, please install FlashInfer.
+INFO 01-04 13:13:46 [gpu_model_runner.py:1329] Starting to load model /data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf...
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+INFO 01-04 13:15:40 [loader.py:458] Loading weights took 114.09 seconds
+INFO 01-04 13:15:41 [gpu_model_runner.py:1347] Model loading took 61.0562 GiB and 114.594255 seconds
+INFO 01-04 13:15:55 [backends.py:420] Using cache directory: /root/.cache/vllm/torch_compile_cache/75e72335d9/rank_0_0 for vLLM's torch.compile
+INFO 01-04 13:15:55 [backends.py:430] Dynamo bytecode transform time: 14.52 s
+INFO 01-04 13:15:59 [backends.py:136] Cache the graph of shape None for later use
+INFO 01-04 13:16:50 [backends.py:148] Compiling a graph for general shape takes 53.09 s
+INFO 01-04 13:17:53 [monitor.py:33] torch.compile takes 67.61 s in total
+INFO 01-04 13:17:54 [kv_cache_utils.py:634] GPU KV cache size: 254,768 tokens
+INFO 01-04 13:17:54 [kv_cache_utils.py:637] Maximum concurrency for 131,072 tokens per request: 1.94x
+INFO 01-04 13:18:34 [gpu_model_runner.py:1686] Graph capturing finished in 40 secs, took 1.21 GiB
+INFO 01-04 13:18:34 [core.py:159] init engine (profile, create kv cache, warmup model) took 172.87 seconds
+INFO 01-04 13:18:34 [core_client.py:439] Core engine process 0 ready.
+WARNING 01-04 13:18:34 [config.py:1239] Default sampling parameters have been overridden by the model's Hugging Face generation config recommended from the model creator. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
+INFO 01-04 13:18:34 [serving_chat.py:118] Using default chat sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
+INFO 01-04 13:18:34 [serving_completion.py:61] Using default completion sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
+INFO 01-04 13:18:34 [api_server.py:1090] Starting vLLM API server on http://127.0.0.1:8002
+INFO 01-04 13:18:34 [launcher.py:28] Available routes are:
+INFO 01-04 13:18:34 [launcher.py:36] Route: /openapi.json, Methods: HEAD, GET
+INFO 01-04 13:18:34 [launcher.py:36] Route: /docs, Methods: HEAD, GET
+INFO 01-04 13:18:34 [launcher.py:36] Route: /docs/oauth2-redirect, Methods: HEAD, GET
+INFO 01-04 13:18:34 [launcher.py:36] Route: /redoc, Methods: HEAD, GET
+INFO 01-04 13:18:34 [launcher.py:36] Route: /health, Methods: GET
+INFO 01-04 13:18:34 [launcher.py:36] Route: /load, Methods: GET
+INFO 01-04 13:18:34 [launcher.py:36] Route: /ping, Methods: POST, GET
+INFO 01-04 13:18:34 [launcher.py:36] Route: /tokenize, Methods: POST
+INFO 01-04 13:18:34 [launcher.py:36] Route: /detokenize, Methods: POST
+INFO 01-04 13:18:34 [launcher.py:36] Route: /v1/models, Methods: GET
+INFO 01-04 13:18:34 [launcher.py:36] Route: /version, Methods: GET
+INFO 01-04 13:18:34 [launcher.py:36] Route: /v1/chat/completions, Methods: POST
+INFO 01-04 13:18:34 [launcher.py:36] Route: /v1/completions, Methods: POST
+INFO 01-04 13:18:34 [launcher.py:36] Route: /v1/embeddings, Methods: POST
+INFO 01-04 13:18:34 [launcher.py:36] Route: /pooling, Methods: POST
+INFO 01-04 13:18:34 [launcher.py:36] Route: /score, Methods: POST
+INFO 01-04 13:18:34 [launcher.py:36] Route: /v1/score, Methods: POST
+INFO 01-04 13:18:34 [launcher.py:36] Route: /v1/audio/transcriptions, Methods: POST
+INFO 01-04 13:18:34 [launcher.py:36] Route: /rerank, Methods: POST
+INFO 01-04 13:18:34 [launcher.py:36] Route: /v1/rerank, Methods: POST
+INFO 01-04 13:18:34 [launcher.py:36] Route: /v2/rerank, Methods: POST
+INFO 01-04 13:18:34 [launcher.py:36] Route: /invocations, Methods: POST
+INFO 01-04 13:18:34 [launcher.py:36] Route: /metrics, Methods: GET
+INFO:     Started server process [1315813]
+INFO:     Waiting for application startup.
+INFO:     Application startup compINFO 0INFO 01-04 13:59:57 [loggers.py:111] Engine 000: Avg promINFO 01-04 14:00:00 [loggers.py:111] Engine 000: Avg prompt throughput: 44.1 tokens/s, Avg generation throughput: 37.1 tokens/s, Running: 1 reqs, Waiting: INFO 0INFO 01-04 14:00:07 [loggers.py:111] Engine 000: Avg proINFO 01-04 14:00:10 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1 reqs, Waiting: INFO 01-04 14:00:15 [loggers.py:111] Engine 000: Avg prompt thINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:00:20 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg INFO 01-04 14:00:25 [loggers.py:111] Engine 000: Avg prompt throughput: INFO 01-04 14:00:27 [loggers.py:111] Engine 000: Avg pINFO 01-04 14:00:30 [loggers.py:111] Engine 000: Avg prompt throughput: 48.6 tokens/s, Avg gINFO:     10.46.17.192:0 - "GET /v1/models HTTP/1.1" 200 OK
+INFO:     1INFO 01-04 14:00:37 [loggers.py:111] Engine 000: Avg pINFO 01-04 14:00:40 [loggers.py:111] Engine 000: Avg prompt throughput: 64.6 tokens/s, Avg generation throughput: 71.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.8%, Prefix cache hit rate: 1.0%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:00:50 [loggers.py:111] Engine 000: Avg prompt throughput: 60.4 tokens/s, Avg generation throughput: 97.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.9%, Prefix cache hit rate: 0.7%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:01:00 [loggers.py:111] Engine 000: Avg prompt throughput: 82.0 tokens/s, Avg generation throughput: 97.5 tokens/s, Running: 1 reqs, WaitINFO:   INFO 01-04 14:01:07 [loggers.py:111] Engine 000: Avg promptINFO 01-04 14:01:10 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.6%, Prefix cache hit rate: 1.1%IINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:01:20 [loggers.py:111] Engine 000: Avg prompt throughput: 85.1 tokens/s, Avg generation throughput: 75.3 tokens/s, Running: 1 reqs, WaitiINFINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 INFO 01-INFO 01-04 14:01:27 [loggers.py:111] Engine 000: Avg proINFO 01-04 14:01:30 [loggers.py:111] Engine 000: Avg prompt throughput: 96.6 tokens/s, INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:01:40 [loggers.py:111] Engine 000: Avg prompt throughput: 66.1 tokens/s, Avg generation throughput: 84.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.1%, Prefix cache hit rate: 0.6%
+ININFO:   INFO 01-04 14:01:47 [loggers.py:111] Engine 000: Avg pINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:01:50 [loggers.py:111] Engine 000: Avg prompt throughput: 96.2 tokens/s, Avg generINFO 01-04 14:01:57 [loggers.py:111] Engine 000: Avg prompt througINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.INFO 01-04 14:02:00 [loggers.pyINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" INFO 01INFO 01-04 14:02:07 [loggers.py:111] Engine 000: Avg prompt thrINFO 01-04 14:02:07 [loggers.py:111] Engine 000: Avg prompt INFO 01-04 14:02:10 [loggers.py:111] Engine 000: Avg prompt throughput: 96.3 tokens/s, Avg genINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:02:17 [loggers.py:111] Engine 000: Avg prompt thrINFO 01-04 14:02:17 [loggers.py:111] Engine 000: Avg prompt thrINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:02:20 [loINFINFO 01-04 14:02:27 [loggers.py:111] Engine 000: Avg prompt throughput: 96.0 tokens/s, Avg generation throughput: 86.9 tokens/sINFO 01-04 14:02:27 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04 14:02:30 [loggeINFO 01-04 14:02:37 [loggers.py:111] Engine 000: Avg prompt throughput: 113.8 tokens/s, Avg generation throughput: 138.8 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, PINFO 01-04 14:02:40 [loggerINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:02:47 [loggers.py:111] Engine 000: Avg prompt throughput: 133.4 tokens/s, Avg generation throughput: 181.1 tokeINFO 01-04 14:02:47 [loggers.py:111] Engine 000: Avg prompt throughput: 62.3 tokens/s, Avg genINFO:     10.46.50.192:0 - "POST /v1/coINFO 01-04 14:02:50 INFO:  INFO 01-04 14:02:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 107.0 tokeINFO 01-04 14:02:57 [loggers.py:111] Engine 000: Avg prompt througINFO 01-04 14:03:00 [loggers.py:111] Engine 000: Avg prompt throughput: 138.3 tokens/s, INFO 0INFO 01-04 14:03:07 [loggers.py:111] Engine 000: Avg promptINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OINFO 01-04 14:03:10 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.8 tokens/s, Running: 1 reqs, Waiting:INFO 01-04 14:03:15 [loggers.py:111] Engine 000: Avg prompt throughpINFO 01-04 14:03:17 [loggers.py:111] Engine 000: Avg prompt tINFO 01-04 14:03:20 [loggers.py:111] Engine 000: Avg prompt throughput: 137.2 tokens/s, Avg generation throughput: 59.5 tokens/s, Running: 1 reqs, Waiting: INFO 01-04 14:03:25 [loggers.py:111] Engine 000: Avg prompt thINFO 01-04 14:03:30 [loggers.py:111] Engine 000: Avg prompt throughput: 154.1 tokens/s, Avg generation throughput: 88.1 tokens/s, Running: 2 reqs, Waiting: INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200INFO:     INFO:     10.45.190.192:0 - "POST /v1/completions HTTPINFO 01-04 14:03:40 [loggers.py:111] Engine 000: Avg prompt throughput: 260.7 tokens/s, Avg generation throughput: 95.0 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 5.4%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:50 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 129.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 5.4%
+ININFO:     INFO 01-04 14:03:57 [loggers.py:111] Engine 000: AvgINFO 01-04 14:04:00 [loggers.py:111] Engine 000: Avg prompt throughput: 155.5 tokens/s, Avg generation throughput: 70.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/INFO:     10INFO 01-04 14:04:07 [loggers.py:111] Engine 000: AvgINFO 01-04 14:04:10 [loggers.py:111] Engine 000: Avg prompt throughput: 111.4 tokens/s, Avg generation throughput: 88.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/INFO:    INFINFO 01-04 14:04:17 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04 14:04:17 [loggers.py:111] Engine 000: Avg pINFO 01-04 14:04:20 [loggers.py:111] Engine 000: Avg prompt throughput: 159.6 tokens/s, Avg generatiINFO 01-04 14:04:27 [loggers.py:111] Engine 000: Avg prompt throughput: 154.5 tokens/s, Avg generation throughput: 78.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: INFO 01-04 14:04:30 [loggers.py:11INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:04:37 [loggers.py:111] Engine 000: Avg prompt throughput: 169.7 tokens/s, Avg generation throughput: 116.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: INFO 01-04 14:04:40 [loggers.py:11INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/INFO 01-04 1INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:04:47 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04 14:04:47 [loggers.py:111] Engine 000: Avg promINFO 01-04 14:04:50 [loggers.py:111] Engine 000: Avg prompt throughput: 107.0 tokens/s,INFO 01-04INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OINFO 01-04 14:04:57 [loggers.py:111] Engine 000: Avg proINFO 01-04 14:05:00 [loggers.py:111] Engine 000: Avg prompt throughput: 314.6 tokens/s,INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 INFO 01-04 14:05:07 [loggers.py:111] Engine 000: Avg prompt throughput: 336.8 tokens/s, Avg generation throughput: 42.1 toINFO 01-04 14:05:10 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generaINFO 01-04 14:05:17 [loggers.py:111] Engine 000: Avg prompt throughput: 164.8 tokens/s, Avg generation throughput: 98.4 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, PrefixINFO 01-04 14:05:17 [lINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1INFO 01-04 INFO 01-04 14:05:27 [loggers.py:111] Engine 000: Avg prompt throughput: 227.0 tokens/s, Avg generation throughput: 98.1 tINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:30 [INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1INFO 01-0INFO 01-04 14:05:37 [loggers.py:111] Engine 000: AvINFO 01-04 14:05:40 [loggers.py:111] Engine 000: Avg prompt throughput: 322.2 tokens/s, Avg generation throughput: 48.5 tokens/s, Running: 2 reqs, Waiting: INFO 01-0INFO 01-04 14:05:47 [loggers.py:111] Engine 000: Avg INFO 01-04 14:05:50 [loggers.py:111] Engine 000: Avg prompt throughput: 193.6 tokens/s, Avg generation throughput: 101.2 tokens/s, Running: 3 reqs, Waiting: 0 reqs, INFO 01-04 14:05:57 [loggers.py:111] Engine 000: Avg prompt throughput: 176.7 tokens/s, Avg generation throughput: 80.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1INFO:     10.43.30.3:0 - "POST /vINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:07 [loggers.py:111] Engine 000: Avg prompt throughput: 274.3 tokens/s, Avg generation throughput: 132.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usagINFO 01-04 14:06:10 [loggers.py:111] INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:17 [loggers.py:111] Engine 000: Avg INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:20 [loggers.py:111] Engine 000: Avg prompt throughput: 317.3 tokens/s, Avg generatINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:27 [loggers.py:111] Engine 000: Avg prompt throughput: 275.0 tokens/s, Avg generation throughput: 74.9 INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:30 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg gINFO INFO 01-04 14:06:37 [loggers.py:111] Engine 000: Avg prompt throughput: 305.9 tokens/s, Avg generation throughput: 49.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usaINFO INFO 01-04 14:06:40 [loggers.py:1INFO 01-04 14:06:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 95.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1INFO:     10.43.30.5:0 - "POST /vINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 20INFO INFO 01-04 14:06:57 [loggers.py:111] Engine 000: Avg prompt throughput: 461.4 tokens/s, Avg generation throughput: 113.0 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usagINFO 01-04 14:07:00 [loggers.py:111] Engine 000: Avg prompt throughput: 143.9 tokens/s, Avg generatINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 0INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:07 [loggers.py:111] Engine 000: AvINFO 01-04 14:07:10 [loggers.py:111] Engine 000: Avg prompt throughput: 247.3 tokens/s, Avg generation throughput: 63.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GININFO 01-04 14:07:17 [loggers.py:111] Engine 000: AvINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:20 [loggers.py:111] Engine 000: Avg prompt throughput: 267.1 tokens/s, Avg generatINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:27 [loggers.py:111] Engine 000: Avg proINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:30 [loggers.py:111] Engine 000: Avg prompt throughput: 195.5 tokens/s, Avg genINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:37 [loggers.py:111] Engine 000: Avg promptINFO 01-04 14:07:40 [loggers.py:111] Engine 000: Avg prompt throughput: 253.5 tokens/s, Avg generation throughput: 110.5 tokens/s, Running: 3 reqs, Waiting: 0 INFOINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:50 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 116.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 1.9%
+INFO:    INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:00 [loggers.py:111] Engine 000: Avg prompt throughput: 624.6 tokens/s, Avg generation throughput: 109.3 tokens/s, Running: 2 reqs, Waiting: 0 reqINFO 01-04 14:08:07 [loggers.py:111] Engine 000: Avg promINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:10 [loggers.py:111] Engine 000: Avg prompt throughput: 317.2 tokens/s, Avg genINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:17 [loggers.py:111] Engine 000: Avg prompINFO 01-04 14:08:20 [loggers.py:111] Engine 000: Avg prompt throughput: 283.7 tokens/s, Avg generation throughput: 97.8 tokens/s, Running: 2 reqs, Waiting: 0 reINFO 01-04 14:08:27 [loggers.py:111] Engine 000: Avg prompINFO 01-04 14:08:30 [loggers.py:111] Engine 000: Avg prompt throughput: 362.9 tokens/s, Avg generation throughput: 97.8 tokens/s, Running: 3 reqs, Waiting: 0 reqs,INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1INFO: INFO 01-04 14:08:37 [loggers.py:111] Engine 000: Avg prompt throughput: 220.4 tokens/s, Avg generation throughput: 32.8 tokens/INFO 01-04 14:08:40 [loggers.py:111] Engine 000: Avg prompt throughput: 271.7 tokens/s, AvgINFINFO 01-04 14:08:44 [loggers.py:111] Engine 000: Avg prompt tINFO 01-04 14:08:47 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04 14:08:50 [loggers.py:111] Engine 000: Avg prompt throughput: 302.4 tokens/s, Avg generation throughput: 97.2 tokens/s, Running: 3 reqs, WaitingINFO 01-04 14:08:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 93.6 tokens/s, RunINFO 01-04 14:09:00 [loggers.py:111] Engine 000: Avg prompt throughput: 295.0 tokens/INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:07 [loggers.py:111] Engine 000: Avg prompt throughput: 445.3 tokens/s, Avg generation throughput: 93.3 tokens/s, RINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:10 [INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:17 [loggers.py:111] Engine 000: Avg prompt throughput: 270.6 tokens/s, Avg generation throughput: 73.4 tokens/s, RINFO 01-04 14:09:20 [loggers.py:111] Engine 000: Avg prompt throughput: 412.2 tokens/s,INFO 01-04 14:09:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 93.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cINFO 01-04 14:09:30 [loggers.py:111] Engine 000: Avg prompt throughput: 278.8 tokens/sINFO 01-04 14:09:37 [loggers.py:111] Engine 000: Avg prompt throughput: 119.1 tokens/s, Avg generation throughput: 77.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix caINFO 01-04 14:09:40INFO 01-04 14:09:47 [loggers.py:111] Engine 000: Avg prompt throughput: 678.1 tokens/s, Avg generation throughput: 133.0 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.2%, Prefix cache hit rate: 10.2%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:57 [loggers.py:111] Engine 000: Avg prompt throughput: 361.0 tokens/s, Avg generation throughput: 108.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 9.8%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:  INFO 01-04 14:10:04 [loggers.py:111] Engine 000: Avg promptINFO 01-04 14:10:07 [loggers.py:111] Engine 000: Avg prompt throughput: 365.5 tokens/s, Avg generation throughput: 52.3 tokens/s, RuINFO:     10.45.190.192:0 - INFO:     10.46.50.192:0 - "POST /v1/coINFO 01-04 14:10:10 INFO 01-04 14:10:17 [loggers.py:111] Engine 000: Avg prompt throughput: 292.4 tokens/s, Avg generation throughput: 60.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cINFO:     10.46.17.19INFO 01-04 14:10:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 93.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cache hit rate: 12.2%
+INFO: INFO 01-04 14:10:34 [loggers.py:111] Engine 000: Avg prompt throughput: 226.3 tokens/s, Avg generation throughput: 114.3 tokINFO 01-04 14:10:37 [loggers.py:111] Engine 000: Avg prompt throughput: 336.0 tokens/s, Avg genINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 30.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cINFO 01-04 14:10:50 [INFO 01-04 14:10:57 [loggers.py:111] Engine 000: Avg prompt throughput: 306.9 tokens/s, Avg generation throughput: 11.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix INFO:     10.46.50.192INFO 01-04 14:11:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.4%, Prefix cache hit rate: 11.5%
+INFO:     10.46.50.192:0 - "POST /v1/compleINFO 01-04 14:11:10 [loINFO 01-04 14:11:17 [loggers.py:111] Engine 000: Avg prompt throughput: 384.9 tokens/s, Avg generation throughput: 44.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: INFO 01-INFO 01-04 14:11:20 [loggerINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OKIINFO 01-04 14:11:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 19.4 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usageINFO 01-04 1INFO 01-04 14:11:30 [lINFO 01-04 14:11:44 [loggers.py:111] Engine 000: Avg prompt throughput: 327.4 tokens/s, Avg generation throughput: 8.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: 8.8%
+INFO 01-04 14:11:54 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, PrefINFO 01-04 14:12:00 [loggers.py:111] Engine 000: Avg prompt throughput: 442.4 tokens/s, AvgINFO 01-04 14:12:04 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 6.4 tokens/s, INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:12:10 [INFO 01-04 14:12:14 [loggers.py:111] Engine 000: Avg prompt throughput: 271.8 tokens/s, Avg generation throughput: 22.1 tokINFO 01-04 14:12:26 [loggers.py:111] Engine 000: Avg prompt throughput: 277.1 tokens/s, Avg generation throughput: 45.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: 11.5%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:12:36 [loggers.py:111] Engine 000: Avg prompt throughput: 273.1 tokens/s, Avg generation throughput: 2.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.1%, Prefix cache hit rate: 11.3%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:12:46 [loggers.py:111] Engine 000: Avg prompt throughput: 180.5 tokens/s, Avg generation throughput: 50.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cacheINFO:     10.45.190.192:0 - "POST /v1/complINFO 01-04 14:12:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 11.1%
+INFO 01-04 14:13:16 [loggers.py:111] Engine 000: Avg prompt throughput: 340.2 tokens/s, Avg generation throughput: 33.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache hit rate: 10.8%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:13:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 42.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 10.8%
+INFO 01-04 14:13:36 [loggers.py:111] Engine 000: Avg prompt throughput: 713.2 tokens/s, Avg generation throughput: 45.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 10.2%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:13:46 [loggers.py:111] Engine 000: Avg prompt throughput: 282.3 tokens/s, Avg generation throughput: 99.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 10.0%
+INFO 01-04 14:13:56 [loggers.py:111] Engine 000: Avg prompt throughput: 382.0 tokens/s, Avg generation throughput: 106.3 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.7%, Prefix cache hit rate: 11.8%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:06 [loggers.py:111] Engine 000: Avg prompt throughput: 532.0 tokens/s, Avg generation throughput: 65.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.7%, Prefix cache hit rate: 11.4%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 51.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 11.4%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:26 [loggers.py:111] Engine 000: Avg prompt throughput: 743.0 tokens/s, Avg generation throughput: 59.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 10.8%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 59.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache hit rate: 10.8%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:46 [loggers.py:111] Engine 000: Avg prompt throughput: 614.5 tokens/s, Avg generation throughput: 49.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 10.4%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:56 [loggers.py:111] Engine 000: Avg prompt throughput: 413.1 tokens/s, Avg generation throughput: 38.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 10.1%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:06 [loggers.py:111] Engine 000: Avg prompt throughput: 872.1 tokens/s, Avg generation throughput: 74.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 9.9%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 50.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.0%, Prefix cache hit rate: 9.9%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:26 [loggers.py:111] Engine 000: Avg prompt throughput: 535.4 tokens/s, Avg generation throughput: 55.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 9.6%
+INFO 01-04 14:15:36 [loggers.py:111] Engine 000: Avg prompt throughput: 417.3 tokens/s, Avg generation throughput: 68.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.1%, Prefix cache hit rate: 9.4%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:46 [loggers.py:111] Engine 000: Avg prompt throughput: 550.4 tokens/s, Avg generation throughput: 79.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.1%, Prefix cache hit rate: 9.1%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:56 [loggers.py:111] Engine 000: Avg prompt throughput: 605.5 tokens/s, Avg generation throughput: 68.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 11.6%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:06 [loggers.py:111] Engine 000: Avg prompt throughput: 303.1 tokens/s, Avg generation throughput: 26.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.2%, Prefix cache hit rate: 11.4%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:16 [loggers.py:111] Engine 000: Avg prompt throughput: 463.2 tokens/s, Avg generation throughput: 53.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 11.2%
+INFO 01-04 14:16:26 [loggers.py:111] Engine 000: Avg prompt throughput: 362.6 tokens/s, Avg generation throughput: 59.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cache hit rate: 11.0%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:36 [loggers.py:111] Engine 000: Avg prompt throughput: 604.3 tokens/s, Avg generation throughput: 64.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.1%, Prefix cache hit rate: 11.8%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 84.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 11.8%
+INFO 01-04 14:16:56 [loggers.py:111] Engine 000: Avg prompt throughput: 576.9 tokens/s, Avg generation throughput: 82.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.2%, Prefix cache hit rate: 11.4%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:06 [loggers.py:111] Engine 000: Avg prompt throughput: 477.4 tokens/s, Avg generation throughput: 81.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 13.1%
+INFO 01-04 14:17:16 [loggers.py:111] Engine 000: Avg prompt throughput: 929.6 tokens/s, Avg generation throughput: 57.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.9%, Prefix cache hit rate: 14.3%
+INFO 01-04 14:17:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 92.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.3%, Prefix cache hit rate: 14.3%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:36 [loggers.py:111] Engine 000: Avg prompt throughput: 633.9 tokens/s, Avg generation throughput: 68.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 16.5%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:46 [loggers.py:111] Engine 000: Avg prompt throughput: 708.9 tokens/s, Avg generation throughput: 40.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 16.0%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:56 [loggers.py:111] Engine 000: Avg prompt throughput: 520.8 tokens/s, Avg generation throughput: 29.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 16.7%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:06 [loggers.py:111] Engine 000: Avg prompt throughput: 496.4 tokens/s, Avg generation throughput: 33.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 16.4%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:16 [loggers.py:111] Engine 000: Avg prompt throughput: 407.0 tokens/s, Avg generation throughput: 68.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.7%, Prefix cache hit rate: 16.6%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:26 [loggers.py:111] Engine 000: Avg prompt throughput: 524.0 tokens/s, Avg generation throughput: 49.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.3%
+INFO 01-04 14:18:36 [loggers.py:111] Engine 000: Avg prompt throughput: 412.6 tokens/s, Avg generation throughput: 16.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.7%, Prefix cache hit rate: 16.5%
+INFO 01-04 14:18:46 [loggers.py:111] Engine 000: Avg prompt throughput: 741.7 tokens/s, Avg generation throughput: 74.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.9%, Prefix cache hit rate: 16.0%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:56 [loggers.py:111] Engine 000: Avg prompt throughput: 602.2 tokens/s, Avg generation throughput: 104.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.5%, Prefix cache hit rate: 17.2%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:06 [loggers.py:111] Engine 000: Avg prompt throughput: 637.4 tokens/s, Avg generation throughput: 51.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 17.6%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:16 [loggers.py:111] Engine 000: Avg prompt throughput: 353.2 tokens/s, Avg generation throughput: 38.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache hit rate: 17.4%
+INFO 01-04 14:19:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 17.4%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:36 [loggers.py:111] Engine 000: Avg prompt throughput: 511.9 tokens/s, Avg generation throughput: 57.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 17.1%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:46 [loggers.py:111] Engine 000: Avg prompt throughput: 1315.0 tokens/s, Avg generation throughput: 56.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.2%, Prefix cache hit rate: 17.8%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 81.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 17.8%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:06 [loggers.py:111] Engine 000: Avg prompt throughput: 721.5 tokens/s, Avg generation throughput: 37.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 17.4%
+INFO 01-04 14:20:16 [loggers.py:111] Engine 000: Avg prompt throughput: 864.7 tokens/s, Avg generation throughput: 75.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.7%, Prefix cache hit rate: 16.9%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:26 [loggers.py:111] Engine 000: Avg prompt throughput: 616.5 tokens/s, Avg generation throughput: 80.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 16.6%
+INFO 01-04 14:20:36 [loggers.py:111] Engine 000: Avg prompt throughput: 670.8 tokens/s, Avg generation throughput: 65.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.5%, Prefix cache hit rate: 16.3%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:46 [loggers.py:111] Engine 000: Avg prompt throughput: 454.9 tokens/s, Avg generation throughput: 95.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 16.9%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:56 [loggers.py:111] Engine 000: Avg prompt throughput: 554.7 tokens/s, Avg generation throughput: 67.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 16.6%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 22.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.6%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:16 [loggers.py:111] Engine 000: Avg prompt throughput: 704.4 tokens/s, Avg generation throughput: 29.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.2%
+INFO 01-04 14:21:26 [loggers.py:111] Engine 000: Avg prompt throughput: 319.7 tokens/s, Avg generation throughput: 30.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.4%, Prefix cache hit rate: 18.5%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:36 [loggers.py:111] Engine 000: Avg prompt throughput: 397.9 tokens/s, Avg generation throughput: 57.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 19.3%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:46 [loggers.py:111] Engine 000: Avg prompt throughput: 692.2 tokens/s, Avg generation throughput: 44.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 18.9%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:56 [loggers.py:111] Engine 000: Avg prompt throughput: 436.7 tokens/s, Avg generation throughput: 53.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 18.8%
+INFO 01-04 14:22:06 [loggers.py:111] Engine 000: Avg prompt throughput: 502.7 tokens/s, Avg generation throughput: 65.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.1%, Prefix cache hit rate: 18.6%
+INFO 01-04 14:22:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 91.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 18.6%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:26 [loggers.py:111] Engine 000: Avg prompt throughput: 768.5 tokens/s, Avg generation throughput: 102.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.5%, Prefix cache hit rate: 18.2%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:36 [loggers.py:111] Engine 000: Avg prompt throughput: 481.2 tokens/s, Avg generation throughput: 56.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.3%, Prefix cache hit rate: 18.0%
+INFO 01-04 14:22:46 [loggers.py:111] Engine 000: Avg prompt throughput: 699.0 tokens/s, Avg generation throughput: 91.6 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.4%, Prefix cache hit rate: 18.9%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 91.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 18.9%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 27.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.9%
+INFO 01-04 14:23:16 [loggers.py:111] Engine 000: Avg prompt throughput: 585.8 tokens/s, Avg generation throughput: 29.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 18.6%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:26 [loggers.py:111] Engine 000: Avg prompt throughput: 399.2 tokens/s, Avg generation throughput: 29.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 18.4%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:36 [loggers.py:111] Engine 000: Avg prompt throughput: 689.4 tokens/s, Avg generation throughput: 44.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 18.1%
+INFO 01-04 14:23:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 18.1%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:56 [loggers.py:111] Engine 000: Avg prompt throughput: 946.6 tokens/s, Avg generation throughput: 70.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.0%, Prefix cache hit rate: 17.7%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 69.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.1%, Prefix cache hit rate: 17.7%
+INFO 01-04 14:24:16 [loggers.py:111] Engine 000: Avg prompt throughput: 344.0 tokens/s, Avg generation throughput: 50.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 17.6%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 62.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 17.6%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:36 [loggers.py:111] Engine 000: Avg prompt throughput: 782.9 tokens/s, Avg generation throughput: 40.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 18.5%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 22.4 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.5%
+INFO 01-04 14:24:56 [loggers.py:111] Engine 000: Avg prompt throughput: 661.2 tokens/s, Avg generation throughput: 35.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 18.2%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 4.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.2%
+INFO 01-04 14:25:16 [loggers.py:111] Engine 000: Avg prompt throughput: 1472.3 tokens/s, Avg generation throughput: 43.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.0%, Prefix cache hit rate: 17.6%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 52.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 17.6%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:36 [loggers.py:111] Engine 000: Avg prompt throughput: 932.1 tokens/s, Avg generation throughput: 54.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.7%, Prefix cache hit rate: 17.3%
+INFO 01-04 14:25:46 [loggers.py:111] Engine 000: Avg prompt throughput: 567.9 tokens/s, Avg generation throughput: 47.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.1%, Prefix cache hit rate: 17.0%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 90.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 17.0%
+INFO 01-04 14:26:06 [loggers.py:111] Engine 000: Avg prompt throughput: 1074.1 tokens/s, Avg generation throughput: 52.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.9%, Prefix cache hit rate: 18.2%
+INFO 01-04 14:26:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 90.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.2%, Prefix cache hit rate: 18.2%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:26:26 [loggers.py:111] Engine 000: Avg prompt throughput: 827.6 tokens/s, Avg generation throughput: 62.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.2%, Prefix cache hit rate: 17.9%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:26:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 32.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.9%
+INFO 01-04 14:26:46 [loggers.py:111] Engine 000: Avg prompt throughput: 864.0 tokens/s, Avg generation throughput: 23.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cache hit rate: 17.6%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:26:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 42.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.6%
+INFO 01-04 14:27:06 [loggers.py:111] Engine 000: Avg prompt throughput: 763.8 tokens/s, Avg generation throughput: 30.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 17.3%
+INFO 01-04 14:27:16 [loggers.py:111] Engine 000: Avg prompt throughput: 522.4 tokens/s, Avg generation throughput: 49.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.4%, Prefix cache hit rate: 17.9%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:27:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 64.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 17.9%
+INFO 01-04 14:27:36 [loggers.py:111] Engine 000: Avg prompt throughput: 788.4 tokens/s, Avg generation throughput: 76.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.7%, Prefix cache hit rate: 17.6%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:27:46 [loggers.py:111] Engine 000: Avg prompt throughput: 528.2 tokens/s, Avg generation throughput: 85.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.5%, Prefix cache hit rate: 17.5%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:27:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 90.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 17.5%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:28:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 4.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.5%
+INFO 01-04 14:28:16 [loggers.py:111] Engine 000: Avg prompt throughput: 840.4 tokens/s, Avg generation throughput: 40.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cache hit rate: 18.6%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:28:26 [loggers.py:111] Engine 000: Avg prompt throughput: 1025.3 tokens/s, Avg generation throughput: 30.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.2%, Prefix cache hit rate: 19.9%
+INFO 01-04 14:28:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.3%, Prefix cache hit rate: 19.9%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:28:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 11.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.9%
+INFO 01-04 14:28:56 [loggers.py:111] Engine 000: Avg prompt throughput: 907.3 tokens/s, Avg generation throughput: 8.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 19.6%
+INFO 01-04 14:29:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 19.6%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:29:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 31.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.6%
+INFO 01-04 14:29:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.6%
+INFO 01-04 14:29:36 [loggers.py:111] Engine 000: Avg prompt throughput: 721.0 tokens/s, Avg generation throughput: 18.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 19.4%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:29:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 45.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.4%
+INFO 01-04 14:29:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.4%
+INFO 01-04 14:30:16 [loggers.py:111] Engine 000: Avg prompt throughput: 608.1 tokens/s, Avg generation throughput: 37.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 19.1%
+INFO 01-04 14:30:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 19.1%
+INFO 01-04 14:30:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 19.1%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:30:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.9 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.1%
+INFO 01-04 14:30:56 [loggers.py:111] Engine 000: Avg prompt throughput: 773.2 tokens/s, Avg generation throughput: 42.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cache hit rate: 18.9%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:31:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 28.4 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.9%
+INFO 01-04 14:31:16 [loggers.py:111] Engine 000: Avg prompt throughput: 1030.2 tokens/s, Avg generation throughput: 14.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.1%, Prefix cache hit rate: 18.6%
+INFO 01-04 14:31:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.3%, Prefix cache hit rate: 18.6%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:31:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 20.1 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.6%
+INFO 01-04 14:31:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.6%
+INFO 01-04 14:31:56 [loggers.py:111] Engine 000: Avg prompt throughput: 591.5 tokens/s, Avg generation throughput: 30.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 18.4%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:32:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 39.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.4%
+INFO 01-04 14:32:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.4%
+INFO 01-04 14:32:46 [loggers.py:111] Engine 000: Avg prompt throughput: 1263.6 tokens/s, Avg generation throughput: 29.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.1%, Prefix cache hit rate: 19.3%
+INFO 01-04 14:32:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.3%, Prefix cache hit rate: 19.3%
+INFO 01-04 14:33:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.4%, Prefix cache hit rate: 19.3%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:33:16 [loggers.py:111] Engine 000: Avg prompt throughput: 838.5 tokens/s, Avg generation throughput: 56.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 19.0%
+INFO 01-04 14:33:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 19.0%
+INFO:     10.43INFO 01-04 14:33:48 [loggers.py:111] Engine 000: Avg prompt throughput: 558.6 tokens/s, Avg generation throughput: 22.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 15.1%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:33:58 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 28.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cacINFO 01-04 14:34:16 [loggers.py:111] Engine 000: Avg prompt throughput: 757.7 tokens/s, Avg generation throughput: 15.4 tokens/s,INFO 01-04 14:34:27 [loggers.py:111] Engine 000: Avg prompt throughput: 697.8 tokens/s, Avg generation throughput: 19.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 16.5%
+INFO 01-04 14:34:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 16.5%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:34:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 30.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.5%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:34:57 [lINFO 01-04 14:35:26 [loggers.py:111] Engine 000: Avg prompt throughput: 870.9 tokens/s, Avg generation throughput: 6.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cache hit rate: 18.5%
+INFO 01-04 14:35:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix INFO 01-04 14:37:07 [loggers.py:111] Engine 000: Avg prompt throughput: 453.7 tokens/s, Avg generation throughput: 7.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 16.2%
+INFO 01-04 14:37:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.0%, Prefix cache hit rate: 16.2%
+INFO 01-04 14:37:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 16.2%
+INFO 01-04 14:37:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 16.2%
+INFO 01-04 14:37:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 16.2%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:37:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 2.9 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.2%
+INFO 01-04 14:38:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.2%
+INFO 01-04 14:38:47 [loggers.py:111] Engine 000: Avg prompt throughput: 632.5 tokens/s, Avg generation throughput: 24.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 16.0%
+INFO 01-04 14:38:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 16.0%
+INFO 01-04 14:39:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 16.0%
+INFO 01-04 14:39:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 16.0%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:39:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 39.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.0INFO 01-04 14:39:44 [loggers.py:111] Engine 000: Avg prompt throughput: 657.7 tokens/s, Avg generation throughput: 28.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 10.8%INFO 01-04 14:40:17 [loggers.py:111] Engine 000: Avg prompt throughput: 901.7 tokens/s, Avg generation throughput: 25.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 15.8%
+INFO 01-04 14:40:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.0 tokens/s, Running: 1 reqs, WaitingINFO 01-04 14:41:26 [loggers.py:111] Engine 000: Avg prompt throughput: 1032.5 tokens/s, Avg generation throughput: 38.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.2%, Prefix cache hit rate: 18.4%
+INFO 01-04 14:41:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 18.4%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:41:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 13.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.4%
+INFO 01-04 14:41:56 [loggers.py:111] Engine 000: Avg prompt tINFO 01-04 14:41:57 [loggers.py:111] Engine 000: Avg prompt througINFO 01-04 14:42:44 [loggers.py:111] Engine 000: Avg prompt throughput: 657.9 tokens/s, AINFO 01-04 14:44:56 [loggers.py:111] Engine 000: Avg prompt throughput: 719.8 tokens/s, Avg generation throughput: 41.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 18.2%
+INFO 01-04 14:45:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cache hit rate: 18.2%
+INFO 01-04 14:45:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, PrefiINFO 01-04 14:46:44 [loggers.py:111] Engine 000: Avg prompt throughput: 1221.5 tokens/s, Avg generation throughput: 36.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.9%, Prefix cache hit rate: 11.1%
+INFO 01-04 14:46:54 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.1%, Prefix cache hit rate: 11.1%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:47:04 INFO 01-04 14:48:56 [loggers.py:111] Engine 000: Avg prompt throughput: 377.9 tokens/s, Avg generation throughput: 35.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 18.1%
+INFO 01-04 14:49:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, PrefixINFO 01-04 14:51:44 [loggers.py:111] Engine 000: Avg prompt throughput: 892.0 tokens/s, Avg generation throughput: 14.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 11.0%
+INFO 01-04 14:51:54 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.7%, Prefix cache hit rate: 11.0%
+INFO 01-04 14:52:04 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.9%, Prefix cache hit rate: 11.0%
+INFO 01-04 14:52:14 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s,INFO 01-04 15:01:06 [loggers.py:111] Engine 000: Avg prompt throughput: 476.6 tokens/s, Avg generation throughput: 9.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 18.5%
+INFO 01-04 15:01:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, RunINFO 01-04 15:02:05 [loggers.py:111] Engine 000: Avg prompt throughput: 481.1 tokens/s, Avg generation throughput: 22.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.0%, Prefix cache hit rate: 17.INFO 01-04 15:09:17 [loggers.py:111] Engine 000: Avg prompt throughput: 542.4 tokens/s, Avg generation throughput: 14.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 15.6%
+INFO 01-04 15:09:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 15.6%
+INFO 01-04 15:09:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 15.6%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 15:09:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 29.1 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 15.6%
+INFO 01-04 15:09:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 15.6%
+5:04:28 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cache hit rate: 18.2%
+INFO 01-04 15:04:38 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 18.2%
+INFO 01-04 15:04:48 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 18.2%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 15:04:58 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 27.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.2%
+INFO 01-04 15:05:08 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.2%

hf_ip/vllm_gpu2.log

@@ -0,0 +1,327 @@
+/usr/local/lib/python3.10/dist-packages/torch/utils/_pytree.py:185: FutureWarning: optree is installed but the version is too old to support PyTorch Dynamo in C++ pytree. C++ pytree support is disabled. Please consider upgrading optree using `python3 -m pip install --upgrade 'optree>=0.13.0'`.
+  warnings.warn(
+INFO 01-04 13:13:30 [__init__.py:239] Automatically detected platform cuda.
+INFO 01-04 13:13:32 [api_server.py:1043] vLLM API server version 0.8.5
+INFO 01-04 13:13:32 [api_server.py:1044] args: Namespace(host='127.0.0.1', port=8003, uvicorn_log_level='info', disable_uvicorn_access_log=False, allow_credentials=False, allowed_origins=['*'], allowed_methods=['*'], allowed_headers=['*'], api_key=None, lora_modules=None, prompt_adapters=None, chat_template=None, chat_template_content_format='auto', response_role='assistant', ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, enable_ssl_refresh=False, ssl_cert_reqs=0, root_path=None, middleware=[], return_tokens_as_token_ids=False, disable_frontend_multiprocessing=False, enable_request_id_headers=False, enable_auto_tool_choice=False, tool_call_parser=None, tool_parser_plugin='', model='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', task='auto', tokenizer=None, hf_config_path=None, skip_tokenizer_init=False, revision=None, code_revision=None, tokenizer_revision=None, tokenizer_mode='auto', trust_remote_code=True, allowed_local_media_path=None, load_format='auto', download_dir=None, model_loader_extra_config={}, use_tqdm_on_load=True, config_format=<ConfigFormat.AUTO: 'auto'>, dtype='auto', max_model_len=131072, guided_decoding_backend='auto', reasoning_parser=None, logits_processor_pattern=None, model_impl='auto', distributed_executor_backend=None, pipeline_parallel_size=1, tensor_parallel_size=1, data_parallel_size=1, enable_expert_parallel=False, max_parallel_loading_workers=None, ray_workers_use_nsight=False, disable_custom_all_reduce=False, block_size=None, gpu_memory_utilization=0.9, swap_space=4, kv_cache_dtype='auto', num_gpu_blocks_override=None, enable_prefix_caching=None, prefix_caching_hash_algo='builtin', cpu_offload_gb=0, calculate_kv_scales=False, disable_sliding_window=False, use_v2_block_manager=True, seed=None, max_logprobs=20, disable_log_stats=False, quantization=None, rope_scaling=None, rope_theta=None, hf_token=None, hf_overrides=None, enforce_eager=False, max_seq_len_to_capture=8192, tokenizer_pool_size=0, tokenizer_pool_type='ray', tokenizer_pool_extra_config={}, limit_mm_per_prompt={}, mm_processor_kwargs=None, disable_mm_preprocessor_cache=False, enable_lora=None, enable_lora_bias=False, max_loras=1, max_lora_rank=16, lora_extra_vocab_size=256, lora_dtype='auto', long_lora_scaling_factors=None, max_cpu_loras=None, fully_sharded_loras=False, enable_prompt_adapter=None, max_prompt_adapters=1, max_prompt_adapter_token=0, device='auto', speculative_config=None, ignore_patterns=[], served_model_name=['default-model'], qlora_adapter_name_or_path=None, show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, disable_async_output_proc=False, max_num_batched_tokens=None, max_num_seqs=256, max_num_partial_prefills=1, max_long_partial_prefills=1, long_prefill_token_threshold=0, num_lookahead_slots=0, scheduler_delay_factor=0.0, preemption_mode=None, num_scheduler_steps=1, multi_step_stream_outputs=True, scheduling_policy='fcfs', enable_chunked_prefill=None, disable_chunked_mm_input=False, scheduler_cls='vllm.core.scheduler.Scheduler', override_neuron_config=None, override_pooler_config=None, compilation_config=None, kv_transfer_config=None, worker_cls='auto', worker_extension_cls='', generation_config='auto', override_generation_config=None, enable_sleep_mode=False, additional_config=None, enable_reasoning=False, disable_cascade_attn=False, disable_log_requests=True, max_log_len=None, disable_fastapi_docs=False, enable_prompt_tokens_details=False, enable_server_load_tracking=False)
+INFO 01-04 13:13:39 [config.py:717] This model supports multiple tasks: {'reward', 'score', 'classify', 'generate', 'embed'}. Defaulting to 'generate'.
+INFO 01-04 13:13:40 [config.py:2003] Chunked prefill is enabled with max_num_batched_tokens=8192.
+/usr/local/lib/python3.10/dist-packages/torch/utils/_pytree.py:185: FutureWarning: optree is installed but the version is too old to support PyTorch Dynamo in C++ pytree. C++ pytree support is disabled. Please consider upgrading optree using `python3 -m pip install --upgrade 'optree>=0.13.0'`.
+  warnings.warn(
+INFO 01-04 13:13:44 [__init__.py:239] Automatically detected platform cuda.
+INFO 01-04 13:13:47 [core.py:58] Initializing a V1 LLM engine (v0.8.5) with config: model='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', speculative_config=None, tokenizer='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=131072, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto,  device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='auto', reasoning_backend=None), observability_config=ObservabilityConfig(show_hidden_metrics=False, otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=None, served_model_name=default-model, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=True, chunked_prefill_enabled=True, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs=None, pooler_config=None, compilation_config={"level":3,"custom_ops":["none"],"splitting_ops":["vllm.unified_attention","vllm.unified_attention_with_output"],"use_inductor":true,"compile_sizes":[],"use_cudagraph":true,"cudagraph_num_of_warmups":1,"cudagraph_capture_sizes":[512,504,496,488,480,472,464,456,448,440,432,424,416,408,400,392,384,376,368,360,352,344,336,328,320,312,304,296,288,280,272,264,256,248,240,232,224,216,208,200,192,184,176,168,160,152,144,136,128,120,112,104,96,88,80,72,64,56,48,40,32,24,16,8,4,2,1],"max_capture_size":512}
+WARNING 01-04 13:13:47 [utils.py:2522] Methods determine_num_available_blocks,device_config,get_cache_block_size_bytes,initialize_cache not implemented in <vllm.v1.worker.gpu_worker.Worker object at 0x7fbf15272710>
+INFO 01-04 13:13:49 [parallel_state.py:1004] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, TP rank 0
+INFO 01-04 13:13:49 [cuda.py:221] Using Flash Attention backend on V1 engine.
+WARNING 01-04 13:13:49 [topk_topp_sampler.py:69] FlashInfer is not available. Falling back to the PyTorch-native implementation of top-p & top-k sampling. For the best performance, please install FlashInfer.
+INFO 01-04 13:13:49 [gpu_model_runner.py:1329] Starting to load model /data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf...
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+INFO 01-04 13:15:40 [loader.py:458] Loading weights took 111.25 seconds
+INFO 01-04 13:15:41 [gpu_model_runner.py:1347] Model loading took 61.0562 GiB and 111.510422 seconds
+INFO 01-04 13:15:55 [backends.py:420] Using cache directory: /root/.cache/vllm/torch_compile_cache/75e72335d9/rank_0_0 for vLLM's torch.compile
+INFO 01-04 13:15:55 [backends.py:430] Dynamo bytecode transform time: 14.56 s
+INFO 01-04 13:15:59 [backends.py:136] Cache the graph of shape None for later use
+INFO 01-04 13:16:50 [backends.py:148] Compiling a graph for general shape takes 53.12 s
+INFO 01-04 13:17:53 [monitor.py:33] torch.compile takes 67.68 s in total
+INFO 01-04 13:17:54 [kv_cache_utils.py:634] GPU KV cache size: 254,768 tokens
+INFO 01-04 13:17:54 [kv_cache_utils.py:637] Maximum concurrency for 131,072 tokens per request: 1.94x
+INFO 01-04 13:18:33 [gpu_model_runner.py:1686] Graph capturing finished in 39 secs, took 1.21 GiB
+INFO 01-04 13:18:33 [core.py:159] init engine (profile, create kv cache, warmup model) took 172.10 seconds
+INFO 01-04 13:18:33 [core_client.py:439] Core engine process 0 ready.
+WARNING 01-04 13:18:33 [config.py:1239] Default sampling parameters have been overridden by the model's Hugging Face generation config recommended from the model creator. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
+INFO 01-04 13:18:33 [serving_chat.py:118] Using default chat sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
+INFO 01-04 13:18:33 [serving_completion.py:61] Using default completion sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
+INFO 01-04 13:18:33 [api_server.py:1090] Starting vLLM API server on http://127.0.0.1:8003
+INFO 01-04 13:18:33 [launcher.py:28] Available routes are:
+INFO 01-04 13:18:33 [launcher.py:36] Route: /openapi.json, Methods: GET, HEAD
+INFO 01-04 13:18:33 [launcher.py:36] Route: /docs, Methods: GET, HEAD
+INFO 01-04 13:18:33 [launcher.py:36] Route: /docs/oauth2-redirect, Methods: GET, HEAD
+INFO 01-04 13:18:33 [launcher.py:36] Route: /redoc, Methods: GET, HEAD
+INFO 01-04 13:18:33 [launcher.py:36] Route: /health, Methods: GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /load, Methods: GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /ping, Methods: POST, GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /tokenize, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /detokenize, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/models, Methods: GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /version, Methods: GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/chat/completions, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/completions, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/embeddings, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /pooling, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /score, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/score, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/audio/transcriptions, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /rerank, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/rerank, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v2/rerank, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /invocations, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /metrics, Methods: GET
+INFO:     Started server process [1316160]
+INFO:     Waiting for application startup.
+INFO:     Application startup compINFO 0INFO 01-04 13:59:58 [loggers.py:111] Engine 000: Avg prompt throughput: 116.3 tokens/s, Avg generation throughput: 47.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.7%, Prefix cache hit rateIINFO 01-04 14:00:07 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 14:00:08 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 53.6 tokens/s, Running: 1 reqs, IINFOININFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 2IINFINFO 01-04 14:00:18 [loggers.py:111] Engine 000: Avg prompt throughput: 61.2 tokens/s, Avg generation throughput: 80.6 tokens/s, Running: 2 reqs,INFO INFO 01-04 14:00:23 [loggers.py:111] Engine 000: Avg prompt thINFOINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1"INFO 01INFO INFO 01-04 14:00:28 [loggers.py:111] Engine 000: Avg prompt throughput: 86.4 tokens/INFO:     10.43.30.5:0 - "GET /v1/models HTTP/1.1" 200 OK
+INFO 01-04 14:00:33 [loggers.py:111] Engine 000: Avg prompt throughput:INFO 01-04 14:00:37 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 14:00:38 [loggers.py:111] Engine 000: Avg prompt throughput: 82.1 tokens/s, Avg generation throughput: 81.6 tokens/s, Running: 2 reqINFINFO 01-04 14:00:46 [loggers.py:111] Engine 000: Avg prompt througINFO INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:00:48 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokeINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/INFO 01-04 IINFO 01-04 14:00:56 [loggers.py:111] Engine 000: Avg prompt throughput: 76.3 tokens/s, Avg generation throughput: 86.0 tokens/s, RunnINFO 01-04 14:00:58 [loggers.py:111] Engine 000: Avg prompt throughput: INFO:     INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:01:07 [loggers.py:111] Engine 000: Avg prompt throughput: INFO 01-04 14:01:08 [loggers.py:111] Engine 000: Avg prompt throughput: 61.2 tokens/s, Avg generation throughput: 51.3 tokens/s, Running: INFO:   INFOINFO 01-04 14:01:16 [loggers.py:111] Engine 000: Avg prompt thINFO 01-04 14:01:17 [loggers.py:111] Engine 000: Avg prompt throughput: INFO 01-04 14:01:18 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 60.6 tokens/s, Running:IINFO:    INFO: INFO 01-04 14:01:26 [loggers.py:111] Engine 000: AINFO 01-0INFO 0INFO 01-04 14:01:28 [loggers.py:111] Engine 000: Avg prompt throughput: 62.5 tokens/s, Avg generation throughput: 83.0 tokens/s, Running: 2 reqs, WaitingINFO:     10.46.50.192:0 - "POST /v1/completions INFO 01-04 14:01:36 [loggers.py:111] Engine 000: Avg prompt throughput: INFO 01-04 INFO 01-04 14:01:38 [loggers.py:111] Engine 000: Avg prompt throughput: 96.0 tokens/s, Avg generation throughput: 81.8 tokens/s, RunniINFO:     10.45.19INFO 01-04 14:01:46 [loggers.py:111] Engine 000: INFO 01-04 1INFO INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:01:48 [loggers.py:111] Engine 000: Avg prompt throughput: 78.3 tokens/s,INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" INFO 01-04 14:01:56 [loggers.py:111] Engine 000: Avg prompt INFO 01-04 14:01:57 [loggers.py:111] Engine 000: Avg prompt throughput: 86.6 toINFO 01-04 14:01:58 [loggers.py:111] Engine 000: Avg prompt throughput: 95.2 tokens/s, Avg generation throughput: 96.9 tokens/s, RINFO 01-INFO 01-04 14:02:07 [loggers.py:111] Engine 000: Avg prompt throughput: 73.5 tokINFO 01-04 14:02:08 [loggers.py:111] Engine 000: Avg prompt throughput: 76.5 tokens/s, Avg generation throughput: 111.0 tokens/sINFO 01-04 14:INFO 01-04 14:02:16 [loggers.py:111] Engine 000: Avg prompt throughput: 86.9 tokens/s, Avg generation throughput: 91.5 tokens/sINFO 01-04 14:02INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 20INFO:     10.INFO:     10.46.50.192:0 - "POST /v1/completions HTTPINFO 01-04 14INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1"INFOINFO 01-04 14:02:27 [loggers.py:111] Engine 000: Avg prompt throughput: 85.7 tokens/s, Avg generation throughput: 82.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.2%, Prefix cache hit ratINFO:     10.INFO 01-04 14:02:36 [loggers.py:111] Engine 000: Avg prompt throughput: 66.0 tokens/s, Avg generation throughput: 89.0 tokens/INFO 01-04 14:02:37 [loggers.py:111] Engine 000: Avg prompt throughput: 102.7 tokens/s,INFOINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1INFO 0INFO 01-04 14:02:47 [loggers.py:111] Engine 000: Avg prompt throughput: 145.2 tokens/s, Avg generation throughput: 62.3 tokens/s, Running: 2 reqs, WaitinINFO 01-04 14:02:48 [loggers.py:111] Engine 000: Avg prompININFO:   INFO 01-04 14:02:56 [loggers.py:111] Engine 000: Avg prompt thrINFO 01-04 14:02:57 [loggers.py:111] Engine 000: Avg prompt throughput: 104.7 tokens/s, Avg generation throughput: 98.4 tokens/s, Running: 2 reqs, WaINFOINFO 01-04 14:03:06 [loggers.py:111] Engine 000: Avg prompt throughput: 165.1 tokens/s, Avg generation throughput: 75.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, PrefiINFO 01-04 14:03:07 [loINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:16 [loggers.py:111] Engine 000: Avg prompt throughput: 121.6 tokens/s, Avg generation throughput: 106.5 tokens/sINFO 01-04 14:03:17 [loggers.py:111] Engine 000: Avg prompt throughput: 117.8 tokens/sINFO 01-04 14:03:18 [loggers.py:111] Engine 000: Avg prompINFO:     10INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:26 [loggers.py:111] Engine 000: Avg prompt INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:27 [INFO 01-04 14:03:28 [loggers.py:111] Engine 000: Avg prompt throughpINFO 01-04 14:03:36 [loggers.py:111] Engine 000: Avg prompt throughput: 123.7 tokens/s, Avg generation throughput: 47.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.7%, PreINFO 01-04 14:03:37 [loININFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.INFO 01-04INFO 01-04 14:03:46 [loggers.py:111] Engine 000: Avg promINFO 01-04 14:03:46 [loggers.py:111] Engine 000: Avg prompt throughput: 149.4 tokens/s, Avg generation throughput: 75.5 tokens/s, Running: 2 reqs, WaINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 1INFO 01-04 14:03:56 [loggers.py:111] Engine 000: Avg prompt throughput: 283.3 tokens/s, Avg generation throughput: 44.5 tokens/s, Running: 2 reqs, WaiINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-0INFO 01-04 14:04:03 [loggers.py:111] Engine 000: Avg proINFO 01-04 14:04:06 [loggers.py:111] Engine 000: Avg promINFO 01-04 14:04:06 [loggers.py:111] Engine 000: Avg prompt throughput: INFO 01-04 14:04:07 [loggers.py:111] Engine 000: Avg prompt throughput: 114.7 tINFO 01-INFO 01-04 14:04:16 [loggers.py:111] Engine 000: Avg prompt throughput: 166.0 tokens/s, Avg generation throughput: 77.1 tokens/s, RINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:04INFO:  INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:  INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" INFO 01-04 14:04:26 [loggers.py:111] Engine 000: Avg prompt throughput: 180.8 tokens/s, Avg generation throughput: 71.2 tokens/s,INFO 01-04 14:04:27 [logINFO 01-04 14:04:28 [loggers.py:111] Engine 000: Avg prompt throuINFO 01-04 14:04:36 [loggers.py:111] Engine 000: Avg prompt throughput: 178.4 tokens/s, Avg generation throughput: 74.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 3.5%
+IINFO 01-04 14:04:38 [loggers.py:111] Engine 000: Avg promptINFO:  INFOINFO 01-04 14:04:43 [loggers.py:111] Engine 000: Avg prompt thINFO 01-04 14:04:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 30.2 tokeINFO 01-04 14:04:47 [loggers.INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 2INFOINFO 01-04 14:04:56 [loggers.py:111] Engine 000: Avg prompt throughput: 95.0 tokens/s, Avg generation throughput: 24.4 tokININFO 01-04 14:04:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg genINFO 01-04 14:05:06 [loggers.py:111] Engine 000: Avg prompt throughput: 119.1 tokens/s, Avg generation throughput: 78.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%,INFO 01-04 14:05:06 [loggers.INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:16 [loggers.py:111] Engine 000: Avg prompt throughput: 217.8 tokens/s, Avg generation throughput: 115.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 3.0%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:   INFO 01-04 14:05:23 [loggers.py:111] Engine 000: Avg prompINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:26 [loggers.py:111] Engine 000: Avg prompt throughput: 211.6 tokens/s, Avg genINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1INFO 01-04 14:05:36 [loggers.py:111] Engine 000: Avg promptINFO 01-04 14:05:36 [loggers.py:111] Engine 000: Avg prompt throughput: 153.9 tokens/s, Avg generation throughput: 103.7 tokens/s, Running: 3 reqs, Waiting: 0 INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:46 [loggers.py:111] Engine 000: Avg prompt throughput: 191.4 tokens/s, Avg generation throughput: 59.1 tokens/s, Running: 1 reqs, WaitingINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFOINFO 01-04 14:05:53 [loggers.py:111] Engine 000: Avg prompt tINFO 01-04 14:05:56 [loggers.py:111] Engine 000: Avg prompt throughput: 254.5 tokens/s, Avg generation throughput: 18.7 INFO 01-04 14:05:56 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04 14:05:57 [loggers.py:11INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:06 [loggers.py:111] Engine 000: AvINFO 01-04 14:06:07 [loggers.py:111] Engine 000: Avg prompt throughput: 230.6 tokens/s, Avg generationINFO 01-04 14:06:08 [loggers.py:111] Engine 000: Avg promptINFO: INFO 01-04 14:06:16 [loggers.py:111] Engine 000: Avg prompt throughput: 244.9 tokens/s, Avg generation throughput: 22.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1INFO 01-04 14:06:17 [loggers.py:1INFO 01-04 14:06:26 [loggers.py:111] Engine 000: Avg prompt throughput: 158.9 tokens/s, Avg generation throughput: 74.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2INFO:     10.45.190.192:0 - "POSTINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:36 [loggers.py:111] Engine 000: Avg prompt throughput: 223.0 tokens/s, Avg generation throughput: 120.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: INFO:     10.46.17.192:0 - "POST /vINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:46 [loggers.py:111] Engine 000: Avg prompt throughput: 254.9 tokens/s, Avg generation throughput: 128.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 10.4%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/INFO 01-04 1INFO 01-04 14:06:56 [loggers.py:111] Engine 000: Avg prompt throughput: 436.6 tokens/s, Avg generation throughput: 43.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache INFO 01-04 14:06:57 [loggers.py:111] EINFOINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 INFO 01-04 14:07:03 [loggers.py:111] Engine 000: Avg prompt throughpuINFO 01-04 14:07:06 [loggers.py:111] EnginINFO 01-04 14:07:06 [loggers.py:111] Engine 000: Avg prompt throughpINFO 01-04 14:07:07 [loggers.py:111] EINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFININFO 01-04 14:07:16 [loggers.py:111] Engine 000: Avg prompt throughput: 205.2 tokens/s, Avg generation throughput: 42.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cachINFO 01-04 14:07:17 [loggers.py:111] EnINFO 01-04 14:07:23 [loggers.py:111] Engine 000: Avg prompt throughput:INFO 01-04 14:07:26 [loggers.py:111] Engine 000: Avg prompt throughput: 238.8 tokens/s, Avg generation throuINFO 01-04 14:07:27 [loggers.py:111] Engine 000: Avg prompt throughput: 148.8 tokens/s, Avg generation thrINFO INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:36 [loggers.py:111] Engine 000: Avg prompt throughput: 234.1 tokens/s, Avg generation IINFO 01-04 14:07:37 [loggers.py:111] EngineINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200INFOINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 0INFO 01-04 14:07:46 [loggers.py:111] Engine 000: Avg prompt throughput: 269.4 tokens/s, Avg generation throughput: 52.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPINFO 01-04 14:07:47 [loggers.py:111] EngiINFO 0INFO: INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:56 [loggers.py:111]INFO 01-04 14:07:56 [loggers.py:111] Engine 000: Avg prompt throuINFO 01-04 14:07:57 [loggers.py:111] EnINFO 01-04 14:07:58 [loggers.py:111] Engine 000: Avg prompt throughINFO 01-04 1INFO 01-04 14:08:06 [loggers.py:111] Engine 000: Avg prompt throughput: 286.0 tokens/s, Avg generaINFO 01-04 14:08:07 [loggers.py:111] Engine 000: Avg prompt throughput: 194.5 tokens/s, Avg generation throuINFO:     10.INFO:     10.46.50.192:0 - "POST /v1/completINFO:    INFO 01-04 14INFO 01-04 14:08:16 [loggers.py:111] Engine 000: Avg prompt throughput: 291.9 tokens/s, Avg generINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:17 [loggers.py:111] Engine INFO:     10.4INFO:     10.45.190.192:0 - "POST /v1/completions HTINFO 01-04 14:0INFO 01-04 14:08:26 [loggers.py:111] Engine 000: Avg prompt throughput: 276.7 tokens/s, Avg generation throughput: 65.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache INFO 01-04 14:08:33 [loggers.py:111] Engine 000: Avg prompt throughput: 232.9 tokenINFO 01-04 14:08:36 [loggers.py:111] Engine 000: Avg prompt throughput: 389.1 tokens/s, Avg generation throughput: 25.7 tokens/s, RunniINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 2INFO 0INFO 01-04 14:08:43 [loggers.py:111] Engine 000: Avg prompt throughput: 343.1 tokeINFO 01-04 14:08:46 [loggers.py:111] Engine 000: Avg prompt throughput: 205.5 tokens/s, Avg generation throughput: 45.2 tokens/s, RuINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.1INFO 01-04 14:08:56 [loggers.py:111] Engine 000: Avg prompt throughput: 254.5 tokens/s, Avg generation throughput: 76.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 17.9%
+INFO:     10.45.190.192:0 - "POST /v1/completiINFO:     10.43.30.4:INFO 01-04 14:09:06 [loggers.py:111] Engine 000: Avg prompt throughput: 307.6 tokens/s, Avg generation throughput: 68.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 17.8%
+INFO 01-04 14:09:16 [loggers.py:111] Engine 000: Avg prompt throughput: 352.6 tokens/s, Avg generation throughput: 91.8 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.3%, Prefix cache hit rate: 17.1%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 135.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cache hit rate: 17.1%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/compINFO 01-04 14:09:33 [loggINFO 01-04 14:09:36 [loggers.py:111] Engine 000: Avg prompt throughput: 418.5 tokens/s, Avg generation throughput: 57.7 tokens/s, Running: 1 reqs, Waiting: 0INFO 01-04 14:09:36 [loggers.py:111]INFO:     10.46.17.192:0 -INFO:     10.43.30.4:0 - "POST /v1/complINFO 01-04 14:09:43 [logINFO 01-04 14:09:46 [loggers.py:111] Engine 000: Avg prompt throughput: 281.4 tokens/s, Avg generation throughput: 43.8 tokens/s, Running: 1 reqs, Waiting: 0 rINFO 01-04 14:09:46 [loggers.py:111INFO:     10.43.30.4:0 - INFO 01-04 14:09:56 [loggers.py:111] Engine 000: Avg prompt throughput: 337.7 tokens/s, Avg generation throughput: 81.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 19.4%
+INFO:     10.43.30.4:0 - "POST /v1/comINFO 01-04 14:10:03 [loggeINFO 01-04 14:10:06 [loggers.py:111] Engine 000: Avg prompt throughput: 369.8 tokens/s, Avg generation throughput: 96.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cache hit rate: 18.7%
+INFO:     10.46.50.192:0 - "POST /v1/coINFO 01-04 14:10:13 [loggerINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:16 [loggers.py:111] Engine 000: Avg prompt throughput: 674.9 tokens/s, Avg generation throughput: 99.4 tokenINFO 01-04 14:10:23 [loggers.py:111] Engine 000: Avg prompt throughput: 264.6 tokens/s, Avg generation throughput: 127.9 tokens/s, Running: 3 reqs, Waiting: 0INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 INFINFO 01-04 14:10:26 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04 14:10:33 [loggers.py:111] Engine 000: Avg prompt throughput: 205.2 tokens/s, Avg generation throughput: 116.9 tokens/s, Running: 3 reqs, WaitiINFO 01-04 14:10:36 [loggers.py:111] Engine 000: Avg prompt througINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:43 [loggers.py:111] Engine 000: Avg prompt throughput: 327.7 tokens/sINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 41.8 tokens/s, RunniINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:53 INFO 01-04 14:10:56 [loggers.py:111] Engine 000: Avg prompt throughput: 339.0 tokens/s, Avg generation throughput: 3.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.4%, Prefix INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:11:03 [lINFO 01-04 14:11:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg geINFO 01-04 14:11:16 [loggers.py:111] Engine 000: Avg prompt throughput: 479.2 tokens/s, Avg generation throughput: 12.6 tokenINFO 01-04 14:11:16 [loggers.py:111] Engine 000: Avg prompt throughput: 359.1 tokens/s, Avg geINFO 01-04 14:11:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, PrefINFO 01-04 14:11:26 [loggINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:11:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation INFO 01-04 14:11:43 [loggers.py:111] Engine 000: Avg prompt throughput: 357.3 tokens/s, Avg generation throughput: 3INFO 01-04 14:11:46 [loggers.py:111] Engine 000: Avg prompt throughput: 352.1 tokens/s, Avg generatioINFO 01-04 14:11:53 [loggeINFO 01-04 14:11:56 [loggers.py:111] Engine 000: Avg prompt throughput: 277.9 tokens/s, Avg generation throughput: 14.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.2%, INFO:     10.46.50.192:0 - "PINFO:     10.46.50.192:0 - "POST /v1/INFO 01-04 14:12:03 [loggers.INFO 01-04 14:12:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 32.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%,INFO 01-04 14:12:13 [loggers.pINFO 01-04 14:12:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, AvgINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:12:26 [loggers.py:111] Engine 000: Avg prompt tINFO 01-04 14:12:26 [loggers.py:111] Engine 000: Avg prINFO 01-04 14:12:27 [loggers.py:111] Engine 000: Avg prompt throughput: 313.4 tokens/s, Avg generation INFO 01-04 14:12:36 [loggers.py:111] Engine 000: Avg prompt tINFO 01-04 14:12:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 95.0 tokens/s, Running: 2 reqs, Waiting: 0INFO 01-04 14:12:46 [loggers.py:111] Engine 000: Avg prompt throughput: 563.3 tokens/s, Avg generation throughput:INFO 01-04 1INFO 01-04 14:12:46 [loggers.py:111] Engine 000: Avg prompt throughput: 284.4 tokens/s, Avg gINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:12:56 [loggers.py:111] Engine 000: INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:12:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation tINFO 01-04 14:13:06 [loggers.py:111] Engine 000: Avg prompt throughput: 242.2 tokens/s, Avg generation throughput:INFO 01-04 14:13:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throINFO 01-04 14:13:16 [loggers.py:111] Engine 000: Avg prompt throughput: 273.4 tokens/s, Avg generation throughINFO 01-04 14:13:17 [loggers.py:111] Engine 000: Avg prompt throughput: 433.9 tokens/s, Avg generation througINFO 01-04 14:13:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 93.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cacheINFO 01-04 14:13:27 [loggers.py:111] EngineINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:13:36 [loggers.py:111] Engine 000: Avg prompt throughput: 595.4 tokens/s, Avg generation thrINFO 01-04 14:13:37 [loggers.py:111] Engine 000: Avg prompt throughput: 299.7 tokens/s, Avg generation throughputINFO 01-04 14:13:46 [loggers.py:111] Engine 000: Avg prompt throughput: 281.5 tokens/s, Avg generation throughput: 154.6 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.7%, Prefix cache hit rate: 10.1%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/coINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 INFO 01-04 14:13:56 [loggers.py:111] EngiINFO 01-04 14:13:57 [loggers.py:111] Engine 000: Avg prompt throughput: 298.1 tokens/s, Avg generation throughput: 52.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: 5.1%
+INFO:     10.43.30.3:0 - INFO 01-04 14:14:06 [loggers.py:111] EnINFO 01-04 14:14:07 [loggers.py:111] Engine 000: Avg prompt throughput: 379.9 tokens/s, Avg generation throughput: 12.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usINFO:     10.46.17.192:0 - "POST /v1/coINFO:     10.43.30.4:0 - "PINFO 01-04 14:14:16 [loggers.py:111] INFO 01-04 14:14:17 [loggers.py:111] Engine 000: Avg prompt throughput: 369.1 tokens/s, Avg generation throughput: 51.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usINFO 01-04 14:14:26 [loggers.py:111] EnINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:27 [loggers.py:111] Engine 000: Avg prompt throughput: 395.9 tokens/s, Avg generation throughput: 6INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/INFO:     10.46.17.192:0 - "PINFO 01-04 14:14:36 [loggers.py:111] INFO 01-04 14:14:37 [loggers.py:111] Engine 000: Avg prompt throughput: 158.3 tokens/s, Avg generation throughput: 51.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usagINFO:     10.46.17.192:0 - "POST /v1/INFO 01-04 14:14:47 [loggers.py:111] Engine 000: Avg prompt throughput: 600.4 tokens/s, Avg generation throughput: 76.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 5.2%
+INFO:     10.43.30.5:0 - "POSINFO:     10.43.30.5:0 - "POST /v1/INFO 01-04 14:14:57 [loggers.py:111] Engine 000: Avg prompt throughput: 326.7 tokens/s, Avg generation throughput: 95.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 5.1%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:07 [loggers.py:111] Engine 000: Avg prompt throughput: 268.5 tokens/s, Avg generation throughput: 101.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 5.9%
+INFO:     10.43.30.3:0 - "INFO:     10.46.50.192:0 - "POST /v1/cINFO 01-04 14:15:17 [loggers.py:111] Engine 000: Avg prompt throughput: 853.6 tokens/s, Avg generation throughput: 97.8 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.9%, Prefix cache hit rate: 7.9%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:27 [loggers.py:111] Engine 000: Avg prompt throughput: 627.9 tokens/s, Avg generation throughput: 121.5 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.0%, Prefix cache hit rate: 7.6%
+INFO:     10.46.17.192:0 - INFO 01-04 14:15:36 [loggers.py:111] Engine 000: Avg prompt throughput: 339.4 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 10.6%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:46 [loggers.py:111] INFO 01-04 14:15:47 [loggers.py:111] Engine 000: Avg prompt throughput: 336.0 tokens/s, Avg generation throughput: 105.9 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usagINFO:     10.43.30.3:0 - "POST /v1/coINFO:     10.43.30.4:0 - "PINFO 01-04 14:15:56 [loggers.py:111] INFO 01-04 14:15:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 128.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: INFO 01-04 14:16:06 [loggers.py:11INFO 01-04 14:16:07 [loggers.py:111] Engine 000: Avg prompt throughput: 435.4 tokens/s, Avg generation throughput: 115.4 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:16 [loggers.py:111] Engine 000: Avg prompt throughput: 528.1 tokens/s, Avg geneINFO 01-04 14:16:17 [loggers.py:111] Engine 000: Avg prompt throughput: 574.4 tokens/s, Avg generation throughput: 90.7 tokeINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:27 [loggers.py:111] Engine 000: Avg prompt throughput: 563.7 tokens/s, Avg generation throughput: 75.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, PrINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:36 [loINFO 01-04 14:16:37 [loggers.py:111] Engine 000: Avg prompt throughput: 296.2 tokens/s, Avg generation throughput: 36.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, PrefixINFO 01-04 14:16:46 [lINFO 01-04 14:16:47 [loggers.py:111] Engine 000: Avg prompt throughput: 461.9 tokens/s, Avg generation throughput: 56.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, PrefiINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:56 [logINFO 01-04 14:16:57 [loggers.py:111] Engine 000: Avg prompt throughput: 529.9 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache uINFO 01-04 14:16:INFO 01-04 14:17:06 [loINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:07 [loggers.py:111] Engine 000: Avg prompt throughput: 368.6 tokens/s, Avg generation throughput: 85.2 tokens/s, INFO 01-04 14:17:16 [loggers.py:111] Engine 000: Avg prompt throughput: 1126.2 tokens/s,INFO 01-04 14:17:17 [loggers.py:111] Engine 000: Avg prompt throughput: 578.2 tokens/s, Avg generation throughput: 62.1 tokens/s, RuINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:26 INFO 01-04 14:17:27 [loggers.py:111] Engine 000: Avg prompt throughput: 596.8 tokens/s, Avg generation throughput: 58.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix INFO 01-04 14:17:36 [INFO 01-04 14:17:37 [loggers.py:111] Engine 000: Avg prompt throughput: 683.6 tokens/s, Avg generation throughput: 71.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.4%, Prefix cache hit rate: 9.8%
+INFO:     10.43.30.4:0 - "POST /v1/completioINFO 01-04 14:17:46 [loggers.py:111] Engine 000: Avg prompt throughput: 171.9 tokensINFO 01-04 14:17:47 [loggers.py:111] Engine 000: Avg prompt throughput: 953.2 tokens/s, Avg generation throughput: 60.7 tokens/s, RunniINFO 01-04 14:17:56 [loggers.py:111] Engine 000: Avg prompt throughput: 665.4 tokens/s, Avg generation throughput: 59.1 tokens/s, Running: 2 reqs, WaiINFO 01-04 14:17:57 [loggers.py:111] Engine 000: Avg prompt throughpuINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:06 [loggers.py:111] Engine 000: Avg prompt throughput: 308.4 tokens/s, Avg generation throughput: 104.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.1%, Prefix cache hit rate: 13.2%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:16 [loggers.py:111] Engine 000: Avg prompt throughput: 329.6 tokens/s, Avg generation throughput: 77.5 tokens/s, Running: 2 reqs, WaiINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+ININFO 01-04 14:18:26 [loggers.py:111] Engine 000: Avg prompt throughput: 245.1 tokens/s, Avg generation throughput: 128.6 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.3%, Prefix cache hit rate: 12.9%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:36 [loggers.py:111] Engine 000: Avg prompt throughput: 483.9 tokens/s, Avg generation throughput: 116.6 tokens/s, Running: 2 reqs, WaINFO 01-04 14:18:37 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 14:18:46 [loggers.py:111] Engine 000: Avg prompt throughput: 513.4 tokens/s, Avg generation throughput: 123.0 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.2%, Prefix cache hit rate: 13.3%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.INFO 01-04INFO 01-04 14:18:56 [loggers.py:111] Engine 000: Avg prompt throughput: 393.9 tokens/s, Avg generation throughput: 31.2 tokens/s, Running: 1 reqs,INFO 01-04 14:18:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0INFO 01-04 14:19:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 1 reqs,INFO 01-04 14:19:07 [loggers.py:111] Engine 000: Avg prompt throughput: 5INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:16 [loggers.py:111] Engine 000: Avg prompt throughput: 727.4 tINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:17 [loggers.py:111] Engine 000: Avg prompt throughput: 5INFO 01-04 14:19:26 [loggers.py:111] Engine 000: Avg prompt throughput: 322.2 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqsINFO 01-04 14:19:27 [loggers.py:111] Engine 000: Avg prompt throughput: 63INFO 01-04 14:19:36 [loggers.py:111] Engine 000: Avg prompt throughput: 589.9 tokens/s, Avg generation throughput: 67.6 tokens/s, Running: 2 reINFO 01-04 14:19:37 [loggers.py:111] Engine 000: Avg prompt throughput: 437.INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:46 [loggers.py:111] Engine 000: Avg prompt throughput: 287.7 tokens/s, Avg generation throughput: 108.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hitINFO 01-04 14INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:56 [loggers.py:111] Engine 000: Avg prompt throughput: 261.5 tokens/s, Avg generation throughput: 112.4 tokens/s, Running: 2INFO 01-04 14:19:57 [loggers.py:111] Engine 000: Avg prompt throughput: 600.4 tINFO 01-04 14:20:06 [loggers.py:111] Engine 000: Avg prompt throughput: 529.4 tokens/s, Avg generation throughput: 113.6 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.1%, Prefix cache hINFO 01-04 14:2INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:16 [loggers.py:111] Engine 000: Avg prompt throughput: 677.4 tokens/s, Avg generation throughput: 84.8 tokens/s, RunINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:17 INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:26 [loggers.py:111] Engine 000: Avg prompt throughINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:27 [loggers.py:111] Engine 000: Avg prompt throughput: 599.8 tokens/INFO 01-04 14:20:36 [loggers.py:111] Engine 000: Avg prompt throughput: 761.0 tokens/s, Avg generation throughput: 62.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.2%, Prefix cacINFO 01-04 14:20:37INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:46 [loggers.py:111] Engine 000: Avg prompt throughpuINFO 01-04 14:20:47 [loggers.py:111] Engine 000: Avg prompt throughput: 634.7 tokens/s, Avg generation throughput: 54.4 tokens/s, Running: 2 reqs, WaiINFO 01-04 14:20:56 [loggers.py:111] Engine 000: Avg prompt throughput: 266.2 tokens/s, Avg generation throughput: 73.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 12.5%
+INFO:     10.45.190.192:0 - "POST /v1/compleINFO 01-04 14:20:58 [loINFO 01-04 14:21:06 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04 14:21:07 [loggers.py:111] Engine 000: Avg prompt throughput: 625.0 tokens/s, Avg generation throughput: 111.1 tokens/s, Running: 2 reqs, WaitiINFO 01-04 14:21:16 [loggers.py:111] Engine 000: Avg prompt throughput: 549.0 tokens/s, Avg generation throughput: 74.0 tokens/s, RINFO 01-04 14:21:17 [loggers.py:111] Engine 000: Avg prompt throughput: 870.4 tokens/s, INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:26 [loggers.py:111] Engine 000: Avg prompt throughput: 635.4 tokens/s, Avg generation throughput: 101.0 tokens/s, INFO 01-04 14:21:27 [loggers.py:111] Engine 000: Avg prompt throughput: 541.1 tokens/s, AINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:36 [loggers.py:111] Engine 000: Avg prompt throughput: 656.7 tokens/s, Avg generation throughput: 65.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 14.4%
+INFO:     10.45.190.192:0 - "POST /v1/completINFO 01-04 14:21:3INFOINFO 01-04 14:21:46 [loggers.py:111] Engine 000: Avg prompt throughput: 637.5 tokens/s, Avg generation throughput: 30.7 tokensINFO 01-04 14:21:47 [loggers.py:111] Engine 000: Avg prompt throughput:INFO:     10.43.30.3:0INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OKIINFO 01-04 14:21:56 [loggers.py:111] Engine 000: Avg prompt INFO 01-04 14:21:57 [loggers.py:111] Engine 000: Avg prompt throughput: 632.1 tokens/s, Avg generation throughput: 70.0 tokens/s, RunninINFO 01-04 14:21:58 INFINFO 01-04 14:22:06 [loggers.py:111] Engine 000: Avg prompt throughput: 665.3 tokens/s, Avg generation throughput: 50.8 tokeINFO 01-04 14:22:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generINFO 01-04 14:22:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 90.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.4%, INFO 01-04 14:22:17 [loggers.INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:26 [loggers.py:111] Engine 000: Avg prompt throughput: 589.8 tokens/s, Avg generation throughput: 111.5 tokenINFO 01-04 14:22:27 [loggers.py:111] Engine 000: Avg prompt throughput: 1509.7 tokens/s, Avg gINFO 01-04 14:22:36 [loggers.py:111] Engine 000: Avg prompt throughput: 672.7 tokens/s, Avg generation throughput: 96.9 tokensINFO 01-04 14:22:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg genINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 102.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.9%, PrINFO:     10.43.30.3:0 - "PINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:56 [loggers.py:111] Engine 000: Avg prompt throuINFO 01-04 14:22:57 [loggers.py:111] Engine 000: Avg prompt throughput: 810.9 tokens/s, Avg generation throughput: 65.9 tokens/s, Running: 2 reqs, WaitingINFO 01-04 14:23:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, RunnINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:07INFO 01-04 14:23:26 [loggers.py:111] Engine 000: Avg prompt throughput: 1094.5 tokens/s, Avg generation throughput: 44.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.5%, Prefix cache hit rate: 13.9%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:36 [loggers.py:111] Engine 000: Avg prompt throughput: 699.4 tokens/s, Avg generation throughput: 48.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.8%, Prefix cache hit rate: 13.7%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 57.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 13.7%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:56 [loggers.py:111] Engine 000: Avg prompt throughput: 715.5 tokens/s, Avg generation throughput: 41.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 13.4%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:06 [loggers.py:111] Engine 000: Avg prompt throughput: 726.9 tokens/s, Avg generation throughput: 72.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 13.2%
+INFO 01-04 14:24:16 [loggers.py:111] Engine 000: Avg prompt throughput: 883.9 tokens/s, Avg generation throughput: 42.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.7%, Prefix cache hit rate: 12.9%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 55.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.7%, Prefix cache hit rate: 12.9%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:36 [loggers.py:111] Engine 000: Avg prompt throughput: 695.2 tokens/s, Avg generation throughput: 23.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 12.7%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 26.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 12.7%
+INFO 01-04 14:24:56 [loggers.py:111] Engine 000: Avg prompt throughput: 445.6 tokens/s, Avg generation throughput: 25.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 12.6%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 10.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 12.6%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:16 [loggers.py:111] Engine 000: Avg prompt throughput: 728.3 tokens/s, Avg generation throughput: 18.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 12.4%
+INFO 01-04 14:25:26 [loggers.py:111] Engine 000: Avg prompt throughput: 517.9 tokens/s, Avg generation throughput: 47.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 12.2%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:36 [loggers.py:111] Engine 000: Avg prompt throughput: 602.2 tokens/s, Avg generation throughput: 46.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 12.0%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:46 [loggers.py:111] Engine 000: Avg prompt throughput: 627.1 tokens/s, Avg generation throughput: 31.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 11.9%
+INFO 01-04 14:25:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 11.9%
+INFO 01-04 14:26:06 [loggers.py:111] Engine 000: Avg prompt throughput: 705.4 tokens/s, Avg generation throughput: 48.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.6%, Prefix cache hit rate: 11.7%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:26:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 71.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 11.7%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:26:26 [loggers.py:111] Engine 000: Avg prompt throughput: 538.3 tokens/s, Avg generation throughput: 54.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 11.6%
+INFO 01-04 14:26:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 11.6%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:26:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 39.1 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 11.6%
+INFO 01-04 14:26:56 [loggers.py:111] Engine 000: Avg prompt throughput: 574.7 tokens/s, Avg generation throughput: 38.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 11.4%
+INFO 01-04 14:27:06 [loggers.py:111] Engine 000: Avg prompt throughput: 724.1 tokens/s, Avg generation throughput: 68.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.5%, Prefix cache hit rate: 11.2%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:27:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 22.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 11.2%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:27:26 [loggers.py:111] Engine 000: Avg prompt throughput: 800.4 tokens/s, Avg generation throughput: 26.9 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 11.0%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:27:36 [loggers.py:111] Engine 000: Avg prompt throughput: 598.9 tokens/s, Avg generation throughput: 15.3 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 10.9%
+INFO 01-04 14:27:46 [loggers.py:111] Engine 000: Avg prompt throughput: 769.5 tokens/s, Avg generation throughput: 10.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 11.9%
+INFO 01-04 14:27:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 11.9%
+INFO 01-04 14:28:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 11.9%
+INFO 01-04 14:28:16 [loggers.py:111] Engine 000: Avg prompt throughput: 219.4 tokens/s, Avg generation throughput: 79.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.6%, Prefix cache hit rate: 11.9%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:28:26 [loggers.py:111] Engine 000: Avg prompt throughput: 462.0 tokens/s, Avg generation throughput: 54.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 11.8%
+INFO 01-04 14:28:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 92.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 11.8%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:28:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 43.4 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 11.8%
+INFO 01-04 14:28:56 [loggers.py:111] Engine 000: Avg prompt throughput: 875.8 tokens/s, Avg generation throughput: 10.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cache hit rate: 11.6%
+INFO 01-04 14:29:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.7%, Prefix cache hit rate: 11.6%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:29:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 31.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 11.6%
+INFO 01-04 14:29:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 11.6%
+INFO 01-04 14:29:46 [loggers.py:111] Engine 000: Avg prompt throughput: 890.9 tokens/s, Avg generation throughput: 12.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 11.4%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:29:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 34.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 11.4%
+INFO 01-04 14:30:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 11.4%
+INFO 01-04 14:30:26 [loggers.py:111] Engine 000: Avg prompt throughput: 974.1 tokens/s, Avg generation throughput: 37.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.0%, Prefix cache hit rate: 11.2%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:30:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 15.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 11.2%
+INFO 01-04 14:30:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 11.2%
+INFO 01-04 14:30:56 [loggers.py:111] Engine 000: Avg prompt throughput: 640.3 tokens/s, Avg generation throughput: 26.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 11.0%
+INFO 01-04 14:31:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 11.0%
+INFO 01-04 14:31:16 [loggers.py:111] Engine 000: Avg prompt throughput: 422.4 tokens/s, Avg generation throughput: 46.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.7%, Prefix cache hit rate: 10.9%
+INFO 01-04 14:31:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 91.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.0%, Prefix cache hit rate: 10.9%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:31:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 84.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 10.9%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:31:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 11.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 10.9%
+INFO 01-04 14:31:56 [loggers.py:111] Engine 000: Avg prompt throughput: 807.4 tokens/s, Avg generation throughput: 13.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cache hit rate: 11.8%
+INFO 01-04 14:32:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 11.8%
+INFO 01-04 14:32:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 11.8%
+INFO 01-04 14:32:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 11.8%
+INFO 01-04 14:32:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.0%, Prefix cache hit rate: 11.8%
+INFO 01-04 14:32:46 [loggers.py:111] Engine 000: Avg prompt throughput: 540.7 tokens/s, Avg generation throughput: 61.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.3%, Prefix cache hit rate: 11.7%
+INFO 01-04 14:32:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 90.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.7%, Prefix cache hit rate: 11.7%
+INFO 01-04 14:33:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 90.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.0%, Prefix cache hit rate: 11.7%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:33:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.7%, Prefix cache hit rate: 11.7%
+INFO 01-04 14:33:26 [loggers.py:111] Engine 000: Avg prompt throughput: 617.3 tokens/s, Avg generation throughput: 88.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.4%, Prefix cache hit rate: 11.5%
+INFO 01-04 14:33:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 89.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.8%, Prefix cache hit rate: 11.5%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:33:46 [loggers.py:1INFO 01-04 14:33:48 [loggers.py:111] Engine 000: Avg prompt throughput: 820.8 tokens/s, Avg generation throughput: 20.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: INFO 01-04 14:33:56 [loggers.py:111INFO 01-04 14:33:58 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: INFO 01-04 14:34:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation INFO 01-04 14:34:08 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 25.INFO 01-04 14:34:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation INFO 01-04 14:34:18 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0INFO 01-04 14:34:26 [loggers.py:111] Engine 000: Avg prompt throughput: 873.3 tokens/s, Avg generatiINFO 01-04 14:34:48 [loggers.py:111] Engine 000: Avg prompt throughput: 759.6 tokens/s, Avg generation throughput: 45.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cache hit rate: 21.7%
+INFO 01-04 14:34:58 [loggers.py:111] Engine 000: Avg prompINFO 01-04 14:35:03 [loggers.py:111] Engine 000: Avg prompt throughput: 925.1 tokens/s, Avg generation throughput: 36.8 tokens/s, Running: 1 reqs, Waiting: 0 rINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" INFO 01-04 14:35:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.0%, Prefix cache hit rate: 19.3%
+INFO 01-04 14:35:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.2%, Prefix cache hit rate: 19.3%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:35:33INFO 01-04 14:35:46 [loggers.py:111] Engine 000: Avg prompt throughput: 898.0 tokens/s, Avg generation throughput: 30.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.7%, Prefix cache hit rate: 16.4%
+INFO 01-04 14:35:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, RunINFO 01-04 14:36:46 [loggers.py:111] Engine 000: Avg prompt throughput: 1207.6 tokens/s, Avg generation throughput: 26.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.8%, Prefix cache hit rate: 16.0%
+INFO 01-04 14:36:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.0%, Prefix cache hit rate: 16.0%
+INFO:     10.46.50.192:0 - "POSTINFO 01-04 14:36:56 [loggers.py:11INFO 01-04 14:37:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokensINFO 01-04 14:37:26 [loggers.py:111] Engine 000: Avg prompt throughput: 615.0 tokens/s, Avg generation throughput: 40.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 17.1%
+INFO 01-04 14:37:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, RunINFO 01-04 14:38:16 [loggers.py:111] Engine 000: Avg prompt throughput: 521.8 tokens/s, Avg generation throughput: 38.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 15.9%
+INFO 01-04 14:38:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 15.9%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:38:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/INFO 01-04 14:38:46 [loggers.py:111] Engine 000: Avg prompt throughput: 471.6 tokens/s, Avg generation throughput: 8.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 16.9%
+INFO 01-04 14:38:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 48.0 tokens/s, RunnINFO 01-04 14:40:36 [loggers.py:111] Engine 000: Avg prompt throughput: 1101.8 tokens/s, Avg generation throughput: 34.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.5%, Prefix cache hit rate: 17.0%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:40:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 31.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.0%
+INFO 01-04 14:40:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.0%
+INFO 01-04 14:42:16 [loggers.py:111] Engine 000: Avg prompt throughput: 698.6 tokens/s, Avg generation throughput: 32.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 16.8%
+INFO 01-04 14:42:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 16.8%
+INFO 01-04 14:42:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 16.8%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:42:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 19.4 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.8%
+INFO 01-04 14:42:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.8%
+INFO 01-04 14:44:26 [loggers.py:111] Engine 000: Avg prompt throughput: 443.6 tokens/s, Avg generation throughput: 23.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 16.7%
+INFO 01-04 14:44:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 48.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.0%, Prefix cache hit rate: 16.7%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:44:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 23.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0INFO 01-04 14:45:26 [loggers.py:111] Engine 000: Avg prompt throughput: 728.0 tokens/s, Avg generation throughput: 22.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 13.3%
+INFO:     10.46.50.192:0 - "PINFO 01-04 14:46:26 [loggers.py:111] Engine 000: Avg prompt throughput: 355.5 tokens/s, Avg generation throughput: 39.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 16.6%
+INFO 01-04 14:46:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.7%, Prefix cache hit rate: 16.6%
+INFO 01-04 14:46:46 [loggers.py:11INFO 01-04 14:50:26 [loggers.py:111] Engine 000: Avg prompt throughput: 645.3 tokens/s, Avg generation throughput: 45.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 13.2%
+INFO 01-04 14:50:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 13.2%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:50:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 10.1 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 13.2%
+INFO 01-04 14:50:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 13.2%
+INFO 01-04 15:05:16 [loggers.py:111] Engine 000: Avg prompt throughput: 507.4 tokens/s, Avg generation throughput: 36.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 13.0%
+INFO 01-04 15:05:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 13.0%
+INFO 01-04 15:05:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 13.0%
+INFO 01-04 15:05:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 13.0%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 15:05:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 19.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 13.0%
+INFO 01-04 15:06:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 13.0%
+n throughput: 47.5 tokensINFO 01-04 15:04:43 [loggers.py:111] Engine 000: Avg prompt throughput: 503.2 tokens/s, Avg generation throughput: 29.2 tokens/s, Running: 1 reqs, Waiting: 0INFO 01-04 15:08:27 [loggers.py:111] Engine 000: Avg prompt throughput: 530.6 tokens/s, Avg generation throughput: 15.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 12.8%
+INFO 01-04 15:08:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 12.8%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 15:08:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 14.4 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 12.8%
+INFO 01-04 15:08:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 12.8%
+0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.0%
+pt throughput: 0.0 tokens/s, Avg generation throughput: 47.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cache hit rate: 19.3%
+INFO 01-04 15:02:28 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 19.3%
+INFO 01-04 15:02:38 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cache hit rate: 19.3%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 15:02:48 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 28.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.3%
+INFO 01-04 15:02:58 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.3%

hf_ip/vllm_gpu3.log

@@ -0,0 +1,280 @@
+/usr/local/lib/python3.10/dist-packages/torch/utils/_pytree.py:185: FutureWarning: optree is installed but the version is too old to support PyTorch Dynamo in C++ pytree. C++ pytree support is disabled. Please consider upgrading optree using `python3 -m pip install --upgrade 'optree>=0.13.0'`.
+  warnings.warn(
+INFO 01-04 13:13:33 [__init__.py:239] Automatically detected platform cuda.
+INFO 01-04 13:13:36 [api_server.py:1043] vLLM API server version 0.8.5
+INFO 01-04 13:13:36 [api_server.py:1044] args: Namespace(host='127.0.0.1', port=8004, uvicorn_log_level='info', disable_uvicorn_access_log=False, allow_credentials=False, allowed_origins=['*'], allowed_methods=['*'], allowed_headers=['*'], api_key=None, lora_modules=None, prompt_adapters=None, chat_template=None, chat_template_content_format='auto', response_role='assistant', ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, enable_ssl_refresh=False, ssl_cert_reqs=0, root_path=None, middleware=[], return_tokens_as_token_ids=False, disable_frontend_multiprocessing=False, enable_request_id_headers=False, enable_auto_tool_choice=False, tool_call_parser=None, tool_parser_plugin='', model='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', task='auto', tokenizer=None, hf_config_path=None, skip_tokenizer_init=False, revision=None, code_revision=None, tokenizer_revision=None, tokenizer_mode='auto', trust_remote_code=True, allowed_local_media_path=None, load_format='auto', download_dir=None, model_loader_extra_config={}, use_tqdm_on_load=True, config_format=<ConfigFormat.AUTO: 'auto'>, dtype='auto', max_model_len=131072, guided_decoding_backend='auto', reasoning_parser=None, logits_processor_pattern=None, model_impl='auto', distributed_executor_backend=None, pipeline_parallel_size=1, tensor_parallel_size=1, data_parallel_size=1, enable_expert_parallel=False, max_parallel_loading_workers=None, ray_workers_use_nsight=False, disable_custom_all_reduce=False, block_size=None, gpu_memory_utilization=0.9, swap_space=4, kv_cache_dtype='auto', num_gpu_blocks_override=None, enable_prefix_caching=None, prefix_caching_hash_algo='builtin', cpu_offload_gb=0, calculate_kv_scales=False, disable_sliding_window=False, use_v2_block_manager=True, seed=None, max_logprobs=20, disable_log_stats=False, quantization=None, rope_scaling=None, rope_theta=None, hf_token=None, hf_overrides=None, enforce_eager=False, max_seq_len_to_capture=8192, tokenizer_pool_size=0, tokenizer_pool_type='ray', tokenizer_pool_extra_config={}, limit_mm_per_prompt={}, mm_processor_kwargs=None, disable_mm_preprocessor_cache=False, enable_lora=None, enable_lora_bias=False, max_loras=1, max_lora_rank=16, lora_extra_vocab_size=256, lora_dtype='auto', long_lora_scaling_factors=None, max_cpu_loras=None, fully_sharded_loras=False, enable_prompt_adapter=None, max_prompt_adapters=1, max_prompt_adapter_token=0, device='auto', speculative_config=None, ignore_patterns=[], served_model_name=['default-model'], qlora_adapter_name_or_path=None, show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, disable_async_output_proc=False, max_num_batched_tokens=None, max_num_seqs=256, max_num_partial_prefills=1, max_long_partial_prefills=1, long_prefill_token_threshold=0, num_lookahead_slots=0, scheduler_delay_factor=0.0, preemption_mode=None, num_scheduler_steps=1, multi_step_stream_outputs=True, scheduling_policy='fcfs', enable_chunked_prefill=None, disable_chunked_mm_input=False, scheduler_cls='vllm.core.scheduler.Scheduler', override_neuron_config=None, override_pooler_config=None, compilation_config=None, kv_transfer_config=None, worker_cls='auto', worker_extension_cls='', generation_config='auto', override_generation_config=None, enable_sleep_mode=False, additional_config=None, enable_reasoning=False, disable_cascade_attn=False, disable_log_requests=True, max_log_len=None, disable_fastapi_docs=False, enable_prompt_tokens_details=False, enable_server_load_tracking=False)
+INFO 01-04 13:13:43 [config.py:717] This model supports multiple tasks: {'generate', 'score', 'classify', 'embed', 'reward'}. Defaulting to 'generate'.
+INFO 01-04 13:13:44 [config.py:2003] Chunked prefill is enabled with max_num_batched_tokens=8192.
+/usr/local/lib/python3.10/dist-packages/torch/utils/_pytree.py:185: FutureWarning: optree is installed but the version is too old to support PyTorch Dynamo in C++ pytree. C++ pytree support is disabled. Please consider upgrading optree using `python3 -m pip install --upgrade 'optree>=0.13.0'`.
+  warnings.warn(
+INFO 01-04 13:13:48 [__init__.py:239] Automatically detected platform cuda.
+INFO 01-04 13:13:52 [core.py:58] Initializing a V1 LLM engine (v0.8.5) with config: model='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', speculative_config=None, tokenizer='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=131072, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto,  device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='auto', reasoning_backend=None), observability_config=ObservabilityConfig(show_hidden_metrics=False, otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=None, served_model_name=default-model, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=True, chunked_prefill_enabled=True, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs=None, pooler_config=None, compilation_config={"level":3,"custom_ops":["none"],"splitting_ops":["vllm.unified_attention","vllm.unified_attention_with_output"],"use_inductor":true,"compile_sizes":[],"use_cudagraph":true,"cudagraph_num_of_warmups":1,"cudagraph_capture_sizes":[512,504,496,488,480,472,464,456,448,440,432,424,416,408,400,392,384,376,368,360,352,344,336,328,320,312,304,296,288,280,272,264,256,248,240,232,224,216,208,200,192,184,176,168,160,152,144,136,128,120,112,104,96,88,80,72,64,56,48,40,32,24,16,8,4,2,1],"max_capture_size":512}
+WARNING 01-04 13:13:52 [utils.py:2522] Methods determine_num_available_blocks,device_config,get_cache_block_size_bytes,initialize_cache not implemented in <vllm.v1.worker.gpu_worker.Worker object at 0x7f3dc054a8c0>
+INFO 01-04 13:13:53 [parallel_state.py:1004] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, TP rank 0
+INFO 01-04 13:13:53 [cuda.py:221] Using Flash Attention backend on V1 engine.
+WARNING 01-04 13:13:53 [topk_topp_sampler.py:69] FlashInfer is not available. Falling back to the PyTorch-native implementation of top-p & top-k sampling. For the best performance, please install FlashInfer.
+INFO 01-04 13:13:53 [gpu_model_runner.py:1329] Starting to load model /data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf...
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+INFO 01-04 13:15:40 [loader.py:458] Loading weights took 106.29 seconds
+INFO 01-04 13:15:40 [gpu_model_runner.py:1347] Model loading took 61.0562 GiB and 106.583423 seconds
+INFO 01-04 13:15:54 [backends.py:420] Using cache directory: /root/.cache/vllm/torch_compile_cache/75e72335d9/rank_0_0 for vLLM's torch.compile
+INFO 01-04 13:15:54 [backends.py:430] Dynamo bytecode transform time: 14.34 s
+INFO 01-04 13:15:59 [backends.py:136] Cache the graph of shape None for later use
+INFO 01-04 13:16:50 [backends.py:148] Compiling a graph for general shape takes 54.37 s
+INFO 01-04 13:17:53 [monitor.py:33] torch.compile takes 68.72 s in total
+INFO 01-04 13:17:53 [kv_cache_utils.py:634] GPU KV cache size: 254,768 tokens
+INFO 01-04 13:17:53 [kv_cache_utils.py:637] Maximum concurrency for 131,072 tokens per request: 1.94x
+INFO 01-04 13:18:32 [gpu_model_runner.py:1686] Graph capturing finished in 39 secs, took 1.21 GiB
+INFO 01-04 13:18:32 [core.py:159] init engine (profile, create kv cache, warmup model) took 172.43 seconds
+INFO 01-04 13:18:32 [core_client.py:439] Core engine process 0 ready.
+WARNING 01-04 13:18:32 [config.py:1239] Default sampling parameters have been overridden by the model's Hugging Face generation config recommended from the model creator. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
+INFO 01-04 13:18:32 [serving_chat.py:118] Using default chat sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
+INFO 01-04 13:18:32 [serving_completion.py:61] Using default completion sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
+INFO 01-04 13:18:32 [api_server.py:1090] Starting vLLM API server on http://127.0.0.1:8004
+INFO 01-04 13:18:32 [launcher.py:28] Available routes are:
+INFO 01-04 13:18:32 [launcher.py:36] Route: /openapi.json, Methods: HEAD, GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /docs, Methods: HEAD, GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /docs/oauth2-redirect, Methods: HEAD, GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /redoc, Methods: HEAD, GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /health, Methods: GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /load, Methods: GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /ping, Methods: GET, POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /tokenize, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /detokenize, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/models, Methods: GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /version, Methods: GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/chat/completions, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/completions, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/embeddings, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /pooling, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /score, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/score, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/audio/transcriptions, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /rerank, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/rerank, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v2/rerank, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /invocations, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /metrics, Methods: GET
+INFO:     Started server process [1316452]
+INFO:     Waiting for application startup.
+INFO:     Application startup compININFOINFO 01-04 14:00:03 [loggers.py:111] Engine 000: Avg prompt throughput: 120.9 tokens/s, Avg generation throughput: 93.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.9%, Prefix cache hit ratINININFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO INFO 01-04 14:00:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 80.8 tokens/s, Running: 1 reqs, WaiIINFO 01-04 14:00:19 [loggers.py:111] Engine 000: Avg prompt thINFO 01-04 14:00:20 [loggers.py:111] Engine 000: Avg prompt throughput:INFO 01-04 14:00:23 [loggers.py:111] Engine 000: Avg prompt throughput: 63.2 tokensINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 2INININFO 01-04 14:00:30 [loggers.py:111] Engine 000: Avg prompt throughput: INFO 01-04 14:00:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.7 tokens/s, Running: 1 reqsININFO 01-04 14:00:40 [loggers.py:111] Engine 000: Avg prompt throughput: 61.5 tokens/s, Avg generation throughput: 107.7 tokens/s, RuINFO 01-04 14:00:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 1 reqs, WaitIINFO 01-04 14:00:50 [loggers.py:111] Engine 000: Avg prompt throughput: 54.3 tokens/s, Avg generation throughput: 118.6 tokens/s, RuINFO 01-04 14:00:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/sININFO:INFO 01-04 14:00:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 93.6 tokeININFO 01-04 14:01:03 [loggers.py:111] Engine 000: Avg prompt throughput: 81.3 tokens/s, Avg generation throughput: 28.1 tokens/s, Running: 1 reqs, WaitiINFO INFO 01-04 14:01:07 [loggers.py:111] Engine 000: Avg prompt tINFO 01-04 14:01:13 [loggers.py:111] Engine 000: Avg prompt throughput: 100.0 tokens/s, Avg generation throughput: 64.3 tokens/s, Running: 2 reqs, Waiting: 0 INFO 01-04 14:01:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 141.3 tokenINFO 01-04 14:01:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 53.7 tokens/s, Running: 1 reqs, Waiting:INFO 01-04 14:01:26 [loggers.py:111] Engine 000: Avg prompt thrINFO 01-04 14:01:33 [loggers.py:111] Engine 000: Avg prompt throughput: 89.9 tokens/s, Avg generation throughput: 91.5 tokens/s, Running: 2 reqs, Waiting: INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:01:43 [loggers.py:111] Engine 000: Avg prompt throughput: 81.1 tokens/s, Avg generation throughput: 131.3 tokens/s, Running: 2 reqs, Waiting: 0INFINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1"INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:01:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 88.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, INFO:     10.45.190.192:0 - "POST /v1/completions HTTPINFO 01-04 14:02:03 [loggers.py:111] Engine 000: Avg prompt throughput: 163.9 tokens/s, Avg generation throughput: 93.9 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 0.7%
+INFO:     1INFO 01-04 14:02:07 [loggers.py:111] Engine 000: Avg INFO 01-04 14:02:13 [loggers.py:111] Engine 000: Avg prompt throughput: 102.2 tokens/s, Avg generation throughput: 138.4 tokens/s, Running: 3 reqs, Waiting: 0 reqs, INFO 01-04 14:02:17 [loggers.py:111] Engine 000: Avg prompt throughput: 103.5 tokens/s, Avg generation throughput: 126INFO 01-04 14:02:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 95.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: INFO INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:02:33 [loggers.py:111] Engine 000: Avg prompt throughput: 97.1 tokens/s, Avg generation throughput: 95.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, INFO 01-04 14:02:36 [loggers.py:111] Engine 000: Avg pINFO 01-04 14:02:43 [loggers.py:111] Engine 000: Avg prompt throughput: 96.5 tokens/s, Avg generation throughput: 107.8 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 0.5%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:02:53 [loggers.py:111] Engine 000: Avg prompt throughput: 86.3 tokens/s, Avg generation throughput: 128.4 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 0.6%
+INFO:     10.45.INFO 01-04 14:02:57 [loggers.py:111] Engine 000: AvINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 81.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPUINFO 01-04 14:03:07 [loggers.py:111] Engine 000: AvINFO 01-04 14:03:13 [loggers.py:111] Engine 000: Avg prompt throughput: 133.9 tokens/s, Avg generation throughput: 90.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPUINFO:     10.46.17.192:0 - "POST /v1/completions HTINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:    INFO 01-04 14:03:20 [loggers.py:111] Engine 000: Avg prompINFO 01-04 14:03:23 [loggers.py:111] Engine 000: Avg prompt throughput: 134.6 tokens/s, Avg generation throughput: 80.3 tokens/s, Running: 1 reINFO 01-04 14:03:25 [loINFO 01-04 14:03:27 [loggers.py:111] Engine 000: AvgINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:33 [loggers.py:111] Engine 000: Avg prompt throughput: 129.5 tokens/s, Avg generatiINFO 01-04 14:03:37 [loggers.py:111] Engine 000: Avg prompt throughput: 88.4 tokens/s, Avg generation throughput: 87.1 tokens/sINFO 01-04 14:03:40 [loggers.py:111] Engine 000: Avg prINFO 01-04 14:03:43 [loggers.py:111INFO 01-04 14:03:47 [loggers.py:111] Engine 000: Avg prompt throughput: 120.8 tokens/s, Avg generation throughput: 116.6 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usageINFO:     10INFO:     10.43.30.3:0 -INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:57 [loggers.py:111] Engine 000: Avg prompt throughput: 159.7 tokens/s, Avg generation throughput: 164.5 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usaINFO 01-04 14:04:03 [loggers.py:111] Engine 000: Avg prompt throughput: 161.7 tokens/s, Avg generationINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:04:07 [loggers.py:111] Engine 000: AINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     1INFO 01-04 14:04:10 [loggers.py:111] Engine 000: Avg prINFO 01-04 14:04:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation thINFO 01-04 14:04:17 [loggers.py:111] Engine 000: Avg prompt INFO:     10.46.17.192:0 - "POST /v1/completions HTTPINFO 01-04 14:04:23 [loggers.py:111] Engine 000: Avg prompt throughput: 167.1 tokens/s, Avg generation thINFO 01-04 14:04:27 [loggers.py:111] Engine 000: Avg prompt throughput: 174.5 tokens/s, Avg generation throughputINFO 01-04 14:04:33 [loggers.py:111] Engine 000: Avg prompt throughput: 144.0 tokens/s, Avg generation throughput: 68.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 0.6%
+INFO:     10.46.17.19INFO 01-04 14:04:37 [loggers.py:111] Engine 0INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:04:43 [loggers.py:111] Engine 000: Avg prompt throughput: 195.8 tokens/s, Avg generation throughput: 52.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cachINFO:     10.46.17.192:0 - "POST /v1/compleINFO:     10.45.190.192INFO 01-04 14:04:47 [loggers.py:111] Engine INFO 01-04 14:04:53 [loggers.py:111] Engine 000: Avg prompt throughput: 171.5 tokens/s, Avg generation throughput: 34.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cINFO 01-04 14:04:57 [loggers.py:111] Engine 00INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:03 [loggers.py:111] Engine 000: Avg prompt throughput: 183.8 tokens/s, Avg generation throughput: 32.2 tokens/s, Running: 1 reqs, WaitiINFO:     10.46.17.1INFO 01-04 14:05:07 [loggers.py:111] Engine 00INFO 01-04 14:05:13 [loggers.py:111] Engine 000: Avg prompt throughput: 131.3 tokens/s, Avg generation throughput: 61.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cINFO:     10.45.190.192:0 - "POST /v1/completIINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:23 [loggers.py:111] Engine 000: Avg prompt throughput: 204.0 tokens/s, Avg generation throughput: 70.4 tokens/s, Running: 2 reqs, WaitinINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OKINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 68.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.0%, Prefix cache hit rate: 2.2%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:43 [loggers.py:111] Engine 000: Avg prompt throughput: 163.2 tokens/s, Avg generation throughput: 68.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cacheINFO 01-04 14:05:47 [loggers.py:111] EnginINFO 01-04 14:05:53 [loggers.py:111] Engine 000: Avg prompt throughput: 166.6 tokens/s, Avg generation throughput: 77.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 2.1%
+INFO:     10.46.50.192:0 INFO 01-04 14:05:57 [loggers.py:111] EngiINFO 01-04 14:06:03 [loggers.py:111] Engine 000: Avg prompt throughput: 340.7 tokens/s, Avg generation throughput: 107.3 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 6.4%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:13 [loggers.py:111] Engine 000: Avg prompt throughput: 152.7 tokens/s, Avg generation throughput: 92.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV INFO:     10.46.17.192:0 - "POST /v1/completionINFO:     10.45.190INFO 01-04 14:06:17 [loggers.py:111] Engine 000:INFO 01-04 14:06:23 [loggers.py:111] Engine 000: Avg prompt throughput: 168.2 tokens/s, Avg generation throughput: 82.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 9.1%
+INFO 01-04 14:06:33 [loggers.py:111] Engine 000: Avg prompt throughput: 186.5 tokens/s, Avg generation throughput: 100.1 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 8.7%
+INFO:     10.43INFO:     10.46.17.192:0 - "POST /v1/completions INFO 01-04 14:06:43 [loggers.py:111] Engine 000: Avg prompt throughput: 225.2 tokens/s, Avg generation throughput: 96.5 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 8.3%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:53 [loggers.py:111] Engine 000: Avg prompt throughput: 212.9 tokens/s, Avg generation throughput: 138.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU INFO:     10.43.30.5:0 - "POST /v1/completions HTTINFO 01-04 14:07:03 [loggers.py:111] Engine 000: Avg prompt throughput: 225.8 tokens/s, Avg generation throughput: 101.0 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 7.6%
+INFO:     10.INFO 01-04 14:07:07 [loggers.py:111] Engine 000: AvINFO 01-04 14:07:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 113.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU INFO:     10.46.17.192:0 - "POST /v1/completions HINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:23 [loggers.py:111] Engine 000: Avg prompt throughput: 239.1 tokens/s, Avg generation throughput: 114.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 7.3%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:33 [loggers.py:111] Engine 000: Avg prompt throughput: 454.7 tokens/s, Avg generation throughput: 121.9 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 6.7%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:43 [loggers.py:111] Engine 000: Avg prompt throughput: 242.1 tokens/s, Avg generation throughput: 95.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs,INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/INFO 01-04 14:07:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 92.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 6.5%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:03 [loggers.py:111] Engine 000: Avg prompt throughput: 322.9 tokens/s, Avg generation throughput: 92.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 6.1%
+INFO 01-04 14:08:13 [loggers.py:111] Engine 000: Avg prompt throughput: 234.5 tokens/s, Avg generation throughput: 109.7 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.1%, Prefix cache hit rate: 5.9%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:23 [loggers.py:111] Engine 000: Avg prompt throughput: 222.1 tokens/s, Avg generation throughput: 95.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.7%, Prefix cache hit rate: 5.7%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:33 [loggers.py:111] Engine 000: Avg prompt throughput: 371.0 tokens/s, Avg generation throughput: 74.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 6.6%
+INFO:     10.45.190.192:0 - "POST /v1/completions HINFO 01-04 14:08INFO 01-04 14:08:43 [loggers.py:111] Engine 000: Avg prompt throughput: 217.7 tokens/s, Avg generation throughput: 81.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.0%, Prefix cache INFO:     10.43INFO 01-04 14:08:53 [loggers.py:111] Engine 000: Avg prompt throughput: 333.3 tokens/s, Avg generation throughput: 115.1 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.7%, Prefix cache hit rate: 6.1%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:03 [loggers.py:111] Engine 000: Avg prompt throughput: 207.6 tokens/s, Avg generation throughput: 123.8 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.8%, Prefix cache hit rate: 6.0%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 94.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.6%, Prefix cache hit rate: 6.0%
+INFO 01-04 14:09:23 [loggers.py:111] Engine 000: Avg prompt throughput: 251.7 tokens/s, Avg generation throughput: 124.3 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.1%, Prefix cache hit rate: 5.8%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:33 [loggers.py:111] Engine 000: Avg prompt throughput: 306.6 tokens/s, Avg generation throughput: 98.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit INFO:     1INFO 01-04 14:09:43 [loggers.py:111] Engine 000: Avg prompt throughput: 357.1 tokens/s, Avg generation throughput: 110.3 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.5%, Prefix cache hit rate: 7.5%
+INFO:INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:53 [loggers.py:111] Engine 000: Avg prompt throughput: 309.2 tokens/s, Avg generation throughput: 116.7 tokens/s, Running: 1 reqs, Waiting: INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200INFO 01-04 14:10:03 [loggers.py:111] Engine 000: Avg prompt throughput: 379.1 tokens/s, Avg generation throughput: 72.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 9.5%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:13 [loggers.py:111] Engine 000: Avg prompt throughput: 379.0 tokens/s, Avg geINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 20INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:23 [loggers.py:111] Engine 000: Avg prompt throughput: 321.2 tokens/s, Avg generation throughput: 64.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.4%, Prefix cache hit rate: 13.5%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:33 [loggers.py:111] Engine 000: Avg prompt throughput: 475.5 tokens/s, Avg generation throughput: 57.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 12.9%
+INFINFO 01-04 14:10:37 [loggers.py:111] Engine 000: AvINFO:     10INFO 01-04 14:10:43 [loggers.py:111] Engine 000: Avg prompt throughput: 292.6 tokens/s, Avg generation throughput: 44.1 tokens/s, Running: 1 reqs, Waiting:INFO 01-04 14:10:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.7 INFO:     10INFO 01-04 14:10:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:57 [loggers.py:111] Engine 000: Avg prompt throughINFO 01-04 14:11:03 [loggers.py:111] Engine 000: Avg prompt throughput: 350.3 tokens/s, Avg generation throughput: 3.3 tokens/s, Running: 1 reqs, WaitiINFO 01-04 14:11:07 [loggers.py:111] Engine 000: Avg prompt throughINFO 01-04 14:11:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.4 tokens/s, Running: 1 reqs, WaitINFO 01-04 14:11:17 [loggers.py:111] Engine 000: Avg prompt throughpINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:11:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 32.7 tokens/s, Running: 0 reqs, WaitiINFO 01-04 14:11:27 [loggers.py:111] Engine 000: Avg prompt throughINFO 01-04 14:11:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, WaitiINFO 01-04 14:11:37 [loggers.py:111] INFO 01-04 14:11:39 [loggers.pINFO 01-04 14:11:53 [loggers.py:111] Engine 000: Avg prompt throughput: 257.2 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.2%, Prefix cache hit rate: 12.7%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:12:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 8.9 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 12.7%
+INFO 01-04 14:12:13 [loggers.py:111] Engine 000: Avg prompt throughput: 209.4 tokens/s, Avg generation throughput: 1.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.8%, Prefix cache hit rate: 12.5%
+INFO 01-04 14:12:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.0%, Prefix cache hit rate: 12.5%
+INFO 01-04 14:12:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.2%, Prefix cache hit rate: 12.5%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:12:43 [loggers.py:111] Engine 000: Avg prompt throughput: 725.6 tokens/s, Avg generation throughput: 86.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 12.3%
+INFO 01-04 14:12:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 92.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cache hit rate: 12.3%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:13:03 [loggers.py:111] Engine 000: Avg prompt throughput: 232.7 tokens/s, Avg generation throughput: 85.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 12.1%
+INFO 01-04 14:13:13 [loggers.py:111] Engine 000: Avg prompt throughput: 172.1 tokens/s, Avg generation throughput: 120.1 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.9%, Prefix cache hit rate: 11.9%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:13:23 [loggers.py:111] Engine 000: Avg prompt throughput: 288.1 tokens/s, Avg generation throughput: 124.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 11.7%
+INFO:     10.46.17.192:0 - "POST /v1/completionINFO 01-04 14:13:25INFO 01-04 14:13:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 54.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cacheINFO:     10.46.1INFO 01-04 14:13:43 [loggers.py:111] Engine 000: Avg prompt throughput: 393.7 tokens/s, Avg generation throughput: 73.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.1%, Prefix cache hit rate: 11.3%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:13:53 [loggers.py:111] Engine 000: Avg prompt throughput: 376.6 tokens/s, Avg generation throughput: 114.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cache hit rate: 12.2%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:03 [loggers.py:111] Engine 000: Avg prompt throughput: 306.6 tokens/s, Avg generation throughput: 65.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3INFO:     10.43.30.3:0 - "POST /INFO 01-04 14:14:13 [loggers.py:111] Engine 000: Avg prompt throughput: 914.3 tokens/s, Avg generation throughput: 81.5 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.3%, Prefix cache hit rate: 11.2%
+INFO:     10.46.17.192:0 - "POST /v1/compINFO 01-04 14:14:15 [loggINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:23 [loggers.py:111] Engine 000: Avg prompt throughput: 387.3 tokens/s, Avg generation throughput: 110.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usagINFO 01-04 14:14:26 [loggers.py:111] Engine 000: Avg prompt throughput: 712.5 tokens/s, Avg generation tINFO 01-04 14:14:33 [loggers.py:111] Engine 000: Avg prompt throughput: 323.5 tokens/s, Avg generation throughput: 58.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache uINFO 01-04 14:14:36 [loggers.py:111] EngiINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:43 [loggers.py:111] Engine 000: Avg prompt throughput: 299.0 tokens/s, Avg generation throughput: 92.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cacheINFO 01-04 14:14:46 [loggers.py:111] Engine 000: Avg prompt throughput: 402.2 tokens/s, Avg generation throINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:53 [loggers.py:111] Engine 000: Avg prompt throughput: 226.2 tokens/s, Avg generation througINFO:     10.45.1INFO 01-04 14:14:55 [loggers.py:111] Engine 000: AINFO 01-04 14:14:56 [loggers.py:111] EngineINFO 01-04 14:15:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.7 tokens/sINFO 01-04 14:15:05 [loggers.py:111] Engine 000: INFO 01-04 14:15:06 [loggers.py:111] EnginINFO 01-04 14:15:13 [loggers.py:111] Engine 000: Avg prompt throughput: 752.4 tokens/s, Avg generation throughput: 75.6 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.3%, Prefix cache hit rate: 12.6%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:23 [loggers.py:111] Engine 000: Avg prompt throughput: 477.7 tokens/s, Avg generation throughput: 89.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache INFO:     10.43.30.4:0INFO:     10.43.30.5INFO:     10.43.30.5:0INFO 01-04 14:15:26 [loggers.py:111] EnginINFO 01-04 14:15:33 [loggers.py:111] Engine 000: Avg prompt throughput: 428.7 tokens/s, Avg generation throughput: 65.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix caINFO 01-04 14:15:35 INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:43 [loggers.py:111] Engine 000: Avg prompt throughput: 466.7 tokens/s, Avg generation throughput: 91.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cacheINFO 01-04 14:15:46 [loggers.py:111] EngineINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:53 [loggers.py:111] Engine 000: Avg prompt throughput: 354.8 tokens/s, Avg generation throughput: 102.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:56 [loggers.py:111] Engine 000: Avg prompt throughput: 358.2 tokens/s, Avg generation throughINFO 01-04 14:16:03 [loggers.py:111] Engine 000: Avg prompt thINFO 01-04 14:16:05 [loggers.py:111] Engine 000INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:06 [loggers.py:111] Engine INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 28.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cachINFO 01-04 14:16:16 [loggers.py:111] Engine INFO 01-04 14:16:23 [loggers.py:111] Engine 000: Avg prompt throughput: 384.0 tokens/s, Avg generation throughput: 39.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.7%, PrefINFO 01-04 14:16:25 [loggINFO 01-04 14:16:33 [loggers.py:111] Engine 000: Avg prompt throughput: 905.5 tokens/s, Avg generation throughput: 82.3 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.5%, Prefix cache hit rate: 15.6%
+INFO:     10.46.17.192:0 - "POST /v1/compINFO 01-04 14:16:35 [loggINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 72.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache INFO 01-04 14:16:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokenINFO 01-04 14:16:49 [loggerINFO 01-04 14:16:53 [loggers.py:111] Engine 000: Avg prompt throughput: 352.9 tokens/s, Avg generation througINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:56 [loggers.py:111] Engine INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:03 [loggers.py:111] Engine 000: Avg prompt throughput: 176.1 tokens/s, Avg generation througINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:06 [loggers.py:111] Engine INFO 01-04 14:17:13 [loggers.py:111] Engine 000: Avg prompt throughput: 378.8 tokens/s, Avg generation throughput: 52.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cachINFO 01-04 14:17:16 [loggers.py:111] Engine INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:23 [loggers.py:111] Engine 000: Avg prompt throughput: 384.4 tokens/s, Avg generation throINFO 01-04 14:17:26 [loggers.py:111] Engine 000: Avg prompt throughput: 437.5 tokens/s, Avg generation throughpuINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:33 [loggers.py:111] Engine 000: Avg prompt throughput: 390.2 tokens/s, Avg generation throughput: 50.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache INFO 01-04 14:17:36 [loggers.py:111] EnginINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 40.9 tokens/s, RunnINFO 01-04 14:17:45 [loggers.py:111] Engine 00INFO 01-04 14:17:46 [loggers.py:111] EnINFO 01-04 14:17:53 [loggers.py:111] Engine 000: Avg prompt throughput: 1134.6 tokens/s, Avg generation throughput: 39.0 tokens/s, RunINFO 01-04 14:17:55 [loggers.py:111] Engine 00INFO 01-04 14:17:56 [loggers.py:111] EngINFO:     10.46.17.192:0 - "POST /v1INFO 01-04 14:17:59 [loggers.pINFO 01-04 14:18:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 71.3 tokens/s, RunINFO 01-04 14:18:05 [loggers.py:111] Engine 000:INFO 01-04 14:18:06 [loggers.py:111] EINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:13 [loggers.py:111] Engine 000: Avg prompt throughpuINFO 01-04 14:18:15 [loggers.py:111] Engine 000: Avg prompt throughput: 470.8 tokens/s, Avg generation throughput:INFO 01-04 14:18:16 [loggers.py:111] Engine 000: Avg prompt throughput: 397.0 tokens/s, Avg generationINFO 01-04 14:18:23 [loggers.py:111] Engine 000: Avg prompt throughput: 497.0 tokens/s, Avg generation throughput: 21INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/comINFO:     10.46.17.192:0 - "POST /v1/coINFO 01-04 14:18:29 [loggerINFO 01-04 14:18:33 [loggers.py:111] Engine 000: Avg prompt throughput: 758.9 tokens/s, Avg generation throughput: 31.3 tokens/s, RunninINFO 01-04 14:18:35 [loggers.py:111] Engine 000: Avg prompt throughput: 421.0 tokenINFO 01-04 14:18:43 [loggers.py:111] Engine 000: Avg prompt throughput: 251.6 tokens/s, Avg generation throughput: 78.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.3%, Prefix cacINFO:     10.46.50.INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:53 [loggers.py:111] Engine 000: Avg prompt throughput: 613.8 tokens/s, Avg generation throughput: 98.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix cacINFO 01-04 14:18:55INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:03 [loggers.py:111] Engine 000: Avg prompt throughINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:05 [loggers.py:111] Engine 000: Avg prompt throughput: 686.9 tokens/INFO:     10.46.17.192:0 - "POSTINFO 01-04 14:19:09 [loggers.py:11INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFOINFO 01-04 14:19:15 [loggers.py:111] Engine 000: Avg prompt throughput: 520.1 tokens/s, Avg generation througINFO 01-04 14:19:16 [loggers.py:111] Engine 000: Avg prompt throughput: 449.5 tokens/s, Avg generation thrINFO 01-04 14:19:23 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 14:19:25 [loggers.py:111] EngineINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:26 [loggers.py:111] EnINFO 01-04 14:19:33 [loggers.py:111] Engine 000: Avg prompt throughput: 457.7 tokens/s, Avg generation throughput: 69.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.2%, Prefix cacheINFO 01-04 14:19:35 [loggers.py:111] Engine INFO 01INFO 01-04 14:19:39 [loggers.py:INFO 01-04 14:19:43 [loggers.py:111] Engine 000: Avg prompt throughput: 290.5 tokens/s, Avg generation throughput: 54.8 tokens/s, RunninINFO 01-04 14:19:45 [loggers.py:111] EnginINFO 01-04 14:19:46 [loggers.py:111] EngiINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:53 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 14:19:55 [loggers.py:111] Engine 000: Avg prompt throughput: 567.7 tokens/s, Avg generation througINFO 01-04 14:19:56 [loggers.py:111] EINFO 01-04 14:20:03 [loggers.py:111] Engine 000: Avg prompt throughput: 757.3 tokens/s, Avg generation throughput: 73.4 tokens/s, Running:INFO 01-04 14:20:05 [loggers.py:111] Engine 000: Avg prompt throughput: 696.2 tokens/s, Avg generation throughINFO 01-04 14:20:06 [loggers.py:111] Engine 000: Avg prompt throughput:INFO 01-04 14:20:09 [loggers.pINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-0INFO 01-04 14:20:15 [loggers.py:111] Engine 000: Avg prompt throughput: 399.1 tokens/s, Avg generation througINFO 01-04 14:20:16 [loggers.py:111] Engine 000: Avg prompt throughput: 580.1 tokens/s, Avg generatioINFO 01-04 14:20:23 [loggers.py:111] Engine 000: Avg prompt throughput: 458INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:25 [loggers.py:111] EngiINFO 01-04 14:20:26 [loggers.py:111] INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:33 [loggers.py:111] Engine 000: Avg prompt throughput: 560.8 tokens/s, Avg generation throughput: 56.7 tokens/s, Running: 1INFO 01-04 14:20:35 [loggers.py:111] EngineINFO:     10.46.50.192:0 - "POST /v1INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:43 [loggers.py:111] Engine 000: Avg prompt throughput: 7INFO 01-04 14:20:45 [loggers.py:111] Engine 000: Avg prompt throughput: 547.0 tokens/s, Avg generation throughput: 30.1 tokens/s, Running: 1 reqs,INFO 01-04 14:20:53 [loggers.py:111] Engine 000: Avg prompt throughput: 643.3 tokens/s, Avg generation throughput: 54.1 tokens/s, Running: INFO 01-04 14:20:55 [loggers.py:111] Engine 000: Avg prompt throughput: 680.4 toINFO 01-04 14:21:03 [loggers.py:111] Engine 000: Avg prompt throughput: 599.1 tokens/s, Avg generation throughput: 104.5 tokens/s, Running:INFO 01-04 14:21:05 [loggers.py:111] Engine 000: Avg prompt throughput: 608.1 tokINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:13 [loggers.py:111] Engine 000: Avg prompt throughput: 5INFO 01-04 14:21:15 [loggers.py:111] Engine 000: Avg prompt throughput: 862.1 tokens/s, Avg generation throughput: 60.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache INFO 01-04 14:21:16 [loggers.py:111] EnginINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" INFO 01-04 14:21:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0INFO 01-04 14:21:25 [loggers.py:111] Engine 000: Avg prompt throughput: 485.5 tokens/s, Avg generation throughINFO 01-04 14:21:26 [loggers.py:1INFO 01-04 14:21:33 [loggers.py:111] Engine 000: Avg prompt throughput: 617.INFO 01-04 14:21:35 [loggers.py:111] Engine 000: Avg prompt throughput: 426.4 tokens/s, Avg generation throughINFO 01-04 14:21:36 [loggers.py:1INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04INFO 01-04 14:21:45 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 92.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cacheINFO 01-04 14:21:46 [loggers.py:111INFO 01-04 14:21:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:55 [loggers.py:111] Engine 000: Avg prompt throughput: 1617.INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:03 [loggers.py:111] Engine 000: Avg prompt throughput: 555.2 tokens/s, Avg generation throughput: 54.1 tokens/s, Running: 1INFO 01-04 14:22:05 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokINFO 01-04 14:22:13 [loggers.py:111] Engine 000: Avg prompt throughput: 731.4 tokens/s, Avg generation throughput: 47.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.3%, Prefix cacheINFO 01-04 14:22:INFO 01-04 14:22:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 91.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.7%, Prefix cache hit rate: 16.0%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:33 [loggers.py:111] Engine 000: Avg prompt throughput: 604.4 tokens/s, Avg generation throughput: 101.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.3%, Prefix cache hit rate: 17.1%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTINFO 01-04 14:2INFO 01-04 14:22:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 52.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hINFO 01-04 14:2INFO 01-04 14:22:53 [loggers.py:111] Engine 000: Avg prompt throughput: 510.3 tokens/s, Avg generation throughput: 69.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.9%, Prefix cache hit rate: 16.9%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTPINFO 01-04 14INFO 01-04 14:23:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 53.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rINFO 01-04 INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:13 [loggers.py:111] Engine 000: Avg prompt throughput: 341.0 tokens/s, Avg generation throughput: 52.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache hit ratINFO:    INFO 01-04 14:23:23 [loggers.py:111] Engine 000: Avg prompt throughput: 1349.9 tokens/s, Avg generation throughput: 83.5 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.1%, Prefix cache hit rate: 16.9%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 108.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.7%, Prefix cache hit rate: 16.9%
+INFO:     10.46.17.192:0 - "POST INFO 01-04 14:23:36 [loggers.py:1INFO 01-04 14:23:43 [loggers.py:111] Engine 000: Avg prompt throughput: 877.1 tokens/s, Avg generation throughput: 84.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.6%, Prefix cache hitINFO 01-04 14:23:45 [loggers.py:111] Engine 00INFO 01-04 14:23:46 [loggers.py:1INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:53 [loggers.py:111] Engine 000: Avg prompt throughput: 697INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:55 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 toINFO 01-04 14:24:03 [loggers.py:111] Engine 000: Avg prompt throughput: 312.1 tokens/s, Avg generation throughput: 62.8 tokens/s, Running: INFO 01-04 14:24:05 [loggers.py:111] Engine 000: Avg prompt throughput: 1436.9 tINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 INFO 01-04 14:24:15 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughpuINFO 01-04 14:24:16 [loggers.pyINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:23 [loggers.py:111] Engine 000: Avg prompt throughput: 1238.8 tokens/s, Avg generation throughput: 40.0 tokens/s, Running: 1 INFO:     10.43.30.5:0 - "POST /v1/completiINFO 01-04 14:24:26 [loggers.py:111INFO 01-04 14:24:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.INFO:     10.46.50.192:0 - "POST INFO 01-04 14:24:43 [loggers.py:111] Engine 000: Avg prompt throughput: 714.5 tokens/s, Avg generation throughput: 80.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.6%, Prefix cache hINFO 01-04 14:24:45 [loggers.py:111] Engine 000: Avg prompt throughput: 760.1 tINFO 01-04 14:24:53 [loggers.py:111] Engine 000: Avg prompt throughput: 584.7 tokens/s, Avg generation throughput: 84.8 tokens/s, Running: 2INFO 01-04 14:24:55 [loggers.py:111] Engine 000: Avg prompt throughput: 736.1 tINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:03 [loggers.py:111] Engine 000: Avg prompt throughput: 366.1 tokens/s, Avg generation throughput: 93.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.2INFO 01-04 14:25:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generaINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:13 [loggers.py:111] Engine 000: Avg prompt throughput: INFO 01-04 14:25:15 [loggers.py:111] Engine 000: INFO 01-04 14:25:16 [loggers.py:111] Engine 000: Avg prompt throughput: 681.3 tokens/s, Avg geneINFO 01-04 14:25:23 [loggers.py:111] Engine 000: Avg prompt throughput:INFO 01-04 14:25:25 [loggers.py:111] Engine 000: Avg prompt throughput: 349.5 tokens/s, Avg generation throughput: 4INFO 01-04 14:25:26 [loggers.py:INFO 01-04 14:25:33 [loggers.py:111] Engine 000: Avg prompt throughput:INFO 01-04 14:25:35 [loggers.py:111] Engine 000: Avg prompt throughput: 778.8 tokens/s, Avg generation throughput: 8INFO 01-04 14:25:36 [loggers.py:INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO INFO 01-04 14:25:45 [loggers.py:111] Engine 000: Avg prompt throughput: 786.8 tokens/s, Avg generation throughput: 8INFO 01-04 14:25:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation thINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1INFO 01-0INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completiINFO 01-04 14:25:56 [loggers.py:111] Engine 000: Avg prompt throughput: 600.3 tokens/s, Avg generaINFO 01-04 14:26:03 [loggers.py:111] Engine 000: Avg prompt throughput: 1210.4 tokens/s, Avg generation throughput: 56.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.2%, Prefix cache hit INFO 01-04 1INFO 01-04 14:26:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 83.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.5%, Prefix cache hit INFO 01-04 1INFO 01-04 14:26:23 [loggers.py:111] Engine 000: Avg prompt throughput: 768.1 tokens/s, Avg generation throughput: 128.4 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 12.0%, Prefix cacheINFO 01-04 14:26:INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:26:33 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 14:26:35 [loggers.py:111] Engine 000: Avg prompt throughput: 731.0 tokens/s, Avg generation throughputINFO 01-04 14:26:36 [loggers.py:111] INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+ININFO 01-04 14:26:45 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 90.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usINFO 01-04 14:26:46 [loggers.py:111] EngINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 INFINFO 01-04 14:26:55 [loggers.py:111] Engine 000: Avg prompt throughput: 582.9 tokens/s, Avg generation throughINFO 01-04 14:26:56 [loggers.py:111] Engine 000: Avg prompt throughput: 624.1 tokens/s, Avg generation througINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 20INFO 01-04 14:27:03 [loggers.py:111] Engine 000:INFO 01-04 14:27:06 [loggers.py:111] Engine 000: Avg prompt throughput: 587.0 tokens/s, Avg generation throughput: 59.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU INFO 01-04 14:27:13 [loggers.py:111] Engine 000: AvINFO:     10.46.50.19INFO 01-04 14:27:15 [loggers.py:111] Engine 0INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:27:16 [loggers.py:111]INFO 01-04 14:27:23 [loggers.py:111] Engine 000: Avg prompt throughput:INFO 01-04 14:27:25 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.7 tokens/s, Running: 1 reqs, WaiINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:27:33 [loggers.py:111] Engine 000: Avg prompt throughpuINFO 01-04 14:27:35 [loggers.py:111] Engine 000: Avg prompt throughput: 802.7 tokens/s, Avg generation throughput:INFO 01-04 14:27:36 [loggers.py:11INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:27:43 [loggers.py:111] Engine 000: Avg prompt throughpuINFO 01-04 14:27:45 [loggers.py:111] Engine 000: AINFO 01-04 14:27:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generatiINFO 01-04 14:27:53 [loggers.py:111] Engine 000: Avg prompt throughput: 829.6 tokens/s, Avg generation throughput: 31.2 tokens/s, RunningINFO 01-04 14:27:55 [loggers.py:111] Engine 00INFO 01-04 14:27:56 [loggers.py:111]INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:28:03 [loggers.py:111] Engine 000: Avg prompt throughput:INFO 01-04 14:28:05 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: INFO 01-04 14:28:06 [loggers.py:1INFO 01-04 14:28:13 [loggers.py:111] Engine 000: Avg prompt throughput: 615.0 tokens/s, Avg generation throughput: 14.0 tokens/s, RunningINFO 01-04 14:28:15 [loggers.py:111] Engine 000INFO 01-04 14:28:16 [loggers.py:111INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:28:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0INFO 01-04 14:28:25 [loggers.py:111] Engine 000INFO 01-04 14:28:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generatINFO 01-04 14:28:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs,INFO 01-04 14:28:35 [loggers.py:111] Engine INFO 01-04 14:28:36 [loggers.INFO 01-04 14:28:43 [loggers.py:111] Engine 000: Avg prompt throughput: 885.1 tokens/s, Avg generation throughput: 43.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.7%, Prefix cache hit rate: INFO 01-04 14:28:45 [loggers.py:111] EngineINFO 01-04 14:28:46 [loggers.INFO 01-04 14:28:53 [loggers.py:111] Engine 000: Avg prompt throughput: 635.7 tokens/s, Avg generation throughput: 4.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%,INFO 01-04 14:28:56 [loggers.pINFO 01-04 14:29:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%INFO 01-04 14:29:06 [loggers.pyINFO 01-04 14:29:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, INFO 01-04 14:29:16 [loggers.INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:29:23 [loggers.py:111] Engine 000: Avg prompt throughput: 669.3 tokens/s, Avg generation throughput: 48.5 tINFO 01-04 14:29:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generatiINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:29:33 [loggers.py:111] Engine 000: Avg INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:29:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generatiINFO 01-04 14:29:43 [loggers.py:111] Engine 000: Avg prompt throughput: 700.5 tokens/s, Avg generation throughput: 25.4INFO 01-04 14:29:46 [loggers.py:111] Engine 000: Avg prompt throughput: 838.5 tokens/s, Avg generatiINFO 01-04 14:29:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.0 tINFO 01-04 14:29:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPINFO 01-04 14:30:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 toINFO 01-04 14:30:05 [loggers.py:111] EngiINFO 01-04 14:30:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generatiINFO 01-04 14:30:13 [loggers.py:111] Engine 000: Avg prompt throughput: 614.1 tokens/s, Avg generation throughput: 20.4INFO 01-04 14:30:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generationINFO 01-04 14:30:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.INFO 01-04 14:30:26 [loggers.py:111] Engine 000: Avg prompt throughput: 1149.3 tokens/s, Avg generatiINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:30:33 [loggers.py:111] Engine 000: Avg prINFO 01-04 14:30:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, INFO 01-04 14:30:43 [loggers.py:111] Engine 000: Avg prINFO 01-04 14:30:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:30:53 [loggers.py:111] Engine 000: Avg INFO 01-04 14:30:56 [loggers.py:111] Engine 000: Avg prompt throughput: 875.2 tokens/s, Avg generation throughput: 20.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, INFO 01-04 14:31:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokeINFO 01-04 14:31:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generINFO 01-04 14:31:13 [loggers.py:111] Engine 000: Avg prompt throughput: 779.9 tokens/s, Avg generation throughput: 29.9 toINFO 01-04 14:31:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generatINFO 01-04 14:31:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tINFO 01-04 14:31:26 [loggers.py:111] Engine 000: Avg prompt throughput: 799.2 tokens/s, Avg generation throughput: 18.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, INFO 01-04 14:31:33 [loggers.py:111] Engine 000: Avg prINFO 01-04 14:31:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs,INFO 01-04 14:31:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokINFO 01-04 14:31:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generaINFO 01-04 14:31:53 [loggers.py:111] Engine 000: Avg prompt throughput: 688.5 tokens/s, Avg generation throughput: 34.5 INFO 01-04 14:31:56 [loggINFO 01-04 14:32:05 [loggers.py:111] Engine 000: Avg prompt throughput: 650.2 tokens/s, Avg generation throughput: 41.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, PrefINFO 01-04 14:32:06 [loggINFO 01-04 14:32:15 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, PrefINFO 01-04 14:32:16 [loggINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:32:25 [loggers.py:111] Engine 000: Avg prompt throughput: 544.9 tokens/s, Avg generation throughput: 52.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, PINFO 01-04 14:32:26 [loggersINFO 01-04 14:32:35 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPUINFO 01-04 14:32:39 [loggers.py:111] Engine 000: AvgINFO 01-04 14:32:45 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%,INFO 01-04 14:32:56 [loggers.py:111] Engine 000: Avg prompt throughput: 710.3 tokens/s, Avg generation throughput: 33.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 15.6%
+INFO 01-04 14:33:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 15.6%
+INFO:     10.45.190.192:0 - "POST /v1/compleINFO 01-04 14:33:49 [loggers.py:111] Engine 000: Avg prompt throughput: 684.6 tokens/s, Avg generation throughput: 9.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7INFO 01-04 14:33:53 [loggers.py:INFO 01-04 14:33:59 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%INFO 01-04 14:34:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generatINFO 01-04 14:34:09 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 12.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%INFO 01-04 14:34:13 [loggers.pyINFO 01-04 14:34:19 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%,INFO 01-04 14:34:23 [loggers.pINFO 01-04 14:34:59 [loggers.py:111] Engine 000: Avg prompt throughput: 779.7 tokens/s, Avg generation throughput: 44.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cache hit rate: 17.1%
+INFO 01-04 14:35:09 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 17.1%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:35:19 [loggers.py:111] Engine 000:INFO 01-04 14:35:25 [loggers.py:111] Engine 000: Avg prompt throughput: 608.8 tokens/s, Avg generation throughput: 24.4 tokens/s, Running: INFO 01-04 14:35:33 [loggers.py:111] Engine 000: Avg INFO 01-04 14:35:36 [loggers.py:111] Engine 000: Avg prompt throughput: 796.6 tokens/s, Avg generation throughput: 19.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPINFO 01-04 14:35:43 [loggers.py:111] Engine 000: Avg INFO 01-04 14:35:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPINFO:     10.46.50.192:0 - "POINFO:     10.43.30.4:0 INFO 01-04 14:35:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 17.3%
+INFO 01-04 14:36:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 17.3%
+INFO:     10INFO 01-04 14:36:43 [loggers.py:111] Engine 000: Avg prompt throughput: 291.8 tokens/s, Avg generation throughput: 32.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: 16.0%
+INFO 01-04 14:36:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache hit rate: 16.0%
+INFO 01-04 14:37:03 [loggers.py:111] Engine 000: AvgINFO 01-04 14:37:06 [loggers.py:111] Engine 000: Avg prompt throughput: 1216.2 tokens/s, Avg generation throughput: 40.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, INFO 01-04 14:37:13 [loggers.py:111] Engine 000: Avg prINFO 01-04 14:37:16 [loggers.py:111] INFO 01-04 14:37:29 [loggers.py:111] Engine 000: Avg prompt throughput: 297.7 tokens/s, Avg generation throughput: 18.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.2%, Prefix cache hit rate: 17.3%
+INFO 01-04 14:37:39 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.4%, Prefix cache hit rate: 17.3%
+INFO 01-04 14:37:49 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 17.3%
+INFO 01-04 14:37:59 [loggerINFO 01-04 14:38:16 [loggers.py:111] Engine 000: Avg prompt throughput: 363.3 tokens/s, Avg generation throughput: 28.4 tokens/s, Running: 1 reqs, Waiting: 0 reqINFO 01-04 14:38:23 [loggers.py:111] Engine 000: Avg prompINFO 01-04 14:38:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg geneINFO 01-04 14:38:36 [loggers.py:111] Engine 000: Avg prompt throughput: 624.1 tokens/s, Avg generation throughput: 34.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%,INFO 01-04 14:38:36 [loggers.pINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:38:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 28.1 tokINFO 01-04 14:38:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generaINFO 01-04 14:38:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokINFO 01-04 14:39:26 [loggers.py:111] Engine 000: Avg prompt throughput: 649.9 tokens/s, Avg geneINFO 01-04 14:40:46 [loggers.py:111] Engine 000: Avg prompt throughput: 1023.4 tokens/s, Avg generation throughput: 12.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.1%, Prefix cache hit rate: 16.2%
+INFO 01-04 14:40:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.3%, Prefix cache hit rate: 16.2%
+INFO 01-04 14:41:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 16.2%
+INFO 01-04 14:41:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.INFO 01-04 14:42:06 [loggers.py:111] Engine 000: Avg prompt throughput: 1721.1 tokens/s, Avg generation throughput: 23.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.9%, Prefix cache hit rate: 17.7%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:42:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 36.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.7%
+INFO 01-04 14:42:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.7%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:45:56 [loggers.py:111] Engine 000: Avg prompt throughput: 1917.4 tokens/s, Avg generation throughput: 34.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.9%
+INFO 01-04 14:46:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.9%
+INFO 01-04 14:53:06 [loggers.py:111] Engine 000: Avg prompt throughput: 410.1 tokens/s, Avg generation throughput: 42.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 18.8%
+INFO 01-04 14:53:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 48.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.0%, Prefix cache hit rate: 18.8%
+INFO 01-04 14:53:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 18.8%
+INFO 01-04 14:53:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 18.8%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:53:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 11.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.8%
+INFO 01-04 14:53:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.8%
+INFO 01-04 15:05:56 [loggers.py:111] Engine 000: Avg prompt throughput: 513.0 tokens/s, Avg generation throughput: 24.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 18.6%
+INFO 01-04 15:06:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 48.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 18.6%
+INFO 01-04 15:06:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/sINFO 01-04 15:07:55 [loggers.py:111] Engine 000: Avg prompt throughput: 524.6 tokens/s, Avg generation throughput: 38.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 19.4%
+INFO 01-04 15:08:05 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 19.4%
+INFO 01-04 15:08:15 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 19.4%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 15:08:25 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 37.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.4%
+INFO 01-04 15:08:35 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.4%
+ Avg generation throughput: 33.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.6%
+INFO 01-04 15:07:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.6%
+py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 16.7%
+INFO 01-04 15:02:49 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 16.7%
+INFO 01-04 15:02:59 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 16.7%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 15:03:09 [loggers.py:111] Engine 0INFO 01-04 15:10:27 [loggers.py:111] Engine 000: Avg prompt throughput: 553.8 tokens/s, Avg generation throughput: 12.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 12.5%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 15:10:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 28.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 12.5%
+INFO 01-04 15:10:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 12.5%

hf_ip/vllm_gpu4.log

@@ -0,0 +1,305 @@
+/usr/local/lib/python3.10/dist-packages/torch/utils/_pytree.py:185: FutureWarning: optree is installed but the version is too old to support PyTorch Dynamo in C++ pytree. C++ pytree support is disabled. Please consider upgrading optree using `python3 -m pip install --upgrade 'optree>=0.13.0'`.
+  warnings.warn(
+INFO 01-04 13:13:36 [__init__.py:239] Automatically detected platform cuda.
+INFO 01-04 13:13:38 [api_server.py:1043] vLLM API server version 0.8.5
+INFO 01-04 13:13:38 [api_server.py:1044] args: Namespace(host='127.0.0.1', port=8005, uvicorn_log_level='info', disable_uvicorn_access_log=False, allow_credentials=False, allowed_origins=['*'], allowed_methods=['*'], allowed_headers=['*'], api_key=None, lora_modules=None, prompt_adapters=None, chat_template=None, chat_template_content_format='auto', response_role='assistant', ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, enable_ssl_refresh=False, ssl_cert_reqs=0, root_path=None, middleware=[], return_tokens_as_token_ids=False, disable_frontend_multiprocessing=False, enable_request_id_headers=False, enable_auto_tool_choice=False, tool_call_parser=None, tool_parser_plugin='', model='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', task='auto', tokenizer=None, hf_config_path=None, skip_tokenizer_init=False, revision=None, code_revision=None, tokenizer_revision=None, tokenizer_mode='auto', trust_remote_code=True, allowed_local_media_path=None, load_format='auto', download_dir=None, model_loader_extra_config={}, use_tqdm_on_load=True, config_format=<ConfigFormat.AUTO: 'auto'>, dtype='auto', max_model_len=131072, guided_decoding_backend='auto', reasoning_parser=None, logits_processor_pattern=None, model_impl='auto', distributed_executor_backend=None, pipeline_parallel_size=1, tensor_parallel_size=1, data_parallel_size=1, enable_expert_parallel=False, max_parallel_loading_workers=None, ray_workers_use_nsight=False, disable_custom_all_reduce=False, block_size=None, gpu_memory_utilization=0.9, swap_space=4, kv_cache_dtype='auto', num_gpu_blocks_override=None, enable_prefix_caching=None, prefix_caching_hash_algo='builtin', cpu_offload_gb=0, calculate_kv_scales=False, disable_sliding_window=False, use_v2_block_manager=True, seed=None, max_logprobs=20, disable_log_stats=False, quantization=None, rope_scaling=None, rope_theta=None, hf_token=None, hf_overrides=None, enforce_eager=False, max_seq_len_to_capture=8192, tokenizer_pool_size=0, tokenizer_pool_type='ray', tokenizer_pool_extra_config={}, limit_mm_per_prompt={}, mm_processor_kwargs=None, disable_mm_preprocessor_cache=False, enable_lora=None, enable_lora_bias=False, max_loras=1, max_lora_rank=16, lora_extra_vocab_size=256, lora_dtype='auto', long_lora_scaling_factors=None, max_cpu_loras=None, fully_sharded_loras=False, enable_prompt_adapter=None, max_prompt_adapters=1, max_prompt_adapter_token=0, device='auto', speculative_config=None, ignore_patterns=[], served_model_name=['default-model'], qlora_adapter_name_or_path=None, show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, disable_async_output_proc=False, max_num_batched_tokens=None, max_num_seqs=256, max_num_partial_prefills=1, max_long_partial_prefills=1, long_prefill_token_threshold=0, num_lookahead_slots=0, scheduler_delay_factor=0.0, preemption_mode=None, num_scheduler_steps=1, multi_step_stream_outputs=True, scheduling_policy='fcfs', enable_chunked_prefill=None, disable_chunked_mm_input=False, scheduler_cls='vllm.core.scheduler.Scheduler', override_neuron_config=None, override_pooler_config=None, compilation_config=None, kv_transfer_config=None, worker_cls='auto', worker_extension_cls='', generation_config='auto', override_generation_config=None, enable_sleep_mode=False, additional_config=None, enable_reasoning=False, disable_cascade_attn=False, disable_log_requests=True, max_log_len=None, disable_fastapi_docs=False, enable_prompt_tokens_details=False, enable_server_load_tracking=False)
+INFO 01-04 13:13:46 [config.py:717] This model supports multiple tasks: {'score', 'reward', 'generate', 'embed', 'classify'}. Defaulting to 'generate'.
+INFO 01-04 13:13:47 [config.py:2003] Chunked prefill is enabled with max_num_batched_tokens=8192.
+/usr/local/lib/python3.10/dist-packages/torch/utils/_pytree.py:185: FutureWarning: optree is installed but the version is too old to support PyTorch Dynamo in C++ pytree. C++ pytree support is disabled. Please consider upgrading optree using `python3 -m pip install --upgrade 'optree>=0.13.0'`.
+  warnings.warn(
+INFO 01-04 13:13:51 [__init__.py:239] Automatically detected platform cuda.
+INFO 01-04 13:13:54 [core.py:58] Initializing a V1 LLM engine (v0.8.5) with config: model='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', speculative_config=None, tokenizer='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=131072, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto,  device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='auto', reasoning_backend=None), observability_config=ObservabilityConfig(show_hidden_metrics=False, otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=None, served_model_name=default-model, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=True, chunked_prefill_enabled=True, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs=None, pooler_config=None, compilation_config={"level":3,"custom_ops":["none"],"splitting_ops":["vllm.unified_attention","vllm.unified_attention_with_output"],"use_inductor":true,"compile_sizes":[],"use_cudagraph":true,"cudagraph_num_of_warmups":1,"cudagraph_capture_sizes":[512,504,496,488,480,472,464,456,448,440,432,424,416,408,400,392,384,376,368,360,352,344,336,328,320,312,304,296,288,280,272,264,256,248,240,232,224,216,208,200,192,184,176,168,160,152,144,136,128,120,112,104,96,88,80,72,64,56,48,40,32,24,16,8,4,2,1],"max_capture_size":512}
+WARNING 01-04 13:13:55 [utils.py:2522] Methods determine_num_available_blocks,device_config,get_cache_block_size_bytes,initialize_cache not implemented in <vllm.v1.worker.gpu_worker.Worker object at 0x7f17117328f0>
+INFO 01-04 13:13:56 [parallel_state.py:1004] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, TP rank 0
+INFO 01-04 13:13:56 [cuda.py:221] Using Flash Attention backend on V1 engine.
+WARNING 01-04 13:13:56 [topk_topp_sampler.py:69] FlashInfer is not available. Falling back to the PyTorch-native implementation of top-p & top-k sampling. For the best performance, please install FlashInfer.
+INFO 01-04 13:13:56 [gpu_model_runner.py:1329] Starting to load model /data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf...
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+INFO 01-04 13:15:40 [loader.py:458] Loading weights took 104.27 seconds
+INFO 01-04 13:15:41 [gpu_model_runner.py:1347] Model loading took 61.0562 GiB and 104.572266 seconds
+INFO 01-04 13:15:55 [backends.py:420] Using cache directory: /root/.cache/vllm/torch_compile_cache/75e72335d9/rank_0_0 for vLLM's torch.compile
+INFO 01-04 13:15:55 [backends.py:430] Dynamo bytecode transform time: 14.47 s
+INFO 01-04 13:15:59 [backends.py:136] Cache the graph of shape None for later use
+INFO 01-04 13:16:50 [backends.py:148] Compiling a graph for general shape takes 53.37 s
+INFO 01-04 13:17:53 [monitor.py:33] torch.compile takes 67.84 s in total
+INFO 01-04 13:17:54 [kv_cache_utils.py:634] GPU KV cache size: 254,768 tokens
+INFO 01-04 13:17:54 [kv_cache_utils.py:637] Maximum concurrency for 131,072 tokens per request: 1.94x
+INFO 01-04 13:18:32 [gpu_model_runner.py:1686] Graph capturing finished in 39 secs, took 1.21 GiB
+INFO 01-04 13:18:33 [core.py:159] init engine (profile, create kv cache, warmup model) took 171.77 seconds
+INFO 01-04 13:18:33 [core_client.py:439] Core engine process 0 ready.
+WARNING 01-04 13:18:33 [config.py:1239] Default sampling parameters have been overridden by the model's Hugging Face generation config recommended from the model creator. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
+INFO 01-04 13:18:33 [serving_chat.py:118] Using default chat sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
+INFO 01-04 13:18:33 [serving_completion.py:61] Using default completion sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
+INFO 01-04 13:18:33 [api_server.py:1090] Starting vLLM API server on http://127.0.0.1:8005
+INFO 01-04 13:18:33 [launcher.py:28] Available routes are:
+INFO 01-04 13:18:33 [launcher.py:36] Route: /openapi.json, Methods: GET, HEAD
+INFO 01-04 13:18:33 [launcher.py:36] Route: /docs, Methods: GET, HEAD
+INFO 01-04 13:18:33 [launcher.py:36] Route: /docs/oauth2-redirect, Methods: GET, HEAD
+INFO 01-04 13:18:33 [launcher.py:36] Route: /redoc, Methods: GET, HEAD
+INFO 01-04 13:18:33 [launcher.py:36] Route: /health, Methods: GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /load, Methods: GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /ping, Methods: POST, GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /tokenize, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /detokenize, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/models, Methods: GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /version, Methods: GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/chat/completions, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/completions, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/embeddings, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /pooling, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /score, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/score, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/audio/transcriptions, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /rerank, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/rerank, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v2/rerank, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /invocations, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /metrics, Methods: GET
+INFO:     Started server process [1317292]
+INFO:     Waiting for application startup.
+INFO:     Application startup compININFO 01-04 14:00:00 [loggers.py:111] Engine 000: Avg prompt throughput: 71.4 tokens/s, Avg generation throughput: 37.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.4%, Prefix cache hit rate: 0.0IIINFO 01-04 14:00:10 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.6%, Prefix cache hit rate: 0.0ININFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:00:20 [loggers.py:111] Engine 000: Avg prompt throughput: 58.3 tokens/s, Avg generation throughput: 48.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.3%, Prefix cache hit rate: 0INFIINFO 01-04 14:00:30 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.4%, Prefix cache hit rate: 0.0ININFO 01-04 14:00:40 [loggers.py:111] Engine 000: Avg prompt throughput: 122.0 tokens/s, Avg generation throughput: 96.9 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: 0.6%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.INFO 01-04INFO 01-04 14:00:50 [loggers.py:111] Engine 000: Avg prompt throughput: 63.6 tokens/s, Avg generation throughput: 100.8 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: 1.0%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.INFO 01-INFO 01-04 14:00:57 [loggers.py:111] Engine 000: Avg prompt tINFO:   INFO 01-04 14:01:00 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 80.3 tokens/s, Running: 1 rINFO 01-04 14:01:05 [loggers.py:111] Engine 000: Avg prompt throughput: 6INFINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:01:10 [loggers.py:111] Engine 000: Avg prompt throughput: 66.4 tokens/s, Avg generation throughput: 74.0 tokens/s, Running:INFO 01-04 1INFO 01-04 14:01:17 [loggers.py:111] Engine 000: Avg prompt throughIINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:01:20 [loggers.py:111] Engine 000: Avg prompt throughput: 85.7 tokens/INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1"INFO 01-INFO INFO 01-04 14:01:27 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04 14:01:30 [loggers.py:111] Engine 000: Avg prompt throughput: 104.9 tokens/s, Avg generation throughput: 65.9 tokens/s, Running: 2 reqs, WINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 0INFO 01-04 14:01:40 [loggers.py:111] Engine 000: Avg prompt throughput: 95.6 tokens/s, Avg generation throughput: 113.9 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hINFO:    INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1ININFO 01INFO 01-04 14:01:47 [loggers.py:111] Engine 000: Avg prompt INFO 01-04 14INFO 01-04 14:01:50 [loggers.py:111] Engine 000: Avg prompt throughput: 83.3 tokens/s, Avg generation throughput: 90.9 tokens/s, Running: 2 reqsINFO 01-04 14:01:57 [loggers.py:111] Engine 000: Avg prompt INFO 01-04 14:01:57 [loggers.py:111] Engine 000: Avg prompt throughput: 75.7 tokens/s, Avg generation throughput: 85.5 tokens/s, Running: 2 rINFO 01-04 14:02:0INFO 01-04 14:02:06 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04 14:02:07 [loggers.py:111] Engine 000: Avg prompt throughput: 81.INFO 01-04 14:02:07 [loggers.py:111] Engine 000: INFO 01-04 INFO 01-04 14:02:10 [loggers.py:111] Engine 000: Avg prompt throughput: 1INFO 01-0INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-0INFO 01-04 14:02:17 [loggers.py:111] Engine 000: Avg prompt throughput: 73.3 tokens/s, Avg generation throughput: 89.4INFO 01-0INFO 01-04 14:02:20 [loggers.py:111] Engine 000: Avg prompt throughput: INFO:    INFO 01-04INFO 01-04 14:02:27 [loggers.py:111] Engine 000: Avg prompt throughput: 83.8 tokens/s, Avg generation throughput: 122.1 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usageINFO:  INFO 01-04INFO 01-04 14:02:33 [loggers.py:111] Engine 000: Avg prompt throughput: 87.INFO 01-04 14:02:37 [loggers.py:111] Engine 000: Avg prompt throughput: 118.8 tokens/s, Avg generation throughput: 81.1 tokens/s, RunINFO 01-04INFO 01-04 14:02:43 [loggers.py:111] Engine 000: Avg prompt throughput: 105.1 tokens/s, Avg generation throughput: 151.5 tokens/s,INFO 01-0INFO 01-04 14:02:47 [loggers.py:111] Engine 000: Avg prompt throughINFO 01-04 14:02:50 [loggers.py:111] Engine 000: Avg prompt throughput: 101.4 tokens/s, Avg generation throughput: 91.6 tokens/s, Running: 2 reqsINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 2INFO 01-04 INFO 01-04 14:02:56 [loggers.py:111] Engine 000: Avg prompt throughINFO 01-04 14:03:00 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 70.3 tokens/s, Running: 1INFO 01-04 14:03:05 [loggers.py:111] Engine 000: Avg prompt throughput: 111.1 IINFO 01-04 14:03:07 [loggers.py:111] Engine 000: Avg prompt throuINFO 01-04 14:03:10 [loggers.py:111] Engine 000: Avg prompt throughput: 67INFO 01-04 14:03:15 [loggers.py:111] Engine 000: Avg prompt throughput: 73.1 tokens/s, Avg generation throughput: 125.7 tokens/s, Running: 3 reINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:20 [loggers.py:111] Engine 000: Avg prompt throughput: INFO 01-04 14:03:25 [loggers.py:111] Engine 000: Avg prompt throughput: 101.4 tokens/s, Avg generation throughput: 149.7 tokens/s, Running: 3 rIININFO 01-04 14:03:30 [loggers.py:111] Engine 000: Avg prompt throughput: 93.3 tokens/s, Avg generation throughput: 74.1 tokens/s, Running: 2INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-0INFINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:40 [loggers.py:111] Engine 000: Avg prompt throughput: 119.4 tokens/s, Avg generation throughput: 105.1 tokens/s, Running: 2 reqs, WaIINFO 01-04 14:03:47 [loggers.py:111] Engine 000: Avg prompt throughpINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:50 [loggers.py:111] Engine 000: Avg prompt throughput: 134.9 tokensINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1"INFO:   INFO 01-04 14:03:57 [loggers.py:111] Engine 000: Avg prompt throughput: 114.2 tokens/s, Avg generation throughput: 100.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cINFO 01-04 14:04:00 [loggers.py:111] Engine 000: Avg prompt throughput: 141.3 INFO 0INFO 01-04 14:04:07 [loggers.py:111] Engine 000: Avg prompt throughput: 160.8 tokens/s, Avg generation throughput: 90.1 tokens/s, RunnINFO 01-04 14:04:10 [loggers.py:111] Engine 000: Avg prompt throughput: 172.8 tokens/s, Avg generation throughput: 57.7 tokens/s, Running: 2 rINFO 01-04 14:04:15 [loggers.py:111] Engine 000: Avg prompt throughput: INININFO 01-04 14:04:20 [loggers.py:111] Engine 000: Avg prompt throughput: 120.0 tokens/s, Avg generation throughput: 100.1 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hitINFO:     10INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/INFO 01-0INFO 01-04 14:04:27 [loggers.py:111] Engine 000: Avg prompt throughINFO 01-04 14:04:30 [loggers.py:111] Engine 000: Avg prompt throughput: 224.8 tokens/s, Avg generation throughput: 95.4 tokens/s, Running: 2 reqs,INFO INFO 01-04 14:04:37 [loggers.py:111] Engine 000: Avg prompt throughINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:04:40 [loggers.py:111] Engine 000: Avg prompt throughput: 174.6 tokens/s, Avg generation throughput: 92.4 tokens/s, Running: 2 reIINFO 01-INFO 01-04 14:04:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 6INFO:     10.45.190INFO 01-04 14:04:50 [loggers.py:111] Engine 000: Avg prompt throughput: 175.2INFO 01-04 14:04:53 [loggers.py:111] Engine 000: Avg prompt throINFO 01-INFO 01-04 14:04:57 [loggers.py:111] Engine 000: Avg prompt throughpuINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     1INFO 01-04 14:05:03 [loggers.py:111] Engine 000: Avg prompt throughputINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:07 [loggers.py:111] Engine 000: Avg prompt throughput: 138.2 tokens/s, Avg generation throughput: INFO 01-04 14:05:07 [lINFO 01-04 14:05:10 [loggers.py:111] Engine 000: Avg prompt throughput: 195.9 tokINFO 01-04 14:05:17 [loggers.py:111] Engine 000: Avg prompt throughput: 163.3 tokens/s, Avg generation throughput: 83.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usageINFO 01-04 14:05:17 [INFO 01-04 14:05:20 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 1INFO 01-04 14:05:27 [loggers.py:111] Engine 000: Avg prompt throughput: 147.4 tokens/s, Avg generation throughput:INFO 01-04 14:05:27 [loggers.py:111] Engine 000: Avg prompt throughput: 148.2 tokens/sINFO 01-04 14:05:3INFO:     10.43.30.5:0 - "POST /v1/completions HTTPINFO 01-04 14INFO 01-04 14:05:37 [loggers.py:111] Engine 000: Avg prompt throughput: 121.6 tokens/s, Avg generation throughput: 103.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.7%, Prefix caINFO 0INFO:     10.4INFO 01-04 14:05:47 [loggers.py:111] Engine 000: Avg prompt throughput: 144.5 tokens/s, Avg generation throughput: 114.0 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:50 [logINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:57 [loggers.py:111] Engine 000: Avg prompt throughput: 191.0 tokens/s, Avg generation throughput: 100.5 tokens/s, RunninINFOINFO 01-04 14:06:03 [loggers.py:111] Engine 000: Avg prompt throughput: 194.0 tINFO 01-04 14:06:07 [loggers.py:111] Engine 000: Avg prompt throughput: 209.2 tokens/s, Avg generation throughput: 76.9 tokens/s, Running: INFO 01-04 14:06:07 [loggers.py:111] Engine 000: Avg prompt throughput: 210.9 toINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:17 [loggers.py:111] Engine 000: Avg prompt throughput: 226.9 tokens/s, Avg generation throughput: 88.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1INFO 01-04 14:06:20 [loggers.py:111] Engine 000: Avg prompt throughput: 505.5 tokens/s, Avg generaINFO 01-04 14:06:27 [loggers.py:111] Engine 000: Avg prompt throughput: 175.3 tokens/s, Avg generation throughput: 86.4 INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /vINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:37 [loggers.py:111] Engine 000: Avg prompt throughput: 178.1 tokens/s, Avg generation throughput: 107.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KINFO 01-04 14INFO:     10.45.190.192:0 - "POST /v1INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:47 [loggers.py:111] Engine 000: Avg prompt throughput: 244.1 tokens/s, Avg generation throughput: INFO 01-04 14:06:50 [loggers.py:111] Engine 000: Avg prompt throughput: 229.6 tokens/s, Avg generation throughput: 10.1 tokens/s, Running: 1 reqsINFO:     10.46.17.192:0INFO 01-04 14:06:57 [loggers.py:111] Engine 000: INFO 01-04 14:07:00 [loggers.py:111] Engine 000: Avg prompt throughput: 511.7 tokens/s, Avg generation throughput: 87.1 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KVINFO 01-04 14:07:07 [loggers.py:111] Engine 000: Avg prompt throughput: 264.4 tokens/s, Avg generation throughput:INFO 01-04 14:07:10 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completiINFO:     10.46.17.19INFO 01-04 14:07:17 [loggers.py:111] Engine 0INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:20 [loggers.py:111] Engine 000: Avg prompt throughput: 570.5 tokens/s, Avg generation throughINFO:     10.43.30.4:0 - "POST /v1/comINFO 01-04 14:07:25 [loggeINFO 01-04 14:07:27 [loggers.py:111] Engine 00INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:30 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokeINFOINFO:     10.43.30.5:0 - INFO 01-04 14:07:37 [loggers.py:111] Engine 000: Avg prompt throughput: 250.0 tokens/s, Avg generation throughput: 64.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cINFO 01-04 14:07:40 [loggers.py:111] Engine 000INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/coINFO 01-04 14:07:45 [loggerINFO 01-04 14:07:47 [loggers.py:111] Engine 000: Avg prompt throughput: 201.9 tokens/s, Avg generation thrINFO 01-04 14:07:50 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, INFO 01-04 14:07:55 [loggerINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:57 [loggers.py:111] EngINFO 01-04 14:08:00 [loggers.py:111] Engine 000: Avg prompt throughput: 412.5 tokens/s, Avg generation throughput: 79.1 tokens/s, Running: 3 reqs, WaitiINFO:     10.43.30.4:0 - "PINFO 01-04 14:08:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation througINFO 01-04 14:08:10 [loggers.py:111] Engine 000: Avg prompt throughput: 160.1 tokens/s, Avg generation throughput: 102.0 tokens/s, Running: 3 reqs, WaitINFO:     10.46.50.192INFO:     10.46.50.192:0 - "POST /v1/completINFO 01-04 14:08:13 [lINFO 01-04 14:08:17 [loggers.py:111] Engine 000INFO 01-04 14:08:20 [loggers.py:111] Engine 000: Avg prompt throughput: 309.3 tokens/s, Avg generation throughput: 107.1 tokens/s, Running: 3 reqs, WINFO:     10.43.30.3:0 INFO 01-04 14:08:27 [loggers.py:111] Engine 000: Avg prompt throughput: 274.5 tokens/s, Avg generation throughput:INFO 01-04 14:08:30 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/comINFO 01-04 14:08:35 [loggeINFO 01-04 14:08:37 [loggers.py:111] Engine 000: AINFO 01-04 14:08:40 [loggers.py:111] Engine 000: Avg prompt throughput: 281.3 tokens/s, Avg generation throughput: 87.1 tokens/s, Running: 2 reqsINFO 01-04 14:08:43 [loggers.py:111] Engine 000: Avg prompt throughput: 334.6 tokens/s, AvINFO 01-04 14:08:47 [loggers.py:111] Engine 000: AINFO 01-04 14:08:50 [loggers.py:111] Engine 000: Avg prompt throughput: 484.9 tokens/s, Avg generation throughput: 93.2 tokens/s, Running: 3 reqINFO 01-04 14:08:53 [loggINFO:     10.43.30.4:0 - "POST /v1/coINFO:     10.46.50.192:0 - INFO 01-04 14:08:57 [loggers.py:111] Engine 000: Avg prompt throughput: 349.3 tokens/s, Avg generation throughput: 52.8 INFO 01-04 14:09:00 [loggers.py:111] Engine 000: Avg prompt throughput: INFO 01-04 14:09:03 [loggerINFO 01-04 14:09:07 [loggers.py:111] Engine 000: Avg prompt throughput: 224.3 tokens/s, Avg generation throughput: 48.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: INFO:INFO 01-04 14:09:13 [loggers.py:111] Engine 000: Avg prompt throughput: 341.7 tokens/s, Avg geINFO 01-04 14:09:17 [loggers.py:111] Engine 000: Avg prompt throughput: 169.0 tokens/s, Avg generation throughput: 54.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.INFOINFO 01-04 14:09:23 [loggers.py:111] Engine 000: Avg prompt throughput: 409.8 tokens/s, Avg geneINFO 01-04 14:09:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 70.7 tINFINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:33 [loggers.py:111] Engine 000: Avg prompt throughput: 265.8 tokens/s, Avg generINFO 01-04 14:09:37 [loggers.py:111] Engine 000: Avg prompt throughput: 268.1 tokens/s, Avg generation throughput: 25.3INFINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:43 [loggers.py:1INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:47 [loggers.py:111] Engine 000: Avg prompt throughput: 188.7 tokens/s, Avg generation throughput: 68.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: INFO 01-04 14:09:53 [loggers.py:111] Engine 000: Avg prompt throughput: 209.5 tokens/s, Avg generatioINFO 01-04 14:09:57 [loggers.py:111] Engine 000: Avg prompt throughput: 450.7 tokens/s, Avg generation throughput: 55.INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 INFO 01-04 14:10:03 [loggers.py:111] EINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:07 [loggers.py:111] Engine 000: AINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:13 [loggers.py:111] Engine 000: Avg prompt throughput: 336.4 tokens/s, Avg generation INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:17 [loggers.py:111] Engine 000: Avg prompt throughput: 386.2 tokens/s, Avg generation throughput:INFO 01-04 14:10:23 [loggers.py:111] Engine 000: Avg prompt throughput: 253.9 tokens/s, Avg generation thINFO 01-04 14:10:27 [loggers.py:111] Engine 000: Avg prompt throughput: 228.2 tokens/s, Avg generation throughput: 37.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usINFO 01-04 14:10:33 [loggers.py:111] EngINFO 01-04 14:10:37 [loggers.py:111] Engine 000: Avg prompt throughput: 340.4 tokens/s, Avg generation throughput: 52.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache uINFO:INFO 01-04 14:10:40 [loggers.py:111]INFO:     10.46.50.192:0 -INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:43 [loggers.py:111] EnginINFO 01-04 14:10:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 38INFO 01-04 14:10:50 [loggers.py:111] Engine 000: Avg prompt tINFO:     10.46.50.192:0 - "POST /v1/compINFO 01-04 14:10:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.INFO 01-04 14:11:00 [loggers.py:111] Engine 000: Avg prompt throughput: 193.1 tokens/s, Avg generatioINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:11:07 [loggers.py:111] Engine 000: AvgINFO 01-04 14:11:10 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 93.0 tokeINFO 01-04 14:11:13 [loggers.py:111] Engine 0INFO 01-04 14:11:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cachINFO:     10.43.30.5:0 - "POST /v1/completioINFO 01-04 14:11:47 [loggers.py:111] Engine 000: Avg prompt throughput: 424.8 tokens/s, Avg generation throughput: 37.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 13.3%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:11:57 [loggers.py:111] Engine 000: Avg prompt throughput: 307.4 tokens/s, Avg generation throughput: 16.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU INFO 01-04 14:12:03 [loggers.py:111] Engine 000: AvINFO 01-04 14:12:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 48.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KINFO 01-04 14:12:13 [loggers.py:111] Engine 000: AINFO 01-04 14:12:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 48.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU INFO 01-04 14:12:23 [loggers.py:111] Engine 000: AvINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:12:27 [loggers.py:111] Engine 000: Avg prompt throughput: 517.1 tokens/s, Avg generation throughput: 102.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPINFO 01-04 14:12:33 [loggers.py:111] Engine 000: Avg INFO 01-04 14:12:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 95.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 12.4%
+INFO:     10.INFO:     10.46.50.192:0 - "POST /v1/completions HTTPINFO:     10.INFO 01-04 14:12:43 [loggers.py:111] Engine 000: Avg INFO 01-04 14:12:47 [loggers.py:111] Engine 000: Avg prompt throughput: 293.8 tokens/s, Avg generation throughput: 52.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GINFO 01-04 14:12:53 [loggers.py:111] Engine 000: Avg pINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:12:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 30.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, INFO 01-04 14:13:03 [loggers.py:111] Engine 000: Avg prINFO 01-04 14:13:07 [loggers.py:111] Engine 000: Avg prompt throughput: 225.2 tokens/s, Avg generation throughput: 16.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs,INFO 01-04 14:13:13 [loggers.py:111] Engine 000: Avg proINFO 01-04 14:13:17 [loggers.py:111] Engine 000: Avg prompt throughput: 215.6 tokens/s, Avg generation throughput: 70.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs,INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.INFO:     INFO 01-04 14:13:23 [loggers.py:111] Engine 000: Avg proINFO 01-04 14:13:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 69.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.1%, Prefix cache hit rate: 13.2%
+INFO:     10INFO 01-04 14:13:33 [loggers.py:111] Engine 000: Avg pINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:13:37 [loggers.py:111] Engine 000: Avg prompt throughput: 662.8 tokens/s, Avg generINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:13:43 [loggers.py:111] Engine 000: Avg proINFO 01-04 14:13:47 [loggers.py:111] Engine 000: Avg prompt throughput: 294.4 tokens/s, Avg generation throughput: 58.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs,INFO 01-04 14:13:53 [loggers.py:111] Engine 000: Avg prompt throughput: 412.0 tokens/s, Avg generation throughput: 80.9 toINFO 01-04 14:13:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generatINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:03 [loggers.py:111] Engine 000: Avg pINFO 01-04 14:14:07 [loggers.py:111] Engine 000: Avg prompt throughput: 995.4 tokens/s, Avg generation throughput: 60.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:17 [loggers.py:111] Engine 000: Avg prompt throughput: 437.8 tokens/s, Avg generation INFO 01-04 14:14:23 [loggers.py:111] Engine 000: Avg prompt throughput: 556.7 tokens/s, Avg generation throughput: 1INFO 01-04 14:14:27 [loggers.py:111] Engine 000: Avg prompt throughput: 293.1 tokens/s, Avg generation tINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:37 [loggers.py:111] Engine 000: Avg prompt throughput: 382.0 tokens/s, Avg generation throughput: 89.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 12.1%
+INFO:     10.46.1INFO:     10.43.30.4:0 - "POST /v1/completions HTINFO 01-04 14:14:47 [loggers.py:111] Engine 000: Avg prompt throughput: 519.8 tokens/s, Avg generation throughput: 68.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.9%, Prefix cache hit rate: 12.8%
+INFO:     10.46INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:53 [loggers.py:111] Engine 000: Avg pINFO 01-04 14:14:57 [loggers.py:111] Engine 000: Avg prompt throughput: 328.9 tokens/s, Avg generation throughput: 28.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs,INFO 01-04 14:15:03 [loggers.py:111] Engine 000: Avg proINFO 01-04 14:15:07 [loggers.py:111] Engine 000: Avg prompt throughput: 266.3 tokens/s, Avg generation throughput: 60.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 12.3%
+INFO:   INFO 01-04 14:15:13 [loggers.py:111] Engine 000: Avg promptINFO 01-04 14:15:17 [loggers.py:111] Engine 000: Avg prompt throughput: 277.3 tokens/s, Avg generation throughput: 84.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 12.1%
+INFO: INFO 01-04 14:15:23 [loggers.py:111] Engine 000: Avg prompt tINFO 01-04 14:15:27 [loggers.py:111] Engine 000: Avg prompt throughput: 217.5 tokens/s, Avg generation throughput: 101.9 tokens/s, Running: 2 reqs, Waiting: INFO 01-04 14:15:33 [loggers.py:111] Engine 000: Avg prompt thrINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:37 [loggers.py:111] Engine 000: Avg prompt throughput: 472.3 tokens/s, Avg generation throughput: 107.2 tokens/s, Running: 2 reqs, WaitiINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:47 [loggers.py:111] Engine 000: Avg prompt throughput: 443.4 tokens/s, Avg generation throughput: 113.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cache hit rate: 13.8%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:57 [loggers.py:111] Engine 000: Avg prompt throughput: 688.7 tokens/s, Avg generation throughput: 112.5 tokens/s, Running: 1 reqs, WaitiINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+IINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:07 [loggers.py:111] Engine 000: Avg prompt throughput: 682.3 tokens/s, Avg generation throughput: 80.1 tokens/s, Running: 2 reqs, WaiINFO 01-04 14:16:13 [loggers.py:111] Engine 000: Avg prompt throughpuINFO 01-04 14:16:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 94.6 tokens/s, Running: 2 reqs, WaINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:23 [loggers.py:111] Engine 000: Avg prompt throughput: 442.8 tokens/s, Avg generation throughput: 55.0 tokens/s, RunnINFO 01-04 14:16:27 [loggers.py:111] Engine 000: Avg prompt throughput: 396.1 tokens/INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:33 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 14:16:37 [loggers.py:111] Engine 000: Avg prompt throughput: 455.7 tokens/s, Avg generation throughput: 76.7 tokens/s, Running: 2 reqs, WaINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:43 [loggers.py:111] Engine 000: Avg prompt throughput: 736.4 tokens/s, Avg generation throughput: 67.5 tokens/s, RunniINFO 01-04 14:16:47 [loggers.py:111] Engine 000: Avg prompt throughput: 371.7 tokenINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:53 [loggers.py:111] Engine 000: Avg prompt throughput: INFO 01-04 14:16:57 [loggers.py:111] Engine 000: Avg prompt throughput: 706.2 tokens/s, Avg generation throughput: 78.8 tokens/s, Running: 2 reqs, INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:07 [loggers.py:111] Engine 000: Avg prompt throughput: 375.2 tokens/s, Avg generation throughput: 95.7 tokens/s, Running: 2 reqs, INFO 01-04 14:17:13 [loggers.py:111] Engine 000: Avg prompt throughput: 587.3 tokens/s, Avg generation throughput: 48.8 tokens/s, Running: INFO 01-04 14:17:17 [loggers.py:111] Engine 000: Avg prompt throughput: 509.2 toINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:23 [loggers.py:111] Engine 000: Avg prompt throughput: 299INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:27 [loggers.py:111] Engine 000: Avg prompt throughput: 728.8 INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0INFO 01-04 14:17:37 [loggers.py:111] Engine 000: Avg prompt throughput: 261.6 tokens/s, Avg generation throughput: 109.0 tokens/s, Running: 3 INFO 01-04 14:17:43 [loggers.py:111] Engine 000: Avg prompt throughput: 526.8 INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:47 [loggers.py:111] Engine 000: Avg prompt throughput: 66INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:53 [loggers.py:111] Engine 000: Avg prompt throughput: 520.6 tokens/s, Avg generation throughput: 106.3 tokens/s, Running: 2 reqINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:57 [loggers.py:111] Engine 000: Avg prompt throughput: 68INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:03 [loggers.py:111] Engine 000: Avg prompt throughput: 424.7 tokens/s, Avg generation throughput: 60.6 tokens/s, Running: 0 reqINFO 01-04 14:18:07 [loggers.py:111] Engine 000: Avg prompt throughput: 293INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:13 [loggers.py:111] Engine 000: Avg prompt throughput: 717.0 tokens/s, Avg generation throughput: 23.1 tokens/s, Running: 0 rINFO 01-04 14:18:17 [loggers.py:111] Engine 000: Avg prompt throughput: 523.IINFO 01-04 14:18:23 [loggers.py:111] Engine 000: Avg prompt throughput: 406.4 tokens/s, Avg generation throughput: 4.8 tokens/s, Running: 1 reINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:33 [loggers.py:111] Engine 000: Avg prompt throughput: 543.4 tokens/s, Avg generation throughput: 48.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit raINFO 01-0INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:43 [loggers.py:111] Engine 000: Avg prompt throughput: 531.7 tokens/s, Avg generation throughput: 56.5 tokens/s, Running: 1 reqINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:53 [loggers.py:111] Engine 000: Avg prompt throughput: 655.8 tokens/s, Avg generation throughput: 44.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit raINFO 01-0INFO 01-04 14:19:03 [loggers.py:111] Engine 000: Avg prompt throughput: 624.2 tokens/s, Avg generation throughput: 62.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.3%, Prefix cache hit ratINFO:    INFO 01-04 14:19:13 [loggers.py:111] Engine 000: Avg prompt throughput: 702.0 tokens/s, Avg generation throughput: 117.5 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.5%, Prefix cache hit rate: 10.0%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1INFO 01-0INFO 01-04 14:19:23 [loggers.py:111] Engine 000: Avg prompt throughput: 710.5 tokens/s, Avg generation throughput: 115.6 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.8%, Prefix cache hit rate: 9.7%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1"INFO 01-INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:33 [loggers.py:111] Engine 000: Avg prompt throughput: 356.9 tokens/s, Avg generation throughput: 68.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache hit rateINFO 01INFO 01-04 14:19:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 45.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 9INFOINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:53 [loggers.py:111] Engine 000: Avg prompt throughput: 549.1 tokens/s, Avg generation throughput: 45.5 tokens/s, Running: 0 reqs, WINFO 01-04 14:19:57 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 14:20:03 [loggers.py:111] Engine 000: Avg prompt throughput: 580.3 tokens/s, Avg generation throughput: 27.8 tokens/s, Running: 1 reqs, INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO INFO 01-04 14:20:13 [loggers.py:111] Engine 000: Avg prompt throughput: 510.5 tokens/s, Avg generation throughput: 63.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.7%, Prefix cache hit rate: 9INFO 01-04 14:INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1"INFO 01-04 14:20:23 [loggers.py:111] Engine 000: Avg prompt throughput: 472.4 tokens/s, Avg generation throughput: 91.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.2%, Prefix cache hit rate: INFO INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:33 [loggers.py:111] Engine 000: Avg prompt throughput: 724.7 tokens/s, Avg generation throughput: 108.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.4%, Prefix cache hit rate: 8.8ININFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:43 [loggers.py:111] Engine 000: Avg prompt throughput: 567.8 tokens/s, Avg generation throughput: 74.5 tokens/s, Running: 1 reqs, WaINFO 01-04 14:20:47 [loggers.py:111] Engine 000: Avg prompt throughput: 500.0 tokens/s, Avg generation throughput: 88.6 tokens/s, RunniINFO 01-04 14:20:53 [loggers.py:111] Engine 000: Avg prompt throughput: 278.0 tokensINFO 01-04 14:20:57 [loggers.py:111] Engine 000: Avg prompt throughput: 799.4 tokens/s, Avg generation throughput: 95.8 tokens/s, RunniINFO 01-04 14:21:03 [loggers.py:111] Engine 000: Avg prompt throughput: 307.5 tokens/s, Avg generation throughput: 67.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 9.9INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+ININFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 32.7 tokens/s, Running: 0 reqs, Waiting:INFO 01-04 14:21:17 [loggers.py:111] Engine 000: Avg prompt thrINFO 01-04 14:21:23 [loggers.py:111] Engine 000: Avg prompt throughput: 639.5 tokens/s, Avg generation throughput: 40.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 9.7%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:33 [loggers.py:111] Engine 000: Avg prompt throughput: 431.7 tokens/s, Avg generation throughput: 52.3 tokens/s, Running: 1 reqs, Waiting:INFO 01-04 14:21:37 [loggers.py:111] Engine 000: Avg prompt thrINFO 01-04 14:21:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 45.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 9.6%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:53 [loggers.py:111] Engine 000: Avg prompt throughput: 533.9 tokens/s, Avg generation throughput: 73.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 9.4%
+INFO 01-04 14:22:03 [loggers.py:111] Engine 000: Avg prompt throughput: 376.2 tokens/s, Avg generation throughput: 69.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.0%, Prefix cache hit rate: 9.3%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 72.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 9.3%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:23 [loggers.py:111] Engine 000: Avg prompt throughput: 502.6 tokens/s, Avg generation throughput: 63.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 9.2%
+INFO:     INFO 01-04 14:22:27 [loggers.py:111] Engine 000: Avg pINFO 01-04 14:22:33 [loggers.py:111] Engine 000: Avg prompt throughput: 740.4 tokens/s, Avg generation throughput: 29.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GINFO 01-04 14:22:37 [loggers.py:111] Engine 000: Avg prompt throughput: 845.2 tokens/s, Avg generation throughput: 72.8INFO 01-04 14:22:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generationINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:47 [loggers.py:111] Engine 000: AvgINFO 01-04 14:22:53 [loggers.py:111] Engine 000: Avg prompt throughput: 697.8 tokens/s, Avg generation throughput: 0.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 8.9%
+INFO 01-04 14:23:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 8.9%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:13 [loggers.py:111] Engine 000: Avg prompt throughput: 830.0 tokens/s, Avg generation throughput: 30.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 8.7%
+INFO 01-04 14:23:23 [loggers.py:111] Engine 000: Avg prompt throughput: 188.6 tokens/s, Avg generation throughput: 84.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 8.6%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:33 [loggers.py:111] Engine 000: Avg prompt throughput: 822.6 tokens/s, Avg generation throughput: 95.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 8.6%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:43 [loggers.py:111] Engine 000: Avg prompt throughput: 641.4 tokens/s, Avg generation throughput: 86.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.9%, Prefix cache hit rate: 8.4%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:53 [loggers.py:111] Engine 000: Avg prompt throughput: 483.3 tokens/s, Avg generation throughput: 93.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.8%, Prefix cache hit rate: 8.3%
+INFO 01-04 14:24:03 [loggers.py:111] Engine 000: Avg prompt throughput: 496.9 tokens/s, Avg generation throughput: 100.8 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.1%, Prefix cache hit rate: 8.2%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 111.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 8.2%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:23 [loggers.py:111] Engine 000: Avg prompt throughput: 1449.0 tokens/s, Avg generation throughput: 46.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.8%, Prefix cache hit rate: 8.0%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 71.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 8.0%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:43 [loggers.py:111] Engine 000: Avg prompt throughput: 581.9 tokens/s, Avg generation throughput: 49.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cINFO 01-04 14:24:47 [loggers.py:111] Engine 00INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:53 [loggers.py:111] Engine 000: Avg prompt throughput: 759.9 tokens/s, Avg generation throuINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:57 [loggers.py:111] Engine INFO 01-04 14:25:03 [loggers.py:111] Engine 000: Avg prompt throughput: 728.7 tokens/s, Avg generation throughput: 50.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.1%, Prefix cache hit rate: 7.6%
+INFO:     10.46.17.192INFO 01-04 14:25:07 [loggers.py:111] Engine INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughINFO 01-04 14:25:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughINFO 01-04 14:25:23 [loggers.py:111] Engine 000: Avg prompt throughput: 686.7 tokens/s, Avg generation throughput: 36.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cacheINFO 01-04 14:25:27 [loggers.py:111] EnginINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:33 [loggers.py:111] Engine 000: Avg prompt throughput: 893.4 tokens/s, Avg generation throughpINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:37 [loggers.py:111] EngINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: INFO 01-04 14:25:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation thINFO 01-04 14:25:53 [loggers.py:111] Engine 000: Avg prompt throughput: 353.3 tokens/s, Avg generation throughput:INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:57 [loggers.py:111] EngINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:26:03 [loggers.py:111] Engine 000: Avg prompt throughput: 358.7 tokens/s, Avg generation throughputINFO 01-04 14:26:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throINFO 01-04 14:26:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput:INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:26:17 [loggers.py:111] EINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:26:23 [loggers.py:111] Engine 000: Avg prompt throughput: 786.4 tokens/s, Avg generation throughput: INFO 01-04 14:26:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation thINFO 01-04 14:26:33 [loggers.py:111] Engine 000: Avg prompt throughput: 885.5 tokens/s, Avg generation throughput:INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:26:37 [loggers.py:111] Engine 000: Avg prompt throughput: 210.6 tokens/s, Avg generation thINFO 01-04 14:26:43 [loggers.py:111] Engine 000: Avg prompt throughput: 816.2 tokens/s, Avg generation throughput:INFO 01-04 14:26:47 [loggers.py:111] Engine 000: Avg prompt throughput: 546.2 tokens/s, Avg generation throughput: 55.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU INFO 01-04 14:26:53 [loggers.py:111] Engine 000: AvINFO 01-04 14:26:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 94.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPUINFO 01-04 14:27:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.5 tINFO 01-04 14:27:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generatINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:27:13 [loggers.py:111] Engine 000: Avg INFO 01-04 14:27:17 [loggers.py:111] Engine 000: Avg prompt throughput: 643.6 tokens/s, Avg generation throughput: 89.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:27:23 [loggers.py:111] Engine 000: AvINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:27:27 [loggers.py:111] Engine 000: Avg prompt throughput: 655.0 tokens/s, Avg generation INFO 01-04 14:27:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:27:37 [loggers.py:11INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:27:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 22.2INFO 01-04 14:27:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generatioINFO 01-04 14:27:53 [loggers.py:111] Engine 000: Avg prompt throughput: 673.6 tokens/s, Avg generation throughput: 29INFO 01-04 14:27:57 [loggers.py:111] Engine 000: Avg prompt throughput: 895.3 tokens/s, Avg generatioINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:28:03 [loggers.py:111] Engine 000: AINFO 01-04 14:28:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU INFO 01-04 14:28:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.INFO 01-04 14:28:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generationINFO 01-04 14:28:23 [loggers.py:111] Engine 000: Avg prompt throughput: 1382.2 tokens/s, Avg generation throughput: 3INFO 01-04 14:28:27 [loggers.py:111] Engine 000: Avg prompt throughput: 985.3 tokens/s, Avg generation throughput: 35.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KINFO 01-04 14:28:33 [loggers.py:111] Engine 000: AINFO 01-04 14:28:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPUINFO 01-04 14:28:43 [loggers.py:111] Engine 000: Avg prompt throughput: 686.6 tokens/s, Avg generation throughput: 3INFO 01-04 14:28:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation tINFO 01-04 14:28:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46INFO 01-04 14:28:57 [loggers.py:111] Engine 000: Avg prompt throughput: 1087.4 tokens/s, Avg generatiINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:29:03 [loggers.py:111] Engine 000: Avg prINFO 01-04 14:29:07 [loggers.py:111] Engine 000: Avg prompt throughput: 666.7 tokens/s, Avg generation throughput: 44.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs,INFO 01-04 14:29:13 [loggers.py:111] Engine 000: Avg proINFO 01-04 14:29:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 91.0 tokens/s, Running: 2 reqs, Waiting: 0 reqsINFO 01-04 14:29:23 [loggers.py:111] Engine 000: Avg promINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:29:27 [loggers.py:111] Engine 000: Avg prompt throughput: 1102.8 tokens/s, Avg geINFO 01-04 14:29:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 91.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.9%, PreINFO 01-04 14:29:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg gINFO 01-04 14:29:43 [loggers.py:111] Engine 000: Avg prompt throughput: 877.5 tokens/s, Avg generation throughput: 63.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cINFO 01-04 14:29:47 [loggers.py:111] Engine 000: Avg prompt throughput: 481.5 tokens/s, Avg generation throughpuINFO 01-04 14:29:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 60.3 tokensINFO 01-04 14:29:57 [loggers.py:111] Engine 000: AINFO 01-04 14:29:57 [loggers.py:111] EnginINFO 01-04 14:30:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cacheINFO 01-04 14:30:07 [loggers.py:111] Engine 000: Avg prompt throughput: 886.6 tokens/s, Avg generation throuINFO 01-04 14:30:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughputINFO 01-04 14:30:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation thrINFO 01-04 14:30:23 [loggers.py:111] Engine 000: Avg prompt throughput: 459.3 tokens/s, Avg generation throughputINFO 01-04 14:30:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:30:33 [loggers.py:111] Engine 00INFO 01-04 14:30:37 [loggers.py:111] Engine 000: Avg prompt throughput: 662.1 tokens/s, Avg generation throughput: 46.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KVINFO 01-04 14:30:43 [loggers.py:111] Engine 000: Avg prompt throughput: 899.9 tokens/s, Avg generation throughputINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:30:47 [loggers.py:111] EngiINFO 01-04 14:30:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.6 tokens/s, RuINFO 01-04 14:30:57 [loggers.py:111] Engine 00INFO 01-04 14:30:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation thrINFO 01-04 14:31:03 [loggers.py:111] Engine 000: Avg prompt throughpINFO:     10.43.30.4:0 - "POST /v1/completioINFO 01-04 14:31:07 [loggers.py:111] Engine 000: Avg prompt throughput: 731.0 tokens/s, Avg generation INFO 01-04 14:31:13 [loggers.py:111] Engine 000: Avg prompt throughput: 520.4 tokens/s, Avg generation throughput: 1INFO 01-04 14:31:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation tINFO 01-04 14:31:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, RunnINFO 01-04 14:31:27 [loggers.py:111] Engine 000: INFO 01-04 14:31:27 [loggers.py:111INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:31:33 [loggers.py:111] Engine 000: Avg prompt throughpINFO 01-04 14:31:37 [loggers.py:111] Engine 000:INFO 01-04 14:31:37 [loggers.py:111] Engine 000: Avg prompt throughput: 434.5 tokens/s, Avg generatiINFO 01-04 14:31:43 [loggers.py:111] Engine 000: Avg prompt throughpuINFO 01-04 14:31:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.7 tokens/s, Running: 1 reqs, WINFO 01-04 14:31:53 [loggers.py:111] Engine 000: Avg prompt throughput: 962.1 tokens/s, Avg generation throughput: 34.4 INFO:     10.46.5INFO 01-04 14:31:57 [loggers.py:111] Engine 000: INFO 01-04 14:31:57 [loggers.py:1INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:32:03 [loggers.py:111] Engine 000: Avg prompt throughput:INFO:     10.43.30.4:0 - "POST /v1/completions INFO 01-04 14:32:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generatioINFO 01-04 14:32:13 [loggers.py:111] Engine 000: Avg prompt throughput: 597.0 tokens/s, Avg generation throughput: 39INFO 01-04 14:32:37 [loggers.py:111] Engine 000: Avg prompt throughput: 694.9 tokens/s, Avg generation throughput: 26.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 14.3%
+INFO 01-04 14:3INFO 01-04 14:32:57 [loggers.py:111] Engine 000: Avg prompt throughput: 843.1 tokens/s, Avg generation throughput: 18.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 13.7%
+INFO 01-04 14:33:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 43.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 13.7%
+INFO 01-04 14:33:17 [loggers.py:111] Engine 000: Avg prompt throughput: 1062.2 tokens/s, Avg generation throughput: 87.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.1%, Prefix cache hit rate: 13.5%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:33:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 55.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cacINFO 01-04 14:33:27INFO 01-04 14:33:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/INFO 01-04 14:34:03 [loggers.py:111] Engine 000: Avg prompt throughput: 770.6 tokens/s, Avg generation throughput:INFO 01-04 14:34:07INFO 01-04 14:34:47 [loggers.py:111] Engine 000: Avg prompt throughput: 497.2 tokens/s, Avg generation throughput: 36.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 13.4%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:34:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 5.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 13.4%
+INFO 01-04 14:35:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 13.4%
+INFO 01-04 14:36:17 [loggers.py:111] Engine 000: Avg prompt throughput: 927.3 tokens/s, Avg generation throughput: 9.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.7%, Prefix cache hit rate: 14.4%
+INFO 01-04 14:36:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.9%, Prefix cache hit rate: 14.4%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:36:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 8.1 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.4%
+INFO 01-04 14:36:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.4%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:37:37 [loggers.py:111] Engine 000: Avg prompt throughput: 1222.8 tokens/s, Avg generation throughput: 37.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.1%
+INFO 01-04 14:37:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.1%
+INFO 01-04 14:38:57 [loggers.py:111] Engine 000: Avg prompt throughput: 1079.6 tokens/s, Avg generation throughput: 1.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.3%, Prefix cache hit rate: 13.9%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:39:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 20.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 13.9%
+INFO 01-04 14:39:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 13.9%
+INFO 01-04 14:41:07 [loggers.py:111] Engine 000: Avg prompt throughput: 1279.1 tokens/s, Avg generation throughput: 43.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.2%, Prefix cache hit rate: 14.6%
+INFO 01-04 14:41:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.4%, Prefix cache hit rate: 14.6%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:41:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 22.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.6%
+INFO 01-04 14:41:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.6%
+INFO 01-04 14:42:27 [loggers.py:111] Engine 000: Avg prompt throughput: 524.2 tokens/s, Avg generation throughput: 32.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 14.5%
+INFO 01-04 14:42:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 14.5%
+INFO 01-04 14:42:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 48.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 14.5%
+INFO 01-04 14:42:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 14.5%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:43:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 33.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.5%
+INFO 01-04 14:43:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.5%
+INFO 01-04 14:45:47 [loggers.py:111] Engine 000: Avg prompt throughput: 456.2 tokens/s, Avg generation throughput: 40.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.0%, Prefix cache hit rate: 14.4%
+INFO 01-04 14:45:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 48.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 14.4%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:46:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 36.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.4%
+INFO 01-04 14:46:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.4%
+INFO 01-04 14:51:47 [loggers.py:111] Engine 000: Avg prompt throughput: 519.4 tokens/s, Avg generation throughput: 19.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 15.0%
+INFO 01-04 14:51:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 48.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 15.0%
+INFO 01-04 14:52:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 48.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 15.0%
+INFO 01-04 14:52:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 15.0%
+INFO 01-04 14:52:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 15.0%
+INFO 01-04 14:52:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 15.0%
+INFO 01-04 14:52:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 15.0%
+INFO 01-04 14:52:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 15.0%
+INFO 01-04 14:53:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix INFO 01-04 14:58:43 [loggers.py:111] Engine 000: Avg prompt throughput: 459.2 tokens/s, Avg generation throughput: 46.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.0%, Prefix cache hit rate: 11.6%
+INFO 01-04 14:58:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 11.6%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:59:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 34.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 11.6%
+INFO 01-04 14:59:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 11.6%
+00: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.5%, Prefix cache hit rate: 15.0%
+INFO 01-04 14:54:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.7%, Prefix cache hit rate: 15.0%
+INFO:    INFO 01-04 15:08:45 [loggers.py:111] Engine 000: Avg prompt throughput: 536.7 tokens/s, Avg generation throughput: 20.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 18.8%
+INFO 01-04 15:08:55 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 18.8%
+INFO 01-04 15:09:05 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.4 tokens/s, Running: 1 reqs, WaitiINFO 01-04 15:09:46 [loggers.py:111] Engine 000: Avg prompt throughput: 547.7 tokens/s, Avg generation throughput: 10.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 16.9%
+INFO 01-04 15:09:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 16.9%
+INFO 01-04 15:10:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 16.9%
+INFO 01-04 15:10:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 16.9%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 15:10:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 36.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.9%
+INFO 01-04 15:10:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.9%
+tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.2%
+cache hit rate: 20.2%
+INFO 01-04 14:45:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.7%, Prefix cache hit rate: 20.2%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:46:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 19.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 20.2%
+INFO 01-04 14:46:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 20.2%
+INFO 01-04 14:47:26 [loggers.py:111] Engine 000: Avg prompt throughput: 826.0 tokens/s, Avg generation throughput: 26.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 20.0%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:47:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 1.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 20.0%
+INFO 01-04 14:47:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 20.0%
+INFO 01-04 14:57:46 [loggers.py:111] Engine 000: Avg prompt throughput: 561.0 tokens/s, Avg generation throughput: 26.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 20.3%
+INFO 01-04 14:57:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 48.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 20.3%
+INFO 01-04 14:58:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 48.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 20.3%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:58:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 29.4 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 20.3%
+INFO 01-04 14:58:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 20.3%

hf_ip/vllm_gpu5.log

@@ -0,0 +1,273 @@
+/usr/local/lib/python3.10/dist-packages/torch/utils/_pytree.py:185: FutureWarning: optree is installed but the version is too old to support PyTorch Dynamo in C++ pytree. C++ pytree support is disabled. Please consider upgrading optree using `python3 -m pip install --upgrade 'optree>=0.13.0'`.
+  warnings.warn(
+INFO 01-04 13:13:39 [__init__.py:239] Automatically detected platform cuda.
+INFO 01-04 13:13:42 [api_server.py:1043] vLLM API server version 0.8.5
+INFO 01-04 13:13:42 [api_server.py:1044] args: Namespace(host='127.0.0.1', port=8006, uvicorn_log_level='info', disable_uvicorn_access_log=False, allow_credentials=False, allowed_origins=['*'], allowed_methods=['*'], allowed_headers=['*'], api_key=None, lora_modules=None, prompt_adapters=None, chat_template=None, chat_template_content_format='auto', response_role='assistant', ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, enable_ssl_refresh=False, ssl_cert_reqs=0, root_path=None, middleware=[], return_tokens_as_token_ids=False, disable_frontend_multiprocessing=False, enable_request_id_headers=False, enable_auto_tool_choice=False, tool_call_parser=None, tool_parser_plugin='', model='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', task='auto', tokenizer=None, hf_config_path=None, skip_tokenizer_init=False, revision=None, code_revision=None, tokenizer_revision=None, tokenizer_mode='auto', trust_remote_code=True, allowed_local_media_path=None, load_format='auto', download_dir=None, model_loader_extra_config={}, use_tqdm_on_load=True, config_format=<ConfigFormat.AUTO: 'auto'>, dtype='auto', max_model_len=131072, guided_decoding_backend='auto', reasoning_parser=None, logits_processor_pattern=None, model_impl='auto', distributed_executor_backend=None, pipeline_parallel_size=1, tensor_parallel_size=1, data_parallel_size=1, enable_expert_parallel=False, max_parallel_loading_workers=None, ray_workers_use_nsight=False, disable_custom_all_reduce=False, block_size=None, gpu_memory_utilization=0.9, swap_space=4, kv_cache_dtype='auto', num_gpu_blocks_override=None, enable_prefix_caching=None, prefix_caching_hash_algo='builtin', cpu_offload_gb=0, calculate_kv_scales=False, disable_sliding_window=False, use_v2_block_manager=True, seed=None, max_logprobs=20, disable_log_stats=False, quantization=None, rope_scaling=None, rope_theta=None, hf_token=None, hf_overrides=None, enforce_eager=False, max_seq_len_to_capture=8192, tokenizer_pool_size=0, tokenizer_pool_type='ray', tokenizer_pool_extra_config={}, limit_mm_per_prompt={}, mm_processor_kwargs=None, disable_mm_preprocessor_cache=False, enable_lora=None, enable_lora_bias=False, max_loras=1, max_lora_rank=16, lora_extra_vocab_size=256, lora_dtype='auto', long_lora_scaling_factors=None, max_cpu_loras=None, fully_sharded_loras=False, enable_prompt_adapter=None, max_prompt_adapters=1, max_prompt_adapter_token=0, device='auto', speculative_config=None, ignore_patterns=[], served_model_name=['default-model'], qlora_adapter_name_or_path=None, show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, disable_async_output_proc=False, max_num_batched_tokens=None, max_num_seqs=256, max_num_partial_prefills=1, max_long_partial_prefills=1, long_prefill_token_threshold=0, num_lookahead_slots=0, scheduler_delay_factor=0.0, preemption_mode=None, num_scheduler_steps=1, multi_step_stream_outputs=True, scheduling_policy='fcfs', enable_chunked_prefill=None, disable_chunked_mm_input=False, scheduler_cls='vllm.core.scheduler.Scheduler', override_neuron_config=None, override_pooler_config=None, compilation_config=None, kv_transfer_config=None, worker_cls='auto', worker_extension_cls='', generation_config='auto', override_generation_config=None, enable_sleep_mode=False, additional_config=None, enable_reasoning=False, disable_cascade_attn=False, disable_log_requests=True, max_log_len=None, disable_fastapi_docs=False, enable_prompt_tokens_details=False, enable_server_load_tracking=False)
+INFO 01-04 13:13:50 [config.py:717] This model supports multiple tasks: {'reward', 'score', 'embed', 'generate', 'classify'}. Defaulting to 'generate'.
+INFO 01-04 13:13:50 [config.py:2003] Chunked prefill is enabled with max_num_batched_tokens=8192.
+/usr/local/lib/python3.10/dist-packages/torch/utils/_pytree.py:185: FutureWarning: optree is installed but the version is too old to support PyTorch Dynamo in C++ pytree. C++ pytree support is disabled. Please consider upgrading optree using `python3 -m pip install --upgrade 'optree>=0.13.0'`.
+  warnings.warn(
+INFO 01-04 13:13:55 [__init__.py:239] Automatically detected platform cuda.
+INFO 01-04 13:13:58 [core.py:58] Initializing a V1 LLM engine (v0.8.5) with config: model='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', speculative_config=None, tokenizer='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=131072, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto,  device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='auto', reasoning_backend=None), observability_config=ObservabilityConfig(show_hidden_metrics=False, otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=None, served_model_name=default-model, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=True, chunked_prefill_enabled=True, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs=None, pooler_config=None, compilation_config={"level":3,"custom_ops":["none"],"splitting_ops":["vllm.unified_attention","vllm.unified_attention_with_output"],"use_inductor":true,"compile_sizes":[],"use_cudagraph":true,"cudagraph_num_of_warmups":1,"cudagraph_capture_sizes":[512,504,496,488,480,472,464,456,448,440,432,424,416,408,400,392,384,376,368,360,352,344,336,328,320,312,304,296,288,280,272,264,256,248,240,232,224,216,208,200,192,184,176,168,160,152,144,136,128,120,112,104,96,88,80,72,64,56,48,40,32,24,16,8,4,2,1],"max_capture_size":512}
+WARNING 01-04 13:13:58 [utils.py:2522] Methods determine_num_available_blocks,device_config,get_cache_block_size_bytes,initialize_cache not implemented in <vllm.v1.worker.gpu_worker.Worker object at 0x7f9660db2860>
+INFO 01-04 13:13:59 [parallel_state.py:1004] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, TP rank 0
+INFO 01-04 13:13:59 [cuda.py:221] Using Flash Attention backend on V1 engine.
+WARNING 01-04 13:13:59 [topk_topp_sampler.py:69] FlashInfer is not available. Falling back to the PyTorch-native implementation of top-p & top-k sampling. For the best performance, please install FlashInfer.
+INFO 01-04 13:13:59 [gpu_model_runner.py:1329] Starting to load model /data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf...
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+INFO 01-04 13:15:40 [loader.py:458] Loading weights took 100.81 seconds
+INFO 01-04 13:15:41 [gpu_model_runner.py:1347] Model loading took 61.0562 GiB and 101.065262 secondININFO 01-04 13:15:55 [backends.py:420] Using cache directory: /root/.cache/vllm/torch_compile_cache/75e72335d9/rank_0_0 for vLLM's torch.compile
+INFO 01-04 13:15:55 [backends.py:430] Dynamo bytecode transform time: 14.50 s
+INFO 01-04 13:15:59 [backends.py:136] Cache the graph of shape None for later usININFO 01-04 13:16:50 [backends.py:148] Compiling a graph for general shape takes 53.44 ININFO 01-04 13:17:53 [monitor.py:33] torch.compile takes 67.94 s in total
+INFO 01-04 13:17:54 [kv_cache_utils.py:634] GPU KV cache size: 254,768 tokens
+INFO 01-04 13:17:54 [kv_cache_utils.py:637] Maximum concurrency for 131,072 tokens per request: 1.94ININFO 01-04 13:18:33 [gpu_model_runner.py:1686] Graph capturing finished in 39 secs, took 1.21 GiB
+INFO 01-04 13:18:33 [core.py:159] init engine (profile, create kv cache, warmup model) took 171.95 seconds
+INFO 01-04 13:18:33 [core_client.py:439] Core engine process 0 ready.
+WARNING 01-04 13:18:33 [config.py:1239] Default sampling parameters have been overridden by the model's Hugging Face generation config recommended from the model creator. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
+INFO 01-04 13:18:33 [serving_chat.py:118] Using default chat sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
+INFO 01-04 13:18:33 [serving_completion.py:61] Using default completion sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
+INFO 01-04 13:18:33 [api_server.py:1090] Starting vLLM API server on http://127.0.0.1:8006
+INFO 01-04 13:18:33 [launcher.py:28] Available routes are:
+INFO 01-04 13:18:33 [launcher.py:36] Route: /openapi.json, Methods: HEAD, GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /docs, Methods: HEAD, GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /docs/oauth2-redirect, Methods: HEAD, GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /redoc, Methods: HEAD, GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /health, Methods: GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /load, Methods: GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /ping, Methods: POST, GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /tokenize, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /detokenize, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/models, Methods: GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /version, Methods: GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/chat/completions, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/completions, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/embeddings, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /pooling, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /score, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/score, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/audio/transcriptions, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /rerank, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/rerank, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v2/rerank, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /invocations, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /metrics, Methods: GET
+INFO:     Started server process [1317730]
+INFO:     Waiting for application startup.
+INFO:     Application startup compIININFO 01-04 14:00:01 [loggers.py:111] Engine 000: Avg prompt throughput: 55.5 tokens/s, Avg generation throughput: 37.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.4%, Prefix cache hit rate: 0.IIIINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:00:11 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.2 tokens/s, Running: 0 reqs, WaitINFO 01-04 14:00:15 [loggers.py:111] Engine 000: Avg prompt throughINFO 01-04 14:00:21 [loggers.py:111] Engine 000: Avg prompt throughput: 51.9 tokens/s, Avg generation throughput: 5.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.2%, Prefix cache hit rate: 0.0%IINFO 01-04 14:00:31 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.4%, Prefix cache hit rate: 0.0%
+IINFOINFO 01-04 14:00:36 [loggers.py:111] Engine 000: Avg INFO 01-04 14:00:41 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs,ININFO 01-04 14:00:46 [loggers.py:111] Engine 000: Avg INFO 01-04 14:00:51 [loggers.py:111] Engine 000: Avg prompt throughput: 137.8 tokens/s, Avg generation throughput: 82.9 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GINFO 01-04 14:00:56 [loggers.py:111] Engine 000: Avg INFO 01-04 14:01:01 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 141.2 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.0%, Prefix cache hit rate: 0.0%
+INFO:     INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1"INFO:    ININFO 01-04 14:01:06 [loggers.py:111] Engine 000: Avg prINFO 01-04 14:01:11 [loggers.py:111] Engine 000: Avg prompt throughput: 62.2 tokens/s, Avg generation throughput: 71.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/INFO 01-04 14:01:21 [loggers.py:111] Engine 000: Avg prompt throughput: 83.6 tokens/s, Avg generation throughput: 110.3 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 0.0%
+INFO: INFO 01-04 14:01:26 [loggers.py:111] Engine 000: Avg prompt tINFO 01-04 14:01:31 [loggers.py:111] Engine 000: Avg prompt throughput: 66.5 tokens/s, Avg generation throughput: 133.3 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 0.4%
+INFO:INFOINFO:     10.43.30.5:0 - "POST /v1/completions HTTPINFOINFO 01-04 14:01:41 [loggers.py:111] Engine 000: Avg prompt throughput: 102.8 tokens/s, Avg generation throughput: 108.4 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate:INFO: INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:01:51 [loggers.py:111] Engine 000: Avg prompt throughput: 91.3 tokens/s, Avg generation throughput: 149.0 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate:INFO 01-04 14:01:57 [loggers.py:111] Engine 000: Avg prompt througINFOINFO 01-04 14:02:01 [loggers.py:111] Engine 000: Avg prompt throughput: 98.0 tokens/s, Avg generation throughput: 106.6 tokens/s, Running: 3 reqs, INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01INFO 01-04 14:02:11 [loggers.py:111] Engine 000: Avg prompt throughput: 93.4 tokens/s, Avg generation throughput: 102.9 tokens/s, Running: 3 reqs, Waiting: 0 INFO 01-04 14:02:16 [loggers.py:111] Engine 000: Avg INFO 01-04 14:02:17 [loggers.py:111] Engine 000: Avg prompt throughpuININFO 01-04 14:02:21 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg geneINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1INFO:   INFINFO 01-04 14:02:26 [loggers.py:111] Engine 000: Avg ININFO 01-04 14:02:27 [loggers.py:111] Engine 000: Avg prompt throughput:INFO 01-04 14:02:31 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 73.1 tokens/s, Running: 1 reqs, INFO:     INFO 01-04 14:02:36 [loggers.py:111] Engine 000: Avg prompINFINFO 01-04 14:02:41 [loggers.py:111] Engine 000: Avg prompt throughput: 132.5 tokens/s, Avg generation throughput: 83.6 tokens/s, Running: 2 reqs, Waiting: 0INFO 01-04 14:02:45 [loggers.py:111] Engine 000: Avg prompt tINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:02:51 [loggers.py:111] Engine 000: Avg prompt throughput: 116.2 tokens/s, Avg generation throughput: 102.9 tokens/s, Running: 2 reqs, WaitingINFO 01-04 14:02:56 [loggers.py:111] Engine 000: Avg prompt throINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:01 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 79.8 tokens/s, Running: 1 reqs, WaitingINFO 01-04 14:03:06 [loggers.py:111] Engine 000: Avg prompINFO 0INFO 01-04 14:03:11 [loggers.py:111] Engine 000: Avg prompt throughput: 83.3 tokens/s, Avg generation throughput: 81.7 tokens/s, Running: 2 reqs, Waiting:INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200INFOINFO 01-04 14:03:16 [loggers.py:111] Engine 000: Avg pINFO 0INFO 01-04 14:03:21 [loggers.py:111] Engine 000: Avg prompt throughput: 176.4 tokens/s, Avg generation throughput: 70.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.4%, Prefix cache hit rate: 7.3%
+INFO: IINFO 01-04 14:03:26 [loggers.py:111] Engine 000: AvINFO 0INFO 01-04 14:03:31 [loggers.py:111] Engine 000: Avg prompt throughput: 79.2 tokens/s, Avg generation throughput: 71.9 tokens/s, Running: 2 reqs, Waiting: 0 reINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 2INFO: INFINFO 01-04 14:03:36 [loggers.py:111] Engine 000: Avg INFOINFO 01-04 14:03:41 [loggers.py:111] Engine 000: Avg prompt throughput: 118.3 tokens/s, Avg generation throughput: 90.1 tokens/s, Running: 2 reqs, Waiting: 0 rIINFO 01-04 14:03:46 [loggers.py:111] Engine 000: Avg prompt throughput: 110.7 tokens/s, Avg generation throughput: 56.INFOINFO 01-04 14:03:51 [loggers.py:111] Engine 000: Avg prompt throughput: 144.2 tokens/s, Avg geneINFO 01-04 14:03:56 [loggers.py:111] Engine 000: Avg prompt throughput: 143.6 tokens/s, Avg generation throughput: 114.8 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: INFINFO:INFO:     10.46.17.192:0 -INFO:     10.46.50.192:0 - "POST /vINFO 01-04 14:04:01 [loggers.py:111] Engine 000: Avg prompt throughput: 78.9 tokens/s, Avg generINFO 01-04 14:04:05 [loggers.py:111] Engine 000: Avg prompt throughput: 107.3 tokens/s, Avg generation throughput: 127.9 tINFO 01-04 14:04:11 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generatINFO 01-04 14:04:16 [loggers.py:111] Engine 000: Avg prompt throughput: 117.2 tokens/s, Avg generation throughput: 107.3 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: ININFO 01-04 14:04:21 [loggers.py:INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:04:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 140.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usaINFO INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:04:31 [loggers.py:111INFO 01-04 14:04:36 [loggers.py:111] Engine 000: Avg prompt throughput: 145.7 tokens/s, Avg generation throughput: 100.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usINFOINFO 01-04 14:04:41 [loggers.py:111INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:04:46 [loggers.py:111] Engine 000: Avg prompt throughput: 188.8 tokens/s, Avg generation throughINFOINFO 01-04 14:04:51 [loggers.py:111] Engine 000: Avg prompt throughput: 125.8 tokens/s, Avg generation thINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 2INFO 0INFO 01-04 14:04:56 [loggers.py:111] Engine 000INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:01 [loggers.py:111] Engine 000: Avg prompt throughput: 93.6 tokens/s, Avg generation thrINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:06 [loggers.py:111] Engine 00INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:11 [loggers.py:111] Engine 000: Avg prompt throughput: 202.6 tokens/s, Avg generation thrINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:16 [loggers.py:111] Engine 000: Avg prompt throughput: 205.8 tokens/s, Avg generation throughpuINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:21 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generatiINFO 01-INFO 01-04 14:05:26 [loggers.py:111] Engine 000: Avg prompt throughput: 181.3 tokens/s, Avg generation throughINFO 01-04 14:05:31 [loggers.py:111] Engine 000: Avg prompt throughput: 248.7 tokens/s, Avg generaINFO:     INFO:     10.45.190.192:0 - "POST /v1/INFO:     10.43.30.3:0 - "POSINFO 01-04 14:05:36 [loggers.py:111] EngineINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:41 [loggers.py:111] Engine 000: Avg prompt throughput: 128.7 tokens/s, Avg generation INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04INFO 01-04 14:05:46 [loggers.py:111] EngiINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:51 [loggers.py:111] Engine 000: Avg prompt throughput: 161.7 tokens/s, Avg generatioINFO 01-04INFO 01-04 14:05:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation througINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:01 [loggers.py:1INFO:     INFO 01-04 14:06:06 [loggers.py:111] Engine 000: Avg prompt throughput: 294.6 tokens/s, Avg generation throughput: 75.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cachINFO:     10.46.50.192:0 - "POST /v1/compleINFO:     10.46.50.192:INFO 01-04 14:06:11 [loggers.pyINFO 01-04 1INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:16 [loggers.py:111] Engine 000: Avg prompt throughput: 207.7 tokens/s, Avg generation througINFO 01-04 14:06:21 [loggers.py:111] Engine 000: Avg prompt throughput: 375.8 tokens/s, Avg generINFO 01-04 14:06:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 INFO 01-04 14:06:26 [loggers.py:111] Engine 000: Avg prompt throughput: 216.5 tokens/s, Avg generation throughpINFO 01-04 14:06:31 [loggers.pINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:INFO 01-04 14:06:36 [loggers.py:111] Engine 000: AvINFO 01-04 14:06:36 [loggers.py:111] Engine 000: Avg promptINFO 01-04 14:06:41 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughINFO 01-04 14:06:46 [loggers.py:111] Engine 000: Avg prompt throughput: 233.9 tokens/s, Avg generation throughput: 67.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cachINFO 01-04 14:06:51 [loggers.py:111] Engine 000: Avg prompt throughput: 312.3 tokens/s, Avg generation througINFO 01-04 14:06:56 [loggers.py:111] Engine 000: Avg prompt throughput: 241.7 tokens/s, Avg generation throughINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:01 [loggers.py:111] Engine 000: Avg prompt throughput: 210.1 tokens/s, Avg generation throuINFO 01-04 14:07:06 [loggers.py:111] Engine 000: Avg prompt throughput: 208.1 tokens/s, Avg generation throughpINFO 01-04 14:07:06 [loggers.py:111] Engine 000: Avg prompt tINFINFO:     10.46.17.192:0 - "POST /v1/compleINFO:     10.46.17.192:INFO 01-04 14:07:11 [loggers.py:INFO:     1INFO 01-04 14:07:16 [loggers.py:111] Engine 000: Avg prompt throughput: 204.3 tokens/s, Avg generation throughpuINFO 01-04INFO 01-04 14:07:18 [loggers.py:111] Engine 000: Avg INFO 01-04 14:07:21 [loggers.py:111] EngineINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:26 [loggers.py:111] Engine 000INFO 01-04 14:07:26 [loggers.py:111] Engine 000: Avg prompt throughput: 177.INFO:     10.46.17.192:0 - "POST /v1/completions INFO 01-04 14:07:31 [loggers.py:111] Engine INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:36 [loggers.py:111] Engine 000: Avg prompt throughput: 522.4 tokens/s, Avg generation thrINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:41 [loggers.py:111] Engine 000: Avg prompt throughput: 328.3 tokens/s, AvgINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:43 [INFO 01-04 14:07:46 [loggers.py:111] Engine 000INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1"ININFO 01-04 14:07:51 [loggers.py:111] Engine 000: Avg prompt throughput: 208.1 tokens/s, Avg generation throughpINFO:     10.45.190.192:0 - "POST /v1/completINFO 01-04 14:07:53 [lINFO 01-04 14:07:56 [loggers.py:111] EngiINFO 01-04 14:08:01 [loggers.py:111] Engine 000: Avg prompt throughput: 506.5 tokens/s, Avg generation throughput: 74.1 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 14.5%
+INFO:     10.46.50.192:0INFO 01-04 14:08:06 [loggers.py:111] Engine 000: Avg prompt throughput: 303.7 tokens/s, Avg generation throuINFO 01-04 14:08:11 [loggers.py:111] Engine 000: Avg prompt throughput: 299.0 tokens/s, INFO 01-04 14:08:13 [loggers.py:111] Engine 000: Avg prompt throughput: 307.5 tokens/s, INFO 01-04 14:08:16 [loggers.py:111] Engine 000: Avg prompt throughput: 224.7 tokens/s, Avg generationINFO 01-INFO:     10.45INFO INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:23 [logINFO 01-04 14:08:26 [loggers.py:111] Engine 000: Avg prompt throughput: 211.0 tokens/s, Avg generatINFO 01-04 14:08:27 [loggerINFO 01-04 14:08:28 [loggers.py:111] Engine 000: INFO 01-04 14:08:31INFO 01-04 14:08:33 [loggers.py:111] Engine 000: Avg prompt throughput: 257.0 tokens/s, AINFO 01-04 14:08:36 [loggers.py:111] Engine 000: Avg prompt throughput: 331.1 tokens/s, Avg generatioINFO 01-INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:41 INFO 01-04 14:08:43 [loggeINFO 01-04 14:08:46 [loggers.py:111] Engine 000: Avg prompt throughput: 416.6 tokens/s, Avg generation throughput: 60.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV caINFO 01-04 14:08:51 [loggers.py:111] Engine 000: Avg prompt throughput: 571.5 tokens/s, Avg generation throughpuINFO 01-04 14:08:56 [loggers.py:111] Engine 000: Avg prompt throughput: 386.5 tokens/s, Avg generation throughput: 73.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV caINFO 01-04 14:09:01 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/INFO 01-04 14:09:03 [loggersINFO:     10.46.50.192:0 - "POST /v1/completionsINFO 01-04 14:09:0INFO 01-04 14:09:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation thrINFO 01-04 14:09:11 [loggers.py:111] Engine 000: Avg prompt throughput: 436.0 tokensINFO 01-04 14:09:13 [loggersINFO 01-04 14:09:16 [loggers.py:111] Engine 000: Avg prompt throughput: 177.9 tokens/s, Avg generation throughput:INFO 01-04 14:09:16 [loggers.py:111] Engine 000: Avg promINFO 01-04 14:09:21 [loggers.py:111] Engine 000INFO:     10.45.190.192:0 - "POST /vINFO 01-04 14:09:23 [loggers.py:111] Engine 000: Avg prompt throughput: 339.9 tINFO:     10.43.INFO 01-04 14:09:26 [loggers.py:111] EnINFO 01-04 14:09:31 [loggers.py:111] Engine 000: Avg prompt throughput: 183.5 tokensINFO 01-04 14:09:33 [loggers.py:111] Engine 000: Avg prompt throughput: 300.8 tokens/s, Avg geneINFO 01-04 14:09:36 [loggers.py:111] Engine 000: Avg prompt throughput: 255.0 tokens/s, Avg generation INFO 01-04 14:09:41 [loggers.py:111] Engine 000: Avg prompt throughput: 624.3 tokenINFO 01-04 14:09:43 [loggers.py:INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:46 [loggers.py:111] Engine 000: Avg prompt throughput: 263.6 tokens/s, Avg generation throughput: 5INFO 01-04 14:09:46 [loggers.py:111] Engine 000: Avg pINFO:     10.4INFO:     10.43.30.3:0 - "POST /v1/INFO 01-04 14:09:56 [loggers.py:111] Engine 000: Avg prompt throughput: 248.6 tokens/s, Avg generation throughput: 62.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usaINFO:     10.46.17.192:0 - "POST /v1/coINFO 01-04 14:10:06 [loggers.py:111] Engine 000: Avg prompt throughput: 445.9 tokens/s, Avg generation throughput: 105.3 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.5%, Prefix cache hit rate: 9.8%
+INFO:     10.46.17.192:0 - INFO:     10.46.17.192:0 - "POST /v1/coINFO:     10.46.50.192:0 - INFO 01-04 14:10:13 [loggers.py:111] EnINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:16 [loggers.py:111] INFO 01-04INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:23 [loggers.py:111] EngiINFO 01-04 14:10:26 [loggers.py:111] Engine 000: Avg prompt throughput: 401.7 tokens/s, Avg generatioINFO 01-04 14:10:31 [loggers.py:111] Engine 000: Avg prompt throughput: 393.8 tokens/s, Avg generation throughput: 66.INFO:     10.43.30.3:0 - "INFO 01-04 14:10:33 [loggers.py:111] EINFO 01-04 14:10:36 [loggers.py:111] INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:41 [loggers.py:111] Engine 000: Avg prompt throughput: 240.5INFO:     10.46.17.192:0 - "POST /v1/coINFO 01-04 14:10:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation thrINFO 01-04 14:10:51 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: INFO 01-04 14:10:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KVINFO:     10.46.50.192:0 - "POST /v1/completions INFO 01-04 14:11:06 [loggers.py:111] Engine 000: Avg prompt throughput: 401.6 tokens/s, Avg generation throughput: 79.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usagINFO 01-04 14:11:06 [loggers.py:111] EINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:11:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 79.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KINFO 01-04 14:11:21 [loggers.py:111] Engine 000: AINFO:     10.43.30.3:INFO 01-04 14:11:43 [loggers.py:111] Engine 000: Avg prompt throughput: 317.7 tokens/s, Avg generation throughput: 18.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: 8.8%
+INFO 01-04 14:11:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache hit rate: 8.8%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:12:03 [loggers.py:111] Engine 000: Avg prompt throughput: 324.5 tokens/s, Avg generation throughput: 64.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.4%, Prefix cache hit rate: 8.5%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:12:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 1.3 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 8.5%
+INFO 01-04 14:12:23 [loggers.py:111] Engine 000: Avg prompt throughput: 367.4 tokens/s, Avg generation throughput: 29.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 8.2%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:12:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 33.4 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 8INFOINFO 01-04 14:12:43 [loggers.py:111] Engine 000: Avg prompt throughput: 590.1 tokens/s, Avg generation throughput: 57.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 7.8%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:12:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 90.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 7.INFINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:13:03 [loggers.py:111] Engine 000: Avg prompt throughput: 340.6 tokens/s, Avg generation throughput: 41.0 tokens/s, Running: 1 reqs, WaitINFO 01-04 14:13:07 [loggers.py:111] Engine 000: Avg prompt throughINFO 01-04 14:13:13 [loggers.py:111] Engine 000: Avg prompt throughput: 415.2 tokens/s, Avg generation throughput: 57.6 tokens/s, Running: 2 reqs, WaiINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:13:17 [loggers.py:111] Engine 000: Avg prompt throughpINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:13:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s,INFO 01-04 14:13:27 [loggers.py:111] Engine 000: Avg prompt throughput: 369.0 tokens/s, Avg generation throughput: 37.7 tokens/s, RINFO 01-04 14:13:33 [loggers.py:111] Engine 000: Avg prompt throughput: 292.2 tokens/s, Avg generation throughput: 13.8 tokens/s, Running: 1 reqs, WaitingINFO 01-04 14:13:37 [loggers.py:111] Engine 000: Avg prompt throINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:13:43 [loggINFO 01-04 14:13:47 [loggers.py:111] Engine 000: Avg prompt throughput: 520.6 tokens/s, Avg generation throughput: 59.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.7%, PreINFO 01-04 14:13:53 [loggers.py:111] Engine 000: Avg prompt throughput: 255.1 tokens/s, Avg generation throughput: 23.3 tokens/s, Running: 1 reqs, Waiting: 0 INFO 01-04 14:13:57 [loggers.py:111] Engine 000: Avg prompt INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:03 [loggers.py:111] Engine 000: Avg prompt throughput: 410.0 tokens/s, Avg genINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:07 [loggers.py:111] Engine 000: Avg promptINFO 01-04 14:14:13 [loggers.py:111] Engine 000: Avg prompt throughput: 406.3 tokens/s, Avg generation throughput: 72.7 tokens/s, Running: 2 reqs, Waiting: 0 INFO 01-04 14:14:17 [loggers.py:111] Engine 000: Avg prompt throughput: 382.7 tokens/s, Avg generation throughput: 35.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.7%, INFO 01-04 14:14:23 [loggers.INFO 01-04 14:14:27 [loggers.py:111] Engine 000: Avg prompt throughput: 666.4 tokens/s, Avg generation throughput: 94.7 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.6%,INFO 01-04 14:14:33 [loggers.py:111] Engine 000: Avg prompt throughput: 565.0 tokens/s, Avg generation throughput: 85.4 tokens/s, Running: 2 reqs, Waiting: 0 reqINFO 01-04 14:14:37 [loggers.py:111] Engine 000: Avg promINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:43 [loggers.py:111] Engine 000: Avg prompt throughput: 713.0 tokens/s, Avg generINFO 01-04 14:14:47 [loggers.py:111] Engine 000: Avg prompt throughput: 506.8 tokens/s, Avg generation throughput: 69.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%,INFO:     10.46.50.192:0 - "POINFO 01-04 14:14:57 [loggers.py:111] Engine 000: Avg prompt throughput: 605.6 tokens/s, Avg generation throughput: 98.3 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.2%, Prefix cache hit rate: 10.8%
+INFO:     10.46.50.192:0 - "POST /INFO 01-04 14:15:03 [loggers.py:111] Engine 000: Avg prompt throughput: 521.6 tokens/s, Avg geneINFO 01-04 14:15:07 [loggers.py:111] Engine 000: Avg prompt throughput: 390.4 tokens/s, Avg generation throughput: 111.3 toINFO 01-04 14:15:13 [loggers.py:111] Engine 000: Avg prompt throughput: 337.7 tokens/s, Avg generation throughput: 72.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs,INFO 01-04 14:15:17 [loggers.py:111] Engine 000: Avg prINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:23 [loggers.py:111] Engine 000: Avg prompt throughput: 991.8 tokens/s, Avg generation throughput: 109.5 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.7%, Prefix cache hit rate: 9.7%
+INFO:     1INFO 01-04 14:15:27 [loggers.py:111] Engine 000: Avg prINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:33 [loggers.py:111] Engine 000: Avg prompt throughput: 209.0 tokens/s, Avg generation throughput: 82.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, INFO 01-04 14:15:37 [loggers.py:111] Engine 000: Avg pINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:43 [loggers.py:111] Engine 000: Avg prompt throughput: 343.3 tokens/s, Avg generation throughput: 105.4 tokens/s, Running: 2 reqs, Waiting: 0 reqsINFO 01-04 14:15:47 [loggers.py:111] Engine 000: Avg prompt throughput: 284.9 tokens/s, Avg generation throughput: 42.2 toINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:53 [loggers.pyINFO 01-04 14:15:57 [loggers.py:111] Engine 000: Avg prompt throughput: 343.0 tokens/s, Avg generation throughput: 58.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9INFO 01-04 14:16:03 [loggers.py:INFO 01-04 14:16:07 [loggers.py:111] Engine 000: Avg prompt throughput: 287.0 tokens/s, Avg generation throughput: 100.6 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 4INFO 01-04 14:16:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation INFO 01-04 14:16:17 [loggers.py:111] Engine 000: Avg prompt throughput: 676.6 tokens/s, Avg generation throughput: INFO 01-04 14:16:23 [loggers.py:111] Engine 000: Avg prompt throughput: 501.5 tokens/s, Avg generation throughput: 105.9 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:27 [loggers.py:111] Engine 000: AvINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:33 [loggers.py:111] Engine 000: Avg prompt throughput: 657.6 tokens/s, Avg generation INFO 01-04 14:16:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 87.INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:47 [loggers.py:111] Engine 000: Avg prompt throughput: 489.5 tokens/s, Avg generation throughput: 119.6 tokens/s, Running: 2 reqs, Waiting: 0 reqsINFO 01-04 14:16:46 [INFO 01-04 14:16:53 [loggers.py:111]INFO 01-04 14:16:57 [loggers.py:111] Engine 000: Avg prompt throughput: 863.9 tokens/s, Avg generation throughput: 145.6 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usagINFO:     10.43.30.5:0 - "POST /v1/comINFO:     10.46.17.192:0 -INFO 01-04 14:17:03 [loggers.py:111] Engine 000: Avg prompt throughput: 542.5 tokens/s, Avg generation throughput: 74.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV caINFO 01-04 14:17:07 [loggers.py:111] Engine 00INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:13 [loggers.py:111] Engine 000: Avg prompt throughput: 515.9 tokens/s, Avg generation throughput: 73.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cacINFO:     10.43.30.4:0 - "INFO 01-04 14:17:16 [loggers.py:111] Engine 000: Avg prompt throughput: 517.0 tokenINFO 01-04 14:17:23 [loggers.py:111] Engine 000: Avg prompt throughput: 624.5 tokens/s, Avg generation throughput: 56.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cINFO 01-04 14:17:27 [loggers.py:111] Engine 000INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation througINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:37 [loggers.py:111] Engine INFO 01-04 14:17:43 [loggers.py:111] Engine 000: Avg prompt throughput: 452.7 tokens/s, Avg generation throughput: 46.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cachINFO 01-04 14:17:47 [loggers.py:111] Engine 000: Avg prompt throughput: 530.9 tokens/s, Avg generation throuINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:53 [loggers.py:111] Engine 000: Avg prompt throughput: 441.2 tokens/s, Avg generation throughputINFO:   INFO:     10.46.50INFO 01-04 14:17:56 [loggers.py:111] EngINFO 01-04 14:17:57 [loggers.py:111] INFINFO 01-04 14:18:03 [loggers.py:111] Engine 000: Avg prompt throughput: 517.5 tokens/s, Avg generation throughput: 24.8 tokens/s, Running: INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:13 [loggers.py:111] Engine 000: Avg prompt throughput: 617.8 tokens/s, Avg generation throughput: 65.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usagINFO 01-04 14:18:17 [loggers.py:111] EINFO 01-04 14:18:23 [loggers.py:111] Engine 000: Avg prompt throughput: 426.4 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 12.1%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:33 [loggers.py:111] Engine 000: Avg prompt throughput: 429.2 tokens/s, Avg generation throughput: 74.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6INFO:     10.46.50.192:0 - "POSTINFO 01-04 14:18:43 [loggers.py:111] Engine 000: Avg prompt throughput: 668.4 tokens/s, Avg generation throughput: 93.8 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.6%, Prefix cache hit rate: 12.5%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 120.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.9%, Prefix cache hit rate: 12.5%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:03 [loggers.py:111] Engine 000: Avg prompt throughput: 1429.8 tokens/s, Avg generation throughput: 101.2 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.8%, Prefix cache hit rate: 13.0%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:13 [loggers.py:111] Engine 000: Avg prompt throughput: 580.7 tokens/s, Avg generation throughput: 115.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.0%, Prefix cache hit rate: 12.7%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:23 [loggers.py:111] Engine 000: Avg prompt throughput: 664.3 tokens/s, Avg generation throughput: 77.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2INFO 01-04 14:19:25 [loggers.py:11INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:33 [loggers.py:111] Engine 000: Avg prompt throughput: 837.0 tokens/s, Avg generation throughput: 50.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usINFO:     10.43.30.3:0 - "POST /v1/complINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 30.4 tINFO 01-04 14:19:45 INFO 01-04 14:19:46 [loggers.py:111] Engine 000: Avg prompt throughput: 1119.2INFO 01-04 14:19:53 [loggers.py:111] Engine 000: Avg prompt throughput: 595.3 tokens/s, Avg generation throughput: 18.2 tINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:55INFO 01-04 14INFO 01-04 14:20:03 [loggers.py:111] Engine 000: Avg prompt throughput: 760.4 tokens/s, Avg generation throughput: 63.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.6INFO 01-04 14:20:05 [loggers.py:INFO:     10.43.30.5:0 - "PINFO 01-04 14:20:07 [loggINFO 01-04 1INFO 01-04 14:20:13 [loggers.py:111] Engine 000: Avg prompt throughput: 804.7 tokens/s, Avg generation throughput: 65.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.4%INFO 01-04 14:20:15 [loggers.py:111] Engine 000: Avg prompt throughput: 334.0 tokens/s, Avg generINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:23 [loggers.py:111] Engine 000: Avg prompt throughput: 204.6 tokens/s, Avg generation throughput: 60.3 tokenINFO 01-04 14:20:25 [loggers.py:111] Engine 000: Avg prompt INFO 01-04 14:20:27 [loggers.py:11INFO 01-04 14:20:33 [loggers.py:111] Engine 000: Avg prompt throughput: 437.5 tokens/s, Avg generation throughput: 65.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, PrINFO 01-04 14:20:35 [loggerINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:43 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04 14:20:45 [loggers.py:111] Engine 000: Avg prompt throughput: 622.2 tokens/s, Avg generation throughput: 62.4 tokens/s, Running: 2 reqs, Waiting:INFO 01-04 14:20:53 [loggers.py:111] Engine 000: Avg prompt throughput: 559.0 tokens/s, Avg generation throughput: 55.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.9%, PrefINFO 01-04 14:20:55 [loggINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:03 [loggers.py:111] Engine 000: Avg prompt throughput: 632.5 tokens/s, Avg generation throughput: 64.5 tokens/sINFO 01-04 14:21:05 [loggers.py:111] Engine 000: Avg prompt throughput: 512.3 tokens/s, AvgINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 73.2 tokens/s, RINFO 01-04 14:21:15 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, AvINFO:     10.43.30.5:0 - "POST /v1/cINFO 01-04 14:21:17 [loggersINFO 01-04 14:21:23 [loggers.py:111] Engine 000: Avg prompt throuINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:25 [loggers.py:111] Engine 000: Avg prompt INFO:     10.43.30.5:0 - "POINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:33 [loggers.py:111] Engine 000: Avg prompt throughput: 423.7 tokens/s, Avg generation throughput: 25.1 tokens/s, INFO 01-04 14:21:35 [loggers.py:111] Engine 000: Avg prompINFO 01-04 14:21:37 [loggers.pyINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:43 [loggers.py:111] Engine 000: Avg prompt thrINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:45 [loggers.py:111] Engine 000: Avg prompt throughput: 566.0 tokensINFO 0INFO 01-04 14:21:53 [loggers.py:111] Engine 000: Avg prompt throughput: 600.6 tokens/s, Avg generation throughput: 38.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, PreINFO 01-04 14:21:55 [loggeINFO:     10.46.50.192:0 - "POST /v1INFO:     10.46.17.192:0 - "POINFO 01-04 14:22:03 [loggers.py:111] Engine 000: Avg prompt throughput: 313.1 tokens/s, Avg generation throughput: 34.4 tokens/INFO 01-04 14:22:05 [loggers.py:111] Engine 000: Avg prompt throughput: 671.4 tokens/s, Avg INFO 01-04 14:22:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, PrefiINFO 01-04 14:22:15 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/sINFO: INFO 01-04 14:22:23 [loggers.py:111] Engine 000: Avg prompt throughput: 679.5 tokens/s, Avg generation throughput: 46.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, PrefINFO 01-04 14:22:25 [loggers.py:111] Engine 000: Avg prompt throughput: 902.9 tokens/s, Avg INFO 01-04 14:22:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 4.4 tokens/s, INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:35 [lINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:43 [loggers.py:111] Engine 000: Avg prompt throughput: 773.5 tokens/s, Avg generation throughput: 21.2 tokens/s, RINFO 01-04 14:22:45 [loggers.py:111] Engine 000: Avg prompt throughput: 664.9 tokens/s, INFO 01-04 14:22:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, RunnINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:55 [INFO 01-04 14:23:03 [loggers.py:111] Engine 000: Avg prompt throughput: 794.3 tokens/s, Avg generation throughput: 42.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cINFO 01-04 14:23:05 [INFO 01-04 14:23:13 [loggers.py:111] Engine 000: Avg prompt throughput: 561.7 tokens/s, Avg generation throughput: 71.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.8%, PrefixINFO 01-04 14:23:15 [loINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:23 [loggers.py:111] Engine 000: Avg prompt throughput: 767.6 tokens/s, Avg generation throughput: 89.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cachINFO 01-04 14:23:2INFO 01-04 14:23:33 [loggers.py:111] Engine 000: Avg prompt throughput: 509.8 tokens/s, Avg generation throughput: 76.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.5%, Prefix cacINFO 01-04 14:23:35 [loggers.py:111] Engine 000: Avg prompt throughput: 702.6 tokens/INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:43 [loggers.py:111] Engine 000: Avg prompt throughput: 505.9 tokens/s, Avg generation throughput: 48.9 tokens/s, RunnINFO 01-04 14:23:45 [loggers.py:111] Engine 000: Avg prompt throughput: 765.8 tokens/INFO 01-04 14:23:53 [loggers.py:111] Engine 000: Avg prompt throughput: 540.5 tokens/s, Avg generation throughput: 48.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cacheINFO 01-04 14:23:INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:03 [loggers.py:111] Engine 000: Avg prompt throughput: 810.6 tokens/s, Avg generation throughput: 75.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.5%, Prefix cache hINFO 01-04 14:2INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 85.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hINFO:     10.46INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:23 [loggers.py:111] Engine 000: Avg prompt throughput: 870.9 tokens/s, Avg generation throughput: 46.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cache hit rate: 16.5%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:33 [loggers.py:111] Engine 000: Avg prompt throughput: 772.6 tokens/s, Avg generation throughput: 58.9 tokens/s, Running: 0 reINFO 01-04 14:24:35 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 INFO 01-04 14:24:43 [loggers.py:111] Engine 000: Avg prompt throughput: 511.3 tokens/s, Avg generation throughput: 27.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit INFO 01-04 1INFO 01-04 14:24:53 [loggers.py:111] Engine 000: Avg prompt throughput: 758.4 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.3%, Prefix cache hit INFO 01-04 1INFO 01-04 14:25:03 [loggers.py:111] Engine 000: Avg prompt throughput: 539.0 tokens/s, Avg generation throughput: 91.0 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.8%, Prefix cache hiINFO 01-04 14:INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 108.4 tokens/s, Running: 2INFO 01-04 14:25:15 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:23 [loggers.py:111] Engine 000: Avg prompt throughput: 826.9 tokens/s, Avg generation throughput: 113.8 tokens/s, RunningINFO 01-04 14:25:25 [loggers.py:111] Engine 000: Avg prompt throughput: 777.0 tokeINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01INFO 01-04 14:25:35 [loggers.py:111] Engine 000: Avg prompt throughput: 747.1 tokens/s, Avg generation throughput: 102.0 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.2%, Prefix cache hiINFO: INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:43 [loggers.py:111] Engine 000: Avg prompt throughput: INFO 01-04 14:25:45 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 111.7 tokens/s, Running: 1 reINFINFO 01-04 14:25:53 [loggers.py:111] Engine 000: Avg prompt throughput: 6INFO 01-04 14:25:55 [loggers.py:111] Engine 000: Avg prompt throughput: 545.0 tokens/s, Avg generation throughput: 90.4 tokens/s, Running: 2 reqsINFO 01-04 14:26:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 45.4 tokens/s, Running: 1 reINFO 01-04 14:26:05 [loggers.py:111] Engine 000: Avg prompt throughput: 377.INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:26:13 [loggers.py:111] Engine 000: Avg prompt throughput: 48INFO 01-04 14:26:15 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 50.2 tokens/s, Running: 1 rINFO INFO 01-04 14:26:23 [loggers.py:111] Engine 000: Avg prompt throughput: 251.7 tokens/s, Avg generation throughput: 41.4 tokens/s, Running:INFO 01-04 14:26:25 [loggers.py:111] Engine 000: Avg prompt throughput: 671.8 tokINFO 01-04 14:26:33 [loggers.py:111] Engine 000: Avg prompt throughput: 505.7 tokens/s, Avg generation throughput: 55.5 tokens/s, Running:INFO 01-04 14:26:35 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokenINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1INFO 01-0INFO 01-04 14:26:43 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 14:26:45 [loggers.py:111] Engine 000: Avg prompt throughput: 606.1 tokens/s, Avg generation throughput: 77.7 tokens/s, Running: 2INFO 01INFO 01-04 14:26:53 [loggers.py:111] Engine 000: Avg prompt throughput: 950.0 tokens/s, Avg generation throughput: 60.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.5%, Prefix cache INFO 01-04 14:26INFO 01-04 14:27:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 90.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.8%, Prefix cache INFO 01-04 14:27:05 [loggers.py:111] Engine 000: Avg prompt throughput: 5INFO 01-04 14:27:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0INFO 01-04 14:27:13 [loggers.py:111] Engine 000: Avg prompt throughput: 649.2 tokens/s, Avg generation throughput: 44.7 tokens/s, RunningINFO 01-04 14:27:15 [loggers.py:111] Engine 000: Avg prompt throughput:INFO 01-04 INFO 01-04 14:27:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 43.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache INFO 01-04 14:27INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1INFO 01-0INFO 01-04 14:27:33 [loggers.py:111] Engine 000: Avg prompt throughput: 896.8 tokens/s, Avg generation throughput: 51.0 tokens/s, RunningINFO 01-04 14:27:35 [loggers.py:111] Engine 000: Avg prompt throughput: 3INFO 01-0INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:27:43 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 14:27:45 [loggers.py:111] Engine 000: Avg prompt throughput: 842.8 tokens/s, Avg generation throughput: 68.9 tokens/s, Running: INFO 01-INFO 01-04 14:27:53 [loggers.py:111] Engine 000: Avg prompt throughput: 596.3 tokens/s, Avg generation throughput: 21.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hiINFO 01-04 14:INFO 01-04 14:28:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hiINFO 01-04 14:INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:28:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 20.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit ratININFO 01INFO 01-04 14:28:23 [loggers.py:111] Engine 000: Avg prompt throughput: 453.7 tokens/s, Avg generation throughput: 34.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit ratININFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01INFO 01-04 14:28:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 2.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 1INFO INFO 01-04 14:28:43 [loggers.py:111] Engine 000: Avg prompt throughput: 566.7 tokens/s, Avg generation throughput: 13.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 17.6%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:28:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 21.3 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.6%
+INFO 01-04 14:29:03 [loggers.py:111] Engine 000: Avg prompt throughput: 659.1 tokens/s, Avg generation throughput: 25.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 18.1%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 2INFO 0INFO 01-04 14:29:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 14.1 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 1INFO 01-04 14:29:25 [loggers.py:111] Engine 000: Avg prompt throughput: 686.4 tokens/s, Avg generation throughput: 30.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 22.1%
+INFO 01-04 14:29:35 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 22.1%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:29:45 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 33.1 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 22.1%
+IINFO 01-04 14:29:56 [loggers.py:111] Engine 000: Avg prompt throughput: 709.1 tokens/s, Avg generation throughput: 0.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 16.6%IINFO 01-04 14:30:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 16.6%IINFO 01-04 14:30:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cache hit rate: 16.6%IINFO 01-04 14:30:26 [loggers.py:111] Engine 000: Avg prompt throughput: 645.4 tokens/s, Avg generation throughput: 61.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.9%, Prefix cache hit rate: 16.4%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:30:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 85.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 16.4%
+IINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OKINFO 01-04 14:30:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 24.1 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.4%
+INFO 01-04 14:30:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.4%
+INFO 01-04 14:31:06 [loggers.py:111] Engine 000: Avg prompt throughput: 663.8 tokens/s, Avg generation throughput: 13.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 17.3%
+INFO 01-04 14:31:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0INFO 01-04 14:31:23 [loggers.py:111] Engine 000: Avg prompt throughput: 518.0 tokens/s, Avg generation throughput: 39.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rINFO 01-04 INFO 01-04 14:31:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit INFO 01-04 1INFO 01-04 14:31:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hitINFO 01-04 14INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:31:53 [loggers.py:111] Engine 000: Avg prompt throughput: 914.5 tokens/s, Avg generation throughput: 50.9 tokens/s, Running: 1 rINFO 01-04 14:31:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:32:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 34.7 tokens/s, Running: INFO 01-04 14:32:06 [loggers.py:111] Engine 000: Avg prompt throughput: 927.5 toINFO 01-04 14:32:13 [loggers.py:111] Engine 000: Avg prompt throughput: 777.9 tokens/s, Avg generation throughput: 24.0 tokens/s, Running: 1 reINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 2INFO 01-04 14:32INFO 01-04 14:32:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cacheINFO 01-04 14:32:INFO 01-04 14:32:33 [loggersINFO 01-04 14:32:37 [loggers.py:111] Engine 000: Avg prompt throughput: 560.4 tokens/s, Avg generation throughput: 0.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cINFO 01-04 14:32:56 [loggers.py:111] Engine 000: Avg prompt throughput: 611.1 tokens/s, Avg generation throughput: 2.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 17.9%
+INFO 01-04 14:33:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 17.9%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:33:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 43.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.9%
+INFO 01-04 14:33:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.9%
+INFO 01-04 14:33:46 [loggers.py:111] Engine 000: Avg prompt throughput: 1091.1 tokens/INFO 01-04 14:34:03 [loggers.py:111] Engine 000: Avg prompt throughput: 497.1 tokens/s, Avg generation throughput: 15.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.0%, Prefix cache hit rate: 17.9%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:34:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 18.3 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usaINFO 01-04 14:34:17 [loggers.py:111] EnINFO 01-04 14:34:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usagINFO 01-04 14:34:27 [loggers.py:111] EINFO 01-04 14:34:33 [loggers.py:111] Engine 000: Avg prompt throughput: 1340.9 tokens/s, Avg generation throughput: 41.7 tokens/s, RINFO 01-04 14:34:46 [loggers.py:111] Engine 000: Avg prompt throughput: 514.1 tokens/s, Avg generation throughput: 14.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 19.0%
+INFO 01-04INFO 01-04 14:35:08 [loggers.py:111] Engine 000: Avg prompt throughput: 854.2 tokens/s, Avg generation throughput: 46.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cache hit rate: 17.0%
+INFO 01-04 14:35:18 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.7%, Prefix cache hit rate: 17.0%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200INFO 01-04 14:35:35 [loggers.py:111] Engine 000: Avg prompt throughput: 958.6INFO 01-04 14:36:03 [loggers.py:111] Engine 000: Avg prompt throughput: 627.6 tokens/s, Avg generation throughput: 27.8 tokens/s, RunINFO 01-04 14:36:16 [loggers.py:111] Engine 000: Avg prompt throughput: 677.4 tokens/s, Avg generation throughput: 28.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 19.8%
+INFO 01-04 14:36:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 19.8%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:36:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 8.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.8%
+INFO 01-04 14:36:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/sINFO 01-04 14:37:03 [loggers.py:111] Engine 000: Avg prompt throughput: 535.8 tokens/s, Avg generation throughput: 31.4 tokens/s, RuINFO 01-04 14:37:06 [loggers.py:111] Engine 000: Avg prompt throughput: 504.0 tokens/s,INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:37:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 30.1 tokens/s, RunninINFO 01-04 14:37INFO 01-04 14:38:06 [loggers.py:111] Engine 000: Avg prompt throughput: 1079.4 tokens/s, Avg generation throughput: 33.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 18.2%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OKINFO 01-04 14:38:33 [loggers.py:111] Engine 000: Avg prompt throughput: 573.9 tokens/s, Avg generation throughput: 21.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 17.8%
+INFO 01-04 14:38:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 17.8INFO 01-04 14:39:26 [loggers.py:111] Engine 000: Avg prompt throughput: 380.6 tokens/s, Avg generation throughput: 9.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache hit rate: 18.1%
+INFO 01-04 14:39:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.7%, Prefix cache hit rate: 18.1%
+INFO 01-04 14:39:46 [loggers.py:111] Engine 000: Avg prompt throughINFO 01-04 14:40:03 [loggers.py:111] Engine 000: Avg prompt throughput: 489.2 tokens/s, Avg generation throughput: 38.8 tokens/s, RunnINFO 01-04 14:40:06 [loggers.py:111] Engine 000: Avg prompt throughput: 1203.8 tokensINFO 01-04 14:40:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, RunninINFO 01-04 14:40:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/INFO 01-04 14:40:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, RunninINFO 01-04 14:40:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tINFO 01-04 14:41:16 [loggers.py:111] Engine 000: Avg prompt throughput: 506.6 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 18.0%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:41:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 15.9 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.0%
+INFO 01-04 14:41:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running:INFO 01-04 14:42:46 [loggers.py:111] Engine 000: Avg prompt throughput: 952.5 tokens/s, Avg generation throughput: 39.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.9%, Prefix cache hit rate: 20.5%
+INFO 01-04 14:42:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.1%, Prefix cache hit rate: 20.5%
+INFO 01-04 14:43:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.3%, Prefix cache hit rate: 20.5%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:43:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 40.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 20.5%
+INFO 01-04 14:43:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokenINFO 01-04 14:44:23 [loggers.py:111] Engine 000: Avg prompt throughput: 652.5 tokens/s, Avg generation throughput: 1.4 tokens/s, RunninINFO 01-04 14:46:36 [loggers.py:111] Engine 000: Avg prompt throughput: 571.7 tokens/s, Avg generation throughput: 28.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 20.4%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:46:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 30.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 20.4%
+INFO 01-04 14:46:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 20.4%
+INFO 01-04 14:54:36 [loggers.py:111] Engine 000: Avg prompt throughput: 426.4 tokens/s, Avg generation throughput: 37.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 20.2%
+INFO 01-04 14:54:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 INFO 01-04 14:55:35 [loggers.py:111] Engine 000: Avg prompt throughput: 431.5 tokens/s, Avg generation throughput: 15.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 25.8%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:55:45 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 29.9 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 25.8%
+INFO 01-04 14:55:55 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 25.8%
+ Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 20.2%
+INFO 01-04 14:55:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 20.2%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:55:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 INFO 01-04 15:02:53 [loggers.py:111] Engine 000: Avg prompt throughput: 612.3 tokens/s, Avg generation throughput: 38.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 17.8%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 15:03:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 37.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.8%
+INFO 01-04 15:03:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.8%
+ning: 1 reqs, Waiting: 0 reqs, GPU KVINFO 01-04 14:55:51 [loggers.py:111] Engine 000: Avg prompt throughput: 437.3 tokens/s, Avg generation throughput: 43.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 16.5%
+INFO 01-04 14:56:01 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 16.5%
+INFO 01-04 14:56:11 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 16.5%
+INFO 01-04 14:56:21 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 16.5%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:56:31 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 4.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.5%
+INFO 01-04 14:56:41 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.5%
+6 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.5%, Prefix cache hit rate: 18.2%
+INFO 01-04 14:55:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput:INFO 01-04 15:07:05 [loggers.py:111] Engine 000: Avg prompt throughput: 518.9 tokens/s, Avg generation throughput: 5.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 19.0%
+INFO 01-04 15:07:15 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 19.0%
+INFO 01-04 15:07:25 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 19.0%
+INFO 01-04 15:07:35 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 19.0%
+INFO 01-04 15:07:45 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 19.0%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 15:07:55 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 5.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.0%
+INFO 01-04 15:08:05 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.0%

hf_ip/vllm_gpu6.log

@@ -0,0 +1,283 @@
+/usr/local/lib/python3.10/dist-packages/torch/utils/_pytree.py:185: FutureWarning: optree is installed but the version is too old to support PyTorch Dynamo in C++ pytree. C++ pytree support is disabled. Please consider upgrading optree using `python3 -m pip install --upgrade 'optree>=0.13.0'`.
+  warnings.warn(
+INFO 01-04 13:13:42 [__init__.py:239] Automatically detected platform cuda.
+INFO 01-04 13:13:45 [api_server.py:1043] vLLM API server version 0.8.5
+INFO 01-04 13:13:45 [api_server.py:1044] args: Namespace(host='127.0.0.1', port=8007, uvicorn_log_level='info', disable_uvicorn_access_log=False, allow_credentials=False, allowed_origins=['*'], allowed_methods=['*'], allowed_headers=['*'], api_key=None, lora_modules=None, prompt_adapters=None, chat_template=None, chat_template_content_format='auto', response_role='assistant', ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, enable_ssl_refresh=False, ssl_cert_reqs=0, root_path=None, middleware=[], return_tokens_as_token_ids=False, disable_frontend_multiprocessing=False, enable_request_id_headers=False, enable_auto_tool_choice=False, tool_call_parser=None, tool_parser_plugin='', model='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', task='auto', tokenizer=None, hf_config_path=None, skip_tokenizer_init=False, revision=None, code_revision=None, tokenizer_revision=None, tokenizer_mode='auto', trust_remote_code=True, allowed_local_media_path=None, load_format='auto', download_dir=None, model_loader_extra_config={}, use_tqdm_on_load=True, config_format=<ConfigFormat.AUTO: 'auto'>, dtype='auto', max_model_len=131072, guided_decoding_backend='auto', reasoning_parser=None, logits_processor_pattern=None, model_impl='auto', distributed_executor_backend=None, pipeline_parallel_size=1, tensor_parallel_size=1, data_parallel_size=1, enable_expert_parallel=False, max_parallel_loading_workers=None, ray_workers_use_nsight=False, disable_custom_all_reduce=False, block_size=None, gpu_memory_utilization=0.9, swap_space=4, kv_cache_dtype='auto', num_gpu_blocks_override=None, enable_prefix_caching=None, prefix_caching_hash_algo='builtin', cpu_offload_gb=0, calculate_kv_scales=False, disable_sliding_window=False, use_v2_block_manager=True, seed=None, max_logprobs=20, disable_log_stats=False, quantization=None, rope_scaling=None, rope_theta=None, hf_token=None, hf_overrides=None, enforce_eager=False, max_seq_len_to_capture=8192, tokenizer_pool_size=0, tokenizer_pool_type='ray', tokenizer_pool_extra_config={}, limit_mm_per_prompt={}, mm_processor_kwargs=None, disable_mm_preprocessor_cache=False, enable_lora=None, enable_lora_bias=False, max_loras=1, max_lora_rank=16, lora_extra_vocab_size=256, lora_dtype='auto', long_lora_scaling_factors=None, max_cpu_loras=None, fully_sharded_loras=False, enable_prompt_adapter=None, max_prompt_adapters=1, max_prompt_adapter_token=0, device='auto', speculative_config=None, ignore_patterns=[], served_model_name=['default-model'], qlora_adapter_name_or_path=None, show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, disable_async_output_proc=False, max_num_batched_tokens=None, max_num_seqs=256, max_num_partial_prefills=1, max_long_partial_prefills=1, long_prefill_token_threshold=0, num_lookahead_slots=0, scheduler_delay_factor=0.0, preemption_mode=None, num_scheduler_steps=1, multi_step_stream_outputs=True, scheduling_policy='fcfs', enable_chunked_prefill=None, disable_chunked_mm_input=False, scheduler_cls='vllm.core.scheduler.Scheduler', override_neuron_config=None, override_pooler_config=None, compilation_config=None, kv_transfer_config=None, worker_cls='auto', worker_extension_cls='', generation_config='auto', override_generation_config=None, enable_sleep_mode=False, additional_config=None, enable_reasoning=False, disable_cascade_attn=False, disable_log_requests=True, max_log_len=None, disable_fastapi_docs=False, enable_prompt_tokens_details=False, enable_server_load_tracking=False)
+INFO 01-04 13:13:53 [config.py:717] This model supports multiple tasks: {'classify', 'generate', 'embed', 'reward', 'score'}. Defaulting to 'generate'.
+INFO 01-04 13:13:53 [config.py:2003] Chunked prefill is enabled with max_num_batched_tokens=8192.
+/usr/local/lib/python3.10/dist-packages/torch/utils/_pytree.py:185: FutureWarning: optree is installed but the version is too old to support PyTorch Dynamo in C++ pytree. C++ pytree support is disabled. Please consider upgrading optree using `python3 -m pip install --upgrade 'optree>=0.13.0'`.
+  warnings.warn(
+INFO 01-04 13:13:58 [__init__.py:239] Automatically detected platform cuda.
+INFO 01-04 13:14:01 [core.py:58] Initializing a V1 LLM engine (v0.8.5) with config: model='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', speculative_config=None, tokenizer='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=131072, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto,  device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='auto', reasoning_backend=None), observability_config=ObservabilityConfig(show_hidden_metrics=False, otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=None, served_model_name=default-model, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=True, chunked_prefill_enabled=True, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs=None, pooler_config=None, compilation_config={"level":3,"custom_ops":["none"],"splitting_ops":["vllm.unified_attention","vllm.unified_attention_with_output"],"use_inductor":true,"compile_sizes":[],"use_cudagraph":true,"cudagraph_num_of_warmups":1,"cudagraph_capture_sizes":[512,504,496,488,480,472,464,456,448,440,432,424,416,408,400,392,384,376,368,360,352,344,336,328,320,312,304,296,288,280,272,264,256,248,240,232,224,216,208,200,192,184,176,168,160,152,144,136,128,120,112,104,96,88,80,72,64,56,48,40,32,24,16,8,4,2,1],"max_capture_size":512}
+WARNING 01-04 13:14:01 [utils.py:2522] Methods determine_num_available_blocks,device_config,get_cache_block_size_bytes,initialize_cache not implemented in <vllm.v1.worker.gpu_worker.Worker object at 0x7faff3c54e80>
+INFO 01-04 13:14:03 [parallel_state.py:1004] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, TP rank 0
+INFO 01-04 13:14:03 [cuda.py:221] Using Flash Attention backend on V1 engine.
+WARNING 01-04 13:14:03 [topk_topp_sampler.py:69] FlashInfer is not available. Falling back to the PyTorch-native implementation of top-p & top-k sampling. For the best performance, please install FlashInfer.
+INFO 01-04 13:14:03 [gpu_model_runner.py:1329] Starting to load model /data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf...
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+INFO 01-04 13:15:40 [loader.py:458] Loading weights took 97.35 seconds
+INFO 01-04 13:15:41 [gpu_model_runner.py:1347] Model loading took 61.0562 GiB and 97.798136 seconds
+INFO 01-04 13:15:55 [backends.py:420] Using cache directory: /root/.cache/vllm/torch_compile_cache/75e72335d9/rank_0_0 for vLLM's torch.compile
+INFO 01-04 13:15:55 [backends.py:430] Dynamo bytecode transform time: 14.48 s
+INFO 01-04 13:15:59 [backends.py:136] Cache the graph of shape None for later use
+INFO 01-04 13:16:50 [backends.py:148] Compiling a graph for general shape takes 53.24 s
+INFO 01-04 13:17:53 [monitor.py:33] torch.compile takes 67.72 s in total
+INFO 01-04 13:17:53 [kv_cache_utils.py:634] GPU KV cache size: 254,768 tokens
+INFO 01-04 13:17:53 [kv_cache_utils.py:637] Maximum concurrency for 131,072 tokens per request: 1.94x
+INFO 01-04 13:18:32 [gpu_model_runner.py:1686] Graph capturing finished in 39 secs, took 1.21 GiB
+INFO 01-04 13:18:32 [core.py:159] init engine (profile, create kv cache, warmup model) took 171.40 seconds
+INFO 01-04 13:18:32 [core_client.py:439] Core engine process 0 ready.
+WARNING 01-04 13:18:32 [config.py:1239] Default sampling parameters have been overridden by the model's Hugging Face generation config recommended from the model creator. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
+INFO 01-04 13:18:32 [serving_chat.py:118] Using default chat sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
+INFO 01-04 13:18:32 [serving_completion.py:61] Using default completion sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
+INFO 01-04 13:18:32 [api_server.py:1090] Starting vLLM API server on http://127.0.0.1:8007
+INFO 01-04 13:18:32 [launcher.py:28] Available routes are:
+INFO 01-04 13:18:32 [launcher.py:36] Route: /openapi.json, Methods: HEAD, GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /docs, Methods: HEAD, GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /docs/oauth2-redirect, Methods: HEAD, GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /redoc, Methods: HEAD, GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /health, Methods: GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /load, Methods: GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /ping, Methods: POST, GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /tokenize, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /detokenize, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/models, Methods: GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /version, Methods: GET
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/chat/completions, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/completions, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/embeddings, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /pooling, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /score, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/score, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/audio/transcriptions, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /rerank, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v1/rerank, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /v2/rerank, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /invocations, Methods: POST
+INFO 01-04 13:18:32 [launcher.py:36] Route: /metrics, Methods: GET
+INFO:     Started server process [1318205]
+INFO:     Waiting for application startup.
+INFO:     Application startup compleIINFINFO 01-04 14:00:02 [loggers.py:111] Engine 000: Avg prompt throughput: 111.2 tokens/s, Avg generation throughput: 85.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.8%, Prefix cache hit rate: 0INFOINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:00:12 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 89.1 tokens/s, Running: 1 reqs, WIINFO 01-04 14:00:19 [loggers.py:111] Engine 000: Avg prompt througINFINFO 01-04 14:00:22 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1 reqs, WaINFO 01-04 14:00:29 [loggers.py:111] Engine 000: Avg prompt throughININFO 01-04 14:00:32 [loggers.py:111] Engine 000: Avg prompt throughput: 65.5 tokens/s, Avg generation throughput: 84.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.4%, Prefix cache hit INFO 01-04 INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OININFO 01-04 14:00:42 [loggers.py:111] Engine 000: Avg prompt throughput: 63.3 tokens/s, Avg generation throughput: 100.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.4%, Prefix cache hiINFO:  INFO 01-04 14:00:47 [loggers.py:111] Engine 000: Avg prompt INFO 01-04INFO 01-04 14:00:52 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 52.2 tokens/s, Running: 1 reqs, WINFO 01-04 14:00:57 [loggers.py:111] Engine 000: Avg prompt tINFO 01-0INFO 01-04 14:01:02 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04 14:01:02 [loggers.py:111] Engine 000: Avg prompt throughput: 89.3 tokens/s, Avg generation throughput: 34.6 tokens/s, Running: ININFOINFO 01-04 14:01:09 [loggers.py:111] Engine 000: Avg prompt throughput: 0INFINFO 01-04 14:01:12 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 104.2 tokens/s, Running:INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 INFINFO 0INFO INFO 01-04 14:01:22 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 94.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hiINFOINFO INFOINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:01:32 [loggers.py:111] Engine 000: Avg prompt throughput: 87.2 tokens/s, Avg generation throughput: 92.6 tokens/s, RunningINFO 01-04 14:01:36 [loggers.py:111] Engine 000: Avg prompt throughput: 84.INFO 01-04 14:01:37 [loggers.py:111] Engine 000: Avg prompt throughINFINFO 01-04 14:01:42 [loggers.py:111] Engine 000: Avg prompt throughput: 58.0 tokens/s, Avg generation throughput: 73.8 tokens/s, Running:INFO 01-04 14:01:46 [loggers.py:111] Engine 000: Avg prompt throughput:INFOINFO 01-04 14:01:47 [loggers.py:111] Engine 000: Avg prompt throughpINFINFO 01-04 14:01:52 [loggers.py:111] Engine 000: Avg prompt throughput: 0INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     INFO 01-04 14:01:57 [loggers.py:111] Engine 000: Avg prompt throughpuINFO 01-04 14:02:02 [loggers.py:111] Engine 000: Avg prompt throughput: 103.4 tokens/s, Avg generation throughput: 73.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.2%, Prefix cache hitINFO:    INFINFO 01-04 14:02:13 [loggers.py:111] Engine 000: Avg prompt throughput: 90.8 tokens/s, Avg generation throughput: 131.0 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.0%, Prefix cache hit rate: 1.1%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTINFOINFO 01-04INFO 01-04 14:02:17 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04 14:02:23 [loggers.py:111] Engine 000: Avg prompt throughput: 115.3 tokens/s, Avg generation throughput: 90.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit ratINFO 01IINFINFO 01-04 14:02:27 [loggers.py:111] Engine 000: Avg prompt thIINFO 01-04 14:02:33 [loggers.py:111] Engine 000: Avg prompt throughput: 90.0 tokens/s, Avg generation throughput: 69.5 tokens/s, Running: 2 reqs, Waiting: 0INFO 01-04 14:02:37 [loggers.py:111] Engine 000: Avg prompt tINFO 01-04 14:02:42 [loggers.py:111] Engine 000: Avg prompt throughput: 123.1 tokens/s, Avg generation throughput: 110.7 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 6.2INFO:     10.46.17.192:0 - "POST /v1/completions HTTPININFO 01-04 INFINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 INFO 01-04 14:02:53 [loggers.py:111] Engine 000: Avg prompt throughput: 116.6 tokens/s, Avg generation throughput: 96.2 tokens/s, Running: INFO 01-04 14:02INFO 01-04 14:02:57 [loggers.py:111] Engine 000: Avg prompt thrINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 79.3 tokens/s, Running: 1 INFO 01-04 14:03:06 [loggers.py:111] Engine 000: Avg prompt througINFO 01-04 ININFO 01-04 14:03:12 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04 14:03:13 [loggers.py:111] Engine 000: Avg prompt throughput: 91.INFO 01-04 14:03:16 [loggers.py:111] Engine 000: Avg prompt througINFO 01-04 14:03:16 [loggers.py:111] Engine 000: Avg prompt throughput: 102.ININFO 01-04 14:03:22 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04 INFO 01-04 14:03:26 [loggers.py:111] Engine 000: Avg prompt throuINFO 01-04 14:03:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 toINFO 01-04 14:03:32 [loggers.py:111] Engine 000: Avg prompt thrINFO 01-04 14:03:33 [loggers.py:111] Engine 000: Avg prompt throughput: 144INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:36 [loggers.py:111] Engine 000: Avg prompt throughput: 144.7 tINFO 01-04 14:03:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 94.6 tokens/s, Running: 2 INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 0INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:53 [loggers.py:111] Engine 000: Avg prompt throughput: 138INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:56 [loggers.py:111] Engine 000: Avg prompt throughput: 174.INFO:INFO 01-04 14:04:02 [loggers.py:111] Engine 000: Avg promptINFO 01-04 14:04:03 [loggers.py:111] Engine 000: Avg prompt throughput: 100.1 tokens/s, Avg generation throughput: 76.0 tokens/s, Running: 1 reqINFO 01-04 14:04:INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1INFO:    INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1INFO 01-04 14:04:13 [loggers.py:111] Engine 000: Avg prompt throughput: 131.2 tokens/s, Avg generation throughput: 56.6 tokens/s, Running: 1INFO 01-04 14:04:16 [loggers.py:111] Engine 000: Avg prompt throughput: 109INFO 01-04 14INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1INFO 01-04 14:04:23 [loggers.py:111] Engine 000: Avg prompt throughput: 2INFO 01-04 14:04:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 94.8 tokens/s, Running: 2 reqINFINFO 01-04 14:04:33 [loggers.py:111] Engine 000: Avg prompt throughput:INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:04:36 [loggers.py:111] Engine 000: Avg prompt throughput: 132.3 tokens/s, Avg generation throughput: 124.6 tokens/s, Running: 2 rININFO 01-04 14:04:43 [loggers.py:111] Engine 000: Avg prompt throughput: 177.2 tokens/s, Avg generation throughput: 48.3 tokens/s, Running:INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 1IINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:04:53 [loggers.py:111] Engine 000: Avg prompt throughput: 184.1INFO 01-04 14:04:56 [loggers.py:111] Engine 000: Avg prompt throughput: 145.4 tokens/s, Avg generation throughput: 50.6 tokens/s, Running: 2 INFO 01-04INFO 01-04 14:05:02 [loggers.py:111] Engine 000: Avg prompt throughINFO 01-04 14:05:06 [loggers.py:111] Engine 000: Avg prompt throughput: 217.4 tokens/s, Avg generation throughput: 98.8 tokens/s, Running: 3 INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:13 [loggers.py:111] Engine 000: Avg prompt throughput: 182.3INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:16 [loggers.py:111] Engine 000: Avg prompt throughput: 174.4 tokens/s, Avg generation throughput: 119.1 tokens/s, Running: 2 rINFO 01-04 14:05:23 [loggers.py:111] Engine 000: Avg prompt throughput: 186.7 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1INFO:     10INFO 01-04 14:05:25 [loggers.py:111] Engine 000: Avg pINFO 01-04 1INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:33 [loggers.py:111] Engine 000: Avg prompt throughput: 207.8 tokens/s, Avg generation throughput: 66.2 tokens/s, Running: INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 1INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:43 [loggers.py:111] Engine 000: Avg prompt throughput: 216.8 tokens/s, Avg generation throughput: 61.3 tokens/s, Running: 1 rINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04INFO 01-04 14:05:53 [loggers.py:111] Engine 000: Avg prompt throughput: 236.5 tokens/s, Avg generation throughput: 69.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rINFO 01-04INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:03 [loggers.py:111] Engine 000: Avg prompt throughput: 247.4 tokens/s, Avg generation throughput: 75.4 tokens/s, Running: 1 reqs, WaitinINFO 01-04 14:06:05 [loggers.py:111] Engine 000: Avg pINFO 01-04 INFO:     1INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.INFO 01-04 14:06:13 [loggers.py:111] Engine 000: Avg prompt throughput: 174.4 tokens/s, Avg generation throughput: 59.7 tokens/s, Running: 1INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:23 [INFO:     10.46.17.192:0 - "POST /v1/completions HINFO 01-04 14:06:26 [loggers.py:111] Engine 000: Avg prompt throughput: 397.3 tokens/s, Avg generation throughput: 59.9 tokens/s, Running: 2 reqs, INFO 01-04 14:06:33 [loggers.py:111] Engine 000: Avg prompt throughput:INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:36 [loggers.py:111] Engine 000: Avg prompt throughput: 168.3 tokenINFO 01-04 14:06:43 [loggers.py:111] Engine 000: Avg prompt throughput: 245.7 tokens/s, INFO 01-04 14:06:45 [loggers.py:111] Engine 000INFO 01-04 14:06:46 [loggers.py:111] Engine 000: Avg prompt throughput: 227.7 tokensINFO:  INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 20INFO:INFO 01-04 14:06:52 [loggers.py:111] Engine 000: Avg prompt thINFO 01-04 14:06:53 [loggers.py:111] Engine 000: Avg prompt throughINFO 01-04 14:06:56 [loggers.py:111] Engine 000: Avg prompt throughput: 285.1 tokens/s, AvINFO 01-04 14:07:02 [loggers.py:111] Engine 000: Avg prompt throughput: 178.4 tokens/s, Avg generation throughput: 126.1 tokens/sINFO 01-04 14:07:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg INFO:     10.46.50.192:0 - "POST /v1/coINFO 01-04 14:07:06 [loggerINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 2INFO 01-04 14:07:13 [loggers.py:111] Engine 000: Avg prompt throughput: 213.5 tokens/s, Avg generation throughput: 12.2 tokens/s, INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:16 [loggINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:23 [loggers.py:111] Engine 000: Avg prompt throughput: 244.0 tokens/s, Avg generation throughput: 20.6 tokens/s,INFO 01-04 14:07:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg INFO 01-04 14:07:33 [loggers.py:111] Engine 000: Avg prompt throughput: 210.2 tokens/s, Avg generation throughput: 70.1 tokensINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:36 [loggers.py:111] Engine 000: Avg prompt throughput: 504.4 tokens/s, Avg geINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:43 [loggers.py:111] Engine 000: Avg prompt throughput: 274.2 tokens/s, Avg generation throughput: 65.4 tokeINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:46 [loggers.INFO 01-04 14:07:53 [loggers.py:111] Engine 000: Avg prompt throughput: 247.1 tokens/s, Avg generation throughput: 79.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 13.0%
+INFO:     10.46.17.192:0 - "POST /v1INFO 01-04 14:07:56 [loggers.pINFO 01-04 14:08:03 [loggers.py:111] Engine 000: Avg prompt throughput: 368.9 tokens/s, Avg generation throughput: 57.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%INFO:     10.46.17.192:0 - "POSINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 69.4 tokenINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:16 [loggersINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:23 [loggers.py:111] Engine 000: Avg prompt throughput: 596.5 tokens/s, Avg generation throughput: 45.5 tokeINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:26 [loggers.pyINFO 01-04 14:08:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.2%,INFO 01-04 14:08:36 [loggers.py:111] Engine 000: Avg prompt throughput: 405.1 tokens/s, Avg geneINFO 01-04 14:08:43 [loggers.py:111] Engine 000: Avg prompt throughput: 305.0 tINFO 01-04 14:08:42 [loggers.py:111] Engine 000: Avg prompt throughput: 281.7 tokens/s, Avg generation throughput: 77.6 tokens/s, Running: 1INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:53 [loggers.py:111] Engine 000: Avg prompt throughput: 318.9 tokens/s, Avg generation throughput: 78.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.4%,INFO:     10.43.30.4:0 - "POSTINFO:     10.45.190.192:0 - "POST INFO 01-04 14:08:56 [loggers.py:1INFO 01-04 14:09:03 [loggers.py:111] Engine 000: Avg prompt throughput: 235.9 tokens/s, Avg generation throughput: 56.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1INFO 01-04 14:09:06 [loggers.py:11INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:13 [loggers.py:111] Engine 000: Avg prompt throughput: 435.6 tokens/s, Avg generation throughput: 70.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1INFO 01-04 14:09:16 [loggers.py:111] Engine 000: Avg prompt throughput: 186.5 tokens/s, Avg generatiINFO 01-04 14:09:23 [loggers.py:111] Engine 000: Avg prompt throughput: 333.5 tokens/s, Avg generation throughput: 15.INFO 01-04 14:09:26 [loggers.py:111] Engine 000: Avg prompt throughput: 240.7 tokens/s, Avg generatioINFO 01-04 14:09:33 [loggers.py:111] Engine 000: Avg prompt throughput: 391.4 tokens/s, Avg generation throughput: 58INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:36 [loggers.py:111] EngiINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:43 [loggers.py:111] Engine 000: Avg prompt throughput: 360.5 tokens/s, Avg generation throuINFO 01-04 14:09:46 [loggers.py:111] Engine 000: Avg prompt throughput: 673.4 tokens/s, Avg generation throughpINFO 01-04 14:09:53 [loggers.py:111] Engine 000: Avg prompt throughput: 263.7 tokens/s, Avg generation throughput: 47.8 tokens/s, Running:INFO 01-04 14:09:52 [loggers.py:111INFO:     10.45.190.192:0 - "POST /v1/completiINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:03 [loggers.py:111] Engine 000: Avg prompt throughput: 377.1 tokens/s, Avg generation throughput: 82.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 21.2%
+INFO:     10.45.1INFO 01-04 14:10:06 [loggers.py:111] Engine 000: AINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:13 [loggers.py:111] Engine 000: Avg prompt throughput: 199.5 tokens/s, Avg generation tINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:16 [loggers.py:111] Engine 000:INFO 01-04 14:10:23 [loggers.py:111] Engine 000: Avg prompt throughput: 370.4 tokens/s, Avg generation throughput: 74.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 22.0%
+INFO:     10.46.5INFO 01-04 14:10:26 [loggers.py:111] Engine 000: INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation tINFO 01-04 14:10:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:43 [loggers.py:111] Engine 000: Avg prompt throughput: 376.7 tokens/s, Avg generation INFO 01-04 14:10:46 [loggers.py:111] Engine 000: Avg prompt throughput: 138.2 tokens/s, Avg generation throughput: 2INFO 01-04 14:10:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation tINFO 01-04 14:10:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 4INFO 01-04 14:11:03 [loggers.py:111] Engine 000: Avg prompt throughput: 245.0 tokens/s, Avg generation throughput: 33.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPINFO 01-04 14:11:06 [loggers.py:111] Engine 000: Avg INFO 01-04 14:11:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: 20.8%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:11:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 23.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, INFO 01-04 14:11:26 [loggers.py:111] Engine 000: Avg prINFO 01-04 14:11:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, RuINFO 01-04 14:11:46 [loggers.pyINFO 01-04 14:11:46 [loggers.py:111] Engine 000: Avg prompt throughput: 303.1 tokens/s, Avg generation throughput: 26.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:11:56 [loggerINFO 01-04 14:11:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 9.6 tokens/s,INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OINFO 01-04 14:12:06 [loggINFO 01-04 14:12:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, PrefINFO 01-04 14:12:16 [logINFO 01-04 14:12:16 [loggers.py:111] Engine 000: Avg prompt throughput: 424.1 tokens/s, Avg generation throughput: 31.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, PrefINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:12:26 [loggeINFO 01-04 14:12:26 [loggers.py:111] Engine 000: Avg prompt throughput: 757.5 tokens/s, Avg generation throughput: 80.4 tokens/s, RuINFO 01-04 14:12:32 [loggers.py:111] Engine 000: Avg prompINFO 01-04 14:12:36 [loggers.py:111] Engine 000: Avg prompt throughput: 400.8 tokens/s, Avg INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:12:36 [loggers.py:111] Engine 000: Avg prompt INFO:  INFO 01-04 14:12:42 [loggers.py:111] Engine 000: Avg prompt INFO 01-04 14:12:46 [loggers.py:111] Engine 000: Avg prompt throughput: 319.9 tokens/s, AINFO 01-04 14:12:46 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 14:12:52 [loggers.py:111] Engine 000: Avg promptINFO 01-04 14:12:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, INFO 01-04 14:12:56 [loggers.py:111] Engine 000: Avg prompt throughput: INFO 01-04 14:13:02 [loggers.py:111] Engine 000: Avg prompt throughput: 260.8 tokens/s, Avg generation throughput: 58.2 tokeINFO 01-04 14:13:06 [INFO 01-04 14:13:06 [loggers.py:111] Engine 000: Avg prompt throughput: 406.1 tokens/s, Avg generation throughput: 49.6 tokens/s, Running: 2 INFO 01-04 14:13:12 [loggers.py:111] Engine 000: Avg prompt throughput: 260.2INFO 01-04 14:13:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 93.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rateINFO 01-04 14:13:22 [loggers.py:111] Engine 000: Avg pINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:13:INFO 01-04 14:13:26 [loggers.py:111] Engine 000: Avg prompt throughput: 187.4 toINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:13:32 [loggers.py:111] Engine 000: Avg INFO 01-04 14:13:36INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:13:36 [loggers.py:111] Engine 000: Avg prompt throughput: 645.0 tokINFO 01-04 14:13:42 [loggers.py:111] Engine 000: AvgINFO 01-04 14:13:46 [loggers.py:111] Engine 000: Avg prompt throughput: 186.9 tokens/s, Avg generation throughput: 76.2 tokens/s, Running: 2 reqs, WaitINFO:     10.46.INFO 01-04 14:13:52 [loggers.py:111] Engine 000: AvgINFO 01-04 14:13:56 [loggers.py:111] Engine 000: Avg prompt throughput: 350.4 tokens/s, Avg generation throughput: 118.8 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPINFO 01-04 14:14:02 [loggers.py:111] Engine 000: Avg INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:12 [loggers.py:111] Engine 000: Avg prompt throughput: 354.7 tokens/s, Avg generation throughput: 159.INFO 01-04 14:14:16 [loggers.py:111] Engine 000: Avg prompt throughput: 367.6 tokINFO 01-04 14:14:16 [loggers.py:111] Engine 000: Avg prompt throughput: 797.4 tokens/INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:22 [loggers.py:111] Engine 000: AvgINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:26 [loggers.py:111] Engine 000: Avg prompt throughput: 309.4 tokens/s, Avg generation throughput: 64.5 tokens/s, Running: 1 reINFO 01-04 14:14:26 [loINFO:     10.46.50.192:0 - "POST /v1/completions HTTPINFO 01-04 14:14:36 [loggers.py:111] Engine 000: Avg prompt throughput: 372.2 tokens/s, Avg generation throughput: 90.5 tokens/s, Running: 2 rINFO 01-04 14:14:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tINFO:     10INFO 01-04 14:14:42 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.5 tokeINFO 01-04 14:14:46 [loggers.py:111] Engine 000: Avg prompt throughput: INFO 01-04 14:14:46 [lINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:52 [loggers.py:111] Engine 000: Avg prompt tINFO 01-04 14:14:56 [loggers.py:111] Engine 000: Avg prompt throughput:INFO 01-04 14:14:56 [loggers.py:111] Engine 000: Avg prompt throughput: 1015.6 tokens/sINFO 01-04 14:15:02 [loggers.py:111] Engine 000: Avg prompt throughput: 346.2 tokens/s, Avg generation throughput: 108.5 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.6%, INFO 01-04 14:15:06 [loggers.py:111] Engine 000: Avg prompt throughput: INFO 01-04 14:15:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg INFO 01-04 14:15:12 [loggers.py:111] Engine 000: Avg promINFO 01-04 14:15:16 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 14:15:16 [loggers.py:111] Engine 000: Avg prompt throughput: 905.5 tokens/s, Avg generation throughput: 88.9 tokens/s, Running: 3 reqs, INFO:     1INFO 01-04 14:15:22 [loggers.py:111] Engine 000: Avg proINFO 01-04 14:15:26 [loggers.py:111] Engine 000: Avg prompt throughputINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:26 [loggers.py:111] Engine 000: Avg prompt throughput: 227.3 tokenINFO:    INFO 01-04 14:15:32 [loggers.py:111] Engine 000: Avg prompINFO 01-04 14:15:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 69.6 tokens/s, RunningINFO 01-04 14:15:36 [loggers.py:111] Engine 000: Avg prompt throughput: 372.0 tokeINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:46 [loggers.py:111] Engine 000: Avg prompt throughput: 422.2 tokens/s, Avg generation throughput: 52.1 tokens/s, RunniINFO 01-04 14:15:46 [loggers.py:111] Engine 000: Avg prompt throughput: 430.3 tokensINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:56 [loggers.py:111] Engine 000: Avg prompt throughput: 471.9 tokens/s, Avg generation throughput: 66.6 tokens/s, Running: 1 reqs, Waiting: 0 INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 2INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:06 [loggers.py:111] Engine 000: Avg prompt throughput: 286.2 tokens/s, Avg generation throughput: 78.7 tokens/s, RunnINFO 01-04 14:16:06 [loggINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 2INFO 01-04 14:16:16 [loggers.py:111] Engine 000: Avg prompt throughput: 508.3 tokens/s, Avg generation throughput: 75.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cINFO 01-04 14:16:16 [INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:26 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 14:16:26 [loggers.py:111] Engine 000: Avg prompt throughput: 454.6 tokens/s, Avg generation throughput: 114.2 tokens/s, Running: 3 reqs, WINFO: INFO 01-04 14:16:32 [loggers.py:111] Engine 000: Avg prompINFO 01-04 14:16:36 [loggers.py:111] Engine 000: Avg prompt throughput: 510.4 tokens/s, Avg generation throughput: 51.3 tokens/s, RunningINFO 01-04 14:16:36 [logINFO 01-04 14:16:42 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 92.4 tokeINFO 01-04 14:16:46 [loggers.py:111] Engine 000: Avg prompt throughput: 534.8 tokens/s, Avg generation throughput: 38.8 tokens/s, Running: 1 reqs, Waiting: 0 reqsINFO 01-04 14:16:52 [loggers.py:111] Engine 000: Avg promINFO 01-04 14:16:56 [loggers.py:111] Engine 000: Avg prompt throughput: 561.4 tokens/s, Avg generation throughput: 50.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.6%, Prefix cache hit rate: 15.6%
+INFO:    INFO 01-04 14:17:02 [loggers.py:111] Engine 000: Avg promINFO 01-04 14:17:06 [loggers.py:111] Engine 000: Avg prompt throughput: 556.6 tokens/s, Avg generation throughput: 83.4 tokens/s, Running: 2 reqs, Waiting: 0 reqsINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:16 [loggers.py:111] Engine 000: Avg prompt throughput: 648.5 tokens/s, Avg generation throughput: 93.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.0%, Prefix cacheINFO 01-04 14:17:INFO: INFO 01-04 14:17:22 [loggers.py:111] Engine 000: Avg prompt tINFO 01-04 14:17:26 [loggers.py:111] Engine 000: Avg prompt throughput: 525.0 tokens/s, Avg generation throughput: 81.9 tokens/s, RunINFO 01-04 14:17:26 [loggers.py:111] Engine 000: Avg prompt throughput: 724.7 tokens/sINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:36 [loggers.py:111] Engine 000: Avg prompt throughput: 236.9 tokens/s, Avg generation throughput: 59.5 tokens/s, RunnINFO 01-04 14:17:36 [loggers.py:111] Engine 000: Avg prompt throughput: 389.6 tokens/INFO 01-04 14:17:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.8 tokens/s, RunninINFO 01-04 14:17:46 [loggers.py:111] Engine 000: Avg prompt throughput: 882.6 tokenINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:56 [loggers.py:111] Engine 000: Avg prompt throughput: 583.1 tokens/s, Avg generation throughput: 59.7 tokens/s, Running: INFO 01-04 14:17:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokeINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:06 [loggers.py:111] Engine 000: Avg prompt throughput: 516.2 tokens/s, Avg generation throughput: 74.6 tokens/s, RunninINFO 01-04 14:18:06 [logINFO 01-04 14:18:12 [loggers.py:111] Engine 000: Avg promptINFO 01-04 14:18:16 [loggers.py:111] Engine 000: Avg prompt throughput: 388.1 tokens/s, Avg generation throughput: 108.3 tokens/s, Running: 3 reqs, Waiting: 0 rINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:22 [loggers.py:111] Engine 000: Avg prompt thINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:26 [loggers.py:111] Engine 000: Avg prompt throughput: 305.7 tokens/s, Avg generation throughput: 93.2 tokens/s, Running: 2 reqs, Waiting: 0 rINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200INFO 01-04 14:18:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 90.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hitINFO 01-04 14INFO: INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:46 [loggers.py:111] Engine 000: Avg prompt throughput: 945.5 tokens/s, Avg generation throughput: 112.7 tokens/s, Running: 2 reqs, Waiting: 0 INFO 01-04 14:18:52 [loggers.py:111] Engine 000: Avg prompt throughput: 192.7 tokens/s, Avg generation throughput: 63.7 tokenINFO 01-04 14:18:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 48.3 tokens/s, Running: 1 reqs, Waiting: 0 reINFO 01-04 14:19:02 [loggers.py:111] Engine 000: Avg promptINFO 01-04 14:19:06 [loggers.py:111] Engine 000: Avg prompt throughput: 605.3 tokens/s, Avg generation throughput: 83.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.3%, Prefix cache hit rate: 17.7%
+INFO 01-04 14:19:16 [loggers.py:111] Engine 000: Avg prompt throughput: 734.9 tokens/s, Avg generation throughput: 103.2 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.6%, Prefix cache hit rate: 17.2%
+INFO 01-04 14:19:26 [loggers.py:111] Engine 000: Avg prompt throughput: 665.1 tokens/s, Avg generation throughput: 129.9 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 10.7%, Prefix cache hit rate: 16.8%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 151.2 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.1%, Prefix cache hit rate: 16.8%
+INFO:  INFO 01-04 14:19:42 [loggers.py:111] Engine 000: Avg promptINFO 01-04 14:19:46 [loggers.py:111] Engine 000: Avg prompt throughput: 1336.2 tokens/s, Avg generation throughput: 130.0 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 11.0%, Prefix cache hit rate: 18.3%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFOINFO 01-04 14:19:52 [loggers.py:111] Engine 000: Avg prompt thINFO 01-04 14:19:56 [loggers.py:111] Engine 000: Avg prompt throughput: 443.6 tokens/s, Avg generation throughput: 117.0 tokens/s, Running: 3 reqs, WaitingINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OININFO 01-04 14:20:02 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04 14:20:06 [loggers.py:111] Engine 000: Avg prompt throughput: 542.3 tokens/s, Avg generation throughput: 106.1 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.9%, Prefix cache hitINFO 01-04 14IINFO 01-04 14:20:12 [loggers.py:111] Engine 000: Avg prompt throuINFO 01-04 14:20:16 [loggers.py:111] Engine 000: Avg prompt throughput: 610.3 tokens/s, Avg generation throughput: 114.6 tokens/s, Running: 3 reqs, WaitinINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:22 [loggers.py:111] Engine 000: Avg prompt throINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg gINFO 01-04 14:20:32 [loggers.py:111] Engine 000: Avg prompt throughput: 778.7 tokens/s, Avg generation throughput: 60.1 tokens/INFO 01-04 14:20:36 [loggers.py:111] Engine 000: Avg prompt throughput: 1211.9 tokens/s, Avg generation throughput: 69.9 tokens/s, Running: 3 reqs, Waiting:INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:42 [loggers.py:111] Engine 000: Avg prompt throINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0INFO 01-04 14:2INFO 01-04 14:20:52 [loggers.py:111] Engine 000: Avg prompt throughput: 664.6 tokens/s, Avg generation throughput: 58.0 tokens/s, INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:56 [loggers.py:111] Engine 000: Avg prompt throughput: 568.1 tokens/s,INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:02 [loggers.py:111] Engine 000: Avg prompt througINFO 01-04 14:21:06 [loggers.py:111] Engine 000: Avg prompt throughput: 353.6 tokens/s, Avg generation throughput: 71.0 tokens/s, Running: INFO 01-04 1INFO 01-04 14:21:12 [loggers.py:111] Engine 000: Avg prompt throughput: 610.5 tokens/s, Avg generation throughput: 28.6 tokens/s, RunnINFO 01-04 14:21:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.INFO:     1INFO 01-04 14:21:22 [loggers.py:111] Engine 000: Avg prompt throughput: 632.5 tokens/s, Avg generation throughput: 69.1 tokens/s, RuINFO 01-04 14:21:26 [loggers.py:111] Engine 000: Avg prompt throughput: 1078.0 tokens/s, Avg generation throughput: 91.3 tokens/s, Running: 3 reqs, WaitiINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:32 [loggers.py:111] Engine 000: Avg prompt throughINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:36 [loggers.py:111] Engine 000: Avg prompt throughput: 444.3 tokens/sINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:42 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 14:21:46 [loggers.py:111] Engine 000: Avg prompt throughput: 616.4 tokens/s, Avg generation throughput: 87.4 tokens/s, Running: 3 reqs, WaINFO 01-04 14:21:52 [loggers.py:111] Engine 000: Avg prompt throughput: 477.0 tokens/s, Avg generation throughput: 50.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.6%, Prefix cache hit rate: 15.5%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:02 [loggers.py:111] Engine 000: Avg prompt throughput: 818.0 tokens/s, Avg generation throughput: 81.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.1%, Prefix cache hit rate: 15.2%
+INFO 01-04 14:22:12 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 90.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.5%, Prefix cache hit rate: 15.2%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:22 [loggers.py:111] Engine 000: Avg prompt throughput: 302.9 tokens/s, Avg generation throughput: 91.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.5%, Prefix cache hit rate: 15.0%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTPINFO 01-04 14INFO 01-04 14:22:32 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 55.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache hitINFO 01-04 14INFO 01-04 14:22:42 [loggers.py:111] Engine 000: Avg prompt throughput: 846.2 tokens/s, Avg generation throughput: 76.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.1%, Prefix cache hit rate: 14.7%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTPINFO 01-04 14INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:52 [loggers.py:111] Engine 000: Avg prompt throughput: 737.4 tokens/s, Avg generation throughput: 73.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hINFO 01-04 14:22:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0INFO 0INFO 01-04 14:23:02 [loggers.py:111] Engine 000: Avg prompt throughput: 610.5 tokens/s, Avg generation throughput: 33.9 tokens/s, RunninINFO 01-04 14:23:06 [loggers.py:111] Engine 000: Avg prompt throughput: 437.0INFO 0INFO 01-04 14:23:12 [loggers.py:111] Engine 000: Avg prompt throughput: 462.1 tokens/s, Avg generation throughput: 60.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.6%, Prefix cacheINFO 01-04INFO 01-04 14:23:16 [loggers.py:111] Engine 000: Avg prompt throughput:INFO 01-04 14:23:22 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 56.1 tokens/s, Running:INFO 01-04 14:23:26 [loggers.py:111] Engine 000: Avg prompt throughput: 820.2 tokens/s, Avg generation throughput: 18.6 tokens/s, Running: 1 reqINFINFO 01-04 14:23:32 [loggers.py:111] Engine 000: Avg prompt throughput: INFO 01-04 14:23:36 [loggers.py:111] Engine 000: Avg prompt throughput: 706.9 tokens/s, Avg generation throughput: 61.3 tokens/s, Running: 2 reqs, INFO 01-04 14:23:42 [loggers.py:111] Engine 000: Avg prompt throughput: 827.2 tokens/s, Avg generation throughput: 62.7 tokens/s, RunninINFO 01-04 14:23:46 [loggers.py:111] Engine 000: Avg prompt throughput: 702.2 tokenINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:23:52 [loggers.py:111] Engine 000: Avg prompt throughput: 886.0 tokens/s, Avg generation throughput: 64.8 tokens/s, RunnINFO 01-04 14:23:56 [loggers.py:111] Engine 000: Avg prompt throughput: 677.2 tokens/INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:02 [loggers.py:111] Engine 000: Avg prompt throughput: 616.8 tokens/s, Avg generation throughput: 86.3 tokens/s, RuINFO 01-04 14:24:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 52.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 1INFO INFO 01-04 14:24:16 [loggers.py:111] Engine 000: Avg prompt throughput: 617.8 tokens/s, Avg generation throughput: 76.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.7%, Prefix cache hit rate: 1INFO IINFO 01-04 14:24:22 [loggers.py:111] Engine 000: Avg prompt throuINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, INFO 01-04 14:24:32 [loggers.py:111] Engine 000: Avg prompt throughput: 735.9 tokens/s, Avg generation throughput: 66.0 tokens/s, RINFO 01-04 14:24:36 [loggers.py:111] Engine 000: Avg prompt throughput: 931.7 tokens/s, INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+IINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:46 [loggers.py:111] Engine 000: Avg prompt throughput: 700.4 tokens/s, Avg generation throughput: 50.5 tokens/s, Running: 1 reqs, WaINFO 01-04 14:24:52 [loggers.py:111] Engine 000: Avg prompt throughputINFO 01-04 14:24:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, WaitINFO 01-04 14:25:02 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.9 tokens/s, RunningINFO 01-04 14:25:06 [loggers.py:111] Engine 000: Avg prompt throughput: 744.9 tokeINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:12 [loggers.py:111] Engine 000: Avg prompt throughput:INFO 01-04 14:25:16 [loggers.py:111] Engine 000: Avg prompt throughput: 1012.7 tokens/s, Avg generation throughput: 63.6 tokens/s, Running: 2 reqs, INFO 01-04 14:25:22 [loggers.py:111] Engine 000: Avg prompt throughput: INFO 01-04 14:25:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 90.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.6%, Prefix cache hit rate: 17INFOINFO 01-04 14:25:36 [loggers.py:111] Engine 000: Avg prompt throughput: 1362.4 tokens/s, Avg generation throughput: 119.0 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 13.4%, Prefix cache hit rate:INFO 01-04 14:25:42 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 90.6 tokens/s, Running: 2 rINFO 01-04 14:25:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14INFO 01-04 14:25:56 [loggers.py:111] Engine 000: Avg prompt throughput: 559.5 tokens/s, Avg generation throughput: 107.6 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.8%, Prefix cache hiINFO 01-04 14:26:02 [loggers.py:111] Engine 000: Avg prompt throughput: 777.3 tokens/s, Avg generation throughput: 58.6 tokens/s, Running: 2 reqs, INFO 01-04 14:26:06 [loggers.py:111] Engine 000: Avg prompt throughput: INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:26:12 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokenINFO 01-04 14:26:16 [loggers.py:111] Engine 000: Avg prompt throughput: 872.4 tokens/s, Avg generation throughput: 66.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.6%, Prefix cache INFO 01-04 14:26INFO 01-04 14:26:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 88.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.0%, Prefix cacheINFO 01-04 14:26:32 [loggers.py:111] Engine 000: Avg prompt throughput: 806.4 tokensINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:26:36 [loggers.py:111] Engine 000: Avg prompt throughpuINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:26:42 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, INFO 01-04 14:26:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.2 tokens/s, RunINFO 01-04 14:26:52 [loggers.py:111] Engine 000: Avg prompt throughput: 355.6 tokens/sINFO 01-04 14:26:56 [loggers.py:111] Engine 000: Avg prompt throughput: 741.0 tokens/s, Avg generation throughput: 54.7 tokens/s, RunINFO 01-04 14:27:02 [loggers.py:111] Engine 000: Avg prompt throughput: 874.8 tokens/sINFO 01-04 14:27:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 89.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.8%, Prefix cache hit rate: 19.9%
+INFO:     10.45.190.192:0 - "POST /v1/completiINFO 01-04 14:27:12 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, AvINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:27:16 [loggers.py:111] Engine 000: Avg prompt thrINFO 01-04 14:27:22 [loggers.py:111] Engine 000: Avg prompt throughput: 818.3 tokens/s, Avg generation throughput: 22.2 tokens/s, Running: 1 reqs, Waiting: INFO 01-04 14:27:26 [loggers.py:111] Engine 000: Avg prompt thrINFO 01-04 14:27:32 [loggers.py:111] Engine 000: Avg prompt throughput: 627.7 tokens/s, Avg generation throughput: 58.7 tokens/s, Running: 2 reqs, Waiting: INFO 01-04 14:27:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.4 tokens/s, RuINFO 01-04 14:27:42 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 53.6 tokens/s, Running: 1 reqs, Waiting:INFO 01-04 14:27:46 [loggers.py:111] Engine 000: Avg prompt throINFO 01-04 14:27:52 [loggers.py:111] Engine 000: Avg prompt throughput: 842.7 tokens/s, Avg generation throughput: 84.9 tokens/s, Running: 2 reqs, Waiting:INFO 01-04 14:27:56 [loggers.py:111] Engine 000: Avg prompt throINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:28:02 [loggers.py:111] Engine 000: Avg prompt throughput: 771.6 tokens/s, Avg generation throughput: 84.7 tokens/s, Running: 2 reqs, WaitiINFO 01-04 14:28:06 [loggers.py:111] Engine 000: Avg prompt throughINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:28:12 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, AvINFO 01-04 14:28:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 44.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, PrefiINFO 01-04 14:28:22 [loggers.py:111] Engine 000: Avg prompt throughput: 355.3 tokens/s, INFO 01-04 14:28:26 [loggers.py:111] Engine 000: Avg prompt throughput: 619.1 tokens/s, Avg generation throughput: 58.5 tokens/s, RINFO 01-04 14:28:32 [loggers.py:111] Engine 000: Avg prompt throughput: 799.2 tokens/s, INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:28:36 [loggers.py:111] Engine 000: Avg prompt throINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:28:42 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, AvgINFO 01-04 14:28:46 [loggers.py:111] Engine 000: Avg prompt throuINFO 01-04 14:28:46 [loggers.py:111] Engine 000: Avg prompt thrINFO 01-04 14:28:52 [loggers.py:111] Engine 000: Avg prompt throughput: 551.0 tokens/s, AvgINFO 01-04 14:28:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.0 tokens/s, RuINFO 01-04 14:28:56 [loggers.py:111] Engine 000: Avg prompt throINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:29:02 [INFO 01-04 14:29:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 22.6 tokens/s, RunniINFO 01-04 14:29:06 [loggers.py:111] Engine 000: Avg prompt thrINFO 01-04 14:29:12 [INFO 01-04 14:29:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, RunninINFO 01-04 14:29:16 [loggers.py:111] Engine 000: Avg prompt thINFO 01-04 14:29:22 [INFO 01-04 14:29:26 [loggers.py:111] Engine 000: Avg prompt throughput: 714.5 tokens/s, Avg generation throughput: 24.3 tokens/s, RunINFO 01-04 14:29:26 [loggers.py:111] Engine 000: Avg prompt throuINFO 01-04 14:29:32 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, AINFO 01-04 14:29:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 23.1 tokens/s, RuINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:29:42 [INFO 01-04 14:29:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, RunninINFO 01-04 14:29:46 [loggers.py:111] Engine 000: Avg prompt thrINFO 01-04 14:29:52 INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:30:06 [loggers.py:111] Engine 000: Avg prompt throughput: 902.4 tokens/s, Avg generation throughput: 27.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.2%
+INFO 01-04 14:30:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.2%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:30:46 [loggers.py:111] Engine 000: Avg prompt throughput: 865.1 tokens/s, Avg generation throughput: 23.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.9%
+INFO 01-04 14:30:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.9%
+INFO 01-04 14:31:16 [loggers.py:111] Engine 000: Avg prompt throughput: 895.3 tokens/s, Avg generation throughput: 44.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.7%, Prefix cache hit rate: 18.6%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:31:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 25.4 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.6%
+INFO 01-04 14:31:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.6%
+INFO 01-04 14:31:46 [loggers.py:111] Engine 000: Avg prompt throughput: 647.1 tokens/s, Avg generation throughput: 23.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 18.4%
+INFO 01-04 14:31:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 18.4%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:32:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 10.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.4%
+INFO 01-04 14:32:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.4%
+INFO 01-04 14:32:26 [loggers.py:111] Engine 000: Avg prompt throughput: 633.5 tokens/s, Avg generation throughput: 20.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 18.2%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:32:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 28.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.2%
+INFO 01-04 14:32:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.2%
+INFO 01-04 14:33:06 [loggers.py:111] Engine 000: Avg prompt throughput: 468.8 tokens/s, Avg generation throughput: 29.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.0%, Prefix cache hit rate: 18.0%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:33:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 45.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.0%
+INFO 01-04 14:33:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.0%
+INFO 01-04 14:33:46 [loggers.py:111] Engine 000: Avg prompt throughput: 709.4 tokens/s, Avg generation throughput: 15.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 17.8%
+INFO 01-04 14:33:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 17.8%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:34:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 21.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.8%
+INFO 01-04 14:34:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.8%
+INFO 01-04 14:35:06 [loggers.py:111] Engine 000: Avg prompt throughput: 415.5 tokens/s, Avg generation throughput: 18.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.7%, Prefix cache hit rate: 17.7%
+INFO 01-04 14:35:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 17.7%
+INFO 01-04 14:35:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 17.7%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:35:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 45.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.7%
+INFO 01-04 14:35:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.7%
+INFO 01-04 14:36:16 [loggers.py:111] Engine 000: Avg prompt throughput: 1200.1 tokens/s, Avg generation throughput: 17.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.8%, Prefix cache hit rate: 17.3%
+INFO 01-04 14:36:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.0%, Prefix cache hit rate: 17.3%
+INFO 01-04 14:36:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.2%, Prefix cache hit rate: 17.3%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:36:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 16.9 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.3%
+INFO 01-04 14:36:56 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.3%
+INFO 01-04 14:37:06 [loggers.py:111] Engine 000: Avg prompt throughput: 1476.1 tokens/s, Avg genINFO 01-04 14:37:15 [loggers.py:111] Engine 000: Avg prompt throughput: 995.5 tokens/s, Avg generation throughput: 3.7 tokenINFO 01-04 14:37:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg geneINFO 01-04 14:37:25 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 4INFO 01-04 14:38:06 [loggers.py:111] Engine 000: Avg prompt throughput: 834.8 tokens/s, Avg generation throughput: 6.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 14.9%
+INFO 01-04 14:38:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cache hit rate: 14.9%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:38:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generatioINFO 01-04 14:39:05 [loggers.py:111] Engine 000: Avg prompt throughput: 1156.2 tokens/s, Avg generation throughput: 2.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.6%, Prefix cache hit rate: 17.7%
+INFO 01-04 14:39:15 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46INFO 01-04 14:39:56 [loggers.py:111] Engine 000: Avg prompt throughput: 666.3 tokens/s, Avg generation throughput: 37.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 15.6%
+INFO 01-04 14:40:06 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 15.6%
+INFO 01-04 14:40:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GINFO 01-04 14:41:35 [loggers.py:111] Engine 000: Avg prompt throughput: 626.1 tokens/s, Avg generation throughput: 13.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 17.5%
+INFO 01-04 14:41:45 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 17.5%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:41:55 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 13.9 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.5%
+INFO 01-04 14:42:05 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokenINFO 01-04 14:43:16 [loggers.py:111] Engine 000: Avg prompt throughput: 719.8 tokens/s, Avg generation throughput: 5.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 17.5%
+INFO 01-04 14:43:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 17.5%
+INFO 01-04 14:43:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cache hit rate: 17.5%
+INFO 01-04 14:43:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 reqsINFO 01-04 14:45:05 [loggers.py:111] Engine 000: Avg prompt throughput: 1882.9 tokens/s, Avg generation throughput: 35.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.5%, Prefix cache hit rate: 16.9%
+INFO 01-04 14:45:15 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 45.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.7%, Prefix cache hit rate: 16.9%
+INFO 01-04 14:45:25 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 45.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.9%, Prefix cache hit rate: 16.9%
+INFO 01-04 14:45:35 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 45.5 tokens/s, Running: 1 reqs, Waiting: 0INFO 01-04 14:46:09 [loggers.py:111] Engine 000: Avg prompt throughput: 463.5 tokens/s, Avg generation throughput: 15.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 16.6%
+INFO 01-04 14:46:19 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 16.6%
+INFO 01-04 14:46:29 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 16.6%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:46:39 [loggers.py:111] Engine 000: Avg promINFO 01-04 14:48:45 [loggers.py:111] Engine 000:INFO 01-04 14:49:33 [loggers.py:111] Engine 000: Avg prompt throughput: 1074.6 tokens/s, Avg generation throughput: 45.6 tokens/INFO 01-04 14:59:23 [loggers.py:111] Engine 000: Avg prompt throughput: 471.3 tokens/s, Avg generation throughput: 1.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 13.3%
+INFO 01-04 14:59:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 13.3%
+INFO 01-04 14:59:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 13.3%
+INFO 01-04 14:59:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation througINFO 01-04 15:10:35 [loggers.py:111] Engine 000: Avg prompt throughput: 561.0 tokens/s, Avg generation throughput: 8.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 17.1%
+INFO 01-04 15:10:45 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 17.1%
+INFO 01-04 15:10:55 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 17.1%
+INFO 01-04 15:11:05 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 17.1%
+INFO 01-04 15:11:15 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 17.1%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 15:11:25 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 13.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.1%
+INFO 01-04 15:11:35 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.1%
+6.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 15:04:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.3 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.6%
+INFO 01-04 15:04:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.6%
+1-04 15:01:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.3%, Prefix cache hit rate: 13.3%
+INFO 01-04 15:01:43 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 45.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 13.3%
+INFO 01-04 15:01:53 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.6%, Prefix cache hit rate: 13.3%
+INFO 01-04 15:02:03 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.8%, Prefix cache hit rate: 13.3%
+INFO 01-04 15:02:13 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 45.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.0%, Prefix cache hit rate: 13.3%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 15:02:23 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 6.1 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 13.3%
+INFO 01-04 15:02:33 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 13.3%
+] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 30.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.0%
+INFO 01-04 14:58:19 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.0%

hf_ip/vllm_gpu7.log

@@ -0,0 +1,377 @@
+/usr/local/lib/python3.10/dist-packages/torch/utils/_pytree.py:185: FutureWarning: optree is installed but the version is too old to support PyTorch Dynamo in C++ pytree. C++ pytree support is disabled. Please consider upgrading optree using `python3 -m pip install --upgrade 'optree>=0.13.0'`.
+  warnings.warn(
+INFO 01-04 13:13:45 [__init__.py:239] Automatically detected platform cuda.
+INFO 01-04 13:13:48 [api_server.py:1043] vLLM API server version 0.8.5
+INFO 01-04 13:13:48 [api_server.py:1044] args: Namespace(host='127.0.0.1', port=8008, uvicorn_log_level='info', disable_uvicorn_access_log=False, allow_credentials=False, allowed_origins=['*'], allowed_methods=['*'], allowed_headers=['*'], api_key=None, lora_modules=None, prompt_adapters=None, chat_template=None, chat_template_content_format='auto', response_role='assistant', ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, enable_ssl_refresh=False, ssl_cert_reqs=0, root_path=None, middleware=[], return_tokens_as_token_ids=False, disable_frontend_multiprocessing=False, enable_request_id_headers=False, enable_auto_tool_choice=False, tool_call_parser=None, tool_parser_plugin='', model='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', task='auto', tokenizer=None, hf_config_path=None, skip_tokenizer_init=False, revision=None, code_revision=None, tokenizer_revision=None, tokenizer_mode='auto', trust_remote_code=True, allowed_local_media_path=None, load_format='auto', download_dir=None, model_loader_extra_config={}, use_tqdm_on_load=True, config_format=<ConfigFormat.AUTO: 'auto'>, dtype='auto', max_model_len=131072, guided_decoding_backend='auto', reasoning_parser=None, logits_processor_pattern=None, model_impl='auto', distributed_executor_backend=None, pipeline_parallel_size=1, tensor_parallel_size=1, data_parallel_size=1, enable_expert_parallel=False, max_parallel_loading_workers=None, ray_workers_use_nsight=False, disable_custom_all_reduce=False, block_size=None, gpu_memory_utilization=0.9, swap_space=4, kv_cache_dtype='auto', num_gpu_blocks_override=None, enable_prefix_caching=None, prefix_caching_hash_algo='builtin', cpu_offload_gb=0, calculate_kv_scales=False, disable_sliding_window=False, use_v2_block_manager=True, seed=None, max_logprobs=20, disable_log_stats=False, quantization=None, rope_scaling=None, rope_theta=None, hf_token=None, hf_overrides=None, enforce_eager=False, max_seq_len_to_capture=8192, tokenizer_pool_size=0, tokenizer_pool_type='ray', tokenizer_pool_extra_config={}, limit_mm_per_prompt={}, mm_processor_kwargs=None, disable_mm_preprocessor_cache=False, enable_lora=None, enable_lora_bias=False, max_loras=1, max_lora_rank=16, lora_extra_vocab_size=256, lora_dtype='auto', long_lora_scaling_factors=None, max_cpu_loras=None, fully_sharded_loras=False, enable_prompt_adapter=None, max_prompt_adapters=1, max_prompt_adapter_token=0, device='auto', speculative_config=None, ignore_patterns=[], served_model_name=['default-model'], qlora_adapter_name_or_path=None, show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, disable_async_output_proc=False, max_num_batched_tokens=None, max_num_seqs=256, max_num_partial_prefills=1, max_long_partial_prefills=1, long_prefill_token_threshold=0, num_lookahead_slots=0, scheduler_delay_factor=0.0, preemption_mode=None, num_scheduler_steps=1, multi_step_stream_outputs=True, scheduling_policy='fcfs', enable_chunked_prefill=None, disable_chunked_mm_input=False, scheduler_cls='vllm.core.scheduler.Scheduler', override_neuron_config=None, override_pooler_config=None, compilation_config=None, kv_transfer_config=None, worker_cls='auto', worker_extension_cls='', generation_config='auto', override_generation_config=None, enable_sleep_mode=False, additional_config=None, enable_reasoning=False, disable_cascade_attn=False, disable_log_requests=True, max_log_len=None, disable_fastapi_docs=False, enable_prompt_tokens_details=False, enable_server_load_tracking=False)
+INFO 01-04 13:13:56 [config.py:717] This model supports multiple tasks: {'classify', 'generate', 'score', 'reward', 'embed'}. Defaulting to 'generate'.
+INFO 01-04 13:13:56 [config.py:2003] Chunked prefill is enabled with max_num_batched_tokens=8192.
+/usr/local/lib/python3.10/dist-packages/torch/utils/_pytree.py:185: FutureWarning: optree is installed but the version is too old to support PyTorch Dynamo in C++ pytree. C++ pytree support is disabled. Please consider upgrading optree using `python3 -m pip install --upgrade 'optree>=0.13.0'`.
+  warnings.warn(
+INFO 01-04 13:14:01 [__init__.py:239] Automatically detected platform cuda.
+INFO 01-04 13:14:04 [core.py:58] Initializing a V1 LLM engine (v0.8.5) with config: model='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', speculative_config=None, tokenizer='/data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=131072, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto,  device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='auto', reasoning_backend=None), observability_config=ObservabilityConfig(show_hidden_metrics=False, otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=None, served_model_name=default-model, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=True, chunked_prefill_enabled=True, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs=None, pooler_config=None, compilation_config={"level":3,"custom_ops":["none"],"splitting_ops":["vllm.unified_attention","vllm.unified_attention_with_output"],"use_inductor":true,"compile_sizes":[],"use_cudagraph":true,"cudagraph_num_of_warmups":1,"cudagraph_capture_sizes":[512,504,496,488,480,472,464,456,448,440,432,424,416,408,400,392,384,376,368,360,352,344,336,328,320,312,304,296,288,280,272,264,256,248,240,232,224,216,208,200,192,184,176,168,160,152,144,136,128,120,112,104,96,88,80,72,64,56,48,40,32,24,16,8,4,2,1],"max_capture_size":512}
+WARNING 01-04 13:14:05 [utils.py:2522] Methods determine_num_available_blocks,device_config,get_cache_block_size_bytes,initialize_cache not implemented in <vllm.v1.worker.gpu_worker.Worker object at 0x7f5053f564d0>
+INFO 01-04 13:14:06 [parallel_state.py:1004] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, TP rank 0
+INFO 01-04 13:14:06 [cuda.py:221] Using Flash Attention backend on V1 engine.
+WARNING 01-04 13:14:06 [topk_topp_sampler.py:69] FlashInfer is not available. Falling back to the PyTorch-native implementation of top-p & top-k sampling. For the best performance, please install FlashInfer.
+INFO 01-04 13:14:06 [gpu_model_runner.py:1329] Starting to load model /data/minimax-dialogue/users/ado/072025project/02verifiable/ckpt/rl_roleplay_1210_6-gmm1/global_step_160/hf...
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+INFO 01-04 13:15:40 [loader.py:458] Loading weights took 94.24 seconds
+INFO 01-04 13:15:41 [gpu_model_runner.py:1347] Model loading took 61.0562 GiB and 94.501715 seconds
+INFO 01-04 13:15:55 [backends.py:420] Using cache directory: /root/.cache/vllm/torch_compile_cache/75e72335d9/rank_0_0 for vLLM's torch.compile
+INFO 01-04 13:15:55 [backends.py:430] Dynamo bytecode transform time: 14.44 s
+INFO 01-04 13:15:59 [backends.py:136] Cache the graph of shape None for later use
+INFO 01-04 13:16:49 [backends.py:148] Compiling a graph for general shape takes 53.17 s
+INFO 01-04 13:17:53 [monitor.py:33] torch.compile takes 67.61 s in total
+INFO 01-04 13:17:54 [kv_cache_utils.py:634] GPU KV cache size: 254,768 tokens
+INFO 01-04 13:17:54 [kv_cache_utils.py:637] Maximum concurrency for 131,072 tokens per request: 1.94x
+INFO 01-04 13:18:33 [gpu_model_runner.py:1686] Graph capturing finished in 40 secs, took 1.21 GiB
+INFO 01-04 13:18:33 [core.py:159] init engine (profile, create kv cache, warmup model) took 172.44 seconds
+INFO 01-04 13:18:33 [core_client.py:439] Core engine process 0 ready.
+WARNING 01-04 13:18:33 [config.py:1239] Default sampling parameters have been overridden by the model's Hugging Face generation config recommended from the model creator. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
+INFO 01-04 13:18:33 [serving_chat.py:118] Using default chat sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
+INFO 01-04 13:18:33 [serving_completion.py:61] Using default completion sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
+INFO 01-04 13:18:33 [api_server.py:1090] Starting vLLM API server on http://127.0.0.1:8008
+INFO 01-04 13:18:33 [launcher.py:28] Available routes are:
+INFO 01-04 13:18:33 [launcher.py:36] Route: /openapi.json, Methods: GET, HEAD
+INFO 01-04 13:18:33 [launcher.py:36] Route: /docs, Methods: GET, HEAD
+INFO 01-04 13:18:33 [launcher.py:36] Route: /docs/oauth2-redirect, Methods: GET, HEAD
+INFO 01-04 13:18:33 [launcher.py:36] Route: /redoc, Methods: GET, HEAD
+INFO 01-04 13:18:33 [launcher.py:36] Route: /health, Methods: GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /load, Methods: GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /ping, Methods: GET, POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /tokenize, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /detokenize, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/models, Methods: GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /version, Methods: GET
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/chat/completions, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/completions, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/embeddings, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /pooling, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /score, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/score, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/audio/transcriptions, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /rerank, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v1/rerank, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /v2/rerank, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /invocations, Methods: POST
+INFO 01-04 13:18:33 [launcher.py:36] Route: /metrics, Methods: GET
+INFO:     Started server process [1319238]
+INFO:     Waiting for application startup.
+INFO:     Application startup compleIINFO 01-04 14:00:14 [loggers.py:111] Engine 000: Avg prompt throughput: 53.6 tokens/s, Avg generation throughput: 29.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.3%, Prefix cache hit rate: 0.0%
+INFO 01-04 14:00:24 [loggers.py:111] Engine 000: Avg prompt throughput: 57.7 tokens/s, Avg generation throughput: 61.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.8%, Prefix cache hit rate: 1.4%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:00:34 [loggers.py:111] Engine 000: Avg prompt throughput: 58.8 tokens/s, Avg generation throughput: 98.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.9%, Prefix cache hit rate: 1.9%
+INFO:     10.45.190.192:0 - "GET /v1/models HTTP/1.1" 200 OK
+INFO 01-04 14:00:44 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 93.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.2%, Prefix cache hit rate: 1.9%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:00:54 [loggers.py:111] Engine 000: Avg prompt throughput: 64.9 tokens/s, Avg generation throughput: 71.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.0%, Prefix cache hit rate: 2.1%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:01:04 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 84.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.5%, Prefix cache hit rate: 2.1%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:01:14 [loggers.py:111] Engine 000: Avg prompt throughput: 75.3 tokens/s, Avg generation throughput: 74.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.6%, Prefix cache hit rate: 2.1%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:01:24 [loggers.py:111] Engine 000: Avg prompt throughput: 71.1 tokens/s, Avg generation throughput: 50.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.4%, Prefix cache hit rate: 2.1%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:01:34 [loggers.py:111] Engine 000: Avg prompt throughput: 86.2 tokens/s, Avg generation throughput: 12.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.4%, Prefix cache hit rate: 2.1%
+INFO 01-04 14:01:44 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.5%, Prefix cache hit rate: 2.1%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:01:54 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 17.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 2.1%
+INFO 01-04 14:02:04 [loggers.py:111] Engine 000: Avg prompt throughput: 90.2 tokens/s, Avg generation throughput: 33.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.5%, Prefix cache hit rate: 2.0%
+INFO 01-04 14:02:14 [loggers.py:111] Engine 000: Avg prompt throughput: 110.2 tokens/s, Avg generation throughput: 51.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.1%, Prefix cache hit rate: 1.7%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:02:24 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 90.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.6%, Prefix cache hit rate: 1.7%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:02:34 [loggers.py:111] Engine 000: Avg prompt throughput: 163.2 tokens/s, Avg generation throughput: 73.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.9%, Prefix cache hit rate: 1.6%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:02:44 [loggers.py:111] Engine 000: Avg prompt throughput: 80.9 tokens/s, Avg generation throughput: 107.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.9%, Prefix cache hit rate: 1.4%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:02:54 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 60.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 1.4%
+INFO 01-04 14:03:04 [loggers.py:111] Engine 000: Avg prompt throughput: 110.1 tokens/s, Avg generation throughput: 5.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.5%, Prefix cache hit rate: 1.3%
+INFO 01-04 14:03:14 [loggers.py:111] Engine 000: Avg prompt throughput: 124.2 tokens/s, Avg generation throughput: 65.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.2%, Prefix cache hit rate: 1.1%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:24 [loggers.py:111] Engine 000: Avg prompt throughput: 132.8 tokens/s, Avg generation throughput: 85.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.6%, Prefix cache hit rate: 1.1%
+INFO 01-04 14:03:34 [loggers.py:111] Engine 000: Avg prompt throughput: 138.8 tokens/s, Avg generation throughput: 80.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache hit rate: 1.0%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:03:44 [loggers.py:111] Engine 000: Avg prompt throughput: 90.8 tokens/s, Avg generation throughput: 51.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.4%, Prefix cache hit rate: 1.1%
+INFO 01-04 14:03:54 [loggers.py:111] Engine 000: Avg prompt throughput: 120.3 tokens/s, Avg generation throughput: 63.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.1%, Prefix cache hit rate: 1.1%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:04:04 [loggers.py:111] Engine 000: Avg prompt throughput: 119.8 tokens/s, Avg generation throughput: 80.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.5%, Prefix cache hit rate: 5.9%
+INFO 01-04 14:04:14 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 48.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.7%, Prefix cache hit rate: 5.9%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:04:24 [loggers.py:111] Engine 000: Avg prompt throughput: 267.0 tokens/s, Avg generation throughput: 68.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.2%, Prefix cache hit rate: 5.1%
+INFO 01-04 14:04:34 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 94.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 5.1%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:04:44 [loggers.py:111] Engine 000: Avg prompt throughput: 283.8 tokens/s, Avg generation throughput: 114.5 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 4.6%
+INFO 01-04 14:04:54 [loggers.py:111] Engine 000: Avg prompt throughput: 182.6 tokens/s, Avg generation throughput: 139.4 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.7%, Prefix cache hit rate: 4.2%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:04 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 145.4 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 4.2%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:14 [loggers.py:111] Engine 000: Avg prompt throughput: 99.6 tokens/s, Avg generation throughput: 91.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 7.6%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:24 [loggers.py:111] Engine 000: Avg prompt throughput: 202.0 tokens/s, Avg generation throughput: 96.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 13.6%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:34 [loggers.py:111] Engine 000: Avg prompt throughput: 140.5 tokens/s, Avg generation throughput: 121.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 12.9%
+INFO 01-04 14:05:44 [loggers.py:111] Engine 000: Avg prompt throughput: 241.2 tokens/s, Avg generation throughput: 123.4 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.2%, Prefix cache hit rate: 12.0%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:05:54 [loggers.py:111] Engine 000: Avg prompt throughput: 211.4 tokens/s, Avg generation throughput: 96.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 11.3%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:04 [loggers.py:111] Engine 000: Avg prompt throughput: 294.4 tokens/s, Avg generation throughput: 93.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.9%, Prefix cache hit rate: 10.4%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:14 [loggers.py:111] Engine 000: Avg prompt throughput: 236.0 tokens/s, Avg generation throughput: 112.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 15.1%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:24 [loggers.py:111] Engine 000: Avg prompt throughput: 114.3 tokens/s, Avg generation throughput: 91.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cache hit rate: 14.7%
+INFO 01-04 14:06:34 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 92.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.9%, Prefix cache hit rate: 14.7%
+INFO 01-04 14:06:44 [loggers.py:111] Engine 000: Avg prompt throughput: 377.1 tokens/s, Avg generation throughput: 144.7 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.9%, Prefix cache hit rate: 14.6%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:06:54 [loggers.py:111] Engine 000: Avg prompt throughput: 166.6 tokens/s, Avg generation throughput: 148.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 15.3%
+INFO 01-04 14:07:04 [loggers.py:111] Engine 000: Avg prompt throughput: 126.1 tokens/s, Avg generation throughput: 131.7 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 14.9%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:14 [loggers.py:111] Engine 000: Avg prompt throughput: 242.7 tokens/s, Avg generation throughput: 82.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.1%, Prefix cache hit rate: 14.2%
+INFO 01-04 14:07:24 [loggers.py:111] Engine 000: Avg prompt throughput: 207.8 tokens/s, Avg generation throughput: 82.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 14.9%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:07:34 [loggers.py:111] Engine 000: Avg prompt throughput: 277.3 tokens/s, Avg generation throughput: 50.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.2%, Prefix cache hit rate: 14.1%
+INFO 01-04 14:07:44 [loggers.py:111] Engine 000: Avg prompt throughput: 213.5 tokens/s, Avg generation throughput: 48.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 13.6%
+INFO 01-04 14:07:54 [loggers.py:111] Engine 000: Avg prompt throughput: 329.5 tokens/s, Avg generation throughput: 91.3 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.9%, Prefix cache hit rate: 12.9%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:04 [loggers.py:111] Engine 000: Avg prompt throughput: 215.0 tokens/s, Avg generation throughput: 92.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 13.3%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:14 [loggers.py:111] Engine 000: Avg prompt throughput: 189.1 tokens/s, Avg generation throughput: 96.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 12.9%
+INFO 01-04 14:08:24 [loggers.py:111] Engine 000: Avg prompt throughput: 292.2 tokens/s, Avg generation throughput: 99.9 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 16.0%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:34 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 124.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 16.0%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:08:44 [loggers.py:111] Engine 000: Avg prompt throughput: 254.9 tokens/s, Avg generation throughput: 87.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 15.4%
+INFO 01-04 14:08:54 [loggers.py:111] Engine 000: Avg prompt throughput: 200.4 tokens/s, Avg generation throughput: 39.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.0%, Prefix cache hit rate: 15.0%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:04 [loggers.py:111] Engine 000: Avg prompt throughput: 256.8 tokens/s, Avg generation throughput: 22.9 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.5%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:14 [loggers.py:111] Engine 000: Avg prompt throughput: 539.6 tokens/s, Avg generation throughput: 51.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache hit rate: 13.5%
+INFO 01-04 14:09:24 [loggers.py:111] Engine 000: Avg prompt throughput: 264.5 tokens/s, Avg generation throughput: 59.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 13.1%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:34 [loggers.py:111] Engine 000: Avg prompt throughput: 244.4 tokens/s, Avg generation throughput: 45.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.0%, Prefix cache hit rate: 12.7%
+INFO 01-04 14:09:44 [loggers.py:111] Engine 000: Avg prompt throughput: 398.8 tokens/s, Avg generation throughput: 57.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 12.1%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:09:54 [loggers.py:111] Engine 000: Avg prompt throughput: 362.1 tokens/s, Avg generation throughput: 101.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 11.6%
+INFO:   INFO 01-04 14:09:57 [loggers.py:111] Engine 000: Avg proINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:04 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generaINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.INFO 01-04 14:10:14 [loggers.py:111] Engine 000: Avg prompt throughput: 448.8 tokens/s, Avg generation throughput: 19.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 11.1%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:24 [loggers.py:111] Engine 000: Avg prompt throughput: 281.9 tokens/s, Avg generation throughput: 58.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: 10.8%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:34 [loggers.py:111] Engine 000: Avg prompt throughput: 286.5 tokens/s, Avg generation throughput: 31.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.2%, Prefix cache hit rate: 10.5%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:10:44 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 22.1 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 10.5%
+INFO 01-04 14:10:54 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 10.5%
+INFO 01-04 14:11:04 [loggers.py:111] Engine 000: Avg prompt throughput: 267.6 tokens/s, Avg generation throughput: 21.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.1%, Prefix cache hit rate: 10.3%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:11:14 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 36.4 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 10.3%
+INFO 01-04 14:11:24 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 10.3%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:11:34 [loggers.py:111] Engine 000: Avg prompt throughput: 397.6 tokens/s, Avg generation throughput: 34.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 9.9%
+INFO 01-04 14:11:44 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0INFO 01-04 14:11:46 [loggers.pyINFO 01-04 14:11:54 [loggers.py:111] Engine 000: Avg prompt throughput: 305.7 tokens/s, Avg generation throughput: 2.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.2%, Prefix cache hit rate: 9.6%
+INFO 01-04 14:12:04 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.4%, Prefix cache hit rate: 9.6%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:12:14 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 31.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: INFO 01-04 14:12:16 [loggers.py:11INFO 01-04 14:12:24 [loggers.py:111] Engine 000: Avg prompt throughput: 527.5 tokens/s, Avg generation throughpINFO 01-04 14:12:29 [loggers.py:111] Engine 000: Avg prompt throughput: 189.1 tokens/s, Avg generationINFO 01-04 14:12:37 [loggers.py:111] Engine 000: Avg prompt throughput: 322.6 tokens/s, Avg generation throughput: 1INFO 01-04 14:12:39 [loggers.py:111] Engine 000: Avg prompt throughput: 528.0 tokens/s, Avg generation INFO 01-04 14:12:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage:INFO 01-04 14:12:49 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation thINFO 01-04 14:12:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.9 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usaINFO 01-04 14:12:59 [loggers.py:111] EnINFO 01-04 14:13:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usaINFO 01-04 14:13:09 [loggers.py:111] EnINFO 01-04 14:13:17 [loggers.py:111] Engine 000: Avg prompt throughput: 507.0 tokens/s, Avg generation throughput: 19.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usINFO 01-04 14:13:19 [loggers.py:111] EngINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:13:27 [loggers.py:111] Engine 000: Avg prompt throughput: 577.2 tokens/s, Avg generation throughput: 46.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 15.9%
+INFO 01-04 14:13:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 15.9%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:13:47 [loggers.py:111] Engine 000: Avg prompt throughput: 950.4 tokens/s, Avg generation throughput: 58.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.9%, Prefix cache hit rate: 16.5%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:13:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 87.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 16.5%
+INFO:     10.43.30.3:0 -INFO 01-04 14:13:59 [loggers.py:111] EngINFO 01-04 14:14:07 [loggers.py:111] Engine 000: Avg prompt throughput: 286.2 tokens/s, Avg generation throughput: 27.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache uINFO:     10.43.30.4:0 - "POST /v1/compleINFO 01-04 14:14:17 [loggers.py:111] Engine 000: Avg prompt throughput: 391.1 tokens/s, Avg generation throughput: 70.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 17.9%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:27 [loggers.py:111] Engine 000: Avg prompt throughput: 312.5 tokens/s, Avg generation throughput: 115.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.4%, Prefix cache hit rate: 17.5%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:37 [loggers.py:111] Engine 000: Avg prompt throughput: 975.2 tokens/s, Avg generation throughput: 55.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU INFO 01-04 14:14:39 [loggers.py:111] Engine 000: AvINFO 01-04 14:14:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 92.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.3%, Prefix cache hit rate: 16.5%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:14:57 [loggers.py:111] Engine 000: Avg prompt throughput: 656.0 tokens/s, Avg generation throughput: 56.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.1%, Prefix cache hit rate: 17.6%
+INFO 01-04 14:15:07 [loggers.py:111] Engine 000: Avg prompt throughput: 436.1 tokens/s, Avg generation throughput: 78.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 18.2%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:17 [loggers.py:111] Engine 000: Avg prompt throughput: 444.8 tokens/s, Avg generation throughput: 100.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 17.7%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:27 [loggers.py:111] Engine 000: Avg prompt throughput: 528.0 tokens/s, Avg generation throughput: 77.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 17.3%
+INFO:     10.46.INFO:     10.43.30.3:0 - "POST /v1/completions HTTINFO 01-04 14:15:37 [loggers.py:111] Engine 000: Avg prompt throughput: 385.4 tokens/s, Avg generation throughput: 40.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 16.9%
+INFO 01-04 14:15:47 [loggers.py:111] Engine 000: Avg prompt throughput: 353.2 tokens/s, Avg generation throughput: 83.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 17.1%
+INFO: INFO 01-04 14:15:49 [loggers.py:111] Engine 000: Avg prompINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:15:57 [loggers.py:111] Engine 000: Avg prompt throughput: 433.8 tokens/s, Avg generation throughput: 43.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 18.6%
+INFO 01-04 14:16:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 18.6%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:17 [loggers.py:111] Engine 000: Avg prompt throughput: 1148.9 tokens/s, Avg generation throughput: 66.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs,INFO 01-04 14:16:19 [loggers.py:111] Engine 000: Avg proINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:27 [loggers.py:111] Engine 000: Avg prompt throughput: 348.1 tokens/s, Avg geneINFO 01-04 14:16:29 [loggers.py:111] Engine 000: Avg prompt throughput: 559.7 tokens/s, Avg generation throughput: 106.4 toINFO 01-04 14:16:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generatINFO 01-04 14:16:39 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 134.1 tINFO 01-04 14:16:47 [loggers.py:111] Engine 000: Avg prompt throughput: 740.6 tokens/s, Avg generaINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:49 [loggers.py:111] Engine 000: Avg promINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:16:57 [loggers.py:111] Engine 000: Avg prompt throughput: 506.6 tokens/s, Avg generation throughput: 126.7 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.5%, Prefix cache hit rate: 18.2%
+INFO:     10INFO 01-04 14:16:59 [loggers.py:111] Engine 000: AvgINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:07 [loggers.py:111] Engine 000: Avg prompt throughput: 421.8 tokens/s, Avg generation throughput: 88.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.0%, Prefix cache hit rate: 17.9%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.17.192INFO 01-04 14:17:14 [loggers.py:111] Engine INFO 01-04 14:17:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation thINFO 01-04 14:17:19 [loggers.py:111] Engine 000: Avg prompt throughput: 6INFO 01-04 14:17:24 [loggers.py:111] EngiINFO 01-04 14:17:27 [loggers.py:111] Engine 000: Avg prompt throughput: 522.6 tokens/s, Avg generation throughput: 44.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 19.4%
+INFO 01-04 14:17:37 [loggers.py:111] Engine 000: Avg prompt throughput: 908.0 tokens/s, Avg generation throughput: 92.2 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.2%, Prefix cache hit rate: 18.7%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:17:47 [loggers.py:111] Engine 000: Avg prompt throughput: 459.1 tokens/s, Avg generation tINFO 01-04 14:17:49 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.INFO:     10.46.50.192:0 INFO 01-04 14:17:54 [loggers.py:111] EngiINFO 01-04 14:17:57 [loggers.py:111] Engine 000: Avg prompt throughput: 477.8 tokens/s, Avg generatINFO 01-04 14:17:59 [loggers.py:111] Engine 000: Avg prompt throughput: 1344.9 INFO 01-04 14:18:04 [loggers.py:111] EngiINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:07 [loggers.py:111] Engine 000: Avg prompt throughput: 594.1 tokens/s, Avg generation throughpuINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.INFO 01-04INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:14 [loggers.py:111] EnginINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:18:17 [loggers.py:1INFO 01-04 14:18:19 [loggers.py:111] Engine 000: Avg prompt throughput: 967.8 tINFO 01-04 14:18:24 [loggers.py:111] Engine 000: Avg prompt throughput: 480.5 tokens/s, Avg generation throINFO 01-04 14:18:27 [loggers.py:111] Engine 000: Avg prompt throughput: 594.8 tokens/s, Avg generatINFO 01-04 14:18:29 [loggers.py:111] Engine 000: Avg prompt throughput: 537.9 toINFO:     10.46.50.192:0 - "POST /v1/comINFO 01-04 14:18:37 [loggers.py:111] Engine 000: Avg prompt throughput: 747.7 tokens/s, Avg generation throughput: 128.7 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.3%, Prefix cache hit rate: 17.6%
+INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.46.50.192:0 - "INFO 01-04 14:18:44 [loggers.py:111] Engine 000: Avg prompt throughput: 414.9 tokens/s, Avg generationINFO 01-04 14:18:47 [loggers.py:111INFO 01-04 14:18:49 [loggers.py:111] Engine 000: Avg prompt throughput: 554.6 tokeINFO 01-04 14:18:54 [loggers.py:111] Engine 000: Avg prompt throughput: 334.6 tokens/s, Avg generationINFO 01-04 14:18:57 [loggers.py:111INFO 01-04 14:18:59 [loggers.py:111] Engine 000: Avg prompt throughput: 750.9 tokens/s, Avg generation throughput: 62.4 tokens/s, Running: 2 reqs, WaINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:04 [loggers.py:111INFO:     10.45.190.192:0 - "POST /INFO 01-04 14:19:09 [loggers.py:111] Engine 000: Avg prompt throughput: 679.8 tokens/s, Avg generation throughput: 68.7 tokens/s, Running: 2 reqs, WaiINFO 01-04 14:19:14 [loggers.py:111] Engine 000: Avg prompt throughput: 277.5 tokens/s, Avg generatioINFO 01-04 14:19:17 [loggers.py:111] Engine 000: Avg prompt throughput: 412.3 tokens/s, Avg generaINFO 01-04 14:19:19 INFO 01-04 14:19:24 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 137.4 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:27 [loggers.pyINFO 01-04 14:19:INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:19:34 [loggers.py:111] Engine 000: Avg prompt throughput: 609.1 tokens/s, Avg generation INFO 01-04 14:19:37 [loggers.py:1INFO 01-04 14:19:39 [loggers.py:111] Engine 000: Avg prompt throughput: 327.6 tokens/s, Avg generation throughput: 53.4 tokens/s, Running: 2 reqs, WINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFOINFO 01-04 14:19:49 [loggers.py:111] Engine 000: Avg prompt throughput: 630.0 tokens/s, Avg generation throughput: 88.7 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.1%, Prefix cache hit rate: INFO: INFO:     10.43.30.5:0 - "POST /v1INFO:     10.46.17.192:0 - "POINFO 01-04 14:19:59 [loggers.py:111] Engine 000: Avg prompt throughput: 704.9 tokens/s, Avg generation throughput: 98.9 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.7%, Prefix cache hit rate: 11.INFO 01-04 14:20:04 [loggers.py:111] Engine 000: Avg prompt throughpuINFO:     10.43.30.5:0 - "POST /v1INFO 01-04 14:20:07 [loggers.pINFO 01-04 14:20:09 [loggers.py:111] Engine 000: Avg prompt throughput: 534.9 tokens/sINFO 01-04 14:20:14 [loggers.py:111] Engine 000: Avg prompt throughput: 453.0 tokens/s, Avg generation INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:17 [loggers.pINFO 01-04 14:20:19 [loggers.py:111] Engine 000: Avg prompt throughput: 646.6 tokens/INFO 01-04 14:20:24 [loggers.py:111] Engine 000: Avg prompt throughput: 617.5 tokens/s, Avg generation throughput: 75.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KVINFO 01-04 14:20:27 [loggersINFO 01-04 14:20:29 [INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:34 [loggers.py:111] Engine 000: Avg prompt throughput: 482.5 tokens/s, Avg generation INFO 01-04 14:20:37 [loggers.INFO 01-04 14:20:39 [loggers.py:111] Engine 000: Avg prompt throughput: 420.4 tokens/s,INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:44 [loggers.py:111] Engine 000: Avg prompt throuINFO 01-04 14:20:49 [loggers.py:111] Engine 000: Avg prompt throughput: 451.3 tokens/s, Avg generation throughput: 53.8 tokens/s, Running: 2 reqs, WaitingINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OKINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:20:59 [loggers.py:111] Engine 000: Avg prompt throughput: 964.4 tokens/s, Avg generation throughput: 75.3 tokens/s, Running: 2 reqs, WaitingINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+IINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:09 [loggers.py:111] Engine 000: Avg prompt throughput: 697.4 tokens/s, Avg generation throughput: 104.3 tokens/s, Running: 2 reqs, WaINFO 01-04 14:21:14 [loggers.py:111] Engine 000: Avg prompt throughputINFO:     10.43.30.5:0 - "POST /vINFO 01-04 14:21:17 [loggers.pyINFO 01-04 14:21:19 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s,INFO 01-04 14:21:24 [loggers.py:111] Engine 000: Avg prompt throughput: 751.5 tokens/s, Avg generatioINFO 01-04 14:21:27 [loggers.py:111] Engine 000: Avg prompt throughput: 1347.5 tokens/s, Avg geINFO 01-04 14:21:29 [loggers.py:111] Engine 000: Avg prompt throughput: 184.8 tokens/s,INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:34 [loggers.py:111] Engine 000: Avg prompt throughpINFO 01-04 14:21:39 [loggers.py:111] Engine 000: Avg prompt throughput: 641.2 tokens/s, Avg generation throughput: 91.1 tokens/s, Running: 2 reqs, WaitINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFOINFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:21:49 [loggers.py:111] Engine 000: Avg prompt throughput: 711.2 tokens/s, Avg generation throughput: 100.0 tokens/s, Running: 2 reqs, WaiINFO 01-04 14:21:54 [loggers.py:111] Engine 000: Avg prompt throughpuINFO 01-04 14:21:59 [loggers.py:111] Engine 000: Avg prompt throughput: 577.5 tokens/s, Avg generation throughput: 101.3 tokens/s, Running: 3 reqs, WaINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:09 [loggers.py:111] Engine 000: Avg prompt throughput: 222.2 tokens/s, Avg generation throughput: 118.2 tokens/s, Running: 3 reqs, WINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFOINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:19 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 120.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: INFO 0INFO 01-04 14:22:29 [loggers.py:111] Engine 000: Avg prompt throughput: 612.5 tokens/s, Avg generation throughput: 87.3 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.6%, Prefix cache hit rate: INFO 01-04 14:22:34 [loggers.py:111] Engine 000: Avg prompt throughput: INFO 01-04 14:22:39 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 100.2 tokens/s, Running: 2 reqs, WINFO 01-04 14:22:44 [loggers.py:111] Engine 000: Avg prompt throughput:INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:49 [loggers.py:111] Engine 000: Avg prompt throughput: 531.4 tokeINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:22:54 [loggers.py:111] Engine 000: Avg prompt throughput: 829.4 tokens/s, Avg generation throughput: 100.2 tokens/s, Running:INFO 01-04 14:22:59 [loggers.py:111] Engine 000: Avg prompt throughput: 568.1 tokINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 0INFO 01-04 14:23:09 [loggers.py:111] Engine 000: Avg prompt throughput: 657.1 tokens/s, Avg generation throughput: 116.0 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.9%, Prefix cache hit rate:INFO 01-04 14:23:14 [loggers.py:111] Engine 000: Avg prompt throughput: 74INFO 01-04 14:23:19 [loggers.py:111] Engine 000: Avg prompt throughput: 807.2 tokens/s, Avg generation throughput: 97.7 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.7%, Prefix cache hit rateINFO:   INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.INFO 01-04INFO 01-04 14:23:29 [loggers.py:111] Engine 000: Avg prompt throughput: 713.3 tokens/s, Avg generation throughput: 133.0 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.7%, Prefix cache hitINFO 01-04 14INFO 01-04 14:23:39 [loggers.py:111] Engine 000: Avg prompt throughput: 404.1 tokens/s, Avg generation throughput: 138.4 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 10.9%, Prefix cache hit rate: 11.3%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTINFO 01-04 14:INFO 01-04 14:23:49 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 141.0 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.3%, Prefix cache hitINFO 01-04 14:23:54 [loggers.py:111] Engine 000: Avg prompt throughput: 569.3 tINFO 01-04 14:23:59 [loggers.py:111] Engine 000: Avg prompt throughput: 609.9 tokens/s, Avg generation throughput: 138.9 tokens/s, Running: INFO 01-04 14:24:04 [loggers.py:111] Engine 000: Avg prompt throughput: 735.8 toINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:09 [loggers.py:111] Engine 000: Avg prompt throughput: 90INFO 01-04 14:24:14 [loggers.py:111] Engine 000: Avg prompt throughput: 849.5 tokens/s, Avg generation throughput: 98.8 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.2%, Prefix cache hit rate: INFO 01-04 14:24:19 [loggers.py:111] Engine 000: Avg prompt throughput: 652.3 tokens/s, Avg generation throughput: 67.9 tokens/s, RunninINFO 01-04 14:24:24 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/INFO 01-04 14:24:29 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 90.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.7%, Prefix cacINFO 01-04 14:24:34INFO 01-04 14:24:39 [loggers.py:111] Engine 000: Avg prompt throughput: 728.4 tokens/s, Avg generation throughput: 107.7 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 10.0%, Prefix cINFO:     10.46.50.19INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:24:49 [loggers.py:111] Engine 000: Avg prompt throughput: 478.7 tokens/s, Avg generation throughput: 140.1 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.3%, PrefINFO 01-04 14:24:54 [loggers.py:111] Engine 000: Avg prompt throughput: 939.1 tokens/s, AINFO:     10INFO 01-04 14:24:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.INFO 01-04 14:24:59 [loggers.py:111] Engine 000: Avg prompt througINFO 01-04 14:25:04 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation INFO 01-04 14:25:07 [loggers.py:111] Engine 000: Avg INFO 01-04 14:25:09 [loggers.py:111] Engine 000: Avg prompt thINFO 01-04 14:25:14 [loggers.py:111] Engine 000: Avg prompt throughput: 712.9 tokens/s, Avg generation throughput: 25.6 tokens/s, Running: 1 reqs, Waiting: 0INFO 01-04 14:25:19 [loggers.py:111] Engine 000: Avg prompt thINFO 01-04 14:25:24 [loggers.py:111] Engine 000: Avg prompt throughput: 599.7 tokens/s, Avg generation throughput: 47.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KINFO:     10.45.190.192:0 - "POST /v1/completions INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:34 [loggers.py:111] Engine 000: Avg prompt throughput: 511.0 tokens/s, Avg generation throughput: 76.0 tokens/s, Running: 2 reqs, Waiting: 0INFO 01-04 14:25:39 [loggers.py:111] Engine 000: Avg prompt thINFO 01-04 14:25:44 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 91.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.9%, Prefix cache hit rate: 11.5%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:25:54 [loggers.py:111] Engine 000: Avg prompt throughput: 345.0 tokens/s, Avg generation throughput: 81.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 11.4%
+INFO: INFO 01-04 14:25:59 [loggers.py:111] Engine 000: Avg prompt INFO 01-04 14:26:04 [loggers.py:111] Engine 000: Avg prompt throughput: 577.1 tokens/s, Avg generation throughput: 77.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KINFO 01-04 14:26:07 [loggers.py:111] Engine 000: AINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:26:14 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generationINFO 01-04 14:26:17 [loggers.py:111] Engine 000: Avg prompt throughput: 798.1 tokens/s, Avg generation throughput: 84.2 toINFO 01-04 14:26:19 [loggers.py:111] Engine 000: Avg promptINFO 01-04 14:26:24 [loggers.py:111] EINFO 01-04 14:26:27 [loggers.py:111] Engine 000: Avg prompt throughput: 589.4 tokens/s, Avg generation throughput: 119.4INFO 01-04 14:26:29 [loggers.py:111] Engine 000: Avg promptINFO 01-04 14:26:34 [loggers.py:111] Engine 000: Avg prompt throughput: 827.0 tokens/s, Avg generation throINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:26:37 [loggers.py:111] Engine 00INFO 01-04 14:26:44 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cache hit rate: 13.8%IINFO 01-04 14:26:54 [loggers.py:111] Engine 000: Avg prompt throughput: 975.8 tokens/s, Avg generation throughput: 71.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.6%, Prefix cache hit rate: 13.5%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:27:04 [loggers.py:111] Engine 000: Avg prompt throughput: 809.6 tokens/s, Avg generation throughput: 100.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU INFO:     10.46.17.192:0 - "POST /v1/completions HTINFO 01-04 14:27:14 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 89.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.8%, Prefix cache hit rate: 13.3%
+INFO:     10.46.1INFO 01-04 14:27:17 [loggers.py:111] Engine 000: Avg prompt throughput: 825.7 tokens/s, Avg generation throughput: INFO 01-04 14:27:24 [loggers.py:111] Engine 000: Avg prompt throughput: 1172.5 tokens/s, Avg generation throughput: 41.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPINFOINFO 01-04 14:27:27 [loggers.py:111] Engine 000: INFO 01-04 14:27:34 [loggers.py:111] Engine 000: Avg prompt throughput: 805.6 tokens/s, Avg generation throughput: 61.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU INFO:     10.45.190.192:0 - "POST /v1/completions HINFO 01-04 14:27:44 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 89.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.5%, Prefix cache hit rate: 12.7%
+INFO:     10.46.INFO 01-04 14:27:47 [loggers.py:111] Engine 000: AINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:27:54 [loggers.py:111] Engine 000: Avg prompt throughput: 623.8 tokens/s, Avg generation INFO 01-04 14:27:57 [loggers.py:111] Engine 000: AINFO 01-04 14:27:59 [loggers.py:111] Engine 000: Avg prompt througINFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:28:04 [loggers.py:111] Engine 000: Avg prompt throughput: 420.2 tokens/s,INFO:     10.43.30INFO:     10.43.30.5:0 - "POST /v1/completionsINFO 01-04 14:28:09 [loggers.py:111] Engine 000: Avg prompt throughpINFO 01-04 14:28:14 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, WaitinINFO:     10.45.INFO 01-04 14:28:17 [loggers.py:111] Engine 000: AIINFO 01-04 14:28:19 [loggers.py:111] Engine 000: Avg prompt throuINFO 01-04 14:28:24 [loggers.py:111] Engine 000: Avg prompt throughput: 852.0 tokens/s, Avg generation INFO 01-04 14:28:27 [loggers.py:111] Engine 000: INFO 01-04 14:28:29 [loggers.py:111] Engine 000: Avg prompt throughINFO 01-04 14:28:34 [loggers.py:111] Engine 000: Avg prompt throughput: 901.2 tokens/s, Avg generation throughput: 75.6 tokens/s, Running: 2 reqs, WaitiINFO 01-04 14:28:39 [loggers.py:111] Engine 000: Avg INFO 01-04 14:28:45 [loggers.py:111] Engine 000: Avg prompt throughput: 1033.5 tokens/s, Avg generation throughput: 25.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache uINFO 01-04 14:28:47 [loggers.py:111] EngiINFO 01-04 14:28:55 [loggers.py:111] Engine 000: Avg prompt throughput: 620.4 tokens/s, Avg generation throughput: 48.3 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cachINFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:29:07 [loggers.py:111] Engine 000: Avg prompt throughput: 698.3 tokens/s, Avg generation throughput: 30.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.8%
+INFO 01-04 14:29:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.8%
+INFO 01-04 14:29:27 [loggers.py:111] Engine 000: Avg prompt throughput: 818.5 tokens/s, Avg generation throughput: 28.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 18.5%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:29:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 8.3 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 18.5%
+INFO 01-04 14:29:47 [loggers.py:111] Engine 000: Avg prINFO 01-04 14:29:49 [loggers.py:111] Engine 000: Avg prompt throughput: 742.5 tokens/s, Avg generation throughput: 35.9 tokens/s, Running: 1 reqs, Waiting: 0 reqINFO 01-04 14:29:57 [loggers.py:111] Engine 000: Avg prompINFO 01-04 14:29:59 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs,INFO 01-04 14:30:07 [loggers.py:111] Engine 000: Avg proINFO 01-04 14:30:09 [loggers.py:111] Engine 000: Avg prompt throughput: 758.9 tokens/s, Avg generation throughput: 84.0 tokens/s, Running: 2 reqs, Waiting: 0 reqINFO 01-04 14:30:17 [loggers.py:111] Engine 000: Avg prompINFO 01-04 14:30:19 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 90.4 tokens/s, Running: 2 reqs, Waiting: 0 reqINFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:30:27 [loggers.py:111] Engine 000: Avg prompt INFO 01-04 14:30:29 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 58.5 tokens/s, Running: 1 reqs, Waiting: 0 reqINFO 01-04 14:30:37 [loggers.py:111] Engine 000: Avg prompINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:30:39 [loggers.py:111] Engine 000: Avg prompt throughput: 573.3 tokens/s, Avg INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:30:47 [loggers.py:111] Engine 000: Avg prompt tINFO 01-04 14:30:49 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1 reqs, Waiting: 0 INFO 01-04 14:30:57 [loggers.py:111] Engine 000: Avg prompt throughput: 758.2 tokens/s, Avg generation throughput: 37.2 tokens/INFO 01-04 14:30:59 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg geINFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:31:07 [loggers.py:111] Engine 000: Avg promptINFO 01-04 14:31:09 [loggers.py:111] Engine 000: Avg prompt throughput: 778.2 tokens/s, Avg generation throughput: 27.9 tokens/s, Running: 1 reqs, Waiting: 0 INFO 01-04 14:31:17 [loggers.py:111] Engine 000: Avg prompt tINFO 01-04 14:31:19 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.9 tokens/s, Running: 1 reqs, Waiting: 0 rINFO 01-04 14:31:27 [loggers.py:111] Engine 000: Avg prompt INFO 01-04 14:31:29 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.8 tokens/s, Running: 1 reqs, Waiting: 0 rINFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:31:37 [loggers.py:111] Engine 000: Avg prompt INFO 01-04 14:31:39 [loggers.py:111] Engine 000: Avg prompt throughput: 962.2 tokens/s, Avg generation throughput: 37.5 tokens/s, Running: 1 reqs, Waiting: 0INFO 01-04 14:31:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, RINFO 01-04 14:31:49 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, AvINFO 01-04 14:31:57 [loggers.py:111] Engine 000: Avg prompt throughput: 500.0 tokens/s, Avg generation throughput: 30.0 tokens/sINFO 01-04 14:31:59 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg gINFO 01-04 14:32:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/INFO 01-04 14:32:09 [loggers.py:111] Engine 000: Avg prompt throughput: 937.4 tokens/s, Avg INFO:     10.43.30.5:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:32:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 36.5 tokens/s, INFO 01-04 14:32:19 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, AvgINFO 01-04 14:32:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, RINFO 01-04 14:32:29 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, AvINFO 01-04 14:32:47 [loggers.py:111] Engine 000: Avg prompt throughput: 708.9 tokens/s, Avg generation throughput: 33.6 tokens/sINFO 01-04 14:33:29 [loggers.py:111] Engine 000: Avg prompt throughput: 527.0 tokens/s, Avg generation throughput: 23.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 13.7%
+INFO 01-04 14:33:39 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 13.7%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:33:49 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 39.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 13.7%
+INFO 01-04 14:33:59 [loggers.pINFO 01-04 14:34:17 [loggers.py:111] Engine 000: Avg prompt throughput: 988.6 tokens/s, Avg generation throughput: 31.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.0%, Prefix cache hit rate: 15.5%
+INFO 01-04 14:34:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.2%, Prefix cache hit rate: 15.5%
+INFO 01-04 14:34:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 15.5%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:34:47 [loggers.py:111] Engine 000: Avg prompt throughput: 591.2 tokens/s, Avg generation throughput: 22.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 15.4%
+INFO 01-04 14:34:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage:INFO 01-04 14:35:09 [loggers.py:111] Engine 000: Avg prompt throughput: 581.7 tokens/s, Avg generation throughput: 14.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cINFO 01-04 14:35:14 [loggers.py:111] Engine 000INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:35:19 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation througINFO 01-04 14:35:24 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughpuINFO 01-04 14:35:29 [loggers.py:1INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:36:37 [loggers.py:111] Engine 000: Avg prompt throughput: 601.2 tokens/s, Avg generation throughput: 44.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 15.3%
+INFO 01-04 14:36:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 15.3%
+INFO 01-04 14:38:27 [loggers.py:111] Engine 000: Avg prompt throughput: 630.3 tokens/s, Avg generation throughput: 17.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 15.1%
+INFO 01-04 14:38:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 15.1%
+INFO 01-04 14:38:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 15.1%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:38:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 1.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 15.1%
+INFO 01-04 14:39:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 15.1%
+INFO 01-04 14:39:57 [loggers.py:111] Engine 000: Avg prompt throughput: 583.0 tokens/s, Avg generation throughput: 24.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 15.0%
+INFO 01-04 14:40:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 15.0%
+INFO:     10.46.17.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:40:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 43.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 15.0%
+INFO 01-04 14:40:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 15.0%
+INFO 01-04 14:41:57 [loggers.py:111] Engine 000: Avg prompt throughput: 609.1 tokens/s, Avg generation throughput: 45.4 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 14.8%
+INFO 01-04 14:42:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 14.8%
+INFO:     10.43.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:42:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 1.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.8%
+INFO 01-04 14:42:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.8%
+INFO 01-04 14:42:47 [loggers.py:111] Engine 000: Avg prompt throughput: 720.5 tokens/s, Avg generation throughput: 32.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 14.7%
+INFO 01-04 14:42:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 46.9 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 14.7%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:43:07 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 21.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.7%
+INFO 01-04 14INFO 01-04 14:43:25 [loggers.py:111] Engine 000: Avg prompt throughput: 1802.9 tokens/s, Avg generation throughput: 20.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.2%, Prefix cacheINFO 01-04 14:47:27 [loggers.py:111] Engine 000: Avg prompt throughput: 773.2 tokens/s, Avg generation throughput: 38.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cache hit rate: 15.4%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:47:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 22.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 15.4%
+INFO 01-04 14:47:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 15.4%
+INFO 01-04 15:03:07 [loggers.py:111] Engine 000: Avg prompt throughput: 618.6 tokens/s, Avg generation throughput: 25.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 15.3%
+INFO 01-04 15:03:17 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 15.3%
+INFO 01-04 15:03:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 15.3%
+INFO 01-04 15:03:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 15.3%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 15:03:47 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 37.9 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 15.3%
+INFO 01-04 15:03:57 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 15.3%
+.30.3:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:47:37 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 20.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usageINFO 01-04 14:48:45 [loggers.py:111] Engine 000: Avg prompt throughput: 814.9 tokens/s, Avg generation throughput: 33.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 14.9%
+INFO 01-04 14:48:55 [loggers.py:11INFO 01-04 14:54:17 [loggers.py:111] Engine 000: Avg prompt throughput: 420.7 tokens/s, Avg generation throughput: 10.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.7%, Prefix cache hit rate: 19.3%
+INFO 01-04 14:54:27 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, RunINFO 01-04 14:58:09 [loggers.py:111] Engine 000: Avg prompt throughput: 454.0 tokens/s, Avg generation throughput: 16.0 tokens/s, Running: 1 reqs, Waiting: INFO 01-04 14:59:06 [loggers.py:111] Engine 000: Avg prompt throughput: 573.5 tokens/s, Avg generation throughput: 32.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 16.5%
+INFO 01-04 14:59:16 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 16.5%
+INFO 01-04 14:59:26 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 16.5%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:59:36 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 19.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.5%
+INFO 01-04 14:59:46 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.5%
+.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 13.5%
+INFO:     10.46.50.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:50:44 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 10.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 13.5%
+INFO 01-04 14:50:54 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 13.5%
+15.1 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 15.0%
+INFO 01-04 14:48:04 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 15.0%
+INFO 01-04 14:59:34 [loggers.py:111] Engine 000: Avg prompt throughput: 579.9 tokens/s, Avg generation throughput: 14.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 14.8%
+INFO 01-04 14:59:44 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 14.8%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:59:54 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 26.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.8%
+INFO 01-04 15:00:04 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.8%
+y:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 31.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.3%
+INFO 01-04 14:47:11 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.3%
+INFO 01-04 14:48:11 [loggers.py:111] Engine 000: Avg prompt throughput: 797.4 tokens/s, Avg generation throughput: 21.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cache hit rate: 14.1%
+INFO 01-04 14:48:21 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.1 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 14.1%
+INFO:     10.45.190.192:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:48:31 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 13.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.1%
+INFO 01-04 14:48:41 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.1%
+INFO 01-04 14:57:21 [loggers.py:111] Engine 000: Avg prompt throughput: 554.8 tokens/s, Avg generation throughput: 2.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 14.5%
+INFO 01-04 14:57:31 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 47.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 14.5%
+INFO:     10.43.30.4:0 - "POST /v1/completions HTTP/1.1" 200 OK
+INFO 01-04 14:57:41 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 44.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.5%
+INFO 01-04 14:57:51 [loggers.py:111] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 14.5%

hf_ip/vllm_instances.txt

@@ -0,0 +1,232 @@
+127.0.0.1:8001
+127.0.0.1:8002
+127.0.0.1:8003
+127.0.0.1:8004
+127.0.0.1:8005
+127.0.0.1:8006
+127.0.0.1:8007
+127.0.0.1:8008
+127.0.0.1:8001
+127.0.0.1:8002
+127.0.0.1:8003
+127.0.0.1:8004
+127.0.0.1:8005
+127.0.0.1:8006
+127.0.0.1:8007
+127.0.0.1:8008
+127.0.0.1:8001
+127.0.0.1:8002
+127.0.0.1:8003
+127.0.0.1:8004
+127.0.0.1:8005
+127.0.0.1:8006
+127.0.0.1:8007
+127.0.0.1:8008
+127.0.0.1:8001
+127.0.0.1:8002
+127.0.0.1:8001
+127.0.0.1:8003
+127.0.0.1:8002
+127.0.0.1:8004
+127.0.0.1:8003
+127.0.0.1:8005
+127.0.0.1:8004
+127.0.0.1:8001
+127.0.0.1:8001
+127.0.0.1:8006
+127.0.0.1:8005
+127.0.0.1:8002
+127.0.0.1:8001
+127.0.0.1:8002
+127.0.0.1:8007
+127.0.0.1:8001
+127.0.0.1:8006
+127.0.0.1:8003
+127.0.0.1:8002
+127.0.0.1:8003
+127.0.0.1:8008
+127.0.0.1:8002
+127.0.0.1:8007
+127.0.0.1:8004
+127.0.0.1:8003
+127.0.0.1:8001
+127.0.0.1:8004
+127.0.0.1:8003
+127.0.0.1:8008
+127.0.0.1:8005
+127.0.0.1:8001
+127.0.0.1:8004
+127.0.0.1:8002
+127.0.0.1:8005
+127.0.0.1:8004
+127.0.0.1:8006
+127.0.0.1:8002
+127.0.0.1:8005
+127.0.0.1:8003
+127.0.0.1:8006
+127.0.0.1:8005
+127.0.0.1:8007
+127.0.0.1:8003
+127.0.0.1:8006
+127.0.0.1:8004
+127.0.0.1:8007
+127.0.0.1:8006
+127.0.0.1:8008
+127.0.0.1:8004
+127.0.0.1:8007
+127.0.0.1:8005
+127.0.0.1:8008
+127.0.0.1:8007
+127.0.0.1:8005
+127.0.0.1:8008
+127.0.0.1:8006
+127.0.0.1:8008
+127.0.0.1:8006
+127.0.0.1:8007
+127.0.0.1:8007
+127.0.0.1:8008
+127.0.0.1:8008
+127.0.0.1:8001
+127.0.0.1:8001
+127.0.0.1:8001
+127.0.0.1:8002
+127.0.0.1:8002
+127.0.0.1:8002
+127.0.0.1:8003
+127.0.0.1:8003
+127.0.0.1:8003
+127.0.0.1:8004
+127.0.0.1:8004
+127.0.0.1:8004
+127.0.0.1:8005
+127.0.0.1:8005
+127.0.0.1:8005
+127.0.0.1:8006
+127.0.0.1:8006
+127.0.0.1:8006
+127.0.0.1:8007
+127.0.0.1:8007
+127.0.0.1:8007
+127.0.0.1:8008
+127.0.0.1:8008
+127.0.0.1:8008
+127.0.0.1:8001
+127.0.0.1:8002
+127.0.0.1:8003
+127.0.0.1:8004
+127.0.0.1:8005
+127.0.0.1:8006
+127.0.0.1:8001
+127.0.0.1:8007
+127.0.0.1:8002
+127.0.0.1:8008
+127.0.0.1:8003
+127.0.0.1:8004
+127.0.0.1:8001
+127.0.0.1:8005
+127.0.0.1:8002
+127.0.0.1:8006
+127.0.0.1:8001
+127.0.0.1:8003
+127.0.0.1:8007
+127.0.0.1:8002
+127.0.0.1:8001
+127.0.0.1:8004
+127.0.0.1:8008
+127.0.0.1:8003
+127.0.0.1:8002
+127.0.0.1:8005
+127.0.0.1:8004
+127.0.0.1:8003
+127.0.0.1:8006
+127.0.0.1:8005
+127.0.0.1:8004
+127.0.0.1:8007
+127.0.0.1:8006
+127.0.0.1:8005
+127.0.0.1:8008
+127.0.0.1:8007
+127.0.0.1:8006
+127.0.0.1:8008
+127.0.0.1:8007
+127.0.0.1:8008
+127.0.0.1:8001
+127.0.0.1:8002
+127.0.0.1:8003
+127.0.0.1:8004
+127.0.0.1:8005
+127.0.0.1:8006
+127.0.0.1:8007
+127.0.0.1:8008
+127.0.0.1:8001
+127.0.0.1:8002
+127.0.0.1:8003
+127.0.0.1:8004
+127.0.0.1:8005
+127.0.0.1:8006
+127.0.0.1:8007
+127.0.0.1:8008
+127.0.0.1:8001
+127.0.0.1:8002
+127.0.0.1:8003
+127.0.0.1:8004
+127.0.0.1:8005
+127.0.0.1:8006
+127.0.0.1:8007
+127.0.0.1:8008
+127.0.0.1:8001
+127.0.0.1:8001
+127.0.0.1:8002
+127.0.0.1:8002
+127.0.0.1:8003
+127.0.0.1:8003
+127.0.0.1:8004
+127.0.0.1:8004
+127.0.0.1:8005
+127.0.0.1:8005
+127.0.0.1:8006
+127.0.0.1:8006
+127.0.0.1:8007
+127.0.0.1:8007
+127.0.0.1:8008
+127.0.0.1:8008
+127.0.0.1:8001
+127.0.0.1:8001
+127.0.0.1:8002
+127.0.0.1:8002
+127.0.0.1:8003
+127.0.0.1:8001
+127.0.0.1:8003
+127.0.0.1:8004
+127.0.0.1:8002
+127.0.0.1:8004
+127.0.0.1:8005
+127.0.0.1:8003
+127.0.0.1:8005
+127.0.0.1:8006
+127.0.0.1:8004
+127.0.0.1:8006
+127.0.0.1:8007
+127.0.0.1:8005
+127.0.0.1:8007
+127.0.0.1:8008
+127.0.0.1:8006
+127.0.0.1:8008
+127.0.0.1:8007
+127.0.0.1:8008
+127.0.0.1:8001
+127.0.0.1:8002
+127.0.0.1:8001
+127.0.0.1:8003
+127.0.0.1:8002
+127.0.0.1:8004
+127.0.0.1:8003
+127.0.0.1:8005
+127.0.0.1:8004
+127.0.0.1:8006
+127.0.0.1:8005
+127.0.0.1:8007
+127.0.0.1:8006
+127.0.0.1:8008
+127.0.0.1:8007
+127.0.0.1:8008